├── requirements-in.txt ├── docs ├── authors.rst ├── history.rst ├── readme.rst ├── contributing.rst ├── installation.rst ├── index.rst ├── usage.rst ├── Makefile ├── make.bat └── conf.py ├── tests ├── __init__.py └── test_wg_gesucht.py ├── .gitattributes ├── HISTORY.rst ├── requirements.txt ├── tox.ini ├── setup.cfg ├── AUTHORS.rst ├── wg_gesucht ├── __init__.py ├── user_details.py ├── logger.py ├── create_results_folders.py ├── cli.py ├── _version.py └── crawler.py ├── .travis.yml ├── MANIFEST.in ├── .editorconfig ├── .gitignore ├── LICENSE ├── Makefile ├── README.rst ├── setup.py ├── CONTRIBUTING.rst └── versioneer.py /requirements-in.txt: -------------------------------------------------------------------------------- 1 | click 2 | -------------------------------------------------------------------------------- /docs/authors.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../AUTHORS.rst 2 | -------------------------------------------------------------------------------- /docs/history.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../HISTORY.rst 2 | -------------------------------------------------------------------------------- /docs/readme.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../README.rst 2 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | -------------------------------------------------------------------------------- /docs/contributing.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../CONTRIBUTING.rst 2 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | wg-gesucht-crawler-cli/_version.py export-subst 2 | -------------------------------------------------------------------------------- /HISTORY.rst: -------------------------------------------------------------------------------- 1 | .. :changelog: 2 | 3 | History 4 | ------- 5 | 6 | Pre-release 7 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | beautifulsoup4==4.6.0 2 | certifi==2018.4.16 3 | chardet==3.0.4 4 | click==6.7 5 | idna==2.7 6 | requests==2.21.0 7 | urllib3==1.23 8 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py33, py34, py35, py36, pypy 3 | 4 | [testenv] 5 | setenv = 6 | PYTHONPATH = {toxinidir}:{toxinidir}/wg_gesucht 7 | commands = python setup.py test 8 | deps = 9 | -r{toxinidir}/requirements.txt 10 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [versioneer] 2 | VCS = git 3 | style = pep440 4 | versionfile_source = wg_gesucht/_version.py 5 | versionfile_build = wg_gesucht/_version.py 6 | tag_prefix = 7 | parentdir_prefix = wg_gesucht- 8 | 9 | [wheel] 10 | universal = 1 11 | -------------------------------------------------------------------------------- /AUTHORS.rst: -------------------------------------------------------------------------------- 1 | ======= 2 | Credits 3 | ======= 4 | 5 | Maintainer 6 | ---------- 7 | 8 | * Grant Williams 9 | 10 | Contributors 11 | ------------ 12 | 13 | None yet. Why not be the first? See: CONTRIBUTING.rst 14 | -------------------------------------------------------------------------------- /wg_gesucht/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import print_function 3 | from ._version import get_versions 4 | 5 | __author__ = 'Grant Williams' 6 | __email__ = 'grant.williams2986@gmail.com' 7 | __version__ = get_versions()['version'] 8 | del get_versions 9 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "3.3" 4 | - "3.4" 5 | - "3.5" 6 | - "3.6" 7 | - "pypy" 8 | # command to install dependencies 9 | install: 10 | - "pip install ." 11 | - "pip install -r requirements.txt" 12 | # command to run tests 13 | script: python setup.py test 14 | -------------------------------------------------------------------------------- /docs/installation.rst: -------------------------------------------------------------------------------- 1 | ============ 2 | Installation 3 | ============ 4 | 5 | At the command line:: 6 | 7 | $ pip install wg-gesucht-crawler-cli 8 | 9 | Or, if you have virtualenvwrapper installed:: 10 | 11 | $ mkvirtualenv wg-gesucht-crawler-cli 12 | $ pip install wg-gesucht-crawler-cli 13 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include AUTHORS.rst 2 | include CONTRIBUTING.rst 3 | include HISTORY.rst 4 | include LICENSE 5 | include README.rst 6 | 7 | recursive-include tests * 8 | recursive-exclude * __pycache__ 9 | recursive-exclude * *.py[co] 10 | 11 | recursive-include docs *.rst conf.py Makefile make.bat 12 | 13 | include versioneer.py 14 | include wg_gesucht/_version.py 15 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | indent_style = space 7 | indent_size = 4 8 | trim_trailing_whitespace = true 9 | insert_final_newline = true 10 | charset = utf-8 11 | end_of_line = lf 12 | 13 | [*.bat] 14 | indent_style = tab 15 | end_of_line = crlf 16 | 17 | [LICENSE] 18 | insert_final_newline = false 19 | 20 | [Makefile] 21 | indent_style = tab 22 | -------------------------------------------------------------------------------- /tests/test_wg_gesucht.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | test_wg_gesucht 6 | ---------------------------------- 7 | 8 | Tests for `wg_gesucht` module. 9 | """ 10 | 11 | import unittest 12 | 13 | import wg_gesucht 14 | 15 | 16 | class Testwg_gesucht(unittest.TestCase): 17 | 18 | def setUp(self): 19 | pass 20 | 21 | def test_something(self): 22 | assert(wg_gesucht.__version__) 23 | 24 | def tearDown(self): 25 | pass 26 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. wg_gesucht documentation master file, created by 2 | sphinx-quickstart on Tue Jul 9 22:26:36 2013. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to WG Gesucht Crawler CLI's documentation! 7 | ================================================== 8 | 9 | Contents: 10 | 11 | .. toctree:: 12 | :maxdepth: 2 13 | 14 | readme 15 | installation 16 | usage 17 | contributing 18 | authors 19 | history 20 | 21 | Indices and tables 22 | ================== 23 | 24 | * :ref:`genindex` 25 | * :ref:`modindex` 26 | * :ref:`search` 27 | 28 | -------------------------------------------------------------------------------- /wg_gesucht/user_details.py: -------------------------------------------------------------------------------- 1 | import json 2 | import getpass 3 | 4 | 5 | def save_details(file, login_info): 6 | with open(file, 'w', encoding='utf-8') as save: 7 | json.dump(login_info, indent=4, sort_keys=True, fp=save) 8 | 9 | 10 | def change_email(): 11 | email = input('Email address used on wg-gesucht.de? ') 12 | return email 13 | 14 | 15 | def change_password(): 16 | password = getpass.getpass('Password used for wg-gesucht.de? ') 17 | return password 18 | 19 | 20 | def change_phone(): 21 | phone = input('What is your phone number? (optional) ') 22 | return phone 23 | 24 | 25 | def change_all(): 26 | return { 27 | 'email': change_email(), 28 | 'password': change_password(), 29 | 'phone': change_phone() 30 | } 31 | -------------------------------------------------------------------------------- /docs/usage.rst: -------------------------------------------------------------------------------- 1 | ======== 2 | Usage 3 | ======== 4 | 5 | To use WG Gesucht Crawler in a project:: 6 | 7 | from wg_gesucht.crawler import WgGesuchtCrawler 8 | 9 | crawler = WgGesuchtCrawler(login_info, ad_links_folder, offline_ad_folder, logs_folder) 10 | crawler.sign_in() 11 | crawler.search() 12 | 13 | 14 | Paramaters 15 | ---------- 16 | 17 | *login_info* 18 | """""""""""" 19 | dict: containing wg-gesucht login details 20 | 21 | keys: 'email', 'password', 'phone'*(optional)* 22 | 23 | *ad_links_folder* 24 | """"""""""""""""" 25 | path to folder where a 'csv' file will be kept with previously applied for ads 26 | 27 | *offline_ad_folder* 28 | """"""""""""""""""" 29 | path to folder where offline ads will be saved 30 | 31 | *logs_folder* 32 | """"""""""""" 33 | path to folder where the log files will be kept 34 | -------------------------------------------------------------------------------- /wg_gesucht/logger.py: -------------------------------------------------------------------------------- 1 | import os 2 | import logging 3 | 4 | 5 | def get_logger(name, folder, level=logging.INFO): 6 | formatter = logging.Formatter('%(asctime)s::%(name)s::%(levelname)s::%(message)s', datefmt='%Y-%m-%d %H:%M:%S') 7 | logger = logging.getLogger(name) 8 | logger.setLevel(level) 9 | 10 | info_file_handler = logging.FileHandler(os.path.join(folder, 'info.log')) 11 | info_file_handler.setFormatter(formatter) 12 | info_file_handler.setLevel(logging.INFO) 13 | 14 | error_file_handler = logging.FileHandler(os.path.join(folder, 'error.log')) 15 | error_file_handler.setFormatter(formatter) 16 | error_file_handler.setLevel(logging.ERROR) 17 | 18 | stream_handler = logging.StreamHandler() 19 | stream_handler.setLevel(logging.WARNING) 20 | 21 | logger.addHandler(info_file_handler) 22 | logger.addHandler(error_file_handler) 23 | logger.addHandler(stream_handler) 24 | 25 | return logger 26 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | venv/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *,cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | 56 | # Sphinx documentation 57 | docs/_build/ 58 | 59 | # PyBuilder 60 | target/ 61 | 62 | #Ipython Notebook 63 | .ipynb_checkpoints 64 | 65 | .data_files 66 | .vscode 67 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2018 Grant Williams 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /wg_gesucht/create_results_folders.py: -------------------------------------------------------------------------------- 1 | import os 2 | import csv 3 | from .logger import get_logger 4 | 5 | 6 | def create_folders(dirname, logs_folder): 7 | if not os.path.exists(os.path.join(dirname, "WG Ad Links")): 8 | os.makedirs(os.path.join(dirname, "WG Ad Links")) 9 | ad_links_file_location = os.path.join(dirname, "WG Ad Links") 10 | 11 | if not os.path.exists(os.path.join(dirname, "Offline Ad Links")): 12 | os.makedirs(os.path.join(dirname, "Offline Ad Links")) 13 | offline_file_location = os.path.join(dirname, "Offline Ad Links") 14 | 15 | if not os.path.exists(os.path.join(ad_links_file_location, "WG Ad Links.csv")): 16 | with open(os.path.join(ad_links_file_location, "WG Ad Links.csv"), "w", newline="", encoding='utf-8') as write: 17 | csv_write = csv.writer(write) 18 | csv_write.writerow(['WG Links', 'Name', 'Ad Title']) 19 | 20 | logger = get_logger( 21 | __name__, folder=os.path.join(logs_folder)) 22 | logger.warning("\n\nTwo folders have been created, %s'%s'%s contains " 23 | "a 'csv' file which contains the URL's of the apartment ads the " 24 | "program has messaged for you, and \n%s'%s'%s " 25 | "contains a the actual ads, which can be viewed offline, in " 26 | "case the submitter has removed the ad before you get chance to " 27 | "look at it.\n\n", 28 | '\033[92m', ad_links_file_location, '\033[0m', 29 | '\033[92m', offline_file_location, '\033[0m') 30 | 31 | return 32 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: clean-pyc clean-build docs clean 2 | 3 | help: 4 | @echo "clean - remove all build, test, coverage and Python artifacts" 5 | @echo "clean-build - remove build artifacts" 6 | @echo "clean-pyc - remove Python file artifacts" 7 | @echo "clean-test - remove test and coverage artifacts" 8 | @echo "lint - check style with flake8" 9 | @echo "test - run tests quickly with the default Python" 10 | @echo "test-all - run tests on every Python version with tox" 11 | @echo "coverage - check code coverage quickly with the default Python" 12 | @echo "docs - generate Sphinx HTML documentation, including API docs" 13 | @echo "release - package and upload a release" 14 | @echo "dist - package" 15 | @echo "install - install the package to the active Python's site-packages" 16 | 17 | clean: clean-build clean-pyc clean-test 18 | 19 | clean-build: 20 | rm -fr build/ 21 | rm -fr dist/ 22 | rm -fr .eggs/ 23 | find . -name '*.egg-info' -exec rm -fr {} + 24 | find . -name '*.egg' -exec rm -f {} + 25 | 26 | clean-pyc: 27 | find . -name '*.pyc' -exec rm -f {} + 28 | find . -name '*.pyo' -exec rm -f {} + 29 | find . -name '*~' -exec rm -f {} + 30 | find . -name '__pycache__' -exec rm -fr {} + 31 | 32 | clean-test: 33 | rm -fr .tox/ 34 | rm -f .coverage 35 | rm -fr htmlcov/ 36 | 37 | lint: 38 | flake8 wg_gesucht tests 39 | 40 | test: 41 | python setup.py test 42 | 43 | test-all: 44 | tox 45 | 46 | coverage: 47 | coverage run --source wg_gesucht setup.py test 48 | coverage report -m 49 | coverage html 50 | open htmlcov/index.html 51 | 52 | docs: 53 | rm -f docs/wg_gesucht.rst 54 | rm -f docs/modules.rst 55 | sphinx-apidoc -o docs/ wg_gesucht 56 | $(MAKE) -C docs clean 57 | $(MAKE) -C docs html 58 | open docs/_build/html/index.html 59 | 60 | release: clean 61 | python setup.py sdist upload 62 | python setup.py bdist_wheel upload 63 | 64 | dist: clean 65 | python setup.py sdist 66 | python setup.py bdist_wheel 67 | ls -l dist 68 | 69 | install: clean 70 | python setup.py install 71 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | =============================== 2 | WG Gesucht Crawler 3 | =============================== 4 | 5 | .. image:: https://img.shields.io/travis/grantwilliams/wg-gesucht-crawler-cli.svg 6 | :target: https://travis-ci.org/grantwilliams/wg-gesucht-crawler-cli 7 | 8 | .. image:: https://img.shields.io/pypi/v/wg-gesucht-crawler-cli.svg 9 | :target: https://pypi.python.org/pypi/wg-gesucht-crawler-cli 10 | 11 | .. image:: https://readthedocs.org/projects/wg-gesucht-crawler-cli/badge/?version=latest 12 | :target: https://wg-gesucht-crawler-cli.readthedocs.io/en/latest/?badge=latest 13 | :alt: Documentation Status 14 | 15 | Python web crawler / scraper for WG-Gesucht. Crawls the WG-Gesucht site for new apartment listings and send a message to the poster, based off your saved filters and saved text template. 16 | 17 | Installation 18 | ------------ 19 | :: 20 | 21 | $ pip install wg-gesucht-crawler-cli 22 | 23 | Or, if you have virtualenvwrapper installed:: 24 | 25 | $ mkvirtualenv wg-gesucht-crawler-cli 26 | $ pip install wg-gesucht-crawler-cli 27 | 28 | Use 29 | --- 30 | Can be run directly from the command line with:: 31 | 32 | $ wg-gesucht-crawler-cli --help 33 | 34 | Or if you want to use it in your own project: 35 | 36 | .. code-block:: python 37 | 38 | from wg_gesucht.crawler import WgGesuchtCrawler 39 | 40 | Just make sure to save at least one search filter as well as a template text on your wg-gesucht account. 41 | 42 | * Free software: MIT license 43 | * Documentation: https://wg-gesucht-crawler-cli.readthedocs.org. 44 | 45 | Features 46 | -------- 47 | 48 | * Searches https://wg-gesucht.de for new WG ads based off your saved filters 49 | * Sends your saved template message and applies to all matching listings 50 | * Reruns every ~5 minutes 51 | * Run on a RPi or free EC2 micro instance 24/7 to always be one of the first to apply for new listings 52 | 53 | 54 | 55 | **Getting Caught with reCAPTCHA** 56 | 57 | I've made the crawler sleep for 5-8 seconds between each request to try and avoid their reCAPTCHA, but if the crawler does get caught, you can sign into your wg-gesucht account manually through the browser and solve the reCAPTCHA, then start the crawler again. 58 | If it continues to happen, you can also increase the sleep time in the :code:`get_page()` function in :code:`wg_gesucht.py` 59 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """A setuptools based setup module for WG-Gesucht-Crawler-CLI""" 2 | #!/usr/bin/env python 3 | # -*- coding: utf-8 -*- 4 | 5 | from codecs import open 6 | from os import path 7 | from setuptools import setup, find_packages 8 | 9 | import versioneer 10 | 11 | here = path.abspath(path.dirname(__file__)) 12 | 13 | with open(path.join(here, 'README.rst'), encoding='utf-8') as readme_file: 14 | readme = readme_file.read() 15 | 16 | with open(path.join(here, 'HISTORY.rst'), encoding='utf-8') as history_file: 17 | history = history_file.read().replace('.. :changelog:', '') 18 | 19 | requirements = [ 20 | 'beautifulsoup4==4.6.0', 21 | 'certifi==2018.4.16', 22 | 'chardet==3.0.4', 23 | 'click==6.7', 24 | 'idna==2.7', 25 | 'requests==2.21.0', 26 | 'urllib3==1.23', 27 | ] 28 | 29 | test_requirements = [] 30 | 31 | setup( 32 | name='wg-gesucht-crawler-cli', 33 | version=versioneer.get_version(), 34 | cmdclass=versioneer.get_cmdclass(), 35 | description="Python web crawler / scraper for WG-Gesucht. Crawls the WG-Gesucht site for new apartment listings and send a message to the poster, based off your saved filters and saved text template.", 36 | long_description=readme + '\n\n' + history, 37 | author="Grant Williams", 38 | author_email='grant.williams2986@gmail.com', 39 | url='https://github.com/grantwilliams/wg-gesucht-crawler-cli', 40 | packages=find_packages(exclude=['contrib', 'docs', 'tests']), 41 | entry_points={ 42 | 'console_scripts': [ 43 | 'wg-gesucht-crawler-cli=wg_gesucht.cli:cli', 44 | ], 45 | }, 46 | include_package_data=True, 47 | install_requires=requirements, 48 | license="MIT", 49 | classifiers=[ 50 | 'Development Status :: 5 - Production/Stable', 51 | 'Environment :: Console', 52 | 'Intended Audience :: Developers', 53 | 'License :: OSI Approved :: MIT License', 54 | 'Natural Language :: English', 55 | 'Operating System :: OS Independent', 56 | 'Programming Language :: Python :: 3', 57 | 'Programming Language :: Python :: 3.3', 58 | 'Programming Language :: Python :: 3.4', 59 | 'Programming Language :: Python :: 3.5', 60 | 'Programming Language :: Python :: 3.6', 61 | 'Programming Language :: Python :: 3 :: Only', 62 | ], 63 | test_suite='tests', 64 | tests_require=test_requirements, 65 | ) 66 | -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | ============ 2 | Contributing 3 | ============ 4 | 5 | Contributions are welcome, and they are greatly appreciated! Every 6 | little bit helps, and credit will always be given. 7 | 8 | You can contribute in many ways: 9 | 10 | Types of Contributions 11 | ---------------------- 12 | 13 | Report Bugs 14 | ~~~~~~~~~~~ 15 | 16 | Report bugs at https://github.com/grantwilliams/wg-gesucht-crawler-cli/issues. 17 | 18 | If you are reporting a bug, please include: 19 | 20 | * Any details about your local setup that might be helpful in troubleshooting. 21 | * Detailed steps to reproduce the bug. 22 | 23 | Fix Bugs 24 | ~~~~~~~~ 25 | 26 | Look through the GitHub issues for bugs. Anything tagged with "bug" 27 | is open to whoever wants to implement it. 28 | 29 | Implement Features 30 | ~~~~~~~~~~~~~~~~~~ 31 | 32 | Look through the GitHub issues for features. Anything tagged with "feature" 33 | is open to whoever wants to implement it. 34 | 35 | Write Documentation 36 | ~~~~~~~~~~~~~~~~~~~ 37 | 38 | WG Gesucht Crawler CLI could always use more documentation, whether 39 | as part of the official WG Gesucht Crawler CLI docs, in docstrings, 40 | or even on the web in blog posts, articles, and such. 41 | 42 | Submit Feedback 43 | ~~~~~~~~~~~~~~~ 44 | 45 | The best way to send feedback is to file an issue at https://github.com/grantwilliams/wg-gesucht-crawler-cli/issues. 46 | 47 | If you are proposing a feature: 48 | 49 | * Explain in detail how it would work. 50 | * Keep the scope as narrow as possible, to make it easier to implement. 51 | * Remember that this is a volunteer-driven project, and that contributions 52 | are welcome :) 53 | 54 | Get Started! 55 | ------------ 56 | 57 | Ready to contribute? Here's how to set up `WG-Gesucht-Crawler-CLI` for local development. 58 | 59 | 1. Fork the `wg-gesucht-crawler-cli` repo on GitHub. 60 | 2. Clone your fork locally:: 61 | 62 | $ git clone git@github.com:your_name_here/wg-gesucht-crawler-cli.git 63 | 64 | 3. Install your local copy into a virtualenv. Assuming you have virtualenvwrapper installed, this is how you set up your fork for local development:: 65 | 66 | $ mkvirtualenv wg-gesucht-crawler-cli 67 | $ cd wg-gesucht-crawler-cli/ 68 | $ python setup.py develop 69 | 70 | 4. Create a branch for local development:: 71 | 72 | $ git checkout -b name-of-your-bugfix-or-feature 73 | 74 | Now you can make your changes locally. 75 | 76 | 5. When you're done making changes, check that your changes pass flake8 and the tests, including testing other Python versions with tox:: 77 | 78 | $ flake8 wg_gesucht tests 79 | $ python setup.py test 80 | $ tox 81 | 82 | To get flake8 and tox, just pip install them into your virtualenv. 83 | 84 | 6. Commit your changes and push your branch to GitHub:: 85 | 86 | $ git add . 87 | $ git commit -m "Your detailed description of your changes." 88 | $ git push origin name-of-your-bugfix-or-feature 89 | 90 | 7. Submit a pull request through the GitHub website. 91 | 92 | Pull Request Guidelines 93 | ----------------------- 94 | 95 | Before you submit a pull request, check that it meets these guidelines: 96 | 97 | 1. The pull request should include tests. 98 | 2. If the pull request adds functionality, the docs should be updated. Put 99 | your new functionality into a function with a docstring, and add the 100 | feature to the list in README.rst. 101 | 3. The pull request should work for Python 3.3, 3.4, 3.5, 3.6 and for PyPy. Check 102 | https://travis-ci.org/grantwilliams/wg-gesucht-crawler-cli/pull_requests 103 | and make sure that the tests pass for all supported Python versions. 104 | 105 | -------------------------------------------------------------------------------- /wg_gesucht/cli.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import json 4 | import atexit 5 | import click 6 | from .create_results_folders import create_folders 7 | from .logger import get_logger 8 | from . import user_details as user 9 | from .crawler import WgGesuchtCrawler 10 | 11 | 12 | @click.command() 13 | @click.option( 14 | "--filter-names", 15 | help='Names of the filters you want to use (comma separated ie: --filter-names="Filter 1,Filter2"', 16 | ) 17 | @click.option("--template", help="Name of the email template you want to use") 18 | @click.option("--share-email", is_flag=True, help="Share your email address with people you send messages to") 19 | @click.option("--change-email", is_flag=True, help="Change your saved email address") 20 | @click.option("--change-password", is_flag=True, help="Change your saved password") 21 | @click.option("--change-phone", is_flag=True, help="Change your saved phone number") 22 | @click.option("--change-all", is_flag=True, help="Change all you saved user details") 23 | @click.option( 24 | "--no-save", 25 | is_flag=True, 26 | help="The script won't save your wg-gesucht login details for future use", 27 | ) 28 | def cli( 29 | change_email, 30 | change_password, 31 | change_phone, 32 | change_all, 33 | no_save, 34 | template, 35 | filter_names, 36 | share_email 37 | ): 38 | """ 39 | -------------------------Wg-Gesucht crawler-------------------------\n 40 | Searches wg-gesucht.de for new room listings based off your saved filters. 41 | Gets your template text from your account and applys to any new ads that match.\n 42 | Run on a RPi or EC2 instance 24/7 to always be one of the first people to apply 43 | for a free room.\n 44 | Logs files in '/home/YOUR_NAME/Documents/WG Finder' 45 | """ 46 | home_path = "HOMEPATH" if sys.platform == "win32" else "HOME" 47 | dirname = os.path.join(os.environ[home_path], "Documents", "WG Finder") 48 | wg_ad_links = os.path.join(dirname, "WG Ad Links") 49 | offline_ad_links = os.path.join(dirname, "Offline Ad Links") 50 | logs_folder = os.path.join(dirname, "logs") 51 | user_folder = os.path.join(dirname, ".user") 52 | login_info_file = os.path.join(user_folder, ".login_info.json") 53 | 54 | if not os.path.exists(logs_folder): 55 | os.makedirs(os.path.join(dirname, "logs")) 56 | if not os.path.exists(user_folder): 57 | os.makedirs(os.path.join(dirname, ".user")) 58 | 59 | if not os.path.exists(wg_ad_links) or not os.path.exists(offline_ad_links): 60 | create_folders(dirname, logs_folder) 61 | 62 | logger = get_logger(__name__, folder=logs_folder) 63 | 64 | @atexit.register 65 | def exiting(): 66 | logger.warning("Stopped running!") 67 | 68 | login_info = dict() 69 | if os.path.isfile(login_info_file): 70 | with open(login_info_file) as file: 71 | login_info = json.load(file) 72 | 73 | login_info_changed = False 74 | if ( 75 | change_all 76 | or not login_info.get("email", "") 77 | or not login_info.get("password", "") 78 | ): 79 | login_info = user.change_all() 80 | login_info_changed = True 81 | 82 | if change_email: 83 | login_info["email"] = user.change_email() 84 | login_info_changed = True 85 | 86 | if change_password: 87 | login_info["password"] = user.change_password() 88 | login_info_changed = True 89 | 90 | if change_phone: 91 | login_info["phone"] = user.change_phone() 92 | login_info_changed = True 93 | 94 | if login_info_changed and not no_save: 95 | user.save_details(login_info_file, login_info) 96 | logger.info("User login details saved to file") 97 | 98 | if template: 99 | template = template.lower() 100 | 101 | if filter_names: 102 | filter_names = [filter.strip().lower() for filter in filter_names.split(",")] 103 | 104 | wg_gesucht = WgGesuchtCrawler( 105 | login_info, wg_ad_links, offline_ad_links, logs_folder, template, filter_names, share_email 106 | ) 107 | wg_gesucht.sign_in() 108 | logger.warning("Running until canceled, check info.log for details...") 109 | wg_gesucht.search() 110 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " dirhtml to make HTML files named index.html in directories" 28 | @echo " singlehtml to make a single large HTML file" 29 | @echo " pickle to make pickle files" 30 | @echo " json to make JSON files" 31 | @echo " htmlhelp to make HTML files and a HTML help project" 32 | @echo " qthelp to make HTML files and a qthelp project" 33 | @echo " devhelp to make HTML files and a Devhelp project" 34 | @echo " epub to make an epub" 35 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 36 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 37 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 38 | @echo " text to make text files" 39 | @echo " man to make manual pages" 40 | @echo " texinfo to make Texinfo files" 41 | @echo " info to make Texinfo files and run them through makeinfo" 42 | @echo " gettext to make PO message catalogs" 43 | @echo " changes to make an overview of all changed/added/deprecated items" 44 | @echo " xml to make Docutils-native XML files" 45 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 46 | @echo " linkcheck to check all external links for integrity" 47 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 48 | 49 | clean: 50 | rm -rf $(BUILDDIR)/* 51 | 52 | html: 53 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 54 | @echo 55 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 56 | 57 | dirhtml: 58 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 59 | @echo 60 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 61 | 62 | singlehtml: 63 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 64 | @echo 65 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 66 | 67 | pickle: 68 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 69 | @echo 70 | @echo "Build finished; now you can process the pickle files." 71 | 72 | json: 73 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 74 | @echo 75 | @echo "Build finished; now you can process the JSON files." 76 | 77 | htmlhelp: 78 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 79 | @echo 80 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 81 | ".hhp project file in $(BUILDDIR)/htmlhelp." 82 | 83 | qthelp: 84 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 85 | @echo 86 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 87 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 88 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/wg_gesucht.qhcp" 89 | @echo "To view the help file:" 90 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/wg_gesucht.qhc" 91 | 92 | devhelp: 93 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 94 | @echo 95 | @echo "Build finished." 96 | @echo "To view the help file:" 97 | @echo "# mkdir -p $$HOME/.local/share/devhelp/wg_gesucht" 98 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/wg_gesucht" 99 | @echo "# devhelp" 100 | 101 | epub: 102 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 103 | @echo 104 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 105 | 106 | latex: 107 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 108 | @echo 109 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 110 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 111 | "(use \`make latexpdf' here to do that automatically)." 112 | 113 | latexpdf: 114 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 115 | @echo "Running LaTeX files through pdflatex..." 116 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 117 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 118 | 119 | latexpdfja: 120 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 121 | @echo "Running LaTeX files through platex and dvipdfmx..." 122 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 123 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 124 | 125 | text: 126 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 127 | @echo 128 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 129 | 130 | man: 131 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 132 | @echo 133 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 134 | 135 | texinfo: 136 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 137 | @echo 138 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 139 | @echo "Run \`make' in that directory to run these through makeinfo" \ 140 | "(use \`make info' here to do that automatically)." 141 | 142 | info: 143 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 144 | @echo "Running Texinfo files through makeinfo..." 145 | make -C $(BUILDDIR)/texinfo info 146 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 147 | 148 | gettext: 149 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 150 | @echo 151 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 152 | 153 | changes: 154 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 155 | @echo 156 | @echo "The overview file is in $(BUILDDIR)/changes." 157 | 158 | linkcheck: 159 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 160 | @echo 161 | @echo "Link check complete; look for any errors in the above output " \ 162 | "or in $(BUILDDIR)/linkcheck/output.txt." 163 | 164 | doctest: 165 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 166 | @echo "Testing of doctests in the sources finished, look at the " \ 167 | "results in $(BUILDDIR)/doctest/output.txt." 168 | 169 | xml: 170 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 171 | @echo 172 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 173 | 174 | pseudoxml: 175 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 176 | @echo 177 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 178 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=_build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . 10 | set I18NSPHINXOPTS=%SPHINXOPTS% . 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. dirhtml to make HTML files named index.html in directories 23 | echo. singlehtml to make a single large HTML file 24 | echo. pickle to make pickle files 25 | echo. json to make JSON files 26 | echo. htmlhelp to make HTML files and a HTML help project 27 | echo. qthelp to make HTML files and a qthelp project 28 | echo. devhelp to make HTML files and a Devhelp project 29 | echo. epub to make an epub 30 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 31 | echo. text to make text files 32 | echo. man to make manual pages 33 | echo. texinfo to make Texinfo files 34 | echo. gettext to make PO message catalogs 35 | echo. changes to make an overview over all changed/added/deprecated items 36 | echo. xml to make Docutils-native XML files 37 | echo. pseudoxml to make pseudoxml-XML files for display purposes 38 | echo. linkcheck to check all external links for integrity 39 | echo. doctest to run all doctests embedded in the documentation if enabled 40 | goto end 41 | ) 42 | 43 | if "%1" == "clean" ( 44 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 45 | del /q /s %BUILDDIR%\* 46 | goto end 47 | ) 48 | 49 | 50 | %SPHINXBUILD% 2> nul 51 | if errorlevel 9009 ( 52 | echo. 53 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 54 | echo.installed, then set the SPHINXBUILD environment variable to point 55 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 56 | echo.may add the Sphinx directory to PATH. 57 | echo. 58 | echo.If you don't have Sphinx installed, grab it from 59 | echo.http://sphinx-doc.org/ 60 | exit /b 1 61 | ) 62 | 63 | if "%1" == "html" ( 64 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 65 | if errorlevel 1 exit /b 1 66 | echo. 67 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 68 | goto end 69 | ) 70 | 71 | if "%1" == "dirhtml" ( 72 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 73 | if errorlevel 1 exit /b 1 74 | echo. 75 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 76 | goto end 77 | ) 78 | 79 | if "%1" == "singlehtml" ( 80 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 81 | if errorlevel 1 exit /b 1 82 | echo. 83 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 84 | goto end 85 | ) 86 | 87 | if "%1" == "pickle" ( 88 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 89 | if errorlevel 1 exit /b 1 90 | echo. 91 | echo.Build finished; now you can process the pickle files. 92 | goto end 93 | ) 94 | 95 | if "%1" == "json" ( 96 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 97 | if errorlevel 1 exit /b 1 98 | echo. 99 | echo.Build finished; now you can process the JSON files. 100 | goto end 101 | ) 102 | 103 | if "%1" == "htmlhelp" ( 104 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 105 | if errorlevel 1 exit /b 1 106 | echo. 107 | echo.Build finished; now you can run HTML Help Workshop with the ^ 108 | .hhp project file in %BUILDDIR%/htmlhelp. 109 | goto end 110 | ) 111 | 112 | if "%1" == "qthelp" ( 113 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 114 | if errorlevel 1 exit /b 1 115 | echo. 116 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 117 | .qhcp project file in %BUILDDIR%/qthelp, like this: 118 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\wg_gesucht.qhcp 119 | echo.To view the help file: 120 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\wg_gesucht.ghc 121 | goto end 122 | ) 123 | 124 | if "%1" == "devhelp" ( 125 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 126 | if errorlevel 1 exit /b 1 127 | echo. 128 | echo.Build finished. 129 | goto end 130 | ) 131 | 132 | if "%1" == "epub" ( 133 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 134 | if errorlevel 1 exit /b 1 135 | echo. 136 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 137 | goto end 138 | ) 139 | 140 | if "%1" == "latex" ( 141 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 142 | if errorlevel 1 exit /b 1 143 | echo. 144 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 145 | goto end 146 | ) 147 | 148 | if "%1" == "latexpdf" ( 149 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 150 | cd %BUILDDIR%/latex 151 | make all-pdf 152 | cd %BUILDDIR%/.. 153 | echo. 154 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 155 | goto end 156 | ) 157 | 158 | if "%1" == "latexpdfja" ( 159 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 160 | cd %BUILDDIR%/latex 161 | make all-pdf-ja 162 | cd %BUILDDIR%/.. 163 | echo. 164 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 165 | goto end 166 | ) 167 | 168 | if "%1" == "text" ( 169 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 170 | if errorlevel 1 exit /b 1 171 | echo. 172 | echo.Build finished. The text files are in %BUILDDIR%/text. 173 | goto end 174 | ) 175 | 176 | if "%1" == "man" ( 177 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 178 | if errorlevel 1 exit /b 1 179 | echo. 180 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 181 | goto end 182 | ) 183 | 184 | if "%1" == "texinfo" ( 185 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 186 | if errorlevel 1 exit /b 1 187 | echo. 188 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 189 | goto end 190 | ) 191 | 192 | if "%1" == "gettext" ( 193 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 194 | if errorlevel 1 exit /b 1 195 | echo. 196 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 197 | goto end 198 | ) 199 | 200 | if "%1" == "changes" ( 201 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 202 | if errorlevel 1 exit /b 1 203 | echo. 204 | echo.The overview file is in %BUILDDIR%/changes. 205 | goto end 206 | ) 207 | 208 | if "%1" == "linkcheck" ( 209 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 210 | if errorlevel 1 exit /b 1 211 | echo. 212 | echo.Link check complete; look for any errors in the above output ^ 213 | or in %BUILDDIR%/linkcheck/output.txt. 214 | goto end 215 | ) 216 | 217 | if "%1" == "doctest" ( 218 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 219 | if errorlevel 1 exit /b 1 220 | echo. 221 | echo.Testing of doctests in the sources finished, look at the ^ 222 | results in %BUILDDIR%/doctest/output.txt. 223 | goto end 224 | ) 225 | 226 | if "%1" == "xml" ( 227 | %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml 228 | if errorlevel 1 exit /b 1 229 | echo. 230 | echo.Build finished. The XML files are in %BUILDDIR%/xml. 231 | goto end 232 | ) 233 | 234 | if "%1" == "pseudoxml" ( 235 | %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml 236 | if errorlevel 1 exit /b 1 237 | echo. 238 | echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. 239 | goto end 240 | ) 241 | 242 | :end 243 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # wg_gesucht documentation build configuration file, created by 5 | # sphinx-quickstart on Tue Jul 9 22:26:36 2013. 6 | # 7 | # This file is execfile()d with the current directory set to its 8 | # containing dir. 9 | # 10 | # Note that not all possible configuration values are present in this 11 | # autogenerated file. 12 | # 13 | # All configuration values have a default; values that are commented out 14 | # serve to show the default. 15 | 16 | import sys 17 | import os 18 | 19 | # If extensions (or modules to document with autodoc) are in another 20 | # directory, add these directories to sys.path here. If the directory is 21 | # relative to the documentation root, use os.path.abspath to make it 22 | # absolute, like shown here. 23 | #sys.path.insert(0, os.path.abspath('.')) 24 | 25 | # Get the project root dir, which is the parent dir of this 26 | cwd = os.getcwd() 27 | project_root = os.path.dirname(cwd) 28 | 29 | # Insert the project root dir as the first element in the PYTHONPATH. 30 | # This lets us ensure that the source package is imported, and that its 31 | # version is used. 32 | sys.path.insert(0, project_root) 33 | 34 | import wg_gesucht 35 | 36 | # -- General configuration --------------------------------------------- 37 | 38 | # If your documentation needs a minimal Sphinx version, state it here. 39 | #needs_sphinx = '1.0' 40 | 41 | # Add any Sphinx extension module names here, as strings. They can be 42 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 43 | extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode'] 44 | 45 | # Add any paths that contain templates here, relative to this directory. 46 | templates_path = ['_templates'] 47 | 48 | # The suffix of source filenames. 49 | source_suffix = '.rst' 50 | 51 | # The encoding of source files. 52 | #source_encoding = 'utf-8-sig' 53 | 54 | # The master toctree document. 55 | master_doc = 'index' 56 | 57 | # General information about the project. 58 | project = u'WG Gesucht Crawler CLI' 59 | copyright = u'2018, Grant Williams' 60 | 61 | # The version info for the project you're documenting, acts as replacement 62 | # for |version| and |release|, also used in various other places throughout 63 | # the built documents. 64 | # 65 | # The short X.Y version. 66 | version = wg_gesucht.__version__ 67 | # The full version, including alpha/beta/rc tags. 68 | release = wg_gesucht.__version__ 69 | 70 | # The language for content autogenerated by Sphinx. Refer to documentation 71 | # for a list of supported languages. 72 | #language = None 73 | 74 | # There are two options for replacing |today|: either, you set today to 75 | # some non-false value, then it is used: 76 | #today = '' 77 | # Else, today_fmt is used as the format for a strftime call. 78 | #today_fmt = '%B %d, %Y' 79 | 80 | # List of patterns, relative to source directory, that match files and 81 | # directories to ignore when looking for source files. 82 | exclude_patterns = ['_build'] 83 | 84 | # The reST default role (used for this markup: `text`) to use for all 85 | # documents. 86 | #default_role = None 87 | 88 | # If true, '()' will be appended to :func: etc. cross-reference text. 89 | #add_function_parentheses = True 90 | 91 | # If true, the current module name will be prepended to all description 92 | # unit titles (such as .. function::). 93 | #add_module_names = True 94 | 95 | # If true, sectionauthor and moduleauthor directives will be shown in the 96 | # output. They are ignored by default. 97 | #show_authors = False 98 | 99 | # The name of the Pygments (syntax highlighting) style to use. 100 | pygments_style = 'sphinx' 101 | 102 | # A list of ignored prefixes for module index sorting. 103 | #modindex_common_prefix = [] 104 | 105 | # If true, keep warnings as "system message" paragraphs in the built 106 | # documents. 107 | #keep_warnings = False 108 | 109 | 110 | # -- Options for HTML output ------------------------------------------- 111 | 112 | # The theme to use for HTML and HTML Help pages. See the documentation for 113 | # a list of builtin themes. 114 | html_theme = 'default' 115 | 116 | # Theme options are theme-specific and customize the look and feel of a 117 | # theme further. For a list of options available for each theme, see the 118 | # documentation. 119 | #html_theme_options = {} 120 | 121 | # Add any paths that contain custom themes here, relative to this directory. 122 | #html_theme_path = [] 123 | 124 | # The name for this set of Sphinx documents. If None, it defaults to 125 | # " v documentation". 126 | #html_title = None 127 | 128 | # A shorter title for the navigation bar. Default is the same as 129 | # html_title. 130 | #html_short_title = None 131 | 132 | # The name of an image file (relative to this directory) to place at the 133 | # top of the sidebar. 134 | #html_logo = None 135 | 136 | # The name of an image file (within the static path) to use as favicon 137 | # of the docs. This file should be a Windows icon file (.ico) being 138 | # 16x16 or 32x32 pixels large. 139 | #html_favicon = None 140 | 141 | # Add any paths that contain custom static files (such as style sheets) 142 | # here, relative to this directory. They are copied after the builtin 143 | # static files, so a file named "default.css" will overwrite the builtin 144 | # "default.css". 145 | html_static_path = ['_static'] 146 | 147 | # If not '', a 'Last updated on:' timestamp is inserted at every page 148 | # bottom, using the given strftime format. 149 | #html_last_updated_fmt = '%b %d, %Y' 150 | 151 | # If true, SmartyPants will be used to convert quotes and dashes to 152 | # typographically correct entities. 153 | #html_use_smartypants = True 154 | 155 | # Custom sidebar templates, maps document names to template names. 156 | #html_sidebars = {} 157 | 158 | # Additional templates that should be rendered to pages, maps page names 159 | # to template names. 160 | #html_additional_pages = {} 161 | 162 | # If false, no module index is generated. 163 | #html_domain_indices = True 164 | 165 | # If false, no index is generated. 166 | #html_use_index = True 167 | 168 | # If true, the index is split into individual pages for each letter. 169 | #html_split_index = False 170 | 171 | # If true, links to the reST sources are added to the pages. 172 | #html_show_sourcelink = True 173 | 174 | # If true, "Created using Sphinx" is shown in the HTML footer. 175 | # Default is True. 176 | #html_show_sphinx = True 177 | 178 | # If true, "(C) Copyright ..." is shown in the HTML footer. 179 | # Default is True. 180 | #html_show_copyright = True 181 | 182 | # If true, an OpenSearch description file will be output, and all pages 183 | # will contain a tag referring to it. The value of this option 184 | # must be the base URL from which the finished HTML is served. 185 | #html_use_opensearch = '' 186 | 187 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 188 | #html_file_suffix = None 189 | 190 | # Output file base name for HTML help builder. 191 | htmlhelp_basename = 'wg_gesuchtdoc' 192 | 193 | 194 | # -- Options for LaTeX output ------------------------------------------ 195 | 196 | latex_elements = { 197 | # The paper size ('letterpaper' or 'a4paper'). 198 | #'papersize': 'letterpaper', 199 | 200 | # The font size ('10pt', '11pt' or '12pt'). 201 | #'pointsize': '10pt', 202 | 203 | # Additional stuff for the LaTeX preamble. 204 | #'preamble': '', 205 | } 206 | 207 | # Grouping the document tree into LaTeX files. List of tuples 208 | # (source start file, target name, title, author, documentclass 209 | # [howto/manual]). 210 | latex_documents = [ 211 | ('index', 'wg_gesucht.tex', 212 | u'WG Gesucht Crawler CLI Documentation', 213 | u'Grant Williams', 'manual'), 214 | ] 215 | 216 | # The name of an image file (relative to this directory) to place at 217 | # the top of the title page. 218 | #latex_logo = None 219 | 220 | # For "manual" documents, if this is true, then toplevel headings 221 | # are parts, not chapters. 222 | #latex_use_parts = False 223 | 224 | # If true, show page references after internal links. 225 | #latex_show_pagerefs = False 226 | 227 | # If true, show URL addresses after external links. 228 | #latex_show_urls = False 229 | 230 | # Documents to append as an appendix to all manuals. 231 | #latex_appendices = [] 232 | 233 | # If false, no module index is generated. 234 | #latex_domain_indices = True 235 | 236 | 237 | # -- Options for manual page output ------------------------------------ 238 | 239 | # One entry per manual page. List of tuples 240 | # (source start file, name, description, authors, manual section). 241 | man_pages = [ 242 | ('index', 'wg_gesucht', 243 | u'WG Gesucht Crawler CLI Documentation', 244 | [u'Grant Williams'], 1) 245 | ] 246 | 247 | # If true, show URL addresses after external links. 248 | #man_show_urls = False 249 | 250 | 251 | # -- Options for Texinfo output ---------------------------------------- 252 | 253 | # Grouping the document tree into Texinfo files. List of tuples 254 | # (source start file, target name, title, author, 255 | # dir menu entry, description, category) 256 | texinfo_documents = [ 257 | ('index', 'wg_gesucht', 258 | u'WG Gesucht Crawler CLI Documentation', 259 | u'Grant Williams', 260 | 'wg_gesucht', 261 | 'One line description of project.', 262 | 'Miscellaneous'), 263 | ] 264 | 265 | # Documents to append as an appendix to all manuals. 266 | #texinfo_appendices = [] 267 | 268 | # If false, no module index is generated. 269 | #texinfo_domain_indices = True 270 | 271 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 272 | #texinfo_show_urls = 'footnote' 273 | 274 | # If true, do not generate a @detailmenu in the "Top" node's menu. 275 | #texinfo_no_detailmenu = False 276 | -------------------------------------------------------------------------------- /wg_gesucht/_version.py: -------------------------------------------------------------------------------- 1 | 2 | # This file helps to compute a version number in source trees obtained from 3 | # git-archive tarball (such as those provided by githubs download-from-tag 4 | # feature). Distribution tarballs (built by setup.py sdist) and build 5 | # directories (produced by setup.py build) will contain a much shorter file 6 | # that just contains the computed version number. 7 | 8 | # This file is released into the public domain. Generated by 9 | # versioneer-0.15 (https://github.com/warner/python-versioneer) 10 | 11 | import errno 12 | import os 13 | import re 14 | import subprocess 15 | import sys 16 | 17 | 18 | def get_keywords(): 19 | # these strings will be replaced by git during git-archive. 20 | # setup.py/versioneer.py will grep for the variable names, so they must 21 | # each be defined on a line of their own. _version.py will just call 22 | # get_keywords(). 23 | git_refnames = "$Format:%d$" 24 | git_full = "$Format:%H$" 25 | keywords = {"refnames": git_refnames, "full": git_full} 26 | return keywords 27 | 28 | 29 | class VersioneerConfig: 30 | pass 31 | 32 | 33 | def get_config(): 34 | # these strings are filled in when 'setup.py versioneer' creates 35 | # _version.py 36 | cfg = VersioneerConfig() 37 | cfg.VCS = "git" 38 | cfg.style = "pep440" 39 | cfg.tag_prefix = "" 40 | cfg.parentdir_prefix = "wg_gesucht-" 41 | cfg.versionfile_source = "wg_gesucht/_version.py" 42 | cfg.verbose = False 43 | return cfg 44 | 45 | 46 | class NotThisMethod(Exception): 47 | pass 48 | 49 | 50 | LONG_VERSION_PY = {} 51 | HANDLERS = {} 52 | 53 | 54 | def register_vcs_handler(vcs, method): # decorator 55 | def decorate(f): 56 | if vcs not in HANDLERS: 57 | HANDLERS[vcs] = {} 58 | HANDLERS[vcs][method] = f 59 | return f 60 | return decorate 61 | 62 | 63 | def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): 64 | assert isinstance(commands, list) 65 | p = None 66 | for c in commands: 67 | try: 68 | dispcmd = str([c] + args) 69 | # remember shell=False, so use git.cmd on windows, not just git 70 | p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE, 71 | stderr=(subprocess.PIPE if hide_stderr 72 | else None)) 73 | break 74 | except EnvironmentError: 75 | e = sys.exc_info()[1] 76 | if e.errno == errno.ENOENT: 77 | continue 78 | if verbose: 79 | print("unable to run %s" % dispcmd) 80 | print(e) 81 | return None 82 | else: 83 | if verbose: 84 | print("unable to find command, tried %s" % (commands,)) 85 | return None 86 | stdout = p.communicate()[0].strip() 87 | if sys.version_info[0] >= 3: 88 | stdout = stdout.decode() 89 | if p.returncode != 0: 90 | if verbose: 91 | print("unable to run %s (error)" % dispcmd) 92 | return None 93 | return stdout 94 | 95 | 96 | def versions_from_parentdir(parentdir_prefix, root, verbose): 97 | # Source tarballs conventionally unpack into a directory that includes 98 | # both the project name and a version string. 99 | dirname = os.path.basename(root) 100 | if not dirname.startswith(parentdir_prefix): 101 | if verbose: 102 | print("guessing rootdir is '%s', but '%s' doesn't start with " 103 | "prefix '%s'" % (root, dirname, parentdir_prefix)) 104 | raise NotThisMethod("rootdir doesn't start with parentdir_prefix") 105 | return {"version": dirname[len(parentdir_prefix):], 106 | "full-revisionid": None, 107 | "dirty": False, "error": None} 108 | 109 | 110 | @register_vcs_handler("git", "get_keywords") 111 | def git_get_keywords(versionfile_abs): 112 | # the code embedded in _version.py can just fetch the value of these 113 | # keywords. When used from setup.py, we don't want to import _version.py, 114 | # so we do it with a regexp instead. This function is not used from 115 | # _version.py. 116 | keywords = {} 117 | try: 118 | f = open(versionfile_abs, "r") 119 | for line in f.readlines(): 120 | if line.strip().startswith("git_refnames ="): 121 | mo = re.search(r'=\s*"(.*)"', line) 122 | if mo: 123 | keywords["refnames"] = mo.group(1) 124 | if line.strip().startswith("git_full ="): 125 | mo = re.search(r'=\s*"(.*)"', line) 126 | if mo: 127 | keywords["full"] = mo.group(1) 128 | f.close() 129 | except EnvironmentError: 130 | pass 131 | return keywords 132 | 133 | 134 | @register_vcs_handler("git", "keywords") 135 | def git_versions_from_keywords(keywords, tag_prefix, verbose): 136 | if not keywords: 137 | raise NotThisMethod("no keywords at all, weird") 138 | refnames = keywords["refnames"].strip() 139 | if refnames.startswith("$Format"): 140 | if verbose: 141 | print("keywords are unexpanded, not using") 142 | raise NotThisMethod("unexpanded keywords, not a git-archive tarball") 143 | refs = set([r.strip() for r in refnames.strip("()").split(",")]) 144 | # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of 145 | # just "foo-1.0". If we see a "tag: " prefix, prefer those. 146 | TAG = "tag: " 147 | tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) 148 | if not tags: 149 | # Either we're using git < 1.8.3, or there really are no tags. We use 150 | # a heuristic: assume all version tags have a digit. The old git %d 151 | # expansion behaves like git log --decorate=short and strips out the 152 | # refs/heads/ and refs/tags/ prefixes that would let us distinguish 153 | # between branches and tags. By ignoring refnames without digits, we 154 | # filter out many common branch names like "release" and 155 | # "stabilization", as well as "HEAD" and "master". 156 | tags = set([r for r in refs if re.search(r'\d', r)]) 157 | if verbose: 158 | print("discarding '%s', no digits" % ",".join(refs-tags)) 159 | if verbose: 160 | print("likely tags: %s" % ",".join(sorted(tags))) 161 | for ref in sorted(tags): 162 | # sorting will prefer e.g. "2.0" over "2.0rc1" 163 | if ref.startswith(tag_prefix): 164 | r = ref[len(tag_prefix):] 165 | if verbose: 166 | print("picking %s" % r) 167 | return {"version": r, 168 | "full-revisionid": keywords["full"].strip(), 169 | "dirty": False, "error": None 170 | } 171 | # no suitable tags, so version is "0+unknown", but full hex is still there 172 | if verbose: 173 | print("no suitable tags, using unknown + full revision id") 174 | return {"version": "0+unknown", 175 | "full-revisionid": keywords["full"].strip(), 176 | "dirty": False, "error": "no suitable tags"} 177 | 178 | 179 | @register_vcs_handler("git", "pieces_from_vcs") 180 | def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): 181 | # this runs 'git' from the root of the source tree. This only gets called 182 | # if the git-archive 'subst' keywords were *not* expanded, and 183 | # _version.py hasn't already been rewritten with a short version string, 184 | # meaning we're inside a checked out source tree. 185 | 186 | if not os.path.exists(os.path.join(root, ".git")): 187 | if verbose: 188 | print("no .git in %s" % root) 189 | raise NotThisMethod("no .git directory") 190 | 191 | GITS = ["git"] 192 | if sys.platform == "win32": 193 | GITS = ["git.cmd", "git.exe"] 194 | # if there is a tag, this yields TAG-NUM-gHEX[-dirty] 195 | # if there are no tags, this yields HEX[-dirty] (no NUM) 196 | describe_out = run_command(GITS, ["describe", "--tags", "--dirty", 197 | "--always", "--long"], 198 | cwd=root) 199 | # --long was added in git-1.5.5 200 | if describe_out is None: 201 | raise NotThisMethod("'git describe' failed") 202 | describe_out = describe_out.strip() 203 | full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) 204 | if full_out is None: 205 | raise NotThisMethod("'git rev-parse' failed") 206 | full_out = full_out.strip() 207 | 208 | pieces = {} 209 | pieces["long"] = full_out 210 | pieces["short"] = full_out[:7] # maybe improved later 211 | pieces["error"] = None 212 | 213 | # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] 214 | # TAG might have hyphens. 215 | git_describe = describe_out 216 | 217 | # look for -dirty suffix 218 | dirty = git_describe.endswith("-dirty") 219 | pieces["dirty"] = dirty 220 | if dirty: 221 | git_describe = git_describe[:git_describe.rindex("-dirty")] 222 | 223 | # now we have TAG-NUM-gHEX or HEX 224 | 225 | if "-" in git_describe: 226 | # TAG-NUM-gHEX 227 | mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) 228 | if not mo: 229 | # unparseable. Maybe git-describe is misbehaving? 230 | pieces["error"] = ("unable to parse git-describe output: '%s'" 231 | % describe_out) 232 | return pieces 233 | 234 | # tag 235 | full_tag = mo.group(1) 236 | if not full_tag.startswith(tag_prefix): 237 | if verbose: 238 | fmt = "tag '%s' doesn't start with prefix '%s'" 239 | print(fmt % (full_tag, tag_prefix)) 240 | pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" 241 | % (full_tag, tag_prefix)) 242 | return pieces 243 | pieces["closest-tag"] = full_tag[len(tag_prefix):] 244 | 245 | # distance: number of commits since tag 246 | pieces["distance"] = int(mo.group(2)) 247 | 248 | # commit: short hex revision ID 249 | pieces["short"] = mo.group(3) 250 | 251 | else: 252 | # HEX: no tags 253 | pieces["closest-tag"] = None 254 | count_out = run_command(GITS, ["rev-list", "HEAD", "--count"], 255 | cwd=root) 256 | pieces["distance"] = int(count_out) # total number of commits 257 | 258 | return pieces 259 | 260 | 261 | def plus_or_dot(pieces): 262 | if "+" in pieces.get("closest-tag", ""): 263 | return "." 264 | return "+" 265 | 266 | 267 | def render_pep440(pieces): 268 | # now build up version string, with post-release "local version 269 | # identifier". Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you 270 | # get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty 271 | 272 | # exceptions: 273 | # 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] 274 | 275 | if pieces["closest-tag"]: 276 | rendered = pieces["closest-tag"] 277 | if pieces["distance"] or pieces["dirty"]: 278 | rendered += plus_or_dot(pieces) 279 | rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) 280 | if pieces["dirty"]: 281 | rendered += ".dirty" 282 | else: 283 | # exception #1 284 | rendered = "0+untagged.%d.g%s" % (pieces["distance"], 285 | pieces["short"]) 286 | if pieces["dirty"]: 287 | rendered += ".dirty" 288 | return rendered 289 | 290 | 291 | def render_pep440_pre(pieces): 292 | # TAG[.post.devDISTANCE] . No -dirty 293 | 294 | # exceptions: 295 | # 1: no tags. 0.post.devDISTANCE 296 | 297 | if pieces["closest-tag"]: 298 | rendered = pieces["closest-tag"] 299 | if pieces["distance"]: 300 | rendered += ".post.dev%d" % pieces["distance"] 301 | else: 302 | # exception #1 303 | rendered = "0.post.dev%d" % pieces["distance"] 304 | return rendered 305 | 306 | 307 | def render_pep440_post(pieces): 308 | # TAG[.postDISTANCE[.dev0]+gHEX] . The ".dev0" means dirty. Note that 309 | # .dev0 sorts backwards (a dirty tree will appear "older" than the 310 | # corresponding clean one), but you shouldn't be releasing software with 311 | # -dirty anyways. 312 | 313 | # exceptions: 314 | # 1: no tags. 0.postDISTANCE[.dev0] 315 | 316 | if pieces["closest-tag"]: 317 | rendered = pieces["closest-tag"] 318 | if pieces["distance"] or pieces["dirty"]: 319 | rendered += ".post%d" % pieces["distance"] 320 | if pieces["dirty"]: 321 | rendered += ".dev0" 322 | rendered += plus_or_dot(pieces) 323 | rendered += "g%s" % pieces["short"] 324 | else: 325 | # exception #1 326 | rendered = "0.post%d" % pieces["distance"] 327 | if pieces["dirty"]: 328 | rendered += ".dev0" 329 | rendered += "+g%s" % pieces["short"] 330 | return rendered 331 | 332 | 333 | def render_pep440_old(pieces): 334 | # TAG[.postDISTANCE[.dev0]] . The ".dev0" means dirty. 335 | 336 | # exceptions: 337 | # 1: no tags. 0.postDISTANCE[.dev0] 338 | 339 | if pieces["closest-tag"]: 340 | rendered = pieces["closest-tag"] 341 | if pieces["distance"] or pieces["dirty"]: 342 | rendered += ".post%d" % pieces["distance"] 343 | if pieces["dirty"]: 344 | rendered += ".dev0" 345 | else: 346 | # exception #1 347 | rendered = "0.post%d" % pieces["distance"] 348 | if pieces["dirty"]: 349 | rendered += ".dev0" 350 | return rendered 351 | 352 | 353 | def render_git_describe(pieces): 354 | # TAG[-DISTANCE-gHEX][-dirty], like 'git describe --tags --dirty 355 | # --always' 356 | 357 | # exceptions: 358 | # 1: no tags. HEX[-dirty] (note: no 'g' prefix) 359 | 360 | if pieces["closest-tag"]: 361 | rendered = pieces["closest-tag"] 362 | if pieces["distance"]: 363 | rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) 364 | else: 365 | # exception #1 366 | rendered = pieces["short"] 367 | if pieces["dirty"]: 368 | rendered += "-dirty" 369 | return rendered 370 | 371 | 372 | def render_git_describe_long(pieces): 373 | # TAG-DISTANCE-gHEX[-dirty], like 'git describe --tags --dirty 374 | # --always -long'. The distance/hash is unconditional. 375 | 376 | # exceptions: 377 | # 1: no tags. HEX[-dirty] (note: no 'g' prefix) 378 | 379 | if pieces["closest-tag"]: 380 | rendered = pieces["closest-tag"] 381 | rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) 382 | else: 383 | # exception #1 384 | rendered = pieces["short"] 385 | if pieces["dirty"]: 386 | rendered += "-dirty" 387 | return rendered 388 | 389 | 390 | def render(pieces, style): 391 | if pieces["error"]: 392 | return {"version": "unknown", 393 | "full-revisionid": pieces.get("long"), 394 | "dirty": None, 395 | "error": pieces["error"]} 396 | 397 | if not style or style == "default": 398 | style = "pep440" # the default 399 | 400 | if style == "pep440": 401 | rendered = render_pep440(pieces) 402 | elif style == "pep440-pre": 403 | rendered = render_pep440_pre(pieces) 404 | elif style == "pep440-post": 405 | rendered = render_pep440_post(pieces) 406 | elif style == "pep440-old": 407 | rendered = render_pep440_old(pieces) 408 | elif style == "git-describe": 409 | rendered = render_git_describe(pieces) 410 | elif style == "git-describe-long": 411 | rendered = render_git_describe_long(pieces) 412 | else: 413 | raise ValueError("unknown style '%s'" % style) 414 | 415 | return {"version": rendered, "full-revisionid": pieces["long"], 416 | "dirty": pieces["dirty"], "error": None} 417 | 418 | 419 | def get_versions(): 420 | # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have 421 | # __file__, we can work backwards from there to the root. Some 422 | # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which 423 | # case we can only use expanded keywords. 424 | 425 | cfg = get_config() 426 | verbose = cfg.verbose 427 | 428 | try: 429 | return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, 430 | verbose) 431 | except NotThisMethod: 432 | pass 433 | 434 | try: 435 | root = os.path.realpath(__file__) 436 | # versionfile_source is the relative path from the top of the source 437 | # tree (where the .git directory might live) to this file. Invert 438 | # this to find the root from __file__. 439 | for i in cfg.versionfile_source.split('/'): 440 | root = os.path.dirname(root) 441 | except NameError: 442 | return {"version": "0+unknown", "full-revisionid": None, 443 | "dirty": None, 444 | "error": "unable to find root of source tree"} 445 | 446 | try: 447 | pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) 448 | return render(pieces, cfg.style) 449 | except NotThisMethod: 450 | pass 451 | 452 | try: 453 | if cfg.parentdir_prefix: 454 | return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) 455 | except NotThisMethod: 456 | pass 457 | 458 | return {"version": "0+unknown", "full-revisionid": None, 459 | "dirty": None, 460 | "error": "unable to compute version"} 461 | -------------------------------------------------------------------------------- /wg_gesucht/crawler.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | import csv 4 | import sys 5 | import json 6 | import time 7 | import errno 8 | import random 9 | import urllib 10 | import logging 11 | import datetime 12 | import requests 13 | from bs4 import BeautifulSoup 14 | 15 | 16 | class InfoFilter(logging.Filter): 17 | def filter(self, record): 18 | return record.levelno in [20, 30] 19 | 20 | 21 | class WgGesuchtCrawler: 22 | def __init__( 23 | self, 24 | login_info, 25 | ad_links_folder, 26 | offline_ad_folder, 27 | logs_folder, 28 | template, 29 | filter_names, 30 | share_email, 31 | ): 32 | self.login_info = login_info 33 | self.ad_links_folder = ad_links_folder 34 | self.offline_ad_folder = offline_ad_folder 35 | self.logs_folder = logs_folder 36 | self.template_name = template 37 | self.filter_names = filter_names 38 | self.share_email = share_email 39 | self.submit_message_url = ( 40 | "https://www.wg-gesucht.de/ajax/api/Smp/api.php?action=conversations" 41 | ) 42 | self.session = requests.Session() 43 | self.logger = self.get_logger() 44 | self.counter = 1 45 | self.continue_next_page = True 46 | 47 | def get_logger(self): 48 | formatter = logging.Formatter( 49 | "%(asctime)s::%(name)s::%(levelname)s::%(message)s", 50 | datefmt="%Y-%m-%d %H:%M:%S", 51 | ) 52 | 53 | logger = logging.getLogger(__name__) 54 | logger.setLevel(logging.INFO) 55 | 56 | info_file_handler = logging.FileHandler( 57 | os.path.join(self.logs_folder, "info.log") 58 | ) 59 | info_file_handler.setFormatter(formatter) 60 | info_file_handler.addFilter(InfoFilter()) 61 | info_file_handler.setLevel(logging.INFO) 62 | 63 | error_file_handler = logging.FileHandler( 64 | os.path.join(self.logs_folder, "error.log") 65 | ) 66 | error_file_handler.setFormatter(formatter) 67 | error_file_handler.setLevel(logging.ERROR) 68 | 69 | stream_handler = logging.StreamHandler() 70 | stream_handler.setLevel(logging.WARNING) 71 | 72 | logger.addHandler(info_file_handler) 73 | logger.addHandler(error_file_handler) 74 | logger.addHandler(stream_handler) 75 | 76 | return logger 77 | 78 | def sign_in(self): 79 | self.logger.info("Signing into WG-Gesucht...") 80 | 81 | payload = { 82 | "login_email_username": self.login_info["email"], 83 | "login_password": self.login_info["password"], 84 | "login_form_auto_login": "1", 85 | "display_language": "de", 86 | } 87 | 88 | try: 89 | login = self.session.post( 90 | "https://www.wg-gesucht.de/ajax/api/Smp/api.php?action=login", 91 | json=payload, 92 | ) 93 | except requests.exceptions.Timeout: 94 | self.logger.exception("Timed out trying to log in") 95 | sys.exit(1) 96 | except requests.exceptions.ConnectionError: 97 | self.logger.exception("Could not connect to internet") 98 | sys.exit(1) 99 | 100 | if login.json() is True: 101 | self.logger.info("Logged in successfully") 102 | else: 103 | self.logger.warning( 104 | "Could not log into wg-gesucht.de with the given email and password" 105 | ) 106 | sys.exit(1) 107 | 108 | def get_page(self, url): 109 | # randomise time between requests to avoid reCAPTCHA 110 | time.sleep(random.randint(5, 8)) 111 | try: 112 | page = self.session.get(url) 113 | except requests.exceptions.Timeout: 114 | self.logger.exception("Timed out trying to log in") 115 | sys.exit(1) 116 | except requests.exceptions.ConnectionError: 117 | self.logger.exception("Could not connect to internet") 118 | sys.exit(1) 119 | 120 | if self.no_captcha(page): 121 | self.logger.info("%s: requested successfully", url) 122 | return page 123 | return None 124 | 125 | def no_captcha(self, page): 126 | soup = BeautifulSoup(page.content, "html.parser") 127 | recaptcha = soup.find_all("div", {"class": "g-recaptcha"}) 128 | 129 | if recaptcha: 130 | self.logger.warning( 131 | """ 132 | Sorry! A 'reCAPTCHA' has been detected, please sign into you WG-Gesucht 133 | account through a browser and solve the 'reCAPTCHA', you may also have to 134 | wait 15-20 mins before restarting 135 | """ 136 | ) 137 | sys.exit(1) 138 | else: 139 | return True 140 | 141 | def retrieve_email_template(self): 142 | self.logger.info("Retrieving email template...") 143 | 144 | template_page = self.get_page( 145 | "https://www.wg-gesucht.de/mein-wg-gesucht-message-templates.html" 146 | ) 147 | 148 | def no_template_error(): 149 | self.logger.warning( 150 | """ 151 | You have not yet saved an email template in your WG-Gesucht account, please log 152 | into your account and save one at https://www.wg-gesucht.de/mein-wg-gesucht-message-templates.html 153 | """ 154 | ) 155 | sys.exit(1) 156 | 157 | soup = BeautifulSoup(template_page.content, "html.parser") 158 | template_texts = [ 159 | text.find_all("div", {"class": "truncate_title"}) 160 | for text in soup.find_all("div", {"class": "panel-body"}) 161 | ] 162 | try: 163 | if not self.template_name: 164 | chosen_text = template_texts[0][1].text 165 | else: 166 | chosen_text = list( 167 | filter( 168 | lambda text: text[0].text.strip().lower() == self.template_name, 169 | template_texts, 170 | ) 171 | )[0][1].text 172 | except IndexError: 173 | no_template_error() 174 | 175 | if not chosen_text: 176 | no_template_error() 177 | else: 178 | return chosen_text.lstrip().rstrip() 179 | 180 | def fetch_filters(self): 181 | filters_page = self.get_page( 182 | "https://www.wg-gesucht.de/mein-wg-gesucht-filter.html" 183 | ) 184 | 185 | soup = BeautifulSoup(filters_page.content, "html.parser") 186 | 187 | filter_results = soup.find_all(id=re.compile("^filter_name_")) 188 | filters_to_check = [] 189 | if self.filter_names: 190 | filters_to_check = [ 191 | filter.get("href") 192 | for filter in filter_results 193 | if filter.text.strip().lower() in self.filter_names 194 | ] 195 | else: 196 | filters_to_check = [filter.get("href") for filter in filter_results] 197 | 198 | if self.filter_names and len(filters_to_check) != len(self.filter_names): 199 | self.logger.warning( 200 | "Not all filters you wanted were found, maybe you mispelled one?" 201 | ) 202 | 203 | if not filters_to_check: 204 | self.logger.warning( 205 | "No filters found! Please create at least 1 filter on your WG-Gesucht account" 206 | ) 207 | sys.exit(1) 208 | else: 209 | self.logger.info("Filters found: %s", len(filters_to_check)) 210 | return filters_to_check 211 | 212 | def already_sent(self, href): 213 | with open( 214 | os.path.join(self.ad_links_folder, "WG Ad Links.csv"), 215 | "rt", 216 | encoding="utf-8", 217 | ) as file: 218 | wg_links_file_csv = csv.reader(file) 219 | for wg_links_row in wg_links_file_csv: 220 | if wg_links_row[0] == href: 221 | return True 222 | return False 223 | 224 | def change_to_list_details_view(self, soup, list_view_href=None): 225 | view_type_links = soup.find_all("a", href=True, title=True) 226 | if view_type_links[0]["title"] == "Listenansicht": 227 | list_view_href = view_type_links[0]["href"] 228 | 229 | # change gallery view to list details view 230 | if list_view_href: 231 | details_results_page = self.get_page( 232 | "https://www.wg-gesucht.de/{}".format(list_view_href) 233 | ) 234 | soup = BeautifulSoup(details_results_page.content, "html.parser") 235 | return soup 236 | 237 | def process_filter_results(self, filter_results): 238 | url_list = list() 239 | for result in filter_results: 240 | post_date_link = result.find("td", {"class": "ang_spalte_datum"}).find("a") 241 | # ignores ads older than 2 days 242 | try: 243 | post_date = datetime.datetime.strptime( 244 | post_date_link.text.strip(), "%d.%m.%Y" 245 | ).date() 246 | if post_date >= datetime.date.today() - datetime.timedelta(days=2): 247 | complete_href = "https://www.wg-gesucht.de/{}".format( 248 | post_date_link.get("href") 249 | ) 250 | if not self.already_sent(complete_href): 251 | url_list.append(complete_href) 252 | else: 253 | continue 254 | else: 255 | self.continue_next_page = False 256 | except ValueError: # caught if ad is inactive or has no date 257 | self.continue_next_page = False 258 | return url_list 259 | 260 | def fetch_ads(self, filters): 261 | self.logger.info( 262 | "Searching filters for new ads, may take a while, depending on how many filters you " 263 | "have set up." 264 | ) 265 | url_list = list() 266 | for wg_filter in filters: 267 | # resets for each fitler, otherwise will immediately skip other filters 268 | self.continue_next_page = True 269 | while self.continue_next_page: 270 | search_results_page = self.get_page(wg_filter) 271 | 272 | soup = self.change_to_list_details_view( 273 | BeautifulSoup(search_results_page.content, "html.parser") 274 | ) 275 | 276 | link_table = soup.find("table", {"id": "table-compact-list"}) 277 | 278 | pagination = soup.find("ul", {"class": "pagination"}) 279 | if not pagination: 280 | self.continue_next_page = False 281 | else: 282 | next_button_href = pagination.find_all("a")[-1].get("href") 283 | 284 | # gets each row from the search results table 285 | search_results = link_table.find_all( 286 | "tr", {"class": ["listenansicht0", "listenansicht1"]} 287 | ) 288 | 289 | url_list.extend(self.process_filter_results(search_results)) 290 | 291 | if self.continue_next_page: 292 | wg_filter = "https://www.wg-gesucht.de/{}".format(next_button_href) 293 | 294 | self.logger.info("Number of apartments to email: %s", len(set(url_list))) 295 | return set(url_list) 296 | 297 | def get_info_from_ad(self, url): 298 | # cleans up file name to allow saving (removes illegal file name characters) 299 | def text_replace(text): 300 | text = re.sub(r"\bhttps://www.wg-gesucht.de/\b|[:/*?|<>&^%@#!]", "", text) 301 | text = ( 302 | text.replace(":", "") 303 | .replace("/", "") 304 | .replace("\\", "") 305 | .replace("*", "") 306 | .replace("?", "") 307 | .replace("|", "") 308 | .replace("<", "") 309 | .replace(">", "") 310 | .replace("https://www.wg-gesucht.de/", "") 311 | ) 312 | return text.rstrip().lstrip() 313 | 314 | ad_page = self.get_page(url) 315 | 316 | ad_page_soup = BeautifulSoup(ad_page.content, "html.parser") 317 | 318 | ad_title = text_replace(ad_page_soup.find("title").text) 319 | ad_url = text_replace(url) 320 | 321 | return { 322 | "ad_page_soup": ad_page_soup, 323 | "ad_title": ad_title, 324 | "ad_submitter": "N/A", 325 | "ad_url": ad_url, 326 | } 327 | 328 | def update_files(self, url, ad_info): 329 | MAX_FILENAME_LENGTH = 245 330 | 331 | ad_page_soup, ad_title, ad_submitter, ad_url = ( 332 | ad_info["ad_page_soup"], 333 | ad_info["ad_title"], 334 | ad_info["ad_submitter"], 335 | ad_info["ad_url"], 336 | ) 337 | # save url to file, so as not to send a message to them again 338 | with open( 339 | os.path.join(self.ad_links_folder, "WG Ad Links.csv"), 340 | "a", 341 | newline="", 342 | encoding="utf-8", 343 | ) as file_write: 344 | csv_file_write = csv.writer(file_write) 345 | csv_file_write.writerow([url, ad_submitter, ad_title]) 346 | 347 | # save a copy of the ad for offline viewing, in case the ad is deleted before the user can view it online 348 | max_ad_title_length = MAX_FILENAME_LENGTH - len(ad_submitter) - len(ad_url) 349 | if len(ad_title) > max_ad_title_length: 350 | ad_title = ad_title[: max_ad_title_length - 1] + "..." 351 | 352 | file_name = "{}-{}-{}".format(ad_submitter, ad_title, ad_url) 353 | try: 354 | with open( 355 | os.path.join(self.offline_ad_folder, file_name), "w", encoding="utf-8" 356 | ) as outfile: 357 | outfile.write(str(ad_page_soup)) 358 | except OSError as err: 359 | if err.errno == errno.ENAMETOOLONG: 360 | self.logger.exception( 361 | "File name of {} is too long, could not save this ad offline".format( 362 | file_name 363 | ) 364 | ) 365 | 366 | def get_payload(self, submit_form, template_text): 367 | return { 368 | "user_id": submit_form.find(attrs={"name": "user_id"})["value"], 369 | "ad_type": submit_form.find(attrs={"name": "ad_type"})["value"], 370 | "ad_id": submit_form.find(attrs={"name": "ad_id"})["value"], 371 | "csrf_token": submit_form.find(attrs={"name": "csrf_token"})["value"], 372 | "messages": [{"content": template_text, "message_type": "text"}], 373 | } 374 | 375 | def email_apartment(self, url, template_text): 376 | ad_info = self.get_info_from_ad(url) 377 | 378 | try: 379 | send_message_url = ( 380 | ad_info["ad_page_soup"] 381 | .find("a", {"class": "btn btn-block btn-md wgg_orange"}) 382 | .get("href") 383 | ) 384 | except AttributeError: 385 | self.logger.exception( 386 | "Could not find submit form, you have possibly already sent a message to this user" 387 | ) 388 | self.update_files(url, ad_info) 389 | return 390 | 391 | submit_form_page = self.get_page(send_message_url) 392 | submit_form_page_soup = BeautifulSoup(submit_form_page.content, "html.parser") 393 | submit_form = submit_form_page_soup.find("form", {"id": "messenger_form"}) 394 | 395 | if not submit_form: 396 | self.logger.exception( 397 | "Could not find submit form, you have possibly already sent a message to this user" 398 | ) 399 | self.update_files(url, ad_info) 400 | return 401 | 402 | ad_submitter = ( 403 | submit_form_page_soup.find( 404 | attrs={"class": "control-label", "for": "message_input"} 405 | ) 406 | .text.replace("Nachricht an ", "") 407 | .replace(":", "") 408 | .rstrip() 409 | .lstrip() 410 | ) 411 | ad_info["ad_submitter"] = ad_submitter 412 | 413 | headers = { 414 | "Content-Type": "application/json", 415 | "Cache-Control": "no-cache", 416 | "Referer": send_message_url, 417 | "Accept": "application/json, text/javascript, */*", 418 | "Origin": "https://www.wg-gesucht.de", 419 | "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36", 420 | } 421 | 422 | try: 423 | payload = self.get_payload(submit_form, template_text) 424 | except AttributeError: 425 | self.logger.exception( 426 | "Could not find submit form, you have possibly already sent a message to this user" 427 | ) 428 | self.update_files(url, ad_info) 429 | return 430 | 431 | json_data = json.dumps(payload) 432 | 433 | try: 434 | sent_message = self.session.post( 435 | self.submit_message_url, data=json_data, headers=headers 436 | ).json() 437 | except requests.exceptions.Timeout: 438 | self.logger.exception( 439 | "Timed out sending a message to %s, will try again next time", 440 | ad_info["ad_submitter"], 441 | ) 442 | return 443 | 444 | if not sent_message.get("conversation_id", None): 445 | self.logger.warning( 446 | "Failed to send message to %s, will try again next time", 447 | ad_info["ad_submitter"], 448 | ) 449 | return 450 | 451 | self.update_files(url, ad_info) 452 | time_now = datetime.datetime.now().strftime("%H:%M:%S") 453 | self.logger.info("Message Sent to %s at %s!", ad_info["ad_submitter"], time_now) 454 | 455 | def search(self): 456 | if self.counter < 2: 457 | self.logger.debug("Starting...") 458 | else: 459 | self.logger.info("Resuming...") 460 | 461 | template_text = self.retrieve_email_template() 462 | 463 | filters_to_check = self.fetch_filters() 464 | 465 | ad_list = self.fetch_ads(filters_to_check) 466 | 467 | for ad_url in ad_list: 468 | self.email_apartment(ad_url, template_text) 469 | 470 | time_now = datetime.datetime.now().strftime("%H:%M:%S") 471 | self.logger.info("Program paused at %s... Will resume in 4-5 minutes", time_now) 472 | self.logger.info( 473 | "WG-Gesucht checked %s %s since running", 474 | self.counter, 475 | "time" if self.counter <= 1 else "times", 476 | ) 477 | # pauses for 4-5 mins before searching again 478 | time.sleep(random.randint(240, 300)) 479 | self.counter += 1 480 | self.search() 481 | -------------------------------------------------------------------------------- /versioneer.py: -------------------------------------------------------------------------------- 1 | 2 | # Version: 0.15 3 | 4 | """ 5 | The Versioneer 6 | ============== 7 | 8 | * like a rocketeer, but for versions! 9 | * https://github.com/warner/python-versioneer 10 | * Brian Warner 11 | * License: Public Domain 12 | * Compatible With: python2.6, 2.7, 3.2, 3.3, 3.4, and pypy 13 | * [![Latest Version] 14 | (https://pypip.in/version/versioneer/badge.svg?style=flat) 15 | ](https://pypi.python.org/pypi/versioneer/) 16 | * [![Build Status] 17 | (https://travis-ci.org/warner/python-versioneer.png?branch=master) 18 | ](https://travis-ci.org/warner/python-versioneer) 19 | 20 | This is a tool for managing a recorded version number in distutils-based 21 | python projects. The goal is to remove the tedious and error-prone "update 22 | the embedded version string" step from your release process. Making a new 23 | release should be as easy as recording a new tag in your version-control 24 | system, and maybe making new tarballs. 25 | 26 | 27 | ## Quick Install 28 | 29 | * `pip install versioneer` to somewhere to your $PATH 30 | * add a `[versioneer]` section to your setup.cfg (see below) 31 | * run `versioneer install` in your source tree, commit the results 32 | 33 | ## Version Identifiers 34 | 35 | Source trees come from a variety of places: 36 | 37 | * a version-control system checkout (mostly used by developers) 38 | * a nightly tarball, produced by build automation 39 | * a snapshot tarball, produced by a web-based VCS browser, like github's 40 | "tarball from tag" feature 41 | * a release tarball, produced by "setup.py sdist", distributed through PyPI 42 | 43 | Within each source tree, the version identifier (either a string or a number, 44 | this tool is format-agnostic) can come from a variety of places: 45 | 46 | * ask the VCS tool itself, e.g. "git describe" (for checkouts), which knows 47 | about recent "tags" and an absolute revision-id 48 | * the name of the directory into which the tarball was unpacked 49 | * an expanded VCS keyword ($Id$, etc) 50 | * a `_version.py` created by some earlier build step 51 | 52 | For released software, the version identifier is closely related to a VCS 53 | tag. Some projects use tag names that include more than just the version 54 | string (e.g. "myproject-1.2" instead of just "1.2"), in which case the tool 55 | needs to strip the tag prefix to extract the version identifier. For 56 | unreleased software (between tags), the version identifier should provide 57 | enough information to help developers recreate the same tree, while also 58 | giving them an idea of roughly how old the tree is (after version 1.2, before 59 | version 1.3). Many VCS systems can report a description that captures this, 60 | for example `git describe --tags --dirty --always` reports things like 61 | "0.7-1-g574ab98-dirty" to indicate that the checkout is one revision past the 62 | 0.7 tag, has a unique revision id of "574ab98", and is "dirty" (it has 63 | uncommitted changes. 64 | 65 | The version identifier is used for multiple purposes: 66 | 67 | * to allow the module to self-identify its version: `myproject.__version__` 68 | * to choose a name and prefix for a 'setup.py sdist' tarball 69 | 70 | ## Theory of Operation 71 | 72 | Versioneer works by adding a special `_version.py` file into your source 73 | tree, where your `__init__.py` can import it. This `_version.py` knows how to 74 | dynamically ask the VCS tool for version information at import time. 75 | 76 | `_version.py` also contains `$Revision$` markers, and the installation 77 | process marks `_version.py` to have this marker rewritten with a tag name 78 | during the `git archive` command. As a result, generated tarballs will 79 | contain enough information to get the proper version. 80 | 81 | To allow `setup.py` to compute a version too, a `versioneer.py` is added to 82 | the top level of your source tree, next to `setup.py` and the `setup.cfg` 83 | that configures it. This overrides several distutils/setuptools commands to 84 | compute the version when invoked, and changes `setup.py build` and `setup.py 85 | sdist` to replace `_version.py` with a small static file that contains just 86 | the generated version data. 87 | 88 | ## Installation 89 | 90 | First, decide on values for the following configuration variables: 91 | 92 | * `VCS`: the version control system you use. Currently accepts "git". 93 | 94 | * `style`: the style of version string to be produced. See "Styles" below for 95 | details. Defaults to "pep440", which looks like 96 | `TAG[+DISTANCE.gSHORTHASH[.dirty]]`. 97 | 98 | * `versionfile_source`: 99 | 100 | A project-relative pathname into which the generated version strings should 101 | be written. This is usually a `_version.py` next to your project's main 102 | `__init__.py` file, so it can be imported at runtime. If your project uses 103 | `src/myproject/__init__.py`, this should be `src/myproject/_version.py`. 104 | This file should be checked in to your VCS as usual: the copy created below 105 | by `setup.py setup_versioneer` will include code that parses expanded VCS 106 | keywords in generated tarballs. The 'build' and 'sdist' commands will 107 | replace it with a copy that has just the calculated version string. 108 | 109 | This must be set even if your project does not have any modules (and will 110 | therefore never import `_version.py`), since "setup.py sdist" -based trees 111 | still need somewhere to record the pre-calculated version strings. Anywhere 112 | in the source tree should do. If there is a `__init__.py` next to your 113 | `_version.py`, the `setup.py setup_versioneer` command (described below) 114 | will append some `__version__`-setting assignments, if they aren't already 115 | present. 116 | 117 | * `versionfile_build`: 118 | 119 | Like `versionfile_source`, but relative to the build directory instead of 120 | the source directory. These will differ when your setup.py uses 121 | 'package_dir='. If you have `package_dir={'myproject': 'src/myproject'}`, 122 | then you will probably have `versionfile_build='myproject/_version.py'` and 123 | `versionfile_source='src/myproject/_version.py'`. 124 | 125 | If this is set to None, then `setup.py build` will not attempt to rewrite 126 | any `_version.py` in the built tree. If your project does not have any 127 | libraries (e.g. if it only builds a script), then you should use 128 | `versionfile_build = None` and override `distutils.command.build_scripts` 129 | to explicitly insert a copy of `versioneer.get_version()` into your 130 | generated script. 131 | 132 | * `tag_prefix`: 133 | 134 | a string, like 'PROJECTNAME-', which appears at the start of all VCS tags. 135 | If your tags look like 'myproject-1.2.0', then you should use 136 | tag_prefix='myproject-'. If you use unprefixed tags like '1.2.0', this 137 | should be an empty string. 138 | 139 | * `parentdir_prefix`: 140 | 141 | a optional string, frequently the same as tag_prefix, which appears at the 142 | start of all unpacked tarball filenames. If your tarball unpacks into 143 | 'myproject-1.2.0', this should be 'myproject-'. To disable this feature, 144 | just omit the field from your `setup.cfg`. 145 | 146 | This tool provides one script, named `versioneer`. That script has one mode, 147 | "install", which writes a copy of `versioneer.py` into the current directory 148 | and runs `versioneer.py setup` to finish the installation. 149 | 150 | To versioneer-enable your project: 151 | 152 | * 1: Modify your `setup.cfg`, adding a section named `[versioneer]` and 153 | populating it with the configuration values you decided earlier (note that 154 | the option names are not case-sensitive): 155 | 156 | ```` 157 | [versioneer] 158 | VCS = git 159 | style = pep440 160 | versionfile_source = src/myproject/_version.py 161 | versionfile_build = myproject/_version.py 162 | tag_prefix = "" 163 | parentdir_prefix = myproject- 164 | ```` 165 | 166 | * 2: Run `versioneer install`. This will do the following: 167 | 168 | * copy `versioneer.py` into the top of your source tree 169 | * create `_version.py` in the right place (`versionfile_source`) 170 | * modify your `__init__.py` (if one exists next to `_version.py`) to define 171 | `__version__` (by calling a function from `_version.py`) 172 | * modify your `MANIFEST.in` to include both `versioneer.py` and the 173 | generated `_version.py` in sdist tarballs 174 | 175 | `versioneer install` will complain about any problems it finds with your 176 | `setup.py` or `setup.cfg`. Run it multiple times until you have fixed all 177 | the problems. 178 | 179 | * 3: add a `import versioneer` to your setup.py, and add the following 180 | arguments to the setup() call: 181 | 182 | version=versioneer.get_version(), 183 | cmdclass=versioneer.get_cmdclass(), 184 | 185 | * 4: commit these changes to your VCS. To make sure you won't forget, 186 | `versioneer install` will mark everything it touched for addition using 187 | `git add`. Don't forget to add `setup.py` and `setup.cfg` too. 188 | 189 | ## Post-Installation Usage 190 | 191 | Once established, all uses of your tree from a VCS checkout should get the 192 | current version string. All generated tarballs should include an embedded 193 | version string (so users who unpack them will not need a VCS tool installed). 194 | 195 | If you distribute your project through PyPI, then the release process should 196 | boil down to two steps: 197 | 198 | * 1: git tag 1.0 199 | * 2: python setup.py register sdist upload 200 | 201 | If you distribute it through github (i.e. users use github to generate 202 | tarballs with `git archive`), the process is: 203 | 204 | * 1: git tag 1.0 205 | * 2: git push; git push --tags 206 | 207 | Versioneer will report "0+untagged.NUMCOMMITS.gHASH" until your tree has at 208 | least one tag in its history. 209 | 210 | ## Version-String Flavors 211 | 212 | Code which uses Versioneer can learn about its version string at runtime by 213 | importing `_version` from your main `__init__.py` file and running the 214 | `get_versions()` function. From the "outside" (e.g. in `setup.py`), you can 215 | import the top-level `versioneer.py` and run `get_versions()`. 216 | 217 | Both functions return a dictionary with different flavors of version 218 | information: 219 | 220 | * `['version']`: A condensed version string, rendered using the selected 221 | style. This is the most commonly used value for the project's version 222 | string. The default "pep440" style yields strings like `0.11`, 223 | `0.11+2.g1076c97`, or `0.11+2.g1076c97.dirty`. See the "Styles" section 224 | below for alternative styles. 225 | 226 | * `['full-revisionid']`: detailed revision identifier. For Git, this is the 227 | full SHA1 commit id, e.g. "1076c978a8d3cfc70f408fe5974aa6c092c949ac". 228 | 229 | * `['dirty']`: a boolean, True if the tree has uncommitted changes. Note that 230 | this is only accurate if run in a VCS checkout, otherwise it is likely to 231 | be False or None 232 | 233 | * `['error']`: if the version string could not be computed, this will be set 234 | to a string describing the problem, otherwise it will be None. It may be 235 | useful to throw an exception in setup.py if this is set, to avoid e.g. 236 | creating tarballs with a version string of "unknown". 237 | 238 | Some variants are more useful than others. Including `full-revisionid` in a 239 | bug report should allow developers to reconstruct the exact code being tested 240 | (or indicate the presence of local changes that should be shared with the 241 | developers). `version` is suitable for display in an "about" box or a CLI 242 | `--version` output: it can be easily compared against release notes and lists 243 | of bugs fixed in various releases. 244 | 245 | The installer adds the following text to your `__init__.py` to place a basic 246 | version in `YOURPROJECT.__version__`: 247 | 248 | from ._version import get_versions 249 | __version__ = get_versions()['version'] 250 | del get_versions 251 | 252 | ## Styles 253 | 254 | The setup.cfg `style=` configuration controls how the VCS information is 255 | rendered into a version string. 256 | 257 | The default style, "pep440", produces a PEP440-compliant string, equal to the 258 | un-prefixed tag name for actual releases, and containing an additional "local 259 | version" section with more detail for in-between builds. For Git, this is 260 | TAG[+DISTANCE.gHEX[.dirty]] , using information from `git describe --tags 261 | --dirty --always`. For example "0.11+2.g1076c97.dirty" indicates that the 262 | tree is like the "1076c97" commit but has uncommitted changes (".dirty"), and 263 | that this commit is two revisions ("+2") beyond the "0.11" tag. For released 264 | software (exactly equal to a known tag), the identifier will only contain the 265 | stripped tag, e.g. "0.11". 266 | 267 | Other styles are available. See details.md in the Versioneer source tree for 268 | descriptions. 269 | 270 | ## Debugging 271 | 272 | Versioneer tries to avoid fatal errors: if something goes wrong, it will tend 273 | to return a version of "0+unknown". To investigate the problem, run `setup.py 274 | version`, which will run the version-lookup code in a verbose mode, and will 275 | display the full contents of `get_versions()` (including the `error` string, 276 | which may help identify what went wrong). 277 | 278 | ## Updating Versioneer 279 | 280 | To upgrade your project to a new release of Versioneer, do the following: 281 | 282 | * install the new Versioneer (`pip install -U versioneer` or equivalent) 283 | * edit `setup.cfg`, if necessary, to include any new configuration settings 284 | indicated by the release notes 285 | * re-run `versioneer install` in your source tree, to replace 286 | `SRC/_version.py` 287 | * commit any changed files 288 | 289 | ### Upgrading to 0.15 290 | 291 | Starting with this version, Versioneer is configured with a `[versioneer]` 292 | section in your `setup.cfg` file. Earlier versions required the `setup.py` to 293 | set attributes on the `versioneer` module immediately after import. The new 294 | version will refuse to run (raising an exception during import) until you 295 | have provided the necessary `setup.cfg` section. 296 | 297 | In addition, the Versioneer package provides an executable named 298 | `versioneer`, and the installation process is driven by running `versioneer 299 | install`. In 0.14 and earlier, the executable was named 300 | `versioneer-installer` and was run without an argument. 301 | 302 | ### Upgrading to 0.14 303 | 304 | 0.14 changes the format of the version string. 0.13 and earlier used 305 | hyphen-separated strings like "0.11-2-g1076c97-dirty". 0.14 and beyond use a 306 | plus-separated "local version" section strings, with dot-separated 307 | components, like "0.11+2.g1076c97". PEP440-strict tools did not like the old 308 | format, but should be ok with the new one. 309 | 310 | ### Upgrading from 0.11 to 0.12 311 | 312 | Nothing special. 313 | 314 | ### Upgrading from 0.10 to 0.11 315 | 316 | You must add a `versioneer.VCS = "git"` to your `setup.py` before re-running 317 | `setup.py setup_versioneer`. This will enable the use of additional 318 | version-control systems (SVN, etc) in the future. 319 | 320 | ## Future Directions 321 | 322 | This tool is designed to make it easily extended to other version-control 323 | systems: all VCS-specific components are in separate directories like 324 | src/git/ . The top-level `versioneer.py` script is assembled from these 325 | components by running make-versioneer.py . In the future, make-versioneer.py 326 | will take a VCS name as an argument, and will construct a version of 327 | `versioneer.py` that is specific to the given VCS. It might also take the 328 | configuration arguments that are currently provided manually during 329 | installation by editing setup.py . Alternatively, it might go the other 330 | direction and include code from all supported VCS systems, reducing the 331 | number of intermediate scripts. 332 | 333 | 334 | ## License 335 | 336 | To make Versioneer easier to embed, all its code is hereby released into the 337 | public domain. The `_version.py` that it creates is also in the public 338 | domain. 339 | 340 | """ 341 | 342 | from __future__ import print_function 343 | try: 344 | import configparser 345 | except ImportError: 346 | import ConfigParser as configparser 347 | import errno 348 | import json 349 | import os 350 | import re 351 | import subprocess 352 | import sys 353 | 354 | 355 | class VersioneerConfig: 356 | pass 357 | 358 | 359 | def get_root(): 360 | # we require that all commands are run from the project root, i.e. the 361 | # directory that contains setup.py, setup.cfg, and versioneer.py . 362 | root = os.path.realpath(os.path.abspath(os.getcwd())) 363 | setup_py = os.path.join(root, "setup.py") 364 | versioneer_py = os.path.join(root, "versioneer.py") 365 | if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): 366 | # allow 'python path/to/setup.py COMMAND' 367 | root = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0]))) 368 | setup_py = os.path.join(root, "setup.py") 369 | versioneer_py = os.path.join(root, "versioneer.py") 370 | if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): 371 | err = ("Versioneer was unable to run the project root directory. " 372 | "Versioneer requires setup.py to be executed from " 373 | "its immediate directory (like 'python setup.py COMMAND'), " 374 | "or in a way that lets it use sys.argv[0] to find the root " 375 | "(like 'python path/to/setup.py COMMAND').") 376 | raise VersioneerBadRootError(err) 377 | try: 378 | # Certain runtime workflows (setup.py install/develop in a setuptools 379 | # tree) execute all dependencies in a single python process, so 380 | # "versioneer" may be imported multiple times, and python's shared 381 | # module-import table will cache the first one. So we can't use 382 | # os.path.dirname(__file__), as that will find whichever 383 | # versioneer.py was first imported, even in later projects. 384 | me = os.path.realpath(os.path.abspath(__file__)) 385 | if os.path.splitext(me)[0] != os.path.splitext(versioneer_py)[0]: 386 | print("Warning: build in %s is using versioneer.py from %s" 387 | % (os.path.dirname(me), versioneer_py)) 388 | except NameError: 389 | pass 390 | return root 391 | 392 | 393 | def get_config_from_root(root): 394 | # This might raise EnvironmentError (if setup.cfg is missing), or 395 | # configparser.NoSectionError (if it lacks a [versioneer] section), or 396 | # configparser.NoOptionError (if it lacks "VCS="). See the docstring at 397 | # the top of versioneer.py for instructions on writing your setup.cfg . 398 | setup_cfg = os.path.join(root, "setup.cfg") 399 | parser = configparser.SafeConfigParser() 400 | with open(setup_cfg, "r") as f: 401 | parser.readfp(f) 402 | VCS = parser.get("versioneer", "VCS") # mandatory 403 | 404 | def get(parser, name): 405 | if parser.has_option("versioneer", name): 406 | return parser.get("versioneer", name) 407 | return None 408 | cfg = VersioneerConfig() 409 | cfg.VCS = VCS 410 | cfg.style = get(parser, "style") or "" 411 | cfg.versionfile_source = get(parser, "versionfile_source") 412 | cfg.versionfile_build = get(parser, "versionfile_build") 413 | cfg.tag_prefix = get(parser, "tag_prefix") 414 | cfg.parentdir_prefix = get(parser, "parentdir_prefix") 415 | cfg.verbose = get(parser, "verbose") 416 | return cfg 417 | 418 | 419 | class NotThisMethod(Exception): 420 | pass 421 | 422 | # these dictionaries contain VCS-specific tools 423 | LONG_VERSION_PY = {} 424 | HANDLERS = {} 425 | 426 | 427 | def register_vcs_handler(vcs, method): # decorator 428 | def decorate(f): 429 | if vcs not in HANDLERS: 430 | HANDLERS[vcs] = {} 431 | HANDLERS[vcs][method] = f 432 | return f 433 | return decorate 434 | 435 | 436 | def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): 437 | assert isinstance(commands, list) 438 | p = None 439 | for c in commands: 440 | try: 441 | dispcmd = str([c] + args) 442 | # remember shell=False, so use git.cmd on windows, not just git 443 | p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE, 444 | stderr=(subprocess.PIPE if hide_stderr 445 | else None)) 446 | break 447 | except EnvironmentError: 448 | e = sys.exc_info()[1] 449 | if e.errno == errno.ENOENT: 450 | continue 451 | if verbose: 452 | print("unable to run %s" % dispcmd) 453 | print(e) 454 | return None 455 | else: 456 | if verbose: 457 | print("unable to find command, tried %s" % (commands,)) 458 | return None 459 | stdout = p.communicate()[0].strip() 460 | if sys.version_info[0] >= 3: 461 | stdout = stdout.decode() 462 | if p.returncode != 0: 463 | if verbose: 464 | print("unable to run %s (error)" % dispcmd) 465 | return None 466 | return stdout 467 | LONG_VERSION_PY['git'] = ''' 468 | # This file helps to compute a version number in source trees obtained from 469 | # git-archive tarball (such as those provided by githubs download-from-tag 470 | # feature). Distribution tarballs (built by setup.py sdist) and build 471 | # directories (produced by setup.py build) will contain a much shorter file 472 | # that just contains the computed version number. 473 | 474 | # This file is released into the public domain. Generated by 475 | # versioneer-0.15 (https://github.com/warner/python-versioneer) 476 | 477 | import errno 478 | import os 479 | import re 480 | import subprocess 481 | import sys 482 | 483 | 484 | def get_keywords(): 485 | # these strings will be replaced by git during git-archive. 486 | # setup.py/versioneer.py will grep for the variable names, so they must 487 | # each be defined on a line of their own. _version.py will just call 488 | # get_keywords(). 489 | git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s" 490 | git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s" 491 | keywords = {"refnames": git_refnames, "full": git_full} 492 | return keywords 493 | 494 | 495 | class VersioneerConfig: 496 | pass 497 | 498 | 499 | def get_config(): 500 | # these strings are filled in when 'setup.py versioneer' creates 501 | # _version.py 502 | cfg = VersioneerConfig() 503 | cfg.VCS = "git" 504 | cfg.style = "%(STYLE)s" 505 | cfg.tag_prefix = "%(TAG_PREFIX)s" 506 | cfg.parentdir_prefix = "%(PARENTDIR_PREFIX)s" 507 | cfg.versionfile_source = "%(VERSIONFILE_SOURCE)s" 508 | cfg.verbose = False 509 | return cfg 510 | 511 | 512 | class NotThisMethod(Exception): 513 | pass 514 | 515 | 516 | LONG_VERSION_PY = {} 517 | HANDLERS = {} 518 | 519 | 520 | def register_vcs_handler(vcs, method): # decorator 521 | def decorate(f): 522 | if vcs not in HANDLERS: 523 | HANDLERS[vcs] = {} 524 | HANDLERS[vcs][method] = f 525 | return f 526 | return decorate 527 | 528 | 529 | def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): 530 | assert isinstance(commands, list) 531 | p = None 532 | for c in commands: 533 | try: 534 | dispcmd = str([c] + args) 535 | # remember shell=False, so use git.cmd on windows, not just git 536 | p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE, 537 | stderr=(subprocess.PIPE if hide_stderr 538 | else None)) 539 | break 540 | except EnvironmentError: 541 | e = sys.exc_info()[1] 542 | if e.errno == errno.ENOENT: 543 | continue 544 | if verbose: 545 | print("unable to run %%s" %% dispcmd) 546 | print(e) 547 | return None 548 | else: 549 | if verbose: 550 | print("unable to find command, tried %%s" %% (commands,)) 551 | return None 552 | stdout = p.communicate()[0].strip() 553 | if sys.version_info[0] >= 3: 554 | stdout = stdout.decode() 555 | if p.returncode != 0: 556 | if verbose: 557 | print("unable to run %%s (error)" %% dispcmd) 558 | return None 559 | return stdout 560 | 561 | 562 | def versions_from_parentdir(parentdir_prefix, root, verbose): 563 | # Source tarballs conventionally unpack into a directory that includes 564 | # both the project name and a version string. 565 | dirname = os.path.basename(root) 566 | if not dirname.startswith(parentdir_prefix): 567 | if verbose: 568 | print("guessing rootdir is '%%s', but '%%s' doesn't start with " 569 | "prefix '%%s'" %% (root, dirname, parentdir_prefix)) 570 | raise NotThisMethod("rootdir doesn't start with parentdir_prefix") 571 | return {"version": dirname[len(parentdir_prefix):], 572 | "full-revisionid": None, 573 | "dirty": False, "error": None} 574 | 575 | 576 | @register_vcs_handler("git", "get_keywords") 577 | def git_get_keywords(versionfile_abs): 578 | # the code embedded in _version.py can just fetch the value of these 579 | # keywords. When used from setup.py, we don't want to import _version.py, 580 | # so we do it with a regexp instead. This function is not used from 581 | # _version.py. 582 | keywords = {} 583 | try: 584 | f = open(versionfile_abs, "r") 585 | for line in f.readlines(): 586 | if line.strip().startswith("git_refnames ="): 587 | mo = re.search(r'=\s*"(.*)"', line) 588 | if mo: 589 | keywords["refnames"] = mo.group(1) 590 | if line.strip().startswith("git_full ="): 591 | mo = re.search(r'=\s*"(.*)"', line) 592 | if mo: 593 | keywords["full"] = mo.group(1) 594 | f.close() 595 | except EnvironmentError: 596 | pass 597 | return keywords 598 | 599 | 600 | @register_vcs_handler("git", "keywords") 601 | def git_versions_from_keywords(keywords, tag_prefix, verbose): 602 | if not keywords: 603 | raise NotThisMethod("no keywords at all, weird") 604 | refnames = keywords["refnames"].strip() 605 | if refnames.startswith("$Format"): 606 | if verbose: 607 | print("keywords are unexpanded, not using") 608 | raise NotThisMethod("unexpanded keywords, not a git-archive tarball") 609 | refs = set([r.strip() for r in refnames.strip("()").split(",")]) 610 | # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of 611 | # just "foo-1.0". If we see a "tag: " prefix, prefer those. 612 | TAG = "tag: " 613 | tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) 614 | if not tags: 615 | # Either we're using git < 1.8.3, or there really are no tags. We use 616 | # a heuristic: assume all version tags have a digit. The old git %%d 617 | # expansion behaves like git log --decorate=short and strips out the 618 | # refs/heads/ and refs/tags/ prefixes that would let us distinguish 619 | # between branches and tags. By ignoring refnames without digits, we 620 | # filter out many common branch names like "release" and 621 | # "stabilization", as well as "HEAD" and "master". 622 | tags = set([r for r in refs if re.search(r'\d', r)]) 623 | if verbose: 624 | print("discarding '%%s', no digits" %% ",".join(refs-tags)) 625 | if verbose: 626 | print("likely tags: %%s" %% ",".join(sorted(tags))) 627 | for ref in sorted(tags): 628 | # sorting will prefer e.g. "2.0" over "2.0rc1" 629 | if ref.startswith(tag_prefix): 630 | r = ref[len(tag_prefix):] 631 | if verbose: 632 | print("picking %%s" %% r) 633 | return {"version": r, 634 | "full-revisionid": keywords["full"].strip(), 635 | "dirty": False, "error": None 636 | } 637 | # no suitable tags, so version is "0+unknown", but full hex is still there 638 | if verbose: 639 | print("no suitable tags, using unknown + full revision id") 640 | return {"version": "0+unknown", 641 | "full-revisionid": keywords["full"].strip(), 642 | "dirty": False, "error": "no suitable tags"} 643 | 644 | 645 | @register_vcs_handler("git", "pieces_from_vcs") 646 | def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): 647 | # this runs 'git' from the root of the source tree. This only gets called 648 | # if the git-archive 'subst' keywords were *not* expanded, and 649 | # _version.py hasn't already been rewritten with a short version string, 650 | # meaning we're inside a checked out source tree. 651 | 652 | if not os.path.exists(os.path.join(root, ".git")): 653 | if verbose: 654 | print("no .git in %%s" %% root) 655 | raise NotThisMethod("no .git directory") 656 | 657 | GITS = ["git"] 658 | if sys.platform == "win32": 659 | GITS = ["git.cmd", "git.exe"] 660 | # if there is a tag, this yields TAG-NUM-gHEX[-dirty] 661 | # if there are no tags, this yields HEX[-dirty] (no NUM) 662 | describe_out = run_command(GITS, ["describe", "--tags", "--dirty", 663 | "--always", "--long"], 664 | cwd=root) 665 | # --long was added in git-1.5.5 666 | if describe_out is None: 667 | raise NotThisMethod("'git describe' failed") 668 | describe_out = describe_out.strip() 669 | full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) 670 | if full_out is None: 671 | raise NotThisMethod("'git rev-parse' failed") 672 | full_out = full_out.strip() 673 | 674 | pieces = {} 675 | pieces["long"] = full_out 676 | pieces["short"] = full_out[:7] # maybe improved later 677 | pieces["error"] = None 678 | 679 | # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] 680 | # TAG might have hyphens. 681 | git_describe = describe_out 682 | 683 | # look for -dirty suffix 684 | dirty = git_describe.endswith("-dirty") 685 | pieces["dirty"] = dirty 686 | if dirty: 687 | git_describe = git_describe[:git_describe.rindex("-dirty")] 688 | 689 | # now we have TAG-NUM-gHEX or HEX 690 | 691 | if "-" in git_describe: 692 | # TAG-NUM-gHEX 693 | mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) 694 | if not mo: 695 | # unparseable. Maybe git-describe is misbehaving? 696 | pieces["error"] = ("unable to parse git-describe output: '%%s'" 697 | %% describe_out) 698 | return pieces 699 | 700 | # tag 701 | full_tag = mo.group(1) 702 | if not full_tag.startswith(tag_prefix): 703 | if verbose: 704 | fmt = "tag '%%s' doesn't start with prefix '%%s'" 705 | print(fmt %% (full_tag, tag_prefix)) 706 | pieces["error"] = ("tag '%%s' doesn't start with prefix '%%s'" 707 | %% (full_tag, tag_prefix)) 708 | return pieces 709 | pieces["closest-tag"] = full_tag[len(tag_prefix):] 710 | 711 | # distance: number of commits since tag 712 | pieces["distance"] = int(mo.group(2)) 713 | 714 | # commit: short hex revision ID 715 | pieces["short"] = mo.group(3) 716 | 717 | else: 718 | # HEX: no tags 719 | pieces["closest-tag"] = None 720 | count_out = run_command(GITS, ["rev-list", "HEAD", "--count"], 721 | cwd=root) 722 | pieces["distance"] = int(count_out) # total number of commits 723 | 724 | return pieces 725 | 726 | 727 | def plus_or_dot(pieces): 728 | if "+" in pieces.get("closest-tag", ""): 729 | return "." 730 | return "+" 731 | 732 | 733 | def render_pep440(pieces): 734 | # now build up version string, with post-release "local version 735 | # identifier". Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you 736 | # get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty 737 | 738 | # exceptions: 739 | # 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] 740 | 741 | if pieces["closest-tag"]: 742 | rendered = pieces["closest-tag"] 743 | if pieces["distance"] or pieces["dirty"]: 744 | rendered += plus_or_dot(pieces) 745 | rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"]) 746 | if pieces["dirty"]: 747 | rendered += ".dirty" 748 | else: 749 | # exception #1 750 | rendered = "0+untagged.%%d.g%%s" %% (pieces["distance"], 751 | pieces["short"]) 752 | if pieces["dirty"]: 753 | rendered += ".dirty" 754 | return rendered 755 | 756 | 757 | def render_pep440_pre(pieces): 758 | # TAG[.post.devDISTANCE] . No -dirty 759 | 760 | # exceptions: 761 | # 1: no tags. 0.post.devDISTANCE 762 | 763 | if pieces["closest-tag"]: 764 | rendered = pieces["closest-tag"] 765 | if pieces["distance"]: 766 | rendered += ".post.dev%%d" %% pieces["distance"] 767 | else: 768 | # exception #1 769 | rendered = "0.post.dev%%d" %% pieces["distance"] 770 | return rendered 771 | 772 | 773 | def render_pep440_post(pieces): 774 | # TAG[.postDISTANCE[.dev0]+gHEX] . The ".dev0" means dirty. Note that 775 | # .dev0 sorts backwards (a dirty tree will appear "older" than the 776 | # corresponding clean one), but you shouldn't be releasing software with 777 | # -dirty anyways. 778 | 779 | # exceptions: 780 | # 1: no tags. 0.postDISTANCE[.dev0] 781 | 782 | if pieces["closest-tag"]: 783 | rendered = pieces["closest-tag"] 784 | if pieces["distance"] or pieces["dirty"]: 785 | rendered += ".post%%d" %% pieces["distance"] 786 | if pieces["dirty"]: 787 | rendered += ".dev0" 788 | rendered += plus_or_dot(pieces) 789 | rendered += "g%%s" %% pieces["short"] 790 | else: 791 | # exception #1 792 | rendered = "0.post%%d" %% pieces["distance"] 793 | if pieces["dirty"]: 794 | rendered += ".dev0" 795 | rendered += "+g%%s" %% pieces["short"] 796 | return rendered 797 | 798 | 799 | def render_pep440_old(pieces): 800 | # TAG[.postDISTANCE[.dev0]] . The ".dev0" means dirty. 801 | 802 | # exceptions: 803 | # 1: no tags. 0.postDISTANCE[.dev0] 804 | 805 | if pieces["closest-tag"]: 806 | rendered = pieces["closest-tag"] 807 | if pieces["distance"] or pieces["dirty"]: 808 | rendered += ".post%%d" %% pieces["distance"] 809 | if pieces["dirty"]: 810 | rendered += ".dev0" 811 | else: 812 | # exception #1 813 | rendered = "0.post%%d" %% pieces["distance"] 814 | if pieces["dirty"]: 815 | rendered += ".dev0" 816 | return rendered 817 | 818 | 819 | def render_git_describe(pieces): 820 | # TAG[-DISTANCE-gHEX][-dirty], like 'git describe --tags --dirty 821 | # --always' 822 | 823 | # exceptions: 824 | # 1: no tags. HEX[-dirty] (note: no 'g' prefix) 825 | 826 | if pieces["closest-tag"]: 827 | rendered = pieces["closest-tag"] 828 | if pieces["distance"]: 829 | rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) 830 | else: 831 | # exception #1 832 | rendered = pieces["short"] 833 | if pieces["dirty"]: 834 | rendered += "-dirty" 835 | return rendered 836 | 837 | 838 | def render_git_describe_long(pieces): 839 | # TAG-DISTANCE-gHEX[-dirty], like 'git describe --tags --dirty 840 | # --always -long'. The distance/hash is unconditional. 841 | 842 | # exceptions: 843 | # 1: no tags. HEX[-dirty] (note: no 'g' prefix) 844 | 845 | if pieces["closest-tag"]: 846 | rendered = pieces["closest-tag"] 847 | rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) 848 | else: 849 | # exception #1 850 | rendered = pieces["short"] 851 | if pieces["dirty"]: 852 | rendered += "-dirty" 853 | return rendered 854 | 855 | 856 | def render(pieces, style): 857 | if pieces["error"]: 858 | return {"version": "unknown", 859 | "full-revisionid": pieces.get("long"), 860 | "dirty": None, 861 | "error": pieces["error"]} 862 | 863 | if not style or style == "default": 864 | style = "pep440" # the default 865 | 866 | if style == "pep440": 867 | rendered = render_pep440(pieces) 868 | elif style == "pep440-pre": 869 | rendered = render_pep440_pre(pieces) 870 | elif style == "pep440-post": 871 | rendered = render_pep440_post(pieces) 872 | elif style == "pep440-old": 873 | rendered = render_pep440_old(pieces) 874 | elif style == "git-describe": 875 | rendered = render_git_describe(pieces) 876 | elif style == "git-describe-long": 877 | rendered = render_git_describe_long(pieces) 878 | else: 879 | raise ValueError("unknown style '%%s'" %% style) 880 | 881 | return {"version": rendered, "full-revisionid": pieces["long"], 882 | "dirty": pieces["dirty"], "error": None} 883 | 884 | 885 | def get_versions(): 886 | # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have 887 | # __file__, we can work backwards from there to the root. Some 888 | # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which 889 | # case we can only use expanded keywords. 890 | 891 | cfg = get_config() 892 | verbose = cfg.verbose 893 | 894 | try: 895 | return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, 896 | verbose) 897 | except NotThisMethod: 898 | pass 899 | 900 | try: 901 | root = os.path.realpath(__file__) 902 | # versionfile_source is the relative path from the top of the source 903 | # tree (where the .git directory might live) to this file. Invert 904 | # this to find the root from __file__. 905 | for i in cfg.versionfile_source.split('/'): 906 | root = os.path.dirname(root) 907 | except NameError: 908 | return {"version": "0+unknown", "full-revisionid": None, 909 | "dirty": None, 910 | "error": "unable to find root of source tree"} 911 | 912 | try: 913 | pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) 914 | return render(pieces, cfg.style) 915 | except NotThisMethod: 916 | pass 917 | 918 | try: 919 | if cfg.parentdir_prefix: 920 | return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) 921 | except NotThisMethod: 922 | pass 923 | 924 | return {"version": "0+unknown", "full-revisionid": None, 925 | "dirty": None, 926 | "error": "unable to compute version"} 927 | ''' 928 | 929 | 930 | @register_vcs_handler("git", "get_keywords") 931 | def git_get_keywords(versionfile_abs): 932 | # the code embedded in _version.py can just fetch the value of these 933 | # keywords. When used from setup.py, we don't want to import _version.py, 934 | # so we do it with a regexp instead. This function is not used from 935 | # _version.py. 936 | keywords = {} 937 | try: 938 | f = open(versionfile_abs, "r") 939 | for line in f.readlines(): 940 | if line.strip().startswith("git_refnames ="): 941 | mo = re.search(r'=\s*"(.*)"', line) 942 | if mo: 943 | keywords["refnames"] = mo.group(1) 944 | if line.strip().startswith("git_full ="): 945 | mo = re.search(r'=\s*"(.*)"', line) 946 | if mo: 947 | keywords["full"] = mo.group(1) 948 | f.close() 949 | except EnvironmentError: 950 | pass 951 | return keywords 952 | 953 | 954 | @register_vcs_handler("git", "keywords") 955 | def git_versions_from_keywords(keywords, tag_prefix, verbose): 956 | if not keywords: 957 | raise NotThisMethod("no keywords at all, weird") 958 | refnames = keywords["refnames"].strip() 959 | if refnames.startswith("$Format"): 960 | if verbose: 961 | print("keywords are unexpanded, not using") 962 | raise NotThisMethod("unexpanded keywords, not a git-archive tarball") 963 | refs = set([r.strip() for r in refnames.strip("()").split(",")]) 964 | # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of 965 | # just "foo-1.0". If we see a "tag: " prefix, prefer those. 966 | TAG = "tag: " 967 | tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) 968 | if not tags: 969 | # Either we're using git < 1.8.3, or there really are no tags. We use 970 | # a heuristic: assume all version tags have a digit. The old git %d 971 | # expansion behaves like git log --decorate=short and strips out the 972 | # refs/heads/ and refs/tags/ prefixes that would let us distinguish 973 | # between branches and tags. By ignoring refnames without digits, we 974 | # filter out many common branch names like "release" and 975 | # "stabilization", as well as "HEAD" and "master". 976 | tags = set([r for r in refs if re.search(r'\d', r)]) 977 | if verbose: 978 | print("discarding '%s', no digits" % ",".join(refs-tags)) 979 | if verbose: 980 | print("likely tags: %s" % ",".join(sorted(tags))) 981 | for ref in sorted(tags): 982 | # sorting will prefer e.g. "2.0" over "2.0rc1" 983 | if ref.startswith(tag_prefix): 984 | r = ref[len(tag_prefix):] 985 | if verbose: 986 | print("picking %s" % r) 987 | return {"version": r, 988 | "full-revisionid": keywords["full"].strip(), 989 | "dirty": False, "error": None 990 | } 991 | # no suitable tags, so version is "0+unknown", but full hex is still there 992 | if verbose: 993 | print("no suitable tags, using unknown + full revision id") 994 | return {"version": "0+unknown", 995 | "full-revisionid": keywords["full"].strip(), 996 | "dirty": False, "error": "no suitable tags"} 997 | 998 | 999 | @register_vcs_handler("git", "pieces_from_vcs") 1000 | def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): 1001 | # this runs 'git' from the root of the source tree. This only gets called 1002 | # if the git-archive 'subst' keywords were *not* expanded, and 1003 | # _version.py hasn't already been rewritten with a short version string, 1004 | # meaning we're inside a checked out source tree. 1005 | 1006 | if not os.path.exists(os.path.join(root, ".git")): 1007 | if verbose: 1008 | print("no .git in %s" % root) 1009 | raise NotThisMethod("no .git directory") 1010 | 1011 | GITS = ["git"] 1012 | if sys.platform == "win32": 1013 | GITS = ["git.cmd", "git.exe"] 1014 | # if there is a tag, this yields TAG-NUM-gHEX[-dirty] 1015 | # if there are no tags, this yields HEX[-dirty] (no NUM) 1016 | describe_out = run_command(GITS, ["describe", "--tags", "--dirty", 1017 | "--always", "--long"], 1018 | cwd=root) 1019 | # --long was added in git-1.5.5 1020 | if describe_out is None: 1021 | raise NotThisMethod("'git describe' failed") 1022 | describe_out = describe_out.strip() 1023 | full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) 1024 | if full_out is None: 1025 | raise NotThisMethod("'git rev-parse' failed") 1026 | full_out = full_out.strip() 1027 | 1028 | pieces = {} 1029 | pieces["long"] = full_out 1030 | pieces["short"] = full_out[:7] # maybe improved later 1031 | pieces["error"] = None 1032 | 1033 | # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] 1034 | # TAG might have hyphens. 1035 | git_describe = describe_out 1036 | 1037 | # look for -dirty suffix 1038 | dirty = git_describe.endswith("-dirty") 1039 | pieces["dirty"] = dirty 1040 | if dirty: 1041 | git_describe = git_describe[:git_describe.rindex("-dirty")] 1042 | 1043 | # now we have TAG-NUM-gHEX or HEX 1044 | 1045 | if "-" in git_describe: 1046 | # TAG-NUM-gHEX 1047 | mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) 1048 | if not mo: 1049 | # unparseable. Maybe git-describe is misbehaving? 1050 | pieces["error"] = ("unable to parse git-describe output: '%s'" 1051 | % describe_out) 1052 | return pieces 1053 | 1054 | # tag 1055 | full_tag = mo.group(1) 1056 | if not full_tag.startswith(tag_prefix): 1057 | if verbose: 1058 | fmt = "tag '%s' doesn't start with prefix '%s'" 1059 | print(fmt % (full_tag, tag_prefix)) 1060 | pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" 1061 | % (full_tag, tag_prefix)) 1062 | return pieces 1063 | pieces["closest-tag"] = full_tag[len(tag_prefix):] 1064 | 1065 | # distance: number of commits since tag 1066 | pieces["distance"] = int(mo.group(2)) 1067 | 1068 | # commit: short hex revision ID 1069 | pieces["short"] = mo.group(3) 1070 | 1071 | else: 1072 | # HEX: no tags 1073 | pieces["closest-tag"] = None 1074 | count_out = run_command(GITS, ["rev-list", "HEAD", "--count"], 1075 | cwd=root) 1076 | pieces["distance"] = int(count_out) # total number of commits 1077 | 1078 | return pieces 1079 | 1080 | 1081 | def do_vcs_install(manifest_in, versionfile_source, ipy): 1082 | GITS = ["git"] 1083 | if sys.platform == "win32": 1084 | GITS = ["git.cmd", "git.exe"] 1085 | files = [manifest_in, versionfile_source] 1086 | if ipy: 1087 | files.append(ipy) 1088 | try: 1089 | me = __file__ 1090 | if me.endswith(".pyc") or me.endswith(".pyo"): 1091 | me = os.path.splitext(me)[0] + ".py" 1092 | versioneer_file = os.path.relpath(me) 1093 | except NameError: 1094 | versioneer_file = "versioneer.py" 1095 | files.append(versioneer_file) 1096 | present = False 1097 | try: 1098 | f = open(".gitattributes", "r") 1099 | for line in f.readlines(): 1100 | if line.strip().startswith(versionfile_source): 1101 | if "export-subst" in line.strip().split()[1:]: 1102 | present = True 1103 | f.close() 1104 | except EnvironmentError: 1105 | pass 1106 | if not present: 1107 | f = open(".gitattributes", "a+") 1108 | f.write("%s export-subst\n" % versionfile_source) 1109 | f.close() 1110 | files.append(".gitattributes") 1111 | run_command(GITS, ["add", "--"] + files) 1112 | 1113 | 1114 | def versions_from_parentdir(parentdir_prefix, root, verbose): 1115 | # Source tarballs conventionally unpack into a directory that includes 1116 | # both the project name and a version string. 1117 | dirname = os.path.basename(root) 1118 | if not dirname.startswith(parentdir_prefix): 1119 | if verbose: 1120 | print("guessing rootdir is '%s', but '%s' doesn't start with " 1121 | "prefix '%s'" % (root, dirname, parentdir_prefix)) 1122 | raise NotThisMethod("rootdir doesn't start with parentdir_prefix") 1123 | return {"version": dirname[len(parentdir_prefix):], 1124 | "full-revisionid": None, 1125 | "dirty": False, "error": None} 1126 | 1127 | SHORT_VERSION_PY = """ 1128 | # This file was generated by 'versioneer.py' (0.15) from 1129 | # revision-control system data, or from the parent directory name of an 1130 | # unpacked source archive. Distribution tarballs contain a pre-generated copy 1131 | # of this file. 1132 | 1133 | import json 1134 | import sys 1135 | 1136 | version_json = ''' 1137 | %s 1138 | ''' # END VERSION_JSON 1139 | 1140 | 1141 | def get_versions(): 1142 | return json.loads(version_json) 1143 | """ 1144 | 1145 | 1146 | def versions_from_file(filename): 1147 | try: 1148 | with open(filename) as f: 1149 | contents = f.read() 1150 | except EnvironmentError: 1151 | raise NotThisMethod("unable to read _version.py") 1152 | mo = re.search(r"version_json = '''\n(.*)''' # END VERSION_JSON", 1153 | contents, re.M | re.S) 1154 | if not mo: 1155 | raise NotThisMethod("no version_json in _version.py") 1156 | return json.loads(mo.group(1)) 1157 | 1158 | 1159 | def write_to_version_file(filename, versions): 1160 | os.unlink(filename) 1161 | contents = json.dumps(versions, sort_keys=True, 1162 | indent=1, separators=(",", ": ")) 1163 | with open(filename, "w") as f: 1164 | f.write(SHORT_VERSION_PY % contents) 1165 | 1166 | print("set %s to '%s'" % (filename, versions["version"])) 1167 | 1168 | 1169 | def plus_or_dot(pieces): 1170 | if "+" in pieces.get("closest-tag", ""): 1171 | return "." 1172 | return "+" 1173 | 1174 | 1175 | def render_pep440(pieces): 1176 | # now build up version string, with post-release "local version 1177 | # identifier". Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you 1178 | # get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty 1179 | 1180 | # exceptions: 1181 | # 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] 1182 | 1183 | if pieces["closest-tag"]: 1184 | rendered = pieces["closest-tag"] 1185 | if pieces["distance"] or pieces["dirty"]: 1186 | rendered += plus_or_dot(pieces) 1187 | rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) 1188 | if pieces["dirty"]: 1189 | rendered += ".dirty" 1190 | else: 1191 | # exception #1 1192 | rendered = "0+untagged.%d.g%s" % (pieces["distance"], 1193 | pieces["short"]) 1194 | if pieces["dirty"]: 1195 | rendered += ".dirty" 1196 | return rendered 1197 | 1198 | 1199 | def render_pep440_pre(pieces): 1200 | # TAG[.post.devDISTANCE] . No -dirty 1201 | 1202 | # exceptions: 1203 | # 1: no tags. 0.post.devDISTANCE 1204 | 1205 | if pieces["closest-tag"]: 1206 | rendered = pieces["closest-tag"] 1207 | if pieces["distance"]: 1208 | rendered += ".post.dev%d" % pieces["distance"] 1209 | else: 1210 | # exception #1 1211 | rendered = "0.post.dev%d" % pieces["distance"] 1212 | return rendered 1213 | 1214 | 1215 | def render_pep440_post(pieces): 1216 | # TAG[.postDISTANCE[.dev0]+gHEX] . The ".dev0" means dirty. Note that 1217 | # .dev0 sorts backwards (a dirty tree will appear "older" than the 1218 | # corresponding clean one), but you shouldn't be releasing software with 1219 | # -dirty anyways. 1220 | 1221 | # exceptions: 1222 | # 1: no tags. 0.postDISTANCE[.dev0] 1223 | 1224 | if pieces["closest-tag"]: 1225 | rendered = pieces["closest-tag"] 1226 | if pieces["distance"] or pieces["dirty"]: 1227 | rendered += ".post%d" % pieces["distance"] 1228 | if pieces["dirty"]: 1229 | rendered += ".dev0" 1230 | rendered += plus_or_dot(pieces) 1231 | rendered += "g%s" % pieces["short"] 1232 | else: 1233 | # exception #1 1234 | rendered = "0.post%d" % pieces["distance"] 1235 | if pieces["dirty"]: 1236 | rendered += ".dev0" 1237 | rendered += "+g%s" % pieces["short"] 1238 | return rendered 1239 | 1240 | 1241 | def render_pep440_old(pieces): 1242 | # TAG[.postDISTANCE[.dev0]] . The ".dev0" means dirty. 1243 | 1244 | # exceptions: 1245 | # 1: no tags. 0.postDISTANCE[.dev0] 1246 | 1247 | if pieces["closest-tag"]: 1248 | rendered = pieces["closest-tag"] 1249 | if pieces["distance"] or pieces["dirty"]: 1250 | rendered += ".post%d" % pieces["distance"] 1251 | if pieces["dirty"]: 1252 | rendered += ".dev0" 1253 | else: 1254 | # exception #1 1255 | rendered = "0.post%d" % pieces["distance"] 1256 | if pieces["dirty"]: 1257 | rendered += ".dev0" 1258 | return rendered 1259 | 1260 | 1261 | def render_git_describe(pieces): 1262 | # TAG[-DISTANCE-gHEX][-dirty], like 'git describe --tags --dirty 1263 | # --always' 1264 | 1265 | # exceptions: 1266 | # 1: no tags. HEX[-dirty] (note: no 'g' prefix) 1267 | 1268 | if pieces["closest-tag"]: 1269 | rendered = pieces["closest-tag"] 1270 | if pieces["distance"]: 1271 | rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) 1272 | else: 1273 | # exception #1 1274 | rendered = pieces["short"] 1275 | if pieces["dirty"]: 1276 | rendered += "-dirty" 1277 | return rendered 1278 | 1279 | 1280 | def render_git_describe_long(pieces): 1281 | # TAG-DISTANCE-gHEX[-dirty], like 'git describe --tags --dirty 1282 | # --always -long'. The distance/hash is unconditional. 1283 | 1284 | # exceptions: 1285 | # 1: no tags. HEX[-dirty] (note: no 'g' prefix) 1286 | 1287 | if pieces["closest-tag"]: 1288 | rendered = pieces["closest-tag"] 1289 | rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) 1290 | else: 1291 | # exception #1 1292 | rendered = pieces["short"] 1293 | if pieces["dirty"]: 1294 | rendered += "-dirty" 1295 | return rendered 1296 | 1297 | 1298 | def render(pieces, style): 1299 | if pieces["error"]: 1300 | return {"version": "unknown", 1301 | "full-revisionid": pieces.get("long"), 1302 | "dirty": None, 1303 | "error": pieces["error"]} 1304 | 1305 | if not style or style == "default": 1306 | style = "pep440" # the default 1307 | 1308 | if style == "pep440": 1309 | rendered = render_pep440(pieces) 1310 | elif style == "pep440-pre": 1311 | rendered = render_pep440_pre(pieces) 1312 | elif style == "pep440-post": 1313 | rendered = render_pep440_post(pieces) 1314 | elif style == "pep440-old": 1315 | rendered = render_pep440_old(pieces) 1316 | elif style == "git-describe": 1317 | rendered = render_git_describe(pieces) 1318 | elif style == "git-describe-long": 1319 | rendered = render_git_describe_long(pieces) 1320 | else: 1321 | raise ValueError("unknown style '%s'" % style) 1322 | 1323 | return {"version": rendered, "full-revisionid": pieces["long"], 1324 | "dirty": pieces["dirty"], "error": None} 1325 | 1326 | 1327 | class VersioneerBadRootError(Exception): 1328 | pass 1329 | 1330 | 1331 | def get_versions(verbose=False): 1332 | # returns dict with two keys: 'version' and 'full' 1333 | 1334 | if "versioneer" in sys.modules: 1335 | # see the discussion in cmdclass.py:get_cmdclass() 1336 | del sys.modules["versioneer"] 1337 | 1338 | root = get_root() 1339 | cfg = get_config_from_root(root) 1340 | 1341 | assert cfg.VCS is not None, "please set [versioneer]VCS= in setup.cfg" 1342 | handlers = HANDLERS.get(cfg.VCS) 1343 | assert handlers, "unrecognized VCS '%s'" % cfg.VCS 1344 | verbose = verbose or cfg.verbose 1345 | assert cfg.versionfile_source is not None, \ 1346 | "please set versioneer.versionfile_source" 1347 | assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix" 1348 | 1349 | versionfile_abs = os.path.join(root, cfg.versionfile_source) 1350 | 1351 | # extract version from first of: _version.py, VCS command (e.g. 'git 1352 | # describe'), parentdir. This is meant to work for developers using a 1353 | # source checkout, for users of a tarball created by 'setup.py sdist', 1354 | # and for users of a tarball/zipball created by 'git archive' or github's 1355 | # download-from-tag feature or the equivalent in other VCSes. 1356 | 1357 | get_keywords_f = handlers.get("get_keywords") 1358 | from_keywords_f = handlers.get("keywords") 1359 | if get_keywords_f and from_keywords_f: 1360 | try: 1361 | keywords = get_keywords_f(versionfile_abs) 1362 | ver = from_keywords_f(keywords, cfg.tag_prefix, verbose) 1363 | if verbose: 1364 | print("got version from expanded keyword %s" % ver) 1365 | return ver 1366 | except NotThisMethod: 1367 | pass 1368 | 1369 | try: 1370 | ver = versions_from_file(versionfile_abs) 1371 | if verbose: 1372 | print("got version from file %s %s" % (versionfile_abs, ver)) 1373 | return ver 1374 | except NotThisMethod: 1375 | pass 1376 | 1377 | from_vcs_f = handlers.get("pieces_from_vcs") 1378 | if from_vcs_f: 1379 | try: 1380 | pieces = from_vcs_f(cfg.tag_prefix, root, verbose) 1381 | ver = render(pieces, cfg.style) 1382 | if verbose: 1383 | print("got version from VCS %s" % ver) 1384 | return ver 1385 | except NotThisMethod: 1386 | pass 1387 | 1388 | try: 1389 | if cfg.parentdir_prefix: 1390 | ver = versions_from_parentdir(cfg.parentdir_prefix, root, verbose) 1391 | if verbose: 1392 | print("got version from parentdir %s" % ver) 1393 | return ver 1394 | except NotThisMethod: 1395 | pass 1396 | 1397 | if verbose: 1398 | print("unable to compute version") 1399 | 1400 | return {"version": "0+unknown", "full-revisionid": None, 1401 | "dirty": None, "error": "unable to compute version"} 1402 | 1403 | 1404 | def get_version(): 1405 | return get_versions()["version"] 1406 | 1407 | 1408 | def get_cmdclass(): 1409 | if "versioneer" in sys.modules: 1410 | del sys.modules["versioneer"] 1411 | # this fixes the "python setup.py develop" case (also 'install' and 1412 | # 'easy_install .'), in which subdependencies of the main project are 1413 | # built (using setup.py bdist_egg) in the same python process. Assume 1414 | # a main project A and a dependency B, which use different versions 1415 | # of Versioneer. A's setup.py imports A's Versioneer, leaving it in 1416 | # sys.modules by the time B's setup.py is executed, causing B to run 1417 | # with the wrong versioneer. Setuptools wraps the sub-dep builds in a 1418 | # sandbox that restores sys.modules to it's pre-build state, so the 1419 | # parent is protected against the child's "import versioneer". By 1420 | # removing ourselves from sys.modules here, before the child build 1421 | # happens, we protect the child from the parent's versioneer too. 1422 | # Also see https://github.com/warner/python-versioneer/issues/52 1423 | 1424 | cmds = {} 1425 | 1426 | # we add "version" to both distutils and setuptools 1427 | from distutils.core import Command 1428 | 1429 | class cmd_version(Command): 1430 | description = "report generated version string" 1431 | user_options = [] 1432 | boolean_options = [] 1433 | 1434 | def initialize_options(self): 1435 | pass 1436 | 1437 | def finalize_options(self): 1438 | pass 1439 | 1440 | def run(self): 1441 | vers = get_versions(verbose=True) 1442 | print("Version: %s" % vers["version"]) 1443 | print(" full-revisionid: %s" % vers.get("full-revisionid")) 1444 | print(" dirty: %s" % vers.get("dirty")) 1445 | if vers["error"]: 1446 | print(" error: %s" % vers["error"]) 1447 | cmds["version"] = cmd_version 1448 | 1449 | # we override "build_py" in both distutils and setuptools 1450 | # 1451 | # most invocation pathways end up running build_py: 1452 | # distutils/build -> build_py 1453 | # distutils/install -> distutils/build ->.. 1454 | # setuptools/bdist_wheel -> distutils/install ->.. 1455 | # setuptools/bdist_egg -> distutils/install_lib -> build_py 1456 | # setuptools/install -> bdist_egg ->.. 1457 | # setuptools/develop -> ? 1458 | 1459 | from distutils.command.build_py import build_py as _build_py 1460 | 1461 | class cmd_build_py(_build_py): 1462 | def run(self): 1463 | root = get_root() 1464 | cfg = get_config_from_root(root) 1465 | versions = get_versions() 1466 | _build_py.run(self) 1467 | # now locate _version.py in the new build/ directory and replace 1468 | # it with an updated value 1469 | if cfg.versionfile_build: 1470 | target_versionfile = os.path.join(self.build_lib, 1471 | cfg.versionfile_build) 1472 | print("UPDATING %s" % target_versionfile) 1473 | write_to_version_file(target_versionfile, versions) 1474 | cmds["build_py"] = cmd_build_py 1475 | 1476 | if "cx_Freeze" in sys.modules: # cx_freeze enabled? 1477 | from cx_Freeze.dist import build_exe as _build_exe 1478 | 1479 | class cmd_build_exe(_build_exe): 1480 | def run(self): 1481 | root = get_root() 1482 | cfg = get_config_from_root(root) 1483 | versions = get_versions() 1484 | target_versionfile = cfg.versionfile_source 1485 | print("UPDATING %s" % target_versionfile) 1486 | write_to_version_file(target_versionfile, versions) 1487 | 1488 | _build_exe.run(self) 1489 | os.unlink(target_versionfile) 1490 | with open(cfg.versionfile_source, "w") as f: 1491 | LONG = LONG_VERSION_PY[cfg.VCS] 1492 | f.write(LONG % 1493 | {"DOLLAR": "$", 1494 | "STYLE": cfg.style, 1495 | "TAG_PREFIX": cfg.tag_prefix, 1496 | "PARENTDIR_PREFIX": cfg.parentdir_prefix, 1497 | "VERSIONFILE_SOURCE": cfg.versionfile_source, 1498 | }) 1499 | cmds["build_exe"] = cmd_build_exe 1500 | del cmds["build_py"] 1501 | 1502 | # we override different "sdist" commands for both environments 1503 | if "setuptools" in sys.modules: 1504 | from setuptools.command.sdist import sdist as _sdist 1505 | else: 1506 | from distutils.command.sdist import sdist as _sdist 1507 | 1508 | class cmd_sdist(_sdist): 1509 | def run(self): 1510 | versions = get_versions() 1511 | self._versioneer_generated_versions = versions 1512 | # unless we update this, the command will keep using the old 1513 | # version 1514 | self.distribution.metadata.version = versions["version"] 1515 | return _sdist.run(self) 1516 | 1517 | def make_release_tree(self, base_dir, files): 1518 | root = get_root() 1519 | cfg = get_config_from_root(root) 1520 | _sdist.make_release_tree(self, base_dir, files) 1521 | # now locate _version.py in the new base_dir directory 1522 | # (remembering that it may be a hardlink) and replace it with an 1523 | # updated value 1524 | target_versionfile = os.path.join(base_dir, cfg.versionfile_source) 1525 | print("UPDATING %s" % target_versionfile) 1526 | write_to_version_file(target_versionfile, 1527 | self._versioneer_generated_versions) 1528 | cmds["sdist"] = cmd_sdist 1529 | 1530 | return cmds 1531 | 1532 | 1533 | CONFIG_ERROR = """ 1534 | setup.cfg is missing the necessary Versioneer configuration. You need 1535 | a section like: 1536 | 1537 | [versioneer] 1538 | VCS = git 1539 | style = pep440 1540 | versionfile_source = src/myproject/_version.py 1541 | versionfile_build = myproject/_version.py 1542 | tag_prefix = "" 1543 | parentdir_prefix = myproject- 1544 | 1545 | You will also need to edit your setup.py to use the results: 1546 | 1547 | import versioneer 1548 | setup(version=versioneer.get_version(), 1549 | cmdclass=versioneer.get_cmdclass(), ...) 1550 | 1551 | Please read the docstring in ./versioneer.py for configuration instructions, 1552 | edit setup.cfg, and re-run the installer or 'python versioneer.py setup'. 1553 | """ 1554 | 1555 | SAMPLE_CONFIG = """ 1556 | # See the docstring in versioneer.py for instructions. Note that you must 1557 | # re-run 'versioneer.py setup' after changing this section, and commit the 1558 | # resulting files. 1559 | 1560 | [versioneer] 1561 | #VCS = git 1562 | #style = pep440 1563 | #versionfile_source = 1564 | #versionfile_build = 1565 | #tag_prefix = 1566 | #parentdir_prefix = 1567 | 1568 | """ 1569 | 1570 | INIT_PY_SNIPPET = """ 1571 | from ._version import get_versions 1572 | __version__ = get_versions()['version'] 1573 | del get_versions 1574 | """ 1575 | 1576 | 1577 | def do_setup(): 1578 | root = get_root() 1579 | try: 1580 | cfg = get_config_from_root(root) 1581 | except (EnvironmentError, configparser.NoSectionError, 1582 | configparser.NoOptionError) as e: 1583 | if isinstance(e, (EnvironmentError, configparser.NoSectionError)): 1584 | print("Adding sample versioneer config to setup.cfg", 1585 | file=sys.stderr) 1586 | with open(os.path.join(root, "setup.cfg"), "a") as f: 1587 | f.write(SAMPLE_CONFIG) 1588 | print(CONFIG_ERROR, file=sys.stderr) 1589 | return 1 1590 | 1591 | print(" creating %s" % cfg.versionfile_source) 1592 | with open(cfg.versionfile_source, "w") as f: 1593 | LONG = LONG_VERSION_PY[cfg.VCS] 1594 | f.write(LONG % {"DOLLAR": "$", 1595 | "STYLE": cfg.style, 1596 | "TAG_PREFIX": cfg.tag_prefix, 1597 | "PARENTDIR_PREFIX": cfg.parentdir_prefix, 1598 | "VERSIONFILE_SOURCE": cfg.versionfile_source, 1599 | }) 1600 | 1601 | ipy = os.path.join(os.path.dirname(cfg.versionfile_source), 1602 | "__init__.py") 1603 | if os.path.exists(ipy): 1604 | try: 1605 | with open(ipy, "r") as f: 1606 | old = f.read() 1607 | except EnvironmentError: 1608 | old = "" 1609 | if INIT_PY_SNIPPET not in old: 1610 | print(" appending to %s" % ipy) 1611 | with open(ipy, "a") as f: 1612 | f.write(INIT_PY_SNIPPET) 1613 | else: 1614 | print(" %s unmodified" % ipy) 1615 | else: 1616 | print(" %s doesn't exist, ok" % ipy) 1617 | ipy = None 1618 | 1619 | # Make sure both the top-level "versioneer.py" and versionfile_source 1620 | # (PKG/_version.py, used by runtime code) are in MANIFEST.in, so 1621 | # they'll be copied into source distributions. Pip won't be able to 1622 | # install the package without this. 1623 | manifest_in = os.path.join(root, "MANIFEST.in") 1624 | simple_includes = set() 1625 | try: 1626 | with open(manifest_in, "r") as f: 1627 | for line in f: 1628 | if line.startswith("include "): 1629 | for include in line.split()[1:]: 1630 | simple_includes.add(include) 1631 | except EnvironmentError: 1632 | pass 1633 | # That doesn't cover everything MANIFEST.in can do 1634 | # (http://docs.python.org/2/distutils/sourcedist.html#commands), so 1635 | # it might give some false negatives. Appending redundant 'include' 1636 | # lines is safe, though. 1637 | if "versioneer.py" not in simple_includes: 1638 | print(" appending 'versioneer.py' to MANIFEST.in") 1639 | with open(manifest_in, "a") as f: 1640 | f.write("include versioneer.py\n") 1641 | else: 1642 | print(" 'versioneer.py' already in MANIFEST.in") 1643 | if cfg.versionfile_source not in simple_includes: 1644 | print(" appending versionfile_source ('%s') to MANIFEST.in" % 1645 | cfg.versionfile_source) 1646 | with open(manifest_in, "a") as f: 1647 | f.write("include %s\n" % cfg.versionfile_source) 1648 | else: 1649 | print(" versionfile_source already in MANIFEST.in") 1650 | 1651 | # Make VCS-specific changes. For git, this means creating/changing 1652 | # .gitattributes to mark _version.py for export-time keyword 1653 | # substitution. 1654 | do_vcs_install(manifest_in, cfg.versionfile_source, ipy) 1655 | return 0 1656 | 1657 | 1658 | def scan_setup_py(): 1659 | found = set() 1660 | setters = False 1661 | errors = 0 1662 | with open("setup.py", "r") as f: 1663 | for line in f.readlines(): 1664 | if "import versioneer" in line: 1665 | found.add("import") 1666 | if "versioneer.get_cmdclass()" in line: 1667 | found.add("cmdclass") 1668 | if "versioneer.get_version()" in line: 1669 | found.add("get_version") 1670 | if "versioneer.VCS" in line: 1671 | setters = True 1672 | if "versioneer.versionfile_source" in line: 1673 | setters = True 1674 | if len(found) != 3: 1675 | print("") 1676 | print("Your setup.py appears to be missing some important items") 1677 | print("(but I might be wrong). Please make sure it has something") 1678 | print("roughly like the following:") 1679 | print("") 1680 | print(" import versioneer") 1681 | print(" setup( version=versioneer.get_version(),") 1682 | print(" cmdclass=versioneer.get_cmdclass(), ...)") 1683 | print("") 1684 | errors += 1 1685 | if setters: 1686 | print("You should remove lines like 'versioneer.VCS = ' and") 1687 | print("'versioneer.versionfile_source = ' . This configuration") 1688 | print("now lives in setup.cfg, and should be removed from setup.py") 1689 | print("") 1690 | errors += 1 1691 | return errors 1692 | 1693 | if __name__ == "__main__": 1694 | cmd = sys.argv[1] 1695 | if cmd == "setup": 1696 | errors = do_setup() 1697 | errors += scan_setup_py() 1698 | if errors: 1699 | sys.exit(1) 1700 | --------------------------------------------------------------------------------