├── tests ├── tox.ini ├── __init__.py ├── entities │ ├── __init__.py │ └── test_arxiv_document.py ├── interface_adapters │ ├── __init__.py │ ├── serializers │ │ ├── __init__.py │ │ └── test_arxiv_document_serializer.py │ ├── rest_adapters │ │ ├── test_request_object.py │ │ ├── test_rest_adapters.py │ │ ├── test_rest_interface.py │ │ └── test_response_object.py │ └── test_process_arxiv_request.py ├── external_interfaces │ └── flask_server │ │ └── rest │ │ └── test_arxiv_document.py ├── conftest.py └── use_case │ └── test_arxiv_repo.py ├── webminer ├── __init__.py ├── entities │ ├── __init__.py │ ├── domain_model.py │ └── arxiv_document.py ├── use_cases │ ├── __init__.py │ └── request_arxiv │ │ ├── __init__.py │ │ └── arxiv_repo.py ├── interface_adapters │ ├── __init__.py │ ├── process_arxiv_request.py │ ├── serializers │ │ └── json │ │ │ └── arxiv_document_serializer.py │ └── rest_adapters │ │ ├── request_object.py │ │ ├── rest_interface.py │ │ ├── request_objects.py │ │ └── response_object.py ├── external_interfaces │ └── flask_server │ │ ├── __init__.py │ │ ├── rest │ │ ├── __init__.py │ │ └── arxiv_document.py │ │ ├── settings.py │ │ └── app.py └── scrapy.cfg ├── Procfile ├── Procfile.windows ├── pytest.ini ├── setup.cfg ├── .deepsource.toml ├── .coveragerc ├── app.py ├── deploy ├── docker-compose.yml ├── Dockerfile └── base.Dockerfile ├── .vscode └── settings.json ├── Dockerfile ├── codecov.yml ├── app.json ├── Pipfile ├── .github └── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md ├── .travis.yml ├── LICENSE ├── .circleci └── config.yml ├── PULL_REQUEST_TEMPLATE.md ├── .gitignore ├── CONTRIBUTING.md ├── CODE_OF_CONDUCT.md ├── README.md ├── .pylintrc └── Pipfile.lock /tests/tox.ini: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /webminer/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/entities/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /webminer/entities/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /webminer/use_cases/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/interface_adapters/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /webminer/interface_adapters/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/interface_adapters/serializers/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /webminer/use_cases/request_arxiv/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | web: gunicorn -w 4 -b 0.0.0.0:${PORT} app:app -------------------------------------------------------------------------------- /webminer/external_interfaces/flask_server/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /webminer/external_interfaces/flask_server/rest/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Procfile.windows: -------------------------------------------------------------------------------- 1 | web: python manage.py runserver 0.0.0.0:5000 2 | -------------------------------------------------------------------------------- /tests/external_interfaces/flask_server/rest/test_arxiv_document.py: -------------------------------------------------------------------------------- 1 | """Testing the REST interface 2 | """ 3 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | minversion = 2.0 3 | norecursedirs = .git .tox venv* requirements* 4 | python_files = test*.py -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | universal = 1 3 | 4 | [flake8] 5 | ignore = D203 6 | exclude = .git, venv*, docs 7 | max-complexity = 10 8 | max-line-length = 100 -------------------------------------------------------------------------------- /.deepsource.toml: -------------------------------------------------------------------------------- 1 | version = 1 2 | 3 | test_patterns = [ 4 | 5 | ] 6 | 7 | exclude_patterns = [ 8 | 9 | ] 10 | 11 | [[analyzers]] 12 | name = 'python' 13 | enabled = true 14 | runtime_version = '3.x.x' 15 | 16 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = True 3 | source = . 4 | 5 | [report] 6 | exclude_lines = 7 | if self.debug: 8 | pragma: no cover 9 | raise NotImplementedError 10 | if __name__ == .__main__.: 11 | ignore_errors = True 12 | omit = 13 | tests/* 14 | 15 | -------------------------------------------------------------------------------- /app.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """Executes the app when running using gunicorn 4 | """ 5 | 6 | 7 | from webminer.external_interfaces.flask_server.app import create_app 8 | app = create_app() 9 | 10 | if __name__ == '__main__': 11 | app.run() 12 | -------------------------------------------------------------------------------- /deploy/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | engine: 4 | build: 5 | context: ../ 6 | dockerfile: deploy/Dockerfile 7 | image: keep-current/web-miner 8 | tty: true 9 | ports: 10 | - "5000:5000" 11 | environment: 12 | - "PORT=5000" -------------------------------------------------------------------------------- /webminer/scrapy.cfg: -------------------------------------------------------------------------------- 1 | # Automatically created by: scrapy startproject 2 | # 3 | # For more information about the [deploy] section see: 4 | # https://scrapyd.readthedocs.io/en/latest/deploy.html 5 | 6 | [settings] 7 | default = tutorial.settings 8 | 9 | [deploy] 10 | #url = http://localhost:6800/ 11 | project = tutorial 12 | -------------------------------------------------------------------------------- /webminer/entities/domain_model.py: -------------------------------------------------------------------------------- 1 | """ Create a DomainModel providing a metaclass""" 2 | 3 | from abc import ABCMeta 4 | 5 | 6 | class DomainModel(metaclass=ABCMeta): 7 | """An ABC can be subclassed directly, and then acts as a mix-in class 8 | metaclass (class, optional): Defaults to ABCMeta. 9 | """ 10 | 11 | pass 12 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | """This tests the basic configuration and creating an app 2 | """ 3 | 4 | import pytest 5 | 6 | 7 | from webminer.external_interfaces.flask_server.app import create_app 8 | from webminer.external_interfaces.flask_server.settings import TestConfig 9 | 10 | 11 | @pytest.fixture(scope='function') 12 | def app(): 13 | return create_app(TestConfig) 14 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "python.venvPath": "${workspaceFolder}/.venv/bin/python", 3 | "python.linting.pylintEnabled": true, 4 | "python.linting.enabled": true, 5 | "python.linting.pylintArgs": ["--disable=C0330"], 6 | "python.formatting.provider": "black", 7 | "python.formatting.blackPath": "black", 8 | "python.pythonPath": "${workspaceFolder}/.venv/bin/python", 9 | "autoDocstring.docstringFormat": "google", 10 | } 11 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.7.0 AS web-miner 2 | MAINTAINER "https://github.com/Keep-Current/web-miner" 3 | LABEL Maintainer="Liad Magen https://www.github.com/keep-current/web-miner" 4 | RUN mkdir -p /webminer 5 | WORKDIR /webminer 6 | # Copies Everything 7 | COPY . . 8 | RUN CGO_ENABLED=0 GOOS=linux pip install --upgrade pip \ 9 | && pip install pipenv \ 10 | && pipenv install 11 | CMD [ "flask", "run", "--host=0.0.0.0" ] 12 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | codecov: 2 | token: 7395b350-ac40-4482-af86-a2e22064d323 3 | notify: 4 | require_ci_to_pass: yes 5 | 6 | coverage: 7 | precision: 2 8 | round: down 9 | range: "70...100" 10 | 11 | status: 12 | project: yes 13 | patch: yes 14 | changes: no 15 | 16 | parsers: 17 | gcov: 18 | branch_detection: 19 | conditional: yes 20 | loop: yes 21 | method: no 22 | macro: no 23 | 24 | comment: 25 | layout: "header, diff" 26 | behavior: default 27 | require_changes: no -------------------------------------------------------------------------------- /app.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Keep Current - Web Miner and Document finder", 3 | "description": 4 | "It opens an API for cron to trigger a search or update the query", 5 | "image": "heroku/python", 6 | "repository": "https://github.com/Keep-Current/Web-Miner", 7 | "keywords": ["python", "crawler", "miner", "flask"], 8 | "stack": "heroku-16", 9 | "environments": { 10 | "test": { 11 | "scripts": { 12 | "test-setup": "python app.py collectstatic --noinput", 13 | "test": "python app.py test" 14 | } 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /Pipfile: -------------------------------------------------------------------------------- 1 | [[source]] 2 | url = "https://pypi.org/simple" 3 | verify_ssl = true 4 | name = "pypi" 5 | 6 | [packages] 7 | logzero = ">=1.3.1" 8 | feedparser = ">5.1" 9 | pytest = "*" 10 | tox = "*" 11 | coverage = "*" 12 | pytest-cov = "*" 13 | pytest-flask = "*" 14 | codecov = "*" 15 | gunicorn = "*" 16 | "flake8" = "*" 17 | Flask = "*" 18 | Sphinx = "*" 19 | requests = "*" 20 | responses = "*" 21 | 22 | [dev-packages] 23 | pylint = "*" 24 | black = "*" 25 | responses = "*" 26 | 27 | [requires] 28 | python_version="3.6" 29 | 30 | [pipenv] 31 | allow_prereleases=true 32 | 33 | [environment] 34 | PIPENV_VENV_IN_PROJECT=true 35 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | 5 | --- 6 | 7 | **Describe the bug** 8 | A clear and concise description of what the bug is. 9 | 10 | **To Reproduce** 11 | Steps to reproduce the behavior: (for example) 12 | 1. Go to '...' 13 | 2. Click on '....' 14 | 3. Scroll down to '....' 15 | 4. See error 16 | 17 | **Expected behavior** 18 | A clear and concise description of what you expected to happen. 19 | 20 | **Screenshots** 21 | If applicable, add screenshots to help explain your problem. 22 | 23 | **Additional context** 24 | Add any other context about the problem here. 25 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | 5 | --- 6 | 7 | **Is your feature request related to a problem? Please describe.** 8 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 9 | 10 | **Describe the solution you'd like** 11 | A clear and concise description of what you want to happen. 12 | 13 | **Describe alternatives you've considered** 14 | A clear and concise description of any alternative solutions or features you've considered. 15 | 16 | **Additional context** 17 | Add any other context or screenshots about the feature request here. 18 | -------------------------------------------------------------------------------- /webminer/interface_adapters/process_arxiv_request.py: -------------------------------------------------------------------------------- 1 | """Connects the use case with the response object 2 | 3 | Returns: 4 | class: For processing arxiv document list and apply the filtering on a repo 5 | """ 6 | 7 | from webminer.interface_adapters.rest_adapters import rest_interface as uc 8 | from webminer.interface_adapters.rest_adapters import response_object as res 9 | 10 | 11 | class ProcessArxivDocuments(uc.RestInterface): 12 | def __init__(self, repo): 13 | self.repo = repo 14 | 15 | def process_request(self, request_object): 16 | arxiv_documents = self.repo.list(filters=request_object.filters) 17 | return res.ResponseSuccess(arxiv_documents) 18 | -------------------------------------------------------------------------------- /deploy/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM wearedevelopers/alpine-ml:v0.6 2 | 3 | LABEL MAINTAINER="https://github.com/Keep-Current/web-miner" 4 | 5 | WORKDIR /usr/local/bin 6 | WORKDIR /usr/local/engine 7 | 8 | COPY ./requirements.txt ./ 9 | 10 | RUN apk update && \ 11 | apk --no-cache add libstdc++ openssl libressl-dev ca-certificates && \ 12 | apk --no-cache add --virtual builddeps g++ gfortran musl-dev lapack-dev gcc make && \ 13 | pip install -r requirements.txt && \ 14 | apk del builddeps && \ 15 | rm -rf /root/.cache 16 | 17 | # Copies Everything 18 | COPY ./ ./ 19 | 20 | #--log-level=info --log-file=./logs/gunicorn.log --access-logfile=./logs/gunicorn-access.log 21 | CMD gunicorn -w 4 -b 0.0.0.0:${PORT} wsgi:app -------------------------------------------------------------------------------- /tests/interface_adapters/rest_adapters/test_request_object.py: -------------------------------------------------------------------------------- 1 | from webminer.interface_adapters.rest_adapters import request_object as req 2 | 3 | 4 | def test_invalid_request_object_is_false(): 5 | request = req.InvalidRequestObject() 6 | 7 | assert bool(request) is False 8 | 9 | 10 | def test_invalid_request_object_accepts_errors(): 11 | request = req.InvalidRequestObject() 12 | request.add_error(parameter="aparam", message="wrong value") 13 | request.add_error(parameter="anotherparam", message="wrong type") 14 | 15 | assert request.has_errors() is True 16 | assert len(request.errors) == 2 17 | 18 | 19 | def test_valid_request_object_is_true(): 20 | request = req.ValidRequestObject() 21 | 22 | assert bool(request) is True 23 | -------------------------------------------------------------------------------- /webminer/external_interfaces/flask_server/settings.py: -------------------------------------------------------------------------------- 1 | """Declare configuration settings and return as class""" 2 | import os 3 | 4 | 5 | class Config(): 6 | """Basic configuration that can be extended""" 7 | 8 | APP_DIR = os.path.abspath(os.path.dirname(__file__)) # This directory 9 | PROJECT_ROOT = os.path.abspath(os.path.join(APP_DIR, os.pardir)) 10 | 11 | 12 | class ProdConfig(Config): 13 | """Production configuration.""" 14 | 15 | ENV = "prod" 16 | DEBUG = False 17 | 18 | 19 | class DevConfig(Config): 20 | """Development configuration.""" 21 | 22 | ENV = "dev" 23 | DEBUG = True 24 | 25 | 26 | class TestConfig(Config): 27 | """Test configuration.""" 28 | 29 | ENV = "test" 30 | TESTING = True 31 | DEBUG = True 32 | -------------------------------------------------------------------------------- /webminer/external_interfaces/flask_server/app.py: -------------------------------------------------------------------------------- 1 | """It creates the flask server with an environment and returns the server""" 2 | 3 | from flask import Flask 4 | 5 | from webminer.external_interfaces.flask_server.rest import arxiv_document 6 | from webminer.external_interfaces.flask_server.settings import DevConfig 7 | 8 | 9 | def create_app(config_object=DevConfig): 10 | """Creates the server 11 | 12 | Args: 13 | config_object (object, optional): Defaults to DevConfig. 14 | Adds a config when creating the app server 15 | 16 | Returns: 17 | class 'flask.app.Flask': A Flask app 18 | """ 19 | 20 | app = Flask(__name__) 21 | app.config.from_object(config_object) 22 | app.register_blueprint(arxiv_document.blueprint) 23 | return app 24 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "3.6" 4 | before_install: 5 | - sudo apt-get -qq update 6 | - sudo apt-get install -y swig 7 | - sudo apt-get install -y libpulse-dev 8 | install: 9 | - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; 10 | - bash miniconda.sh -b -p $HOME/miniconda 11 | - export PATH="$HOME/miniconda/bin:$PATH" 12 | - hash -r 13 | - conda config --set always_yes yes --set changeps1 no 14 | - conda update -q conda 15 | # Useful for debugging any issues with conda 16 | - conda info -a 17 | 18 | - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION 19 | - source activate test-environment 20 | # python setup.py install 21 | - pip install pipenv 22 | - pipenv install 23 | # command to run the tests: 24 | script: 25 | - py.test --cov=./ 26 | notifications: 27 | slack: vdsg:YeRk6YBgJuTEtaRT1pPp76pb 28 | after_success: 29 | - codecov 30 | 31 | -------------------------------------------------------------------------------- /webminer/interface_adapters/serializers/json/arxiv_document_serializer.py: -------------------------------------------------------------------------------- 1 | """ 2 | Serializers allow complex data such as querysets and model instances to be converted 3 | to native Python datatypes that can then be easily rendered into JSON 4 | """ 5 | 6 | import json 7 | 8 | 9 | class ArxivDocEncoder(json.JSONEncoder): 10 | """Encodes the arxiv document 11 | 12 | Args: 13 | json (obj): The JSON encoded object 14 | 15 | Returns: 16 | dict: The serialized format 17 | """ 18 | 19 | def default(self, o): # pylint: disable=E0202 20 | try: 21 | to_serialize = { 22 | "id": o.doc_id, 23 | "url": o.url, 24 | "title": o.title, 25 | "abstract": o.abstract, 26 | "authors": ",".join(o.authors), 27 | "publish_date": o.publish_date, 28 | "pdf": o.pdf_url, 29 | } 30 | return to_serialize 31 | 32 | except AttributeError: 33 | return super().default(o) 34 | -------------------------------------------------------------------------------- /deploy/base.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.6-alpine3.8 2 | 3 | LABEL MAINTAINER="Liad Magen, liad.magen@gmail.com" 4 | 5 | ENV CFLAGS="-fPIC" 6 | 7 | WORKDIR /usr/local/include 8 | 9 | RUN apk --no-cache add lapack && \ 10 | apk --no-cache add --virtual builddeps g++ gfortran musl-dev lapack-dev gcc make && \ 11 | # musl-dev python3-dev openblas-dev && \ 12 | \ 13 | wget https://mupdf.com/downloads/archive/mupdf-1.13.0-source.tar.gz -O - | tar -xz && \ 14 | mv mupdf-1.13.0-source mupdf && \ 15 | cd mupdf && \ 16 | make HAVE_X11=no HAVE_GLFW=no HAVE_GLUT=no prefix=/usr/local && \ 17 | make HAVE_X11=no HAVE_GLFW=no HAVE_GLUT=no prefix=/usr/local install && \ 18 | mv /usr/local/include/mupdf/thirdparty /usr/local/thirdparty 19 | 20 | WORKDIR /opt/app 21 | 22 | RUN pip install numpy==1.15.4 && \ 23 | pip install scipy==1.1.0 && \ 24 | pip install scikit-learn==0.20 && \ 25 | pip install pandas==0.23.4 && \ 26 | pip install -U spacy==2.0.17.dev1 && \ 27 | python -m spacy download en && \ 28 | apk del builddeps && \ 29 | rm -rf /root/.cache 30 | -------------------------------------------------------------------------------- /webminer/interface_adapters/rest_adapters/request_object.py: -------------------------------------------------------------------------------- 1 | """Create a class for invalid request objects 2 | 3 | Raises: 4 | NotImplementedError: If a request object is not valid 5 | 6 | Returns: 7 | classes: For requested objects 8 | """ 9 | 10 | 11 | class InvalidRequestObject(object): 12 | """Checks the requested object for errors 13 | 14 | Args: 15 | object (obj): Base object to be extended 16 | 17 | Returns: 18 | class methods: For handling (missing) errors 19 | """ 20 | 21 | def __init__(self): 22 | self.errors = [] 23 | 24 | def add_error(self, parameter, message): 25 | self.errors.append({"parameter": parameter, "message": message}) 26 | 27 | def has_errors(self): 28 | return len(self.errors) > 0 29 | 30 | def __nonzero__(self): 31 | return False 32 | 33 | __bool__ = __nonzero__ 34 | 35 | 36 | class ValidRequestObject(object): 37 | @classmethod 38 | def from_dict(cls, adict): 39 | raise NotImplementedError 40 | 41 | def __nonzero__(self): 42 | return True 43 | 44 | __bool__ = __nonzero__ 45 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Liad 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /webminer/interface_adapters/rest_adapters/rest_interface.py: -------------------------------------------------------------------------------- 1 | """Create a class for applying use cases 2 | 3 | Raises: 4 | NotImplementedError: If a use case cannot be implemented 5 | 6 | Returns: 7 | class: UseCase class 8 | """ 9 | 10 | from webminer.interface_adapters.rest_adapters import response_object as res 11 | 12 | 13 | class RestInterface(object): 14 | """Creates the UseCase class 15 | 16 | Args: 17 | object (obj): A base object 18 | 19 | Raises: 20 | NotImplementedError: 21 | If a a requested object cannot be implemented by the UseCase class 22 | 23 | Returns: 24 | class: Methods for executing and processing incoming requests 25 | """ 26 | 27 | def execute(self, request_object): 28 | if not request_object: 29 | return res.ResponseFailure.build_from_invalid_request_object(request_object) 30 | try: 31 | return self.process_request(request_object) 32 | except Exception as exc: # pylint: disable=W0703 33 | return res.ResponseFailure.build_system_error( 34 | f"{exc.__class__.__name__}: {exc}" 35 | ) 36 | 37 | def process_request(self, request_object): 38 | raise NotImplementedError("process_request() not implemented by UseCase class") 39 | -------------------------------------------------------------------------------- /webminer/interface_adapters/rest_adapters/request_objects.py: -------------------------------------------------------------------------------- 1 | """Takes the valid request object and returns a filtered arxiv document""" 2 | 3 | import collections 4 | from webminer.interface_adapters.rest_adapters import request_object as req 5 | 6 | 7 | class ArxivDocumentListRequestObject(req.ValidRequestObject): 8 | """Creates an arxiv document list out of a request object 9 | 10 | Args: 11 | req (obj): Validated requested object 12 | 13 | Returns: 14 | class: For transforming the request into a dictionary 15 | """ 16 | 17 | def __init__(self, filters=None): 18 | self.filters = filters 19 | 20 | @classmethod 21 | def from_dict(cls, adict): 22 | """Convert dictionary to Arxiv document object 23 | 24 | Args: 25 | adict (dict): a python dictionary 26 | 27 | Returns: 28 | dict: a filtered list object 29 | """ 30 | 31 | invalid_req = req.InvalidRequestObject() 32 | 33 | if "filters" in adict and not isinstance(adict["filters"], collections.Mapping): 34 | invalid_req.add_error("filters", "Is not iterable") 35 | 36 | if invalid_req.has_errors(): 37 | return invalid_req 38 | 39 | return ArxivDocumentListRequestObject(filters=adict.get("filters", None)) 40 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | # Python CircleCI 2.0 configuration file 2 | # 3 | # Check https://circleci.com/docs/2.0/language-python/ for more details 4 | # 5 | version: 2 6 | jobs: 7 | build: 8 | docker: 9 | # specify the version you desire here 10 | # use `-browsers` prefix for selenium tests, e.g. `3.6.1-browsers` 11 | - image: circleci/python:3.6.1 12 | 13 | # Specify service dependencies here if necessary 14 | # CircleCI maintains a library of pre-built images 15 | # documented at https://circleci.com/docs/2.0/circleci-images/ 16 | # - image: circleci/postgres:9.4 17 | 18 | working_directory: ~/repo 19 | 20 | steps: 21 | - checkout 22 | 23 | # Download and cache dependencies 24 | - restore_cache: 25 | keys: 26 | - cache-{{ checksum "Pipfile.lock" }} 27 | - cache- 28 | 29 | - run: 30 | name: install dependencies 31 | command: | 32 | sudo pip install pipenv 33 | pipenv sync --dev 34 | 35 | - save_cache: 36 | key: cache-{{ checksum "Pipfile.lock" }} 37 | paths: 38 | - ~/.local 39 | - ~/.cache 40 | 41 | - run: 42 | name: run tests 43 | command: | 44 | pipenv run pytest tests --cov=./ 45 | 46 | - store_artifacts: 47 | path: test-reports 48 | destination: test-reports 49 | -------------------------------------------------------------------------------- /PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## Proposed changes 2 | 3 | Describe the big picture of your changes here to communicate to the maintainers why we should accept this pull request. If it fixes a bug or resolves a feature request, be sure to link to that issue. 4 | 5 | ## Types of changes 6 | 7 | What types of changes does your code introduce to Appium? 8 | _Put an `x` in the boxes that apply_ 9 | 10 | - [ ] Bugfix (non-breaking change which fixes an issue) 11 | - [ ] New feature (non-breaking change which adds functionality) 12 | - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) 13 | 14 | ## Checklist 15 | 16 | _Put an `x` in the boxes that apply. You can also fill these out after creating the PR. If you're unsure about any of them, don't hesitate to ask. We're here to help! This is simply a reminder of what we are going to look for before merging your code._ 17 | 18 | - [ ] I have read the [CONTRIBUTING](https://github.com/Keep-Current/web-miner/blob/master/CONTRIBUTING.md) doc 19 | - [ ] Lint and unit tests pass locally with my changes 20 | - [ ] I have added tests that prove my fix is effective or that my feature works 21 | - [ ] I have added necessary documentation (if appropriate) 22 | - [ ] Any dependent changes have been merged and published 23 | 24 | ## Further comments 25 | 26 | If this is a relatively large or complex change, kick off the discussion by explaining why you chose the solution you did and what alternatives you considered, etc... 27 | -------------------------------------------------------------------------------- /webminer/entities/arxiv_document.py: -------------------------------------------------------------------------------- 1 | """Creates an arxiv document class and registers it as domain model 2 | 3 | Returns: 4 | class: ArxivDocument class 5 | """ 6 | 7 | from webminer.entities.domain_model import DomainModel 8 | 9 | 10 | class ArxivDocument(): 11 | """Create an arxiv document 12 | 13 | Returns: 14 | class: Transformed and extended document 15 | """ 16 | 17 | def __init__(self, **kwargs): 18 | self.__dict__.update(kwargs) 19 | 20 | @classmethod 21 | def from_dict(cls, feedparser_dict): 22 | """Checks the feedparser dictionary and returns it as a document 23 | 24 | Args: 25 | feedparser_dict (feedParserDict): Dictionary from parsing the feed 26 | 27 | Returns: 28 | class object: ArxivDocument object 29 | """ 30 | 31 | pdf = "" 32 | for link in feedparser_dict["links"]: 33 | try: 34 | if link["title"] == "pdf": 35 | pdf = link["href"] 36 | except AttributeError: 37 | pass 38 | 39 | document = ArxivDocument( 40 | doc_id=feedparser_dict["doc_id"], 41 | title=feedparser_dict["title"], 42 | abstract=feedparser_dict["summary"], 43 | authors=[auth["name"] for auth in feedparser_dict["authors"]], 44 | url=feedparser_dict["url"], 45 | publish_date=feedparser_dict["published"], 46 | pdf_url=pdf, 47 | ) 48 | 49 | return document 50 | 51 | 52 | DomainModel.register(ArxivDocument) 53 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | -------------------------------------------------------------------------------- /tests/interface_adapters/rest_adapters/test_rest_adapters.py: -------------------------------------------------------------------------------- 1 | from webminer.interface_adapters.rest_adapters import request_objects as ro 2 | 3 | 4 | def test_build_storageroom_list_request_object_without_parameters(): 5 | req = ro.ArxivDocumentListRequestObject() 6 | 7 | assert req.filters is None 8 | assert bool(req) is True 9 | 10 | 11 | def test_build_file_list_request_object_from_empty_dict(): 12 | req = ro.ArxivDocumentListRequestObject.from_dict({}) 13 | 14 | assert req.filters is None 15 | assert bool(req) is True 16 | 17 | 18 | def test_build_storageroom_list_request_object_with_empty_filters(): 19 | req = ro.ArxivDocumentListRequestObject(filters={}) 20 | 21 | assert req.filters == {} 22 | assert bool(req) is True 23 | 24 | 25 | def test_build_storageroom_list_request_object_from_dict_with_empty_filters(): 26 | req = ro.ArxivDocumentListRequestObject.from_dict({"filters": {}}) 27 | 28 | assert req.filters == {} 29 | assert bool(req) is True 30 | 31 | 32 | def test_build_storageroom_list_request_object_with_filters(): 33 | req = ro.ArxivDocumentListRequestObject(filters={"a": 1, "b": 2}) 34 | 35 | assert req.filters == {"a": 1, "b": 2} 36 | assert bool(req) is True 37 | 38 | 39 | def test_build_storageroom_list_request_object_from_dict_with_filters(): 40 | req = ro.ArxivDocumentListRequestObject.from_dict({"filters": {"a": 1, "b": 2}}) 41 | 42 | assert req.filters == {"a": 1, "b": 2} 43 | assert bool(req) is True 44 | 45 | 46 | def test_build_storageroom_list_request_object_from_dict_with_invalid_filters(): 47 | req = ro.ArxivDocumentListRequestObject.from_dict({"filters": 5}) 48 | 49 | assert req.has_errors() 50 | assert req.errors[0]["parameter"] == "filters" 51 | assert bool(req) is False 52 | -------------------------------------------------------------------------------- /tests/interface_adapters/rest_adapters/test_rest_interface.py: -------------------------------------------------------------------------------- 1 | from unittest import mock 2 | 3 | from webminer.interface_adapters.rest_adapters import request_object as req, response_object as res 4 | from webminer.interface_adapters.rest_adapters import rest_interface as uc 5 | 6 | 7 | def test_use_case_cannot_process_valid_requests(): 8 | valid_request_object = mock.MagicMock() 9 | valid_request_object.__bool__.return_value = True 10 | 11 | use_case = uc.RestInterface() 12 | response = use_case.execute(valid_request_object) 13 | 14 | assert not response 15 | assert response.type == res.ResponseFailure.SYSTEM_ERROR 16 | assert ( 17 | response.message 18 | == "NotImplementedError: process_request() not implemented by UseCase class" 19 | ) 20 | 21 | 22 | def test_use_case_can_process_invalid_requests_and_returns_response_failure(): 23 | invalid_request_object = req.InvalidRequestObject() 24 | invalid_request_object.add_error("someparam", "somemessage") 25 | 26 | use_case = uc.RestInterface() 27 | response = use_case.execute(invalid_request_object) 28 | 29 | assert not response 30 | assert response.type == res.ResponseFailure.PARAMETERS_ERROR 31 | assert response.message == "someparam: somemessage" 32 | 33 | 34 | def test_use_case_can_manage_generic_exception_from_process_request(): 35 | use_case = uc.RestInterface() 36 | 37 | class TestException(Exception): 38 | pass 39 | 40 | use_case.process_request = mock.Mock() 41 | use_case.process_request.side_effect = TestException("somemessage") 42 | response = use_case.execute(mock.Mock) 43 | 44 | assert not response 45 | assert response.type == res.ResponseFailure.SYSTEM_ERROR 46 | assert response.message == "TestException: somemessage" 47 | -------------------------------------------------------------------------------- /webminer/external_interfaces/flask_server/rest/arxiv_document.py: -------------------------------------------------------------------------------- 1 | """Defines the REST functionality and returns a response object""" 2 | 3 | import json 4 | from flask import Blueprint, request, Response 5 | 6 | from webminer.use_cases.request_arxiv import arxiv_repo as ar 7 | from webminer.interface_adapters import process_arxiv_request as uc 8 | from webminer.interface_adapters.rest_adapters import request_objects as req 9 | from webminer.interface_adapters.rest_adapters import response_object as res 10 | from webminer.interface_adapters.serializers.json import arxiv_document_serializer as ser 11 | 12 | blueprint = Blueprint("arxiv", __name__) 13 | 14 | STATUS_CODES = { 15 | res.ResponseSuccess.SUCCESS: 200, 16 | res.ResponseFailure.RESOURCE_ERROR: 404, 17 | res.ResponseFailure.PARAMETERS_ERROR: 400, 18 | res.ResponseFailure.SYSTEM_ERROR: 500, 19 | } 20 | 21 | 22 | @blueprint.route("/arxiv", methods=["GET"]) 23 | def arxiv(): 24 | """ 25 | Defines a GET route for the arxiv API. 26 | Make the request object API ready. 27 | Transform the response into JSON format 28 | 29 | Returns: 30 | object (JSON): A response object for further querying 31 | """ 32 | 33 | qrystr_params = {"filters": {}} 34 | 35 | for arg, values in request.args.items(): 36 | if arg.startswith("filter_"): 37 | qrystr_params["filters"][arg.replace("filter_", "")] = values 38 | 39 | request_object = req.ArxivDocumentListRequestObject.from_dict(qrystr_params) 40 | 41 | repo = ar.ArxivRepo() 42 | use_case = uc.ProcessArxivDocuments(repo) 43 | 44 | response = use_case.execute(request_object) 45 | 46 | return Response( 47 | json.dumps(response.value, cls=ser.ArxivDocEncoder), 48 | mimetype="application/json", 49 | status=STATUS_CODES[response.type], 50 | ) 51 | -------------------------------------------------------------------------------- /tests/interface_adapters/serializers/test_arxiv_document_serializer.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import json 3 | import pytest 4 | 5 | from webminer.interface_adapters.serializers.json import arxiv_document_serializer as srs 6 | from webminer.entities.arxiv_document import ArxivDocument 7 | 8 | test_url = "https://arxiv.org/abs/1801.06605" 9 | test_title = "A Collaborative Filtering Recommender System" 10 | test_abstract = "The use of relevant metrics of software systems could improve various software engineering tasks, but" 11 | test_authors = ["Maral Azizi", "Hyunsook Do"] 12 | test_publish_date = "Sat, 20 Jan 2018 00:11:42" 13 | test_pdf_url = "https://arxiv.org/pdf/1801.06605" 14 | 15 | 16 | def test_serialize_domain_arxiv_document(): 17 | arxiv_doc = ArxivDocument( 18 | doc_id="f853578c-fc0f-4e65-81b8-566c5dffa35a", 19 | url=test_url, 20 | title=test_title, 21 | abstract=test_abstract, 22 | authors=test_authors, 23 | publish_date=test_publish_date, 24 | pdf_url=test_pdf_url, 25 | ) 26 | 27 | expected_json = """ 28 | { 29 | "id" : "f853578c-fc0f-4e65-81b8-566c5dffa35a", 30 | "url" : "https://arxiv.org/abs/1801.06605", 31 | "title" : "A Collaborative Filtering Recommender System", 32 | "abstract" : "The use of relevant metrics of software systems could improve various software engineering tasks, but", 33 | "authors" : "Maral Azizi,Hyunsook Do", 34 | "publish_date" : "Sat, 20 Jan 2018 00:11:42", 35 | "pdf" : "https://arxiv.org/pdf/1801.06605" 36 | } 37 | """ 38 | 39 | print(json.loads(json.dumps(arxiv_doc, cls=srs.ArxivDocEncoder))) 40 | print(json.loads(expected_json)) 41 | 42 | assert json.loads(json.dumps(arxiv_doc, cls=srs.ArxivDocEncoder)) == json.loads( 43 | expected_json 44 | ) 45 | 46 | 47 | def test_serialize_domain_arxivdocument_wrong_type(): 48 | with pytest.raises(TypeError): 49 | json.dumps(datetime.datetime.now(), cls=srs.ArxivDocEncoder) 50 | -------------------------------------------------------------------------------- /tests/entities/test_arxiv_document.py: -------------------------------------------------------------------------------- 1 | """Tests the arxiv model 2 | """ 3 | 4 | import uuid 5 | from webminer.entities import arxiv_document as ad 6 | 7 | test_url = "https://arxiv.org/abs/1801.06605" 8 | test_title = "A Collaborative Filtering Recommender System" 9 | test_abstract = "The use of relevant metrics of software systems " + \ 10 | "could improve various software engineering tasks, but" 11 | test_authors = ["Maral Azizi", "Hyunsook Do"] 12 | test_publish_date = "Sat, 20 Jan 2018 00:11:42" 13 | test_pdf_url = "https://arxiv.org/pdf/1801.06605" 14 | 15 | 16 | def test_arxiv_doc_model_init(): 17 | """Tests a successful creation of a arxiv doc model 18 | """ 19 | 20 | code = uuid.uuid4() 21 | arxiv_doc = ad.ArxivDocument( 22 | doc_id=code, 23 | url=test_url, 24 | title=test_title, 25 | abstract=test_abstract, 26 | authors=test_authors, 27 | publish_date=test_publish_date, 28 | pdf_url=test_pdf_url, 29 | ) 30 | assert_equal(arxiv_doc.doc_id, code) 31 | assert_equal(arxiv_doc.url, test_url) 32 | assert_equal(arxiv_doc.title, test_title) 33 | assert_equal(arxiv_doc.abstract, test_abstract) 34 | assert_equal(arxiv_doc.authors, test_authors) 35 | assert_equal(arxiv_doc.publish_date, test_publish_date) 36 | assert_equal(arxiv_doc.pdf_url, test_pdf_url) 37 | 38 | 39 | def test_arxiv_doc_model_from_dict(): 40 | """Tests a successful creation of a arxiv doc model from a 41 | dictionary object 42 | """ 43 | code = uuid.uuid4() 44 | arxiv_doc = ad.ArxivDocument.from_dict( 45 | { 46 | "doc_id": code, 47 | "url": test_url, 48 | "title": test_title, 49 | "summary": test_abstract, 50 | "authors": [{"name": "Maral Azizi"}, {"name": "Hyunsook Do"}], 51 | "published": test_publish_date, 52 | "links": [{"title": "pdf", "href": "https://arxiv.org/pdf/1801.06605"}], 53 | } 54 | ) 55 | assert_equal(arxiv_doc.doc_id, code) 56 | assert_equal(arxiv_doc.url, test_url) 57 | assert_equal(arxiv_doc.title, test_title) 58 | assert_equal(arxiv_doc.abstract, test_abstract) 59 | assert_equal(arxiv_doc.authors, test_authors) 60 | assert_equal(arxiv_doc.publish_date, test_publish_date) 61 | assert_equal(arxiv_doc.pdf_url, test_pdf_url) 62 | 63 | def assert_equal(arg1, arg2): 64 | if arg1 != arg2: 65 | raise AssertionError("Assert equal failed - values are not equal") 66 | -------------------------------------------------------------------------------- /webminer/interface_adapters/rest_adapters/response_object.py: -------------------------------------------------------------------------------- 1 | """Create classes for different responses""" 2 | 3 | 4 | class ResponseSuccess(object): 5 | """Creates a class for a successful response 6 | 7 | Args: 8 | object (obj): Base object to be extended 9 | 10 | Returns: 11 | boolean: True on success 12 | """ 13 | 14 | SUCCESS = "SUCCESS" 15 | 16 | def __init__(self, value=None): 17 | self.type = self.SUCCESS 18 | self.value = value 19 | 20 | def __nonzero__(self): 21 | return True 22 | 23 | __bool__ = __nonzero__ 24 | 25 | 26 | class ResponseFailure(object): 27 | """Creates a class for a failed response 28 | 29 | Args: 30 | object (obj): Base object to be extended 31 | 32 | Returns: 33 | boolean: False on Failure 34 | """ 35 | 36 | RESOURCE_ERROR = "RESOURCE_ERROR" 37 | PARAMETERS_ERROR = "PARAMETERS_ERROR" 38 | SYSTEM_ERROR = "SYSTEM_ERROR" 39 | 40 | def __init__(self, type_, message): 41 | self.type = type_ 42 | self.message = self._format_message(message) 43 | 44 | def _format_message(self, msg): 45 | if isinstance(msg, Exception): 46 | return "{}: {}".format(msg.__class__.__name__, "{}".format(msg)) 47 | return msg 48 | 49 | @property 50 | def value(self): 51 | return {"type": self.type, "message": self.message} 52 | 53 | def __bool__(self): 54 | return False 55 | 56 | @classmethod 57 | def build_resource_error(cls, message=None): 58 | return cls(cls.RESOURCE_ERROR, message) 59 | 60 | @classmethod 61 | def build_system_error(cls, message=None): 62 | return cls(cls.SYSTEM_ERROR, message) 63 | 64 | @classmethod 65 | def build_parameters_error(cls, message=None): 66 | return cls(cls.PARAMETERS_ERROR, message) 67 | 68 | @classmethod 69 | def build_from_invalid_request_object(cls, invalid_request_object): 70 | """Create an error message from an invalid requested object 71 | 72 | Args: 73 | invalid_request_object (obj): The requested object that is invalid 74 | 75 | Returns: 76 | string: The message consisting of the error parameters and the message 77 | """ 78 | 79 | message = "\n".join( 80 | [ 81 | "{}: {}".format(err["parameter"], err["message"]) 82 | for err in invalid_request_object.errors 83 | ] 84 | ) 85 | 86 | print("_______________") 87 | print(message) 88 | 89 | return cls.build_parameters_error(message) 90 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Welcome! 2 | 3 | We are so glad you're thinking about contributing to the Keep-Current project! If you're unsure about something, please don't hesitate to ask us. 4 | 5 | We want to ensure a welcoming environment for all the Keep-Current different repositories. Please follow the [Code of Conduct](CODE_OF_CONDUCT.md). 6 | 7 | We encourage you to read the [License](LICENSE) and the [Readme](README.md). 8 | 9 | ## How to Contribute 10 | 11 | You can find our Project board here on [GitHub](https://github.com/Keep-Current/web-miner/projects) and we use [Slack](https://keep-current.slack.com) as our communication channel. If you're new, you can join using [this link](https://join.slack.com/t/keep-current/shared_invite/enQtMzY4MTA0OTQ0NTAzLTcxY2U5NmIwNmM0NmU2MmMyMWQ0YTIyMTg4MWRjMWUyYmVlNWQxMzU3ZWJlNjM4NzVmNTFhM2FjYjkzZDU3YWM). 12 | 13 | We welcome anyone who would like to join and contribute. 14 | 15 | We meet regularly every month in Vienna through 16 | 17 | - the [Data Science Cafe meetup of the VDSG](https://www.meetup.com/Vienna-Data-Science-Group-Meetup/) or 18 | - the [WeAreDevelopers :: Keep-Current meetup](https://www.meetup.com/WeAreDevelopers/) 19 | 20 | to show our progress and discuss the next steps. 21 | 22 | ### Project Architecture 23 | 24 | We follow the [clean architecture style](https://8thlight.com/blog/uncle-bob/2012/08/13/the-clean-architecture.html) and structure the codebase accordingly. 25 | 26 | ![cleanArchitecture image](https://cdn-images-1.medium.com/max/1600/1*B7LkQDyDqLN3rRSrNYkETA.jpeg) 27 | 28 | _Image credit to [Uncle Bob](https://8thlight.com/blog/uncle-bob/)_ 29 | 30 | Most important rule: 31 | 32 | > Source code dependencies can only point inwards. Nothing in an inner circle can know anything at all about something in an outer circle. In particular, the name of something declared in an outer circle must not be mentioned by the code in the an inner circle. That includes, functions, classes. variables, or any other named software entity. 33 | 34 | ### Git workflow 35 | 36 | Currently we use an adapted version of [Gitflow](https://www.atlassian.com/git/tutorials/comparing-workflows/gitflow-workflow). We are currently not using the develop branch as the project structure does not require it. Everything else is the same. 37 | 38 | ![gitflow image](https://www.bluesource.at/fileadmin/user_upload/bluesource/Wissen/Detailseite/git-model.jpg) 39 | 40 | _Image credit to [nvie.com](https://nvie.com/posts/a-successful-git-branching-model/)_ 41 | 42 | ### Issues / Pull Request 43 | 44 | We have created templates for issues and pull requests to ensure a coherent workflow. 45 | 46 | As a general rule, always make sure that: 47 | 48 | - you are formatting your code according to the black formatter 49 | - you are linting your code with pylint and our pylint settings 50 | - all tests pass 51 | 52 | We have several integration tools hooked up for reviewing pull requests. Make sure that those tests are also passing, and if not provide a detailled explanation why not and why it is not necessary to comply. 53 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. 6 | 7 | ## Our Standards 8 | 9 | Examples of behavior that contributes to creating a positive environment include: 10 | 11 | * Using welcoming and inclusive language 12 | * Being respectful of differing viewpoints and experiences 13 | * Gracefully accepting constructive criticism 14 | * Focusing on what is best for the community 15 | * Showing empathy towards other community members 16 | 17 | Examples of unacceptable behavior by participants include: 18 | 19 | * The use of sexualized language or imagery and unwelcome sexual attention or advances 20 | * Trolling, insulting/derogatory comments, and personal or political attacks 21 | * Public or private harassment 22 | * Publishing others' private information, such as a physical or electronic address, without explicit permission 23 | * Other conduct which could reasonably be considered inappropriate in a professional setting 24 | 25 | ## Our Responsibilities 26 | 27 | Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. 28 | 29 | Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. 30 | 31 | ## Scope 32 | 33 | This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. 34 | 35 | ## Enforcement 36 | 37 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at liad.magen@gmail.com. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. 38 | 39 | Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. 40 | 41 | ## Attribution 42 | 43 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version] 44 | 45 | [homepage]: http://contributor-covenant.org 46 | [version]: http://contributor-covenant.org/version/1/4/ 47 | -------------------------------------------------------------------------------- /tests/interface_adapters/rest_adapters/test_response_object.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from webminer.interface_adapters.rest_adapters import response_object as res, request_object as req 4 | 5 | 6 | @pytest.fixture 7 | def response_value(): 8 | return {"key": ["value1", "value2"]} 9 | 10 | 11 | @pytest.fixture 12 | def response_type(): 13 | return "ResponseError" 14 | 15 | 16 | @pytest.fixture 17 | def response_message(): 18 | return "This is a response error" 19 | 20 | 21 | def test_valid_request_object_cannot_be_used(): 22 | with pytest.raises(NotImplementedError): 23 | req.ValidRequestObject.from_dict({}) 24 | 25 | 26 | def test_response_success_is_true(response_value): 27 | assert bool(res.ResponseSuccess(response_value)) is True 28 | 29 | 30 | def test_response_failure_is_false(response_type, response_message): 31 | assert bool(res.ResponseFailure(response_type, response_message)) is False 32 | 33 | 34 | def test_response_success_contains_value(response_value): 35 | response = res.ResponseSuccess(response_value) 36 | 37 | assert response.value == response_value 38 | 39 | 40 | def test_response_failure_has_type_and_message(response_type, response_message): 41 | response = res.ResponseFailure(response_type, response_message) 42 | 43 | assert response.type == response_type 44 | assert response.message == response_message 45 | 46 | 47 | def test_response_failure_contains_value(response_type, response_message): 48 | response = res.ResponseFailure(response_type, response_message) 49 | 50 | assert response.value == {"type": response_type, "message": response_message} 51 | 52 | 53 | def test_response_failure_initialization_with_exception(): 54 | response = res.ResponseFailure(response_type, Exception("Just an error message")) 55 | 56 | assert bool(response) is False 57 | assert response.type == response_type 58 | assert response.message == "Exception: Just an error message" 59 | 60 | 61 | def test_response_failure_from_invalid_request_object(): 62 | response = res.ResponseFailure.build_from_invalid_request_object( 63 | req.InvalidRequestObject() 64 | ) 65 | 66 | assert bool(response) is False 67 | 68 | 69 | def test_response_failure_from_invalid_request_object_with_errors(): 70 | request_object = req.InvalidRequestObject() 71 | request_object.add_error("path", "Is mandatory") 72 | request_object.add_error("path", "can't be blank") 73 | 74 | response = res.ResponseFailure.build_from_invalid_request_object(request_object) 75 | 76 | assert bool(response) is False 77 | assert response.type == res.ResponseFailure.PARAMETERS_ERROR 78 | assert response.message == "path: Is mandatory\npath: can't be blank" 79 | 80 | 81 | def test_response_failure_build_resource_error(): 82 | response = res.ResponseFailure.build_resource_error("test message") 83 | 84 | assert bool(response) is False 85 | assert response.type == res.ResponseFailure.RESOURCE_ERROR 86 | assert response.message == "test message" 87 | 88 | 89 | def test_response_failure_build_parameters_error(): 90 | response = res.ResponseFailure.build_parameters_error("test message") 91 | 92 | assert bool(response) is False 93 | assert response.type == res.ResponseFailure.PARAMETERS_ERROR 94 | assert response.message == "test message" 95 | 96 | 97 | def test_response_failure_build_system_error(): 98 | response = res.ResponseFailure.build_system_error("test message") 99 | 100 | assert bool(response) is False 101 | assert response.type == res.ResponseFailure.SYSTEM_ERROR 102 | assert response.message == "test message" 103 | -------------------------------------------------------------------------------- /tests/interface_adapters/test_process_arxiv_request.py: -------------------------------------------------------------------------------- 1 | """Testing the serialization of the use-case ouput 2 | """ 3 | 4 | from unittest import mock 5 | import pytest 6 | 7 | from webminer.entities.arxiv_document import ArxivDocument 8 | from webminer.interface_adapters.rest_adapters import response_object as res 9 | from webminer.interface_adapters.rest_adapters import request_objects as req 10 | from webminer.interface_adapters import process_arxiv_request as uc 11 | 12 | 13 | @pytest.fixture 14 | def domain_arxivdocs(): 15 | """Creates a fixture for the returned objects 16 | """ 17 | 18 | arxiv_doc_1 = ArxivDocument( 19 | doc_id="url_1", 20 | url="url_1", 21 | title="title_1", 22 | abstract="abstract_1", 23 | authors=["author1", "author2"], 24 | publish_date="publish_date_1", 25 | pdf_url="pfg_url1", 26 | ) 27 | 28 | arxiv_doc_2 = ArxivDocument( 29 | doc_id="url_2", 30 | url="url_2", 31 | title="title_2", 32 | abstract="abstract_2", 33 | authors=["author2", "author2"], 34 | publish_date="publish_date_2", 35 | pdf_url="pfg_url2", 36 | ) 37 | 38 | arxiv_doc_3 = ArxivDocument( 39 | doc_id="url_3", 40 | url="url_3", 41 | title="title_3", 42 | abstract="abstract_3", 43 | authors=["author3", "author2"], 44 | publish_date="publish_date_3", 45 | pdf_url="pfg_url3", 46 | ) 47 | 48 | arxiv_doc_4 = ArxivDocument( 49 | doc_id="url_4", 50 | url="url_4", 51 | title="title_4", 52 | abstract="abstract_4", 53 | authors=["author4", "author2"], 54 | publish_date="publish_date_4", 55 | pdf_url="pfg_url4", 56 | ) 57 | 58 | return [arxiv_doc_1, arxiv_doc_2, arxiv_doc_3, arxiv_doc_4] 59 | 60 | 61 | def test_arxiv_doc_list_without_parameters(domain_arxivdocs): 62 | """Tests calling the ProcessArxivDocuments method without any params 63 | 64 | Args: 65 | domain_arxivdocs ([type]): the expected results 66 | 67 | Raises: 68 | AssertionError: If nothing was returned 69 | AssertionError: If the response is not the expected one 70 | """ 71 | 72 | repo = mock.Mock() 73 | repo.list.return_value = domain_arxivdocs 74 | 75 | arxiv_doc_list_use_case = uc.ProcessArxivDocuments(repo) 76 | request_object = req.ArxivDocumentListRequestObject.from_dict({}) 77 | 78 | response_object = arxiv_doc_list_use_case.execute(request_object) 79 | 80 | if not bool(response_object): 81 | raise AssertionError("response_object is empty") 82 | repo.list.assert_called_with(filters=None) 83 | 84 | if response_object.value != domain_arxivdocs: 85 | raise AssertionError("respons differs form expected") 86 | 87 | 88 | def test_arxiv_doc_list_with_filters(domain_arxivdocs): 89 | """Tests calling the usecase filter with parameters 90 | TODO - implement that part. 91 | 92 | Args: 93 | domain_arxivdocs ([type]): The expected filtered documents 94 | """ 95 | 96 | repo = mock.Mock() 97 | repo.list.return_value = domain_arxivdocs 98 | 99 | arxiv_doc_list_use_case = uc.ProcessArxivDocuments(repo) 100 | qry_filters = {"a": 5} 101 | request_object = req.ArxivDocumentListRequestObject.from_dict( 102 | {"filters": qry_filters} 103 | ) 104 | 105 | response_object = arxiv_doc_list_use_case.execute(request_object) 106 | 107 | if not bool(response_object): 108 | raise AssertionError("response_object is empty") 109 | repo.list.assert_called_with(filters=qry_filters) 110 | if response_object.value != domain_arxivdocs: 111 | raise AssertionError("respons differs form expected") 112 | 113 | 114 | def test_arxiv_doc_list_handles_generic_error(): 115 | """Tests handling of a generic error when the request is empty 116 | """ 117 | 118 | repo = mock.Mock() 119 | repo.list.side_effect = Exception("Just an error message") 120 | 121 | arxiv_doc_list_use_case = uc.ProcessArxivDocuments(repo) 122 | request_object = req.ArxivDocumentListRequestObject.from_dict({}) 123 | 124 | response_object = arxiv_doc_list_use_case.execute(request_object) 125 | 126 | if bool(response_object): 127 | raise AssertionError("response_object supposed to be empty") 128 | if response_object.value != { 129 | "type": res.ResponseFailure.SYSTEM_ERROR, 130 | "message": "Exception: Just an error message", 131 | }: 132 | raise AssertionError("error response differs from expected") 133 | 134 | 135 | def test_arxiv_doc_list_handles_bad_request(): 136 | """Tests handling a usecase with a bad request 137 | """ 138 | 139 | repo = mock.Mock() 140 | 141 | arxiv_doc_list_use_case = uc.ProcessArxivDocuments(repo) 142 | request_object = req.ArxivDocumentListRequestObject.from_dict({"filters": 5}) 143 | 144 | response_object = arxiv_doc_list_use_case.execute(request_object) 145 | 146 | if bool(response_object): 147 | raise AssertionError("response_object supposed to be empty") 148 | if response_object.value != { 149 | "type": res.ResponseFailure.PARAMETERS_ERROR, 150 | "message": "filters: Is not iterable", 151 | }: 152 | raise AssertionError("error response differs from expected") 153 | -------------------------------------------------------------------------------- /webminer/use_cases/request_arxiv/arxiv_repo.py: -------------------------------------------------------------------------------- 1 | """ 2 | This is a helper class to parse arxiv.org site. 3 | It uses the arxiv.org REST API to search for articles. 4 | 5 | based on karpathy's arxiv-sanity: 6 | https://github.com/karpathy/arxiv-sanity-preserver/blob/master/fetch_papers.py 7 | and arxiv example: 8 | https://arxiv.org/help/api/examples/python_arXiv_parsing_example.txt 9 | """ 10 | 11 | import requests 12 | import feedparser 13 | 14 | from webminer.entities import arxiv_document as ad 15 | 16 | 17 | class ArxivRepo: 18 | """ 19 | The helper class to parse arxiv.org site. 20 | 21 | Raises: 22 | ValueError: If formatting operator is not supported 23 | AssertionError: If url parsing went wrong 24 | 25 | Returns: 26 | dict: Dictionary of results 27 | """ 28 | 29 | # Base api query url 30 | base_url = "http://export.arxiv.org/api/query?" 31 | 32 | # Expose both of the open source metadata namespaces in feedparser 33 | feedparser._FeedParserMixin.namespaces[ # pylint: disable=W0212 34 | "http://a9.com/-/spec/opensearch/1.1/" 35 | ] = "opensearch" 36 | feedparser._FeedParserMixin.namespaces[ # pylint: disable=W0212 37 | "http://arxiv.org/schemas/atom" 38 | ] = "arxiv" 39 | 40 | def __init__(self, entries=None): 41 | self._entries = [] 42 | if entries: 43 | self._entries.extend(entries) 44 | 45 | def _check(self, element, key, value): 46 | """Checks elements and formats them 47 | 48 | Args: 49 | element (obj): Document object 50 | key (string): Key in document object 51 | value (string): Value of the corresponding document object 52 | 53 | Raises: 54 | ValueError: If some operators are not supported 55 | 56 | Returns: 57 | string: Value of the attribute of an object 58 | """ 59 | 60 | if "__" not in key: 61 | key = key + "__eq" 62 | 63 | key, operator = key.split("__") 64 | 65 | if operator not in ["eq", "lt", "gt"]: 66 | raise ValueError(f"Operator {operator} is not supported") 67 | 68 | operator = "__{}__".format(operator) 69 | 70 | return getattr(element[key], operator)(value) 71 | 72 | def list(self, filters=None): 73 | """Apply filters on entries and return a filtered list 74 | filters (dict, optional): Defaults to None. Parameters to filter entries 75 | 76 | Returns: 77 | list: List of arxiv document objects 78 | """ 79 | 80 | if not filters: 81 | return self._entries 82 | 83 | self._entries = self.fetch_papers() 84 | 85 | result = [] 86 | result.extend(self._entries) 87 | 88 | for key, value in filters.items(): 89 | result = [e for e in result if self._check(e, key, value)] 90 | 91 | return [ad.ArxivDocument.from_dict(r) for r in result] 92 | 93 | # @classmethod 94 | # def encode_feedparser_dict(cls, fp_dict): 95 | # """ 96 | # recursive function to convert the internal feedparse object to a simple dict 97 | # """ 98 | # if isinstance(fp_dict, feedparser.FeedParserDict) or isinstance(fp_dict, dict): 99 | # ret_dict = {} 100 | # for key in fp_dict.keys(): 101 | # ret_dict[key] = self.encode_feedparser_dict(fp_dict[key]) 102 | # return ret_dict 103 | # elif isinstance(fp_dict, list): 104 | # dict_list = [] 105 | # for key in fp_dict: 106 | # dict_list.append(self.encode_feedparser_dict(key)) 107 | # return dict_list 108 | # else: 109 | # return fp_dict 110 | 111 | @classmethod 112 | def extract_relevant_info(cls, fp_dict): 113 | """Extracts the relevant info 114 | 115 | Args: 116 | fp_dict (dict): Dictionary that provides the information 117 | 118 | Returns: 119 | dict: The dictionary with only relevant information 120 | """ 121 | 122 | ret_dict = {} 123 | ret_dict["publish_date"] = fp_dict["published"] 124 | ret_dict["authors"] = [auth["name"] for auth in fp_dict["authors"]] 125 | ret_dict["title"] = fp_dict["title"] 126 | ret_dict["abstract"] = fp_dict["summary"] 127 | ret_dict["id"] = fp_dict["id"] 128 | ret_dict["link"] = fp_dict["link"] 129 | 130 | for link in fp_dict["links"]: 131 | try: 132 | if link.title == "pdf": 133 | ret_dict["pdf"] = link.href 134 | except AttributeError: 135 | pass 136 | 137 | return ret_dict 138 | 139 | def run_query(self, search_query, start=0, max_results=10): 140 | """Queries url and returns a parsed response 141 | 142 | Args: 143 | search_query (string): The terms we are looking for 144 | start (int, optional): Defaults to 0. Start at results page 145 | max_results (int, optional): 146 | Defaults to 10. How many results shall be retrieved 147 | 148 | Returns: 149 | FeedParserDict: A parsed dictionary with the information 150 | """ 151 | 152 | query = f"search_query={search_query}&sortBy=lastUpdatedDate&start={start}&max_results={max_results}" # pylint: disable=C0301 153 | r = requests.get(self.base_url + query) 154 | parsed_response = feedparser.parse(r.text) 155 | 156 | return parsed_response 157 | 158 | @classmethod 159 | def parse_arxiv_url(cls, url): 160 | """ 161 | extracts the raw id and the version 162 | examples is http://arxiv.org/abs/1512.08756v2 163 | """ 164 | ix = url.rfind("/") 165 | idversion = url[ix + 1 :] # extract just the id (and the version) 166 | parts = idversion.split("v") 167 | if not len(parts) == 2: 168 | raise AssertionError("error parsing url " + url) 169 | return parts[0], int(parts[1]) 170 | 171 | default_search_query = ( 172 | "cat:cs.CV+OR+cat:cs.AI+OR+cat:cs.LG+OR+cat:cs.CL+OR+cat:cs.NE+OR+cat:stat.ML" 173 | ) 174 | 175 | def fetch_papers( 176 | self, 177 | search_query=default_search_query, 178 | start_index=0, 179 | max_index=100, 180 | results_per_iteration=100, 181 | # wait_time=5.0, 182 | # break_on_no_added=True, 183 | ): 184 | """loops according to the results_per_iteration and fetch results pages from arxiv.org 185 | 186 | Keyword Arguments: 187 | search_query {str} -- [arxiv.org topics to query] 188 | (default: default_search_query) 189 | start_index {int} -- [pagination start index] (default: {0}) 190 | max_index {int} -- 191 | [upper bound on paper index we will fetch] (default: {10000}) 192 | results_per_iteration {int} -- [passed to arxiv API] (default: {100}) 193 | wait_time {float} -- [pause in seconds between requests] (default: {5.0}) 194 | break_on_no_added {bool} -- 195 | [break out early if all returned query papers are already in db] 196 | (default: {True}) 197 | """ 198 | 199 | for i in range(start_index, max_index, results_per_iteration): 200 | parsed_response = self.run_query(search_query, i, results_per_iteration) 201 | 202 | results = [] 203 | 204 | for entry in parsed_response.entries: 205 | dict_entry = self.extract_relevant_info(entry) 206 | 207 | rawid, version = self.parse_arxiv_url(dict_entry["link"]) 208 | dict_entry["_rawid"] = rawid 209 | dict_entry["_version"] = version 210 | 211 | results.append(dict_entry) 212 | 213 | return results 214 | -------------------------------------------------------------------------------- /tests/use_case/test_arxiv_repo.py: -------------------------------------------------------------------------------- 1 | """Tests the arxiv repository 2 | """ 3 | 4 | import pytest 5 | import responses 6 | 7 | from webminer.entities.arxiv_document import ArxivDocument 8 | from webminer.use_cases.request_arxiv import arxiv_repo as a_repo 9 | 10 | arxiv_doc_1 = ArxivDocument( 11 | doc_id="url_1", 12 | url="url_1", 13 | title="title_1", 14 | abstract="abstract_1", 15 | authors=["author1", "author2"], 16 | publish_date="publish_date_1", 17 | pdf_url="pfg_url1", 18 | ) 19 | 20 | arxiv_doc_2 = ArxivDocument( 21 | doc_id="url_2", 22 | url="url_2", 23 | title="title_2", 24 | abstract="abstract_2", 25 | authors=["author2", "author2"], 26 | publish_date="publish_date_2", 27 | pdf_url="pfg_url2", 28 | ) 29 | 30 | arxiv_doc_3 = ArxivDocument( 31 | doc_id="url_3", 32 | url="url_3", 33 | title="title_3", 34 | abstract="abstract_3", 35 | authors=["author3", "author2"], 36 | publish_date="publish_date_3", 37 | pdf_url="pfg_url3", 38 | ) 39 | 40 | arxiv_doc_4 = ArxivDocument( 41 | doc_id="url_4", 42 | url="url_4", 43 | title="title_4", 44 | abstract="abstract_4", 45 | authors=["author4", "author2"], 46 | publish_date="publish_date_4", 47 | pdf_url="pfg_url4", 48 | ) 49 | 50 | arxiv_response = """ 51 | 52 | 53 | ArXiv Query: search_query=cat:cs.CV OR cat:cs.AI OR cat:cs.LG OR cat:cs.CL OR cat:cs.NE OR cat:stat.ML&id_list=&start=1&max_results=100 54 | http://arxiv.org/api//mx7Y+oW1RP05QqmCZfHNto2duM 55 | 2018-09-23T00:00:00-04:00 56 | 68000 57 | 1 58 | 100 59 | 60 | http://arxiv.org/abs/1809.07759v1 61 | 2018-09-20T17:48:27Z 62 | 2018-09-20T17:48:27Z 63 | Implementing Adaptive Separable Convolution for Video Frame 64 | Interpolation 65 | As Deep Neural Networks are becoming more popular, much of the attention is 66 | being devoted to Computer Vision problems that used to be solved with more 67 | traditional approaches. 68 | 69 | 70 | Mart Kartašev 71 | 72 | 73 | Carlo Rapisarda 74 | 75 | 76 | Dominik Fay 77 | 78 | All authors contributed equally 79 | 80 | 81 | 82 | 83 | 84 | 85 | http://arxiv.org/abs/1806.01482v2 86 | 2018-09-20T17:42:42Z 87 | 2018-06-05T03:49:46Z 88 | SoPhie: An Attentive GAN for Predicting Paths Compliant to Social and 89 | Physical Constraints 90 | This paper addresses the problem of path prediction for multiple interacting 91 | agents in a scene, which is a crucial step for many autonomous platforms such 92 | as self-driving cars and social robots. 93 | 94 | 95 | Amir Sadeghian 96 | 97 | 98 | Vineet Kosaraju 99 | 100 | 101 | Ali Sadeghian 102 | 103 | 104 | Noriaki Hirose 105 | 106 | 107 | S. Hamid Rezatofighi 108 | 109 | 110 | Silvio Savarese 111 | 112 | 113 | 114 | 115 | 116 | """ 117 | 118 | arxiv_result = [ 119 | { 120 | "publish_date": "2018-09-20T17:48:27Z", 121 | "authors": ["Mart KartaÅ¡ev", "Carlo Rapisarda", "Dominik Fay"], 122 | "title": "Implementing Adaptive Separable Convolution for Video" 123 | + " Frame\n Interpolation", 124 | "abstract": "As Deep Neural Networks are becoming more popular," 125 | + " much of the attention is\nbeing devoted to Computer Vision problems" 126 | + " that used to be solved with more\ntraditional approaches.", 127 | "id": "http://arxiv.org/abs/1809.07759v1", 128 | "link": "http://arxiv.org/abs/1809.07759v1", 129 | "pdf": "http://arxiv.org/pdf/1809.07759v1", 130 | "_rawid": "1809.07759", 131 | "_version": 1, 132 | }, 133 | { 134 | "publish_date": "2018-06-05T03:49:46Z", 135 | "authors": [ 136 | "Amir Sadeghian", 137 | "Vineet Kosaraju", 138 | "Ali Sadeghian", 139 | "Noriaki Hirose", 140 | "S. Hamid Rezatofighi", 141 | "Silvio Savarese", 142 | ], 143 | "title": "SoPhie: An Attentive GAN for Predicting Paths Compliant" 144 | + " to Social and\n Physical Constraints", 145 | "abstract": "This paper addresses the problem of path prediction for" 146 | + " multiple interacting\nagents in a scene, which is a crucial step" 147 | + " for many autonomous platforms such\nas self-driving cars and " 148 | + "social robots.", 149 | "id": "http://arxiv.org/abs/1806.01482v2", 150 | "link": "http://arxiv.org/abs/1806.01482v2", 151 | "pdf": "http://arxiv.org/pdf/1806.01482v2", 152 | "_rawid": "1806.01482", 153 | "_version": 2, 154 | }, 155 | ] 156 | 157 | 158 | @pytest.fixture 159 | def domain_arxivdocs(): 160 | """Creates a fixture for the returned objects 161 | """ 162 | return [arxiv_doc_1, arxiv_doc_2, arxiv_doc_3, arxiv_doc_4] 163 | 164 | 165 | def assert_equal(arg1, arg2): 166 | if arg1 != arg2: 167 | print("arg1: ", arg1) 168 | print("arg2: ", arg1) 169 | raise AssertionError("Assert equal failed - values are not equal") 170 | 171 | 172 | def _check_results(domain_models_list, data_list): 173 | assert_equal(len(domain_models_list), len(data_list)) 174 | if not all([isinstance(dm, DomainModel) for dm in domain_models_list]): 175 | raise AssertionError("not all domain model returned true") 176 | assert_equal( 177 | set([dm.doc_id for dm in domain_models_list]), 178 | set([d["doc_id"] for d in data_list]), 179 | ) 180 | 181 | 182 | def test_repository_list_without_parameters(domain_arxivdocs): 183 | repo = a_repo.ArxivRepo(domain_arxivdocs) 184 | 185 | assert_equal(repo.list(), domain_arxivdocs) 186 | 187 | 188 | @responses.activate 189 | def test_extract_relevant_info(): 190 | url = "http://export.arxiv.org/api/query?search_query=cat:cs.CV+OR+cat:cs.AI+OR+cat:cs.LG+OR+cat:cs.CL+OR+cat:cs.NE+OR+cat:stat.ML&sortBy=lastUpdatedDate&start=0&max_results=100" 191 | responses.add(method=responses.GET, url=url, body=arxiv_response, status=200) 192 | 193 | repo = a_repo.ArxivRepo() 194 | result = repo.fetch_papers() 195 | 196 | assert_equal(len(result), 2) 197 | assert_equal(result, arxiv_result) 198 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Keep-Current - The Web Miner 2 | 3 | 4 | 5 | [![Codacy Badge](https://api.codacy.com/project/badge/Grade/883c8e93b4934566b9dfdc6b91fa85e7)](https://app.codacy.com/app/Keep-Current/web-miner?utm_source=github.com&utm_medium=referral&utm_content=Keep-Current/web-miner&utm_campaign=badger) 6 | [![Build Status](https://travis-ci.org/Keep-Current/web-miner.svg?branch=master)](https://travis-ci.org/Keep-Current/web-miner) 7 | [![CircleCI](https://circleci.com/gh/Keep-Current/web-miner.svg?style=svg)](https://circleci.com/gh/Keep-Current/web-miner) 8 | [![BCH compliance](https://bettercodehub.com/edge/badge/Keep-Current/web-miner?branch=master)](https://bettercodehub.com/) 9 | [![codecov](https://codecov.io/gh/Keep-Current/web-miner/branch/master/graph/badge.svg)](https://codecov.io/gh/Keep-Current/web-miner) 10 | [![codebeat badge](https://codebeat.co/badges/03da69a3-74cf-468d-80f9-bc62651323f7)](https://codebeat.co/projects/github-com-keep-current-web-miner-master) 11 | 12 | ## Web Miner 13 | 14 | This repository is the web miner of the [Keep-Current project](#keep-current-project). 15 | 16 | The goal is to deploy a web crawler, that given a specific set of sources (URLs), should locate new documents (web-pages) and save them in the DB for future processing. 17 | When possible and legal, an API can be used. 18 | For example, for [arxiv.org](https://arxiv.org/help/api/index). 19 | 20 | ### Potential tools to implement 21 | 22 | We lean heavily on existing tools as well as developing our own new methods. 23 | 24 | - [scrapy](https://scrapy.org/) which later we hope to host on [scrapy-cloud](https://scrapinghub.com/scrapy-cloud) 25 | - [scrapy-splash](https://github.com/scrapy-plugins/scrapy-splash) which can render JS-based pages before storing them. 26 | - [Textract](https://github.com/deanmalmgren/textract) can be used to extract the content (the text) to be saved. 27 | 28 | ### Getting started 29 | 30 | for running this project locally, you need first to install the dependency packages. 31 | To install them, you can use 32 | 33 | - [pipenv](https://docs.pipenv.org/) 34 | - [anaconda](https://anaconda.org/) 35 | - [virtualenv](https://virtualenv.pypa.io/en/stable/) 36 | 37 | #### Installation using pipenv (which combines virtualenv with pip) 38 | 39 | Install pipenv 40 | 41 | ```bash 42 | sudo easy_install pip # if you haven't installed pip 43 | pip install pipenv # install pipenv 44 | 45 | brew install pipenv # with homebrew (on macOS) 46 | ``` 47 | 48 | Install the packages and run the server 49 | 50 | ```bash 51 | pipenv install # install all packages 52 | 53 | pipenv run flask run # run the server 54 | ``` 55 | 56 | If you are on Windows OS, some packages may not be installed. Specifically - feedparser. In case the web server doesn't run, please install these packages manually using 57 | 58 | ```bash 59 | pip install feedparser 60 | ``` 61 | 62 | #### Installing using Anaconda 63 | 64 | If you have anaconda installed, it's recommended to create an environment for the project, and install the dependencies in it. 65 | 66 | ```bash 67 | conda create -q -n web-miner python=3.6 # create the environment 68 | 69 | source activate web-miner # activate the environment 70 | 71 | pip install pipenv 72 | 73 | pipenv install 74 | ``` 75 | 76 | and test your installation by running the web server: 77 | 78 | ```bash 79 | flask run # start server 80 | ``` 81 | 82 | #### Installing using virtualenv and pip 83 | 84 | ```bash 85 | sudo easy_install pip # installl pip if you haven't 86 | 87 | pip3 install --upgrade virtualenv # install virtualenv 88 | 89 | virtualenv --python3 # create the environment 90 | 91 | source /./bin/activate # activate the virtualenv 92 | 93 | pip install pipenv 94 | 95 | pipenv install 96 | 97 | flask run # start server 98 | ``` 99 | 100 | ### Architecture 101 | 102 | ### Project Architecture 103 | 104 | We follow the [clean architecture style](https://8thlight.com/blog/uncle-bob/2012/08/13/the-clean-architecture.html) and structure the codebase accordingly. 105 | 106 | ![cleanArchitecture image](https://cdn-images-1.medium.com/max/1600/1*B7LkQDyDqLN3rRSrNYkETA.jpeg) 107 | 108 | _Image credit to [Uncle Bob](https://8thlight.com/blog/uncle-bob/)_ 109 | 110 | Most important rule: 111 | 112 | > Source code dependencies can only point inwards. Nothing in an inner circle can know anything at all about something in an outer circle. In particular, the name of something declared in an outer circle must not be mentioned by the code in the an inner circle. That includes, functions, classes. variables, or any other named software entity. 113 | 114 | ## Who are we? 115 | 116 | This project intends to be a shared work of meetup members, with the purpose, beside the obvious result, to also be used as a learning platform, while advancing the Natural Language Processing / Machine Learning field by exploring, comparing and hacking different models. 117 | 118 | Please visit 119 | 120 | - the project board on [Github](https://github.com/orgs/Keep-Current/projects) 121 | - the repository board on [Github](https://github.com/Keep-Current/web-miner/projects) 122 | - our chat room on [Slack](https://keep-current.slack.com). If you're new, you can join using [this link](https://join.slack.com/t/keep-current/shared_invite/enQtMzY3Mzk1NjE2MzIzLWZlZWFjMDM1YWYxYmI5ZWE4YmZjNWYzMmNjMzlhMDYzOTIxZDViODhmNTMzZDI0NThmZWVlOTRjNjczZGJiOWE) 123 | - our [facebook group](https://www.facebook.com/groups/308893846340861/) where we discuss and share current topics also outside of the project 124 | 125 | for more. 126 | 127 | ## How to Contribute 128 | 129 | You can find our Project board here on [GitHub](https://github.com/Keep-Current/web-miner/projects) and we use [Slack](https://keep-current.slack.com) as our communication channel. If you're new, you can join using [this link](https://join.slack.com/t/keep-current/shared_invite/enQtMzY4MTA0OTQ0NTAzLTcxY2U5NmIwNmM0NmU2MmMyMWQ0YTIyMTg4MWRjMWUyYmVlNWQxMzU3ZWJlNjM4NzVmNTFhM2FjYjkzZDU3YWM) 130 | 131 | We welcome anyone who would like to join and contribute. 132 | 133 | Please see our [contribute guide](CONTRIBUTING.md). 134 | 135 | We meet regularly every month in Vienna through 136 | 137 | - the [Data Science Cafe meetup of the VDSG](https://www.meetup.com/Vienna-Data-Science-Group-Meetup/) or 138 | - the [WeAreDevelopers :: Keep-Current meetup](https://www.meetup.com/WeAreDevelopers/) 139 | 140 | to show our progress and discuss the next steps. 141 | 142 | ## Keep-Current Project 143 | 144 | After studying a topic, keeping current with the news, published papers, advanced technologies and such proved to be a hard work. 145 | One must attend conventions, subscribe to different websites and newsletters, go over different emails, alerts and such while filtering the relevant data out of these sources. 146 | 147 | In this project, we aspire to create a platform for students, researchers, professionals and enthusiasts to discover news on relevant topics. The users are encouraged to constantly give a feedback on the suggestions, in order to adapt and personalize future results. 148 | 149 | The goal is to create an automated system that scans the web, through a list of trusted sources, classify and categorize the documents it finds, and match them to the different users, according to their interest. It then presents it as a timely summarized digest to the user, whether by email or within a site. 150 | 151 | This repository is the web miner. It encourage you to learn about software architecture, mining the web, setting up web-spiders, scheduling CRON Jobs, creating pipelines, etc. 152 | 153 | If you wish to assist in different aspects (Data Engineering / Web development / DevOps), we have divided the project to several additional repositories focusing on these topics: 154 | 155 | - The machine-learning engine can be found in our [Main repository](https://github.com/Keep-Current/Engine) 156 | - Web Development & UI/UX experiments can be found in our [App repository](https://github.com/Keep-Current/WebApp) 157 | - Data Engineering tasks are more than welcomed in our [Data Engineering repository](https://github.com/Keep-Current/Data-Engineering) 158 | - Devops tasks are all across the project. This project is developed mostly in a serverless architecture. Using Docker and Kubernetes enables freedom in deploying it on different hosting providers and plans. 159 | 160 | _Feel free to join the discussion and provide your input!_ 161 | 162 | [travis-badge-url]: https://travis-ci.org/Keep-Current/web-miner.svg?branch=master 163 | -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- 1 | [MASTER] 2 | 3 | # Specify a configuration file. 4 | #rcfile= 5 | 6 | # Python code to execute, usually for sys.path manipulation such as 7 | # pygtk.require(). 8 | #init-hook= 9 | 10 | # Profiled execution. 11 | profile=no 12 | 13 | # Add files or directories to the blacklist. They should be base names, not 14 | # paths. 15 | ignore=CVS 16 | 17 | # Pickle collected data for later comparisons. 18 | persistent=yes 19 | 20 | # List of plugins (as comma separated values of python modules names) to load, 21 | # usually to register additional checkers. 22 | load-plugins= 23 | 24 | 25 | [MESSAGES CONTROL] 26 | 27 | # Enable the message, report, category or checker with the given id(s). You can 28 | # either give multiple identifier separated by comma (,) or put this option 29 | # multiple time. See also the "--disable" option for examples. 30 | enable=indexing-exception,old-raise-syntax 31 | 32 | # Disable the message, report, category or checker with the given id(s). You 33 | # can either give multiple identifiers separated by comma (,) or put this 34 | # option multiple times (only on the command line, not in the configuration 35 | # file where it should appear only once).You can also use "--disable=all" to 36 | # disable everything first and then reenable specific checks. For example, if 37 | # you want to run only the similarities checker, you can use "--disable=all 38 | # --enable=similarities". If you want to run only the classes checker, but have 39 | # no Warning level messages displayed, use"--disable=all --enable=classes 40 | # --disable=W" 41 | disable=design,similarities,no-self-use,attribute-defined-outside-init,locally-disabled,star-args,pointless-except,bad-option-value,global-statement,fixme,suppressed-message,useless-suppression,locally-enabled,no-member,no-name-in-module,import-error,unsubscriptable-object,unbalanced-tuple-unpacking,undefined-variable,not-context-manager 42 | 43 | 44 | # Set the cache size for astng objects. 45 | cache-size=500 46 | 47 | 48 | [REPORTS] 49 | 50 | # Set the output format. Available formats are text, parseable, colorized, msvs 51 | # (visual studio) and html. You can also give a reporter class, eg 52 | # mypackage.mymodule.MyReporterClass. 53 | output-format=text 54 | 55 | # Put messages in a separate file for each module / package specified on the 56 | # command line instead of printing them on stdout. Reports (if any) will be 57 | # written in a file name "pylint_global.[txt|html]". 58 | files-output=no 59 | 60 | # Tells whether to display a full report or only the messages 61 | reports=no 62 | 63 | # Python expression which should return a note less than 10 (10 is the highest 64 | # note). You have access to the variables errors warning, statement which 65 | # respectively contain the number of errors / warnings messages and the total 66 | # number of statements analyzed. This is used by the global evaluation report 67 | # (RP0004). 68 | evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) 69 | 70 | # Add a comment according to your evaluation note. This is used by the global 71 | # evaluation report (RP0004). 72 | comment=no 73 | 74 | # Template used to display messages. This is a python new-style format string 75 | # used to format the message information. See doc for all details 76 | #msg-template= 77 | 78 | 79 | [TYPECHECK] 80 | 81 | # Tells whether missing members accessed in mixin class should be ignored. A 82 | # mixin class is detected if its name ends with "mixin" (case insensitive). 83 | ignore-mixin-members=yes 84 | 85 | # List of classes names for which member attributes should not be checked 86 | # (useful for classes with attributes dynamically set). 87 | ignored-classes=SQLObject 88 | 89 | # When zope mode is activated, add a predefined set of Zope acquired attributes 90 | # to generated-members. 91 | zope=no 92 | 93 | # List of members which are set dynamically and missed by pylint inference 94 | # system, and so shouldn't trigger E0201 when accessed. Python regular 95 | # expressions are accepted. 96 | generated-members=REQUEST,acl_users,aq_parent 97 | 98 | # List of decorators that create context managers from functions, such as 99 | # contextlib.contextmanager. 100 | contextmanager-decorators=contextlib.contextmanager,contextlib2.contextmanager 101 | 102 | 103 | [VARIABLES] 104 | 105 | # Tells whether we should check for unused import in __init__ files. 106 | init-import=no 107 | 108 | # A regular expression matching the beginning of the name of dummy variables 109 | # (i.e. not used). 110 | dummy-variables-rgx=^\*{0,2}(_$|unused_|dummy_) 111 | 112 | # List of additional names supposed to be defined in builtins. Remember that 113 | # you should avoid to define new builtins when possible. 114 | additional-builtins= 115 | 116 | 117 | [BASIC] 118 | 119 | # Required attributes for module, separated by a comma 120 | required-attributes= 121 | 122 | # List of builtins function names that should not be used, separated by a comma 123 | bad-functions=apply,input,reduce 124 | 125 | 126 | # Disable the report(s) with the given id(s). 127 | # All non-Google reports are disabled by default. 128 | disable-report=R0001,R0002,R0003,R0004,R0101,R0102,R0201,R0202,R0220,R0401,R0402,R0701,R0801,R0901,R0902,R0903,R0904,R0911,R0912,R0913,R0914,R0915,R0921,R0922,R0923 129 | 130 | # Regular expression which should only match correct module names 131 | module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ 132 | 133 | # Regular expression which should only match correct module level names 134 | const-rgx=^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$ 135 | 136 | # Regular expression which should only match correct class names 137 | class-rgx=^_?[A-Z][a-zA-Z0-9]*$ 138 | 139 | # Regular expression which should only match correct function names 140 | function-rgx=^(?:(?P_?[A-Z][a-zA-Z0-9]*)|(?P_?[a-z][a-z0-9_]*))$ 141 | 142 | # Regular expression which should only match correct method names 143 | method-rgx=^(?:(?P__[a-z0-9_]+__|next)|(?P_{0,2}[A-Z][a-zA-Z0-9]*)|(?P_{0,2}[a-z][a-z0-9_]*))$ 144 | 145 | # Regular expression which should only match correct instance attribute names 146 | attr-rgx=^_{0,2}[a-z][a-z0-9_]*$ 147 | 148 | # Regular expression which should only match correct argument names 149 | argument-rgx=^[a-z][a-z0-9_]*$ 150 | 151 | # Regular expression which should only match correct variable names 152 | variable-rgx=^[a-z][a-z0-9_]*$ 153 | 154 | # Regular expression which should only match correct attribute names in class 155 | # bodies 156 | class-attribute-rgx=^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$ 157 | 158 | # Regular expression which should only match correct list comprehension / 159 | # generator expression variable names 160 | inlinevar-rgx=^[a-z][a-z0-9_]*$ 161 | 162 | # Good variable names which should always be accepted, separated by a comma 163 | good-names=main,_ 164 | 165 | # Bad variable names which should always be refused, separated by a comma 166 | bad-names= 167 | 168 | # Regular expression which should only match function or class names that do 169 | # not require a docstring. 170 | no-docstring-rgx=(__.*__|main) 171 | 172 | # Minimum line length for functions/classes that require docstrings, shorter 173 | # ones are exempt. 174 | docstring-min-length=10 175 | 176 | 177 | [FORMAT] 178 | 179 | # Maximum number of characters on a single line. 180 | max-line-length=88 181 | 182 | # Regexp for a line that is allowed to be longer than the limit. 183 | ignore-long-lines=(?x) 184 | (^\s*(import|from)\s 185 | |\$Id:\s\/\/depot\/.+#\d+\s\$ 186 | |^[a-zA-Z_][a-zA-Z0-9_]*\s*=\s*("[^"]\S+"|'[^']\S+') 187 | |^\s*\#\ LINT\.ThenChange 188 | |^[^#]*\#\ type:\ [a-zA-Z_][a-zA-Z0-9_.,[\] ]*$ 189 | |pylint 190 | |""" 191 | |\# 192 | |lambda 193 | |(https?|ftp):) 194 | 195 | # Allow the body of an if to be on the same line as the test if there is no 196 | # else. 197 | single-line-if-stmt=y 198 | 199 | # List of optional constructs for which whitespace checking is disabled 200 | no-space-check= 201 | 202 | # Maximum number of lines in a module 203 | max-module-lines=99999 204 | 205 | # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 206 | # tab). 207 | indent-string=' ' 208 | 209 | 210 | [SIMILARITIES] 211 | 212 | # Minimum lines number of a similarity. 213 | min-similarity-lines=4 214 | 215 | # Ignore comments when computing similarities. 216 | ignore-comments=yes 217 | 218 | # Ignore docstrings when computing similarities. 219 | ignore-docstrings=yes 220 | 221 | # Ignore imports when computing similarities. 222 | ignore-imports=no 223 | 224 | 225 | [MISCELLANEOUS] 226 | 227 | # List of note tags to take in consideration, separated by a comma. 228 | notes= 229 | 230 | 231 | [IMPORTS] 232 | 233 | # Deprecated modules which should not be used, separated by a comma 234 | deprecated-modules=regsub,TERMIOS,Bastion,rexec,sets 235 | 236 | # Create a graph of every (i.e. internal and external) dependencies in the 237 | # given file (report RP0402 must not be disabled) 238 | import-graph= 239 | 240 | # Create a graph of external dependencies in the given file (report RP0402 must 241 | # not be disabled) 242 | ext-import-graph= 243 | 244 | # Create a graph of internal dependencies in the given file (report RP0402 must 245 | # not be disabled) 246 | int-import-graph= 247 | 248 | 249 | [CLASSES] 250 | 251 | # List of interface methods to ignore, separated by a comma. This is used for 252 | # instance to not check methods defines in Zope's Interface base class. 253 | ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by 254 | 255 | # List of method names used to declare (i.e. assign) instance attributes. 256 | defining-attr-methods=__init__,__new__,setUp 257 | 258 | # List of valid names for the first argument in a class method. 259 | valid-classmethod-first-arg=cls,class_ 260 | 261 | # List of valid names for the first argument in a metaclass class method. 262 | valid-metaclass-classmethod-first-arg=mcs 263 | 264 | 265 | [DESIGN] 266 | 267 | # Maximum number of arguments for function / method 268 | max-args=5 269 | 270 | # Argument names that match this expression will be ignored. Default to name 271 | # with leading underscore 272 | ignored-argument-names=_.* 273 | 274 | # Maximum number of locals for function / method body 275 | max-locals=15 276 | 277 | # Maximum number of return / yield for function / method body 278 | max-returns=6 279 | 280 | # Maximum number of branch for function / method body 281 | max-branches=12 282 | 283 | # Maximum number of statements in function / method body 284 | max-statements=50 285 | 286 | # Maximum number of parents for a class (see R0901). 287 | max-parents=7 288 | 289 | # Maximum number of attributes for a class (see R0902). 290 | max-attributes=7 291 | 292 | # Minimum number of public methods for a class (see R0903). 293 | min-public-methods=2 294 | 295 | # Maximum number of public methods for a class (see R0904). 296 | max-public-methods=20 297 | 298 | 299 | [EXCEPTIONS] 300 | 301 | # Exceptions that will emit a warning when being caught. Defaults to 302 | # "Exception" 303 | overgeneral-exceptions=Exception,StandardError,BaseException 304 | 305 | 306 | [AST] 307 | 308 | # Maximum line length for lambdas 309 | short-func-length=1 310 | 311 | # List of module members that should be marked as deprecated. 312 | # All of the string functions are listed in 4.1.4 Deprecated string functions 313 | # in the Python 2.4 docs. 314 | deprecated-members=string.atof,string.atoi,string.atol,string.capitalize,string.expandtabs,string.find,string.rfind,string.index,string.rindex,string.count,string.lower,string.split,string.rsplit,string.splitfields,string.join,string.joinfields,string.lstrip,string.rstrip,string.strip,string.swapcase,string.translate,string.upper,string.ljust,string.rjust,string.center,string.zfill,string.replace,sys.exitfunc 315 | 316 | 317 | [DOCSTRING] 318 | 319 | # List of exceptions that do not need to be mentioned in the Raises section of 320 | # a docstring. 321 | ignore-exceptions=AssertionError,NotImplementedError,StopIteration,TypeError 322 | 323 | 324 | 325 | [TOKENS] 326 | 327 | # Number of spaces of indent required when the last token on the preceding line 328 | # is an open (, [, or {. 329 | indent-after-paren=4 330 | 331 | # Regexp for a proper copyright notice. (changed) 332 | copyright=Copyright \d{4} The TensorFlow Authors\. +All [Rr]ights [Rr]eserved\. -------------------------------------------------------------------------------- /Pipfile.lock: -------------------------------------------------------------------------------- 1 | { 2 | "_meta": { 3 | "hash": { 4 | "sha256": "cca7be2c5b96757b058e0c153e1cc373523e056f14b1f272bf1fc3ac7965ca12" 5 | }, 6 | "pipfile-spec": 6, 7 | "requires": { 8 | "python_version": "3.6" 9 | }, 10 | "sources": [ 11 | { 12 | "name": "pypi", 13 | "url": "https://pypi.org/simple", 14 | "verify_ssl": true 15 | } 16 | ] 17 | }, 18 | "default": { 19 | "alabaster": { 20 | "hashes": [ 21 | "sha256:446438bdcca0e05bd45ea2de1668c1d9b032e1a9154c2c259092d77031ddd359", 22 | "sha256:a661d72d58e6ea8a57f7a86e37d86716863ee5e92788398526d58b26a4e4dc02" 23 | ], 24 | "version": "==0.7.12" 25 | }, 26 | "atomicwrites": { 27 | "hashes": [ 28 | "sha256:0312ad34fcad8fac3704d441f7b317e50af620823353ec657a53e981f92920c0", 29 | "sha256:ec9ae8adaae229e4f8446952d204a3e4b5fdd2d099f9be3aaf556120135fb3ee" 30 | ], 31 | "version": "==1.2.1" 32 | }, 33 | "attrs": { 34 | "hashes": [ 35 | "sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69", 36 | "sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb" 37 | ], 38 | "version": "==18.2.0" 39 | }, 40 | "babel": { 41 | "hashes": [ 42 | "sha256:6778d85147d5d85345c14a26aada5e478ab04e39b078b0745ee6870c2b5cf669", 43 | "sha256:8cba50f48c529ca3fa18cf81fa9403be176d374ac4d60738b839122dfaaa3d23" 44 | ], 45 | "version": "==2.6.0" 46 | }, 47 | "certifi": { 48 | "hashes": [ 49 | "sha256:376690d6f16d32f9d1fe8932551d80b23e9d393a8578c5633a2ed39a64861638", 50 | "sha256:456048c7e371c089d0a77a5212fb37a2c2dce1e24146e3b7e0261736aaeaa22a" 51 | ], 52 | "version": "==2018.8.24" 53 | }, 54 | "chardet": { 55 | "hashes": [ 56 | "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", 57 | "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" 58 | ], 59 | "version": "==3.0.4" 60 | }, 61 | "click": { 62 | "hashes": [ 63 | "sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13", 64 | "sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7" 65 | ], 66 | "version": "==7.0" 67 | }, 68 | "codecov": { 69 | "hashes": [ 70 | "sha256:8ed8b7c6791010d359baed66f84f061bba5bd41174bf324c31311e8737602788", 71 | "sha256:ae00d68e18d8a20e9c3288ba3875ae03db3a8e892115bf9b83ef20507732bed4" 72 | ], 73 | "index": "pypi", 74 | "version": "==2.0.15" 75 | }, 76 | "cookies": { 77 | "hashes": [ 78 | "sha256:15bee753002dff684987b8df8c235288eb8d45f8191ae056254812dfd42c81d3", 79 | "sha256:d6b698788cae4cfa4e62ef8643a9ca332b79bd96cb314294b864ae8d7eb3ee8e" 80 | ], 81 | "version": "==2.2.1" 82 | }, 83 | "coverage": { 84 | "hashes": [ 85 | "sha256:05adfd7b9058026377b65af69f14abd8f74c8df99651aafc1b63a252864ebd22", 86 | "sha256:0dcf381f51f589f1f797449602a7fe4e63be8a7963c259c13742af3f30be902e", 87 | "sha256:11a4bb30306def2fa012e3429de44a93ef2ae3b6ad3f6b800f6c578658a5c402", 88 | "sha256:166c957a38b034050a14201f64eec11fc95e17bf2ba31fc07d887db82bae1a47", 89 | "sha256:184e6680f85fcc1b371f67ab732290ecf96a225448198e14ec170986db47b0aa", 90 | "sha256:1904deb72c561a8e445feb190db07ca4b165ee85567894b4b85fdb9bf21a27c0", 91 | "sha256:1f2003b83426cfaadebff8b9bb1fb3650134a15fda3a81434cc8415896d7a7bc", 92 | "sha256:1f462997b1804f8b5d1ee2b262626fc76b746e66023eb64f529af35991167c7c", 93 | "sha256:213697f49eba45b5fb05e77f63bdb7c0d13eed12dcd08e6af43224615b28b524", 94 | "sha256:245a5bde6f777dc6a2e797c2d9cf997e35508ed02bb87105fec4f65550737d3b", 95 | "sha256:2557da232b0daeb55afe2f7e55f7b80c56bfa2981864c6638b32b5691da9f4c3", 96 | "sha256:395a8525f1456439a5d6c248bc1397040491047e3e0e0c4ceb2059155419cd3b", 97 | "sha256:43d6334b35e50e74d034ec075ffd9082c559bca624924af6c7e9d2b8aef0f362", 98 | "sha256:4566c74bde36aaaef0372fb11678edf43dcc73f4eb8dbb6987250658c4a3b95a", 99 | "sha256:4946ee7df3b2223d6be40a3531a869e714abf1f159047ba5d0372e69a79e5d13", 100 | "sha256:5305bc1d8571d1162b9c843229806e4f4ac6da6eeb94dc4a06cae7616854d569", 101 | "sha256:6d39cc527c9c7a30f20bed14b5cf9a7e87ef1f3528c1847d1c81caf75a31ebb6", 102 | "sha256:8bd69d3cba21d885df6fe8728cee779a722da08cf84072558956c148b5ab61e5", 103 | "sha256:95a0f6d78b898865b83d0027ffcff4e8f0b1b7323515f21be4b8be2824b698e3", 104 | "sha256:a1d0fcbbe0735eb66c6622266b12e60ea8d37ada405cb8f73b154c5eec467187", 105 | "sha256:ab706bfbb365f232be01a536a9199ee6bfc80c9b63fb7825fdd5f4ae5cc2a12c", 106 | "sha256:ade570b15380d2752dea759e98aa36be73ea7710703fbd71e070602edd0bf774", 107 | "sha256:afbf4cee68d2f2968b06951cf16c0b18513eb59bb3af0685084de6cacb04e217", 108 | "sha256:bbc8913cd5889df7eab597a4b4074a2c6c5ee6ca9aad58a9ba0f3f847b1a99df", 109 | "sha256:bd5428ab378a7432e43afa52b6bb9c5d48f5029f395a97dc9ebf87fc0f2a9d8b", 110 | "sha256:c3efe0185583443e04f8519818f4772d92fbbdf5f9fa23165f2f2482b20efc37", 111 | "sha256:d40277e918da575d008e2955a0ca6600f870bdb3570b07ee3a754ea9301862e7", 112 | "sha256:d4b6ec6951e20ea3f5d1fefe35b4bcbf692d4306f1b932c28dd2ee4cb167152c", 113 | "sha256:d5837e813ad62c856bc80f988c4e24e0d2b7b22a8a1dad8c1cfcb8ff4d4750a8", 114 | "sha256:d9583ae0e152c5fb0142cb55c3a11e1b13006c00d0c3e8b35ccc2d4ebfc6645e", 115 | "sha256:de5d5284e410957dd99799a59707ed3dd3c462adb9e116abc8abb8177b87b087", 116 | "sha256:e27380cbe4088a1df514e75aa4fe6dc9e98bbd7902cf28ab16e8b2de0f8cb344", 117 | "sha256:e624daef32f8808296312e72190c7e576852cb75c27935b31c1bbbde14ab353c", 118 | "sha256:ef4278e5ac1e47c731ec5e3e48351721e01d2eb4fefa9b97fcdba7495a82cfad", 119 | "sha256:fd1da071003e2d16947262af1adeb39a8d592c198f1c670b0e898f3c944944ac" 120 | ], 121 | "index": "pypi", 122 | "version": "==5.0a2" 123 | }, 124 | "docutils": { 125 | "hashes": [ 126 | "sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6", 127 | "sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274", 128 | "sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6" 129 | ], 130 | "version": "==0.14" 131 | }, 132 | "feedparser": { 133 | "hashes": [ 134 | "sha256:bd030652c2d08532c034c27fcd7c85868e7fa3cb2b17f230a44a6bbc92519bf9", 135 | "sha256:cd2485472e41471632ed3029d44033ee420ad0b57111db95c240c9160a85831c", 136 | "sha256:ce875495c90ebd74b179855449040003a1beb40cd13d5f037a0654251e260b02" 137 | ], 138 | "index": "pypi", 139 | "version": "==5.2.1" 140 | }, 141 | "flake8": { 142 | "hashes": [ 143 | "sha256:7253265f7abd8b313e3892944044a365e3f4ac3fcdcfb4298f55ee9ddf188ba0", 144 | "sha256:c7841163e2b576d435799169b78703ad6ac1bbb0f199994fc05f700b2a90ea37" 145 | ], 146 | "index": "pypi", 147 | "version": "==3.5.0" 148 | }, 149 | "flask": { 150 | "hashes": [ 151 | "sha256:2271c0070dbcb5275fad4a82e29f23ab92682dc45f9dfbc22c02ba9b9322ce48", 152 | "sha256:a080b744b7e345ccfcbc77954861cb05b3c63786e93f2b3875e0913d44b43f05" 153 | ], 154 | "index": "pypi", 155 | "version": "==1.0.2" 156 | }, 157 | "gunicorn": { 158 | "hashes": [ 159 | "sha256:aa8e0b40b4157b36a5df5e599f45c9c76d6af43845ba3b3b0efe2c70473c2471", 160 | "sha256:fa2662097c66f920f53f70621c6c58ca4a3c4d3434205e608e121b5b3b71f4f3" 161 | ], 162 | "index": "pypi", 163 | "version": "==19.9.0" 164 | }, 165 | "idna": { 166 | "hashes": [ 167 | "sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e", 168 | "sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16" 169 | ], 170 | "version": "==2.7" 171 | }, 172 | "imagesize": { 173 | "hashes": [ 174 | "sha256:3f349de3eb99145973fefb7dbe38554414e5c30abd0c8e4b970a7c9d09f3a1d8", 175 | "sha256:f3832918bc3c66617f92e35f5d70729187676313caa60c187eb0f28b8fe5e3b5" 176 | ], 177 | "version": "==1.1.0" 178 | }, 179 | "itsdangerous": { 180 | "hashes": [ 181 | "sha256:cbb3fcf8d3e33df861709ecaf89d9e6629cff0a217bc2848f1b41cd30d360519" 182 | ], 183 | "version": "==0.24" 184 | }, 185 | "jinja2": { 186 | "hashes": [ 187 | "sha256:74c935a1b8bb9a3947c50a54766a969d4846290e1e788ea44c1392163723c3bd", 188 | "sha256:f84be1bb0040caca4cea721fcbbbbd61f9be9464ca236387158b0feea01914a4" 189 | ], 190 | "version": "==2.10.1" 191 | }, 192 | "logzero": { 193 | "hashes": [ 194 | "sha256:34fa1e2e436dfa9f37e5ff8750e932bafe0c5abbb42e1f669e4cf5ce1f179142", 195 | "sha256:818072e4fcb53a3f6fb4114a92f920e1135fe6f47bffd9dc2b6c4d10eedacf27" 196 | ], 197 | "index": "pypi", 198 | "version": "==1.5.0" 199 | }, 200 | "markupsafe": { 201 | "hashes": [ 202 | "sha256:a6be69091dac236ea9c6bc7d012beab42010fa914c459791d627dad4910eb665" 203 | ], 204 | "version": "==1.0" 205 | }, 206 | "mccabe": { 207 | "hashes": [ 208 | "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", 209 | "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" 210 | ], 211 | "version": "==0.6.1" 212 | }, 213 | "more-itertools": { 214 | "hashes": [ 215 | "sha256:c187a73da93e7a8acc0001572aebc7e3c69daf7bf6881a2cea10650bd4420092", 216 | "sha256:c476b5d3a34e12d40130bc2f935028b5f636df8f372dc2c1c01dc19681b2039e", 217 | "sha256:fcbfeaea0be121980e15bc97b3817b5202ca73d0eae185b4550cbfce2a3ebb3d" 218 | ], 219 | "version": "==4.3.0" 220 | }, 221 | "packaging": { 222 | "hashes": [ 223 | "sha256:0886227f54515e592aaa2e5a553332c73962917f2831f1b0f9b9f4380a4b9807", 224 | "sha256:f95a1e147590f204328170981833854229bb2912ac3d5f89e2a8ccd2834800c9" 225 | ], 226 | "version": "==18.0" 227 | }, 228 | "pluggy": { 229 | "hashes": [ 230 | "sha256:6e3836e39f4d36ae72840833db137f7b7d35105079aee6ec4a62d9f80d594dd1", 231 | "sha256:95eb8364a4708392bae89035f45341871286a333f749c3141c20573d2b3876e1" 232 | ], 233 | "version": "==0.7.1" 234 | }, 235 | "py": { 236 | "hashes": [ 237 | "sha256:06a30435d058473046be836d3fc4f27167fd84c45b99704f2fb5509ef61f9af1", 238 | "sha256:50402e9d1c9005d759426988a492e0edaadb7f4e68bcddfea586bc7432d009c6" 239 | ], 240 | "version": "==1.6.0" 241 | }, 242 | "pycodestyle": { 243 | "hashes": [ 244 | "sha256:682256a5b318149ca0d2a9185d365d8864a768a28db66a84a2ea946bcc426766", 245 | "sha256:6c4245ade1edfad79c3446fadfc96b0de2759662dc29d07d80a6f27ad1ca6ba9" 246 | ], 247 | "version": "==2.3.1" 248 | }, 249 | "pyflakes": { 250 | "hashes": [ 251 | "sha256:08bd6a50edf8cffa9fa09a463063c425ecaaf10d1eb0335a7e8b1401aef89e6f", 252 | "sha256:8d616a382f243dbf19b54743f280b80198be0bca3a5396f1d2e1fca6223e8805" 253 | ], 254 | "version": "==1.6.0" 255 | }, 256 | "pygments": { 257 | "hashes": [ 258 | "sha256:78f3f434bcc5d6ee09020f92ba487f95ba50f1e3ef83ae96b9d5ffa1bab25c5d", 259 | "sha256:dbae1046def0efb574852fab9e90209b23f556367b5a320c0bcb871c77c3e8cc" 260 | ], 261 | "version": "==2.2.0" 262 | }, 263 | "pyparsing": { 264 | "hashes": [ 265 | "sha256:bc6c7146b91af3f567cf6daeaec360bc07d45ffec4cf5353f4d7a208ce7ca30a", 266 | "sha256:d29593d8ebe7b57d6967b62494f8c72b03ac0262b1eed63826c6f788b3606401" 267 | ], 268 | "version": "==2.2.2" 269 | }, 270 | "pytest": { 271 | "hashes": [ 272 | "sha256:7e258ee50338f4e46957f9e09a0f10fb1c2d05493fa901d113a8dafd0790de4e", 273 | "sha256:9332147e9af2dcf46cd7ceb14d5acadb6564744ddff1fe8c17f0ce60ece7d9a2" 274 | ], 275 | "index": "pypi", 276 | "version": "==3.8.2" 277 | }, 278 | "pytest-cov": { 279 | "hashes": [ 280 | "sha256:513c425e931a0344944f84ea47f3956be0e416d95acbd897a44970c8d926d5d7", 281 | "sha256:e360f048b7dae3f2f2a9a4d067b2dd6b6a015d384d1577c994a43f3f7cbad762" 282 | ], 283 | "index": "pypi", 284 | "version": "==2.6.0" 285 | }, 286 | "pytest-flask": { 287 | "hashes": [ 288 | "sha256:2e64ba176ccc00e84adb88b38a4a44a032a09f98992881f09b0baab9ab7d06a6", 289 | "sha256:959f01a2e6121d4208263f571bf2de24aa89ebf2f752b15824e4e597fa35bb7e" 290 | ], 291 | "index": "pypi", 292 | "version": "==0.13.0" 293 | }, 294 | "pytz": { 295 | "hashes": [ 296 | "sha256:a061aa0a9e06881eb8b3b2b43f05b9439d6583c206d0a6c340ff72a7b6669053", 297 | "sha256:ffb9ef1de172603304d9d2819af6f5ece76f2e85ec10692a524dd876e72bf277" 298 | ], 299 | "version": "==2018.5" 300 | }, 301 | "requests": { 302 | "hashes": [ 303 | "sha256:63b52e3c866428a224f97cab011de738c36aec0185aa91cfacd418b5d58911d1", 304 | "sha256:ec22d826a36ed72a7358ff3fe56cbd4ba69dd7a6718ffd450ff0e9df7a47ce6a" 305 | ], 306 | "index": "pypi", 307 | "version": "==2.20.0" 308 | }, 309 | "responses": { 310 | "hashes": [ 311 | "sha256:c6082710f4abfb60793899ca5f21e7ceb25aabf321560cc0726f8b59006811c9", 312 | "sha256:f23a29dca18b815d9d64a516b4a0abb1fbdccff6141d988ad8100facb81cf7b3" 313 | ], 314 | "index": "pypi", 315 | "version": "==0.9.0" 316 | }, 317 | "six": { 318 | "hashes": [ 319 | "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", 320 | "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb" 321 | ], 322 | "version": "==1.11.0" 323 | }, 324 | "snowballstemmer": { 325 | "hashes": [ 326 | "sha256:919f26a68b2c17a7634da993d91339e288964f93c274f1343e3bbbe2096e1128", 327 | "sha256:9f3bcd3c401c3e862ec0ebe6d2c069ebc012ce142cce209c098ccb5b09136e89" 328 | ], 329 | "version": "==1.2.1" 330 | }, 331 | "sphinx": { 332 | "hashes": [ 333 | "sha256:652eb8c566f18823a022bb4b6dbc868d366df332a11a0226b5bc3a798a479f17", 334 | "sha256:d222626d8356de702431e813a05c68a35967e3d66c6cd1c2c89539bb179a7464" 335 | ], 336 | "index": "pypi", 337 | "version": "==1.8.1" 338 | }, 339 | "sphinxcontrib-websupport": { 340 | "hashes": [ 341 | "sha256:68ca7ff70785cbe1e7bccc71a48b5b6d965d79ca50629606c7861a21b206d9dd", 342 | "sha256:9de47f375baf1ea07cdb3436ff39d7a9c76042c10a769c52353ec46e4e8fc3b9" 343 | ], 344 | "version": "==1.1.0" 345 | }, 346 | "toml": { 347 | "hashes": [ 348 | "sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c", 349 | "sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e", 350 | "sha256:f1db651f9657708513243e61e6cc67d101a39bad662eaa9b5546f789338e07a3" 351 | ], 352 | "version": "==0.10.0" 353 | }, 354 | "tox": { 355 | "hashes": [ 356 | "sha256:7f802b37fffd3b5ef2aab104943fa5dad24bf9564bb7e732e54b8d0cfec2fca0", 357 | "sha256:cc97859bd7f38aa5b3b8ba55ffe7ee9952e7050faad1aedc0829cd3db2fb61d6" 358 | ], 359 | "index": "pypi", 360 | "version": "==3.4.0" 361 | }, 362 | "urllib3": { 363 | "hashes": [ 364 | "sha256:a68ac5e15e76e7e5dd2b8f94007233e01effe3e50e8daddf69acfd81cb686baf", 365 | "sha256:b5725a0bd4ba422ab0e66e89e030c806576753ea3ee08554382c14e685d117b5" 366 | ], 367 | "version": "==1.24.2" 368 | }, 369 | "virtualenv": { 370 | "hashes": [ 371 | "sha256:2ce32cd126117ce2c539f0134eb89de91a8413a29baac49cbab3eb50e2026669", 372 | "sha256:ca07b4c0b54e14a91af9f34d0919790b016923d157afda5efdde55c96718f752" 373 | ], 374 | "version": "==16.0.0" 375 | }, 376 | "werkzeug": { 377 | "hashes": [ 378 | "sha256:c3fd7a7d41976d9f44db327260e263132466836cef6f91512889ed60ad26557c", 379 | "sha256:d5da73735293558eb1651ee2fddc4d0dedcfa06538b8813a2e20011583c9e49b" 380 | ], 381 | "version": "==0.14.1" 382 | } 383 | }, 384 | "develop": { 385 | "appdirs": { 386 | "hashes": [ 387 | "sha256:9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92", 388 | "sha256:d8b24664561d0d34ddfaec54636d502d7cea6e29c3eaf68f3df6180863e2166e" 389 | ], 390 | "version": "==1.4.3" 391 | }, 392 | "astroid": { 393 | "hashes": [ 394 | "sha256:37f8e89d0e78a649edeb3751b408e96d103e76a1df19d79a0a3b559d0f4f7cd1", 395 | "sha256:39870f07180e50c5a1c73a6de7b7cb487d6db649c0acd9917f154617e09f9e94" 396 | ], 397 | "version": "==2.1.0.dev0" 398 | }, 399 | "attrs": { 400 | "hashes": [ 401 | "sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69", 402 | "sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb" 403 | ], 404 | "version": "==18.2.0" 405 | }, 406 | "black": { 407 | "hashes": [ 408 | "sha256:817243426042db1d36617910df579a54f1afd659adb96fc5032fcf4b36209739", 409 | "sha256:e030a9a28f542debc08acceb273f228ac422798e5215ba2a791a6ddeaaca22a5" 410 | ], 411 | "index": "pypi", 412 | "version": "==18.9b0" 413 | }, 414 | "certifi": { 415 | "hashes": [ 416 | "sha256:376690d6f16d32f9d1fe8932551d80b23e9d393a8578c5633a2ed39a64861638", 417 | "sha256:456048c7e371c089d0a77a5212fb37a2c2dce1e24146e3b7e0261736aaeaa22a" 418 | ], 419 | "version": "==2018.8.24" 420 | }, 421 | "chardet": { 422 | "hashes": [ 423 | "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", 424 | "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" 425 | ], 426 | "version": "==3.0.4" 427 | }, 428 | "click": { 429 | "hashes": [ 430 | "sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13", 431 | "sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7" 432 | ], 433 | "version": "==7.0" 434 | }, 435 | "cookies": { 436 | "hashes": [ 437 | "sha256:15bee753002dff684987b8df8c235288eb8d45f8191ae056254812dfd42c81d3", 438 | "sha256:d6b698788cae4cfa4e62ef8643a9ca332b79bd96cb314294b864ae8d7eb3ee8e" 439 | ], 440 | "version": "==2.2.1" 441 | }, 442 | "idna": { 443 | "hashes": [ 444 | "sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e", 445 | "sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16" 446 | ], 447 | "version": "==2.7" 448 | }, 449 | "isort": { 450 | "hashes": [ 451 | "sha256:1153601da39a25b14ddc54955dbbacbb6b2d19135386699e2ad58517953b34af", 452 | "sha256:b9c40e9750f3d77e6e4d441d8b0266cf555e7cdabdcff33c4fd06366ca761ef8", 453 | "sha256:ec9ef8f4a9bc6f71eec99e1806bfa2de401650d996c59330782b89a5555c1497" 454 | ], 455 | "version": "==4.3.4" 456 | }, 457 | "lazy-object-proxy": { 458 | "hashes": [ 459 | "sha256:0ce34342b419bd8f018e6666bfef729aec3edf62345a53b537a4dcc115746a33", 460 | "sha256:1b668120716eb7ee21d8a38815e5eb3bb8211117d9a90b0f8e21722c0758cc39", 461 | "sha256:209615b0fe4624d79e50220ce3310ca1a9445fd8e6d3572a896e7f9146bbf019", 462 | "sha256:27bf62cb2b1a2068d443ff7097ee33393f8483b570b475db8ebf7e1cba64f088", 463 | "sha256:27ea6fd1c02dcc78172a82fc37fcc0992a94e4cecf53cb6d73f11749825bd98b", 464 | "sha256:2c1b21b44ac9beb0fc848d3993924147ba45c4ebc24be19825e57aabbe74a99e", 465 | "sha256:2df72ab12046a3496a92476020a1a0abf78b2a7db9ff4dc2036b8dd980203ae6", 466 | "sha256:320ffd3de9699d3892048baee45ebfbbf9388a7d65d832d7e580243ade426d2b", 467 | "sha256:50e3b9a464d5d08cc5227413db0d1c4707b6172e4d4d915c1c70e4de0bbff1f5", 468 | "sha256:5276db7ff62bb7b52f77f1f51ed58850e315154249aceb42e7f4c611f0f847ff", 469 | "sha256:61a6cf00dcb1a7f0c773ed4acc509cb636af2d6337a08f362413c76b2b47a8dd", 470 | "sha256:6ae6c4cb59f199d8827c5a07546b2ab7e85d262acaccaacd49b62f53f7c456f7", 471 | "sha256:7661d401d60d8bf15bb5da39e4dd72f5d764c5aff5a86ef52a042506e3e970ff", 472 | "sha256:7bd527f36a605c914efca5d3d014170b2cb184723e423d26b1fb2fd9108e264d", 473 | "sha256:7cb54db3535c8686ea12e9535eb087d32421184eacc6939ef15ef50f83a5e7e2", 474 | "sha256:7f3a2d740291f7f2c111d86a1c4851b70fb000a6c8883a59660d95ad57b9df35", 475 | "sha256:81304b7d8e9c824d058087dcb89144842c8e0dea6d281c031f59f0acf66963d4", 476 | "sha256:933947e8b4fbe617a51528b09851685138b49d511af0b6c0da2539115d6d4514", 477 | "sha256:94223d7f060301b3a8c09c9b3bc3294b56b2188e7d8179c762a1cda72c979252", 478 | "sha256:ab3ca49afcb47058393b0122428358d2fbe0408cf99f1b58b295cfeb4ed39109", 479 | "sha256:bd6292f565ca46dee4e737ebcc20742e3b5be2b01556dafe169f6c65d088875f", 480 | "sha256:cb924aa3e4a3fb644d0c463cad5bc2572649a6a3f68a7f8e4fbe44aaa6d77e4c", 481 | "sha256:d0fc7a286feac9077ec52a927fc9fe8fe2fabab95426722be4c953c9a8bede92", 482 | "sha256:ddc34786490a6e4ec0a855d401034cbd1242ef186c20d79d2166d6a4bd449577", 483 | "sha256:e34b155e36fa9da7e1b7c738ed7767fc9491a62ec6af70fe9da4a057759edc2d", 484 | "sha256:e5b9e8f6bda48460b7b143c3821b21b452cb3a835e6bbd5dd33aa0c8d3f5137d", 485 | "sha256:e81ebf6c5ee9684be8f2c87563880f93eedd56dd2b6146d8a725b50b7e5adb0f", 486 | "sha256:eb91be369f945f10d3a49f5f9be8b3d0b93a4c2be8f8a5b83b0571b8123e0a7a", 487 | "sha256:f460d1ceb0e4a5dcb2a652db0904224f367c9b3c1470d5a7683c0480e582468b" 488 | ], 489 | "version": "==1.3.1" 490 | }, 491 | "mccabe": { 492 | "hashes": [ 493 | "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", 494 | "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" 495 | ], 496 | "version": "==0.6.1" 497 | }, 498 | "pylint": { 499 | "hashes": [ 500 | "sha256:1d6d3622c94b4887115fe5204982eee66fdd8a951cf98635ee5caee6ec98c3ec", 501 | "sha256:31142f764d2a7cd41df5196f9933b12b7ee55e73ef12204b648ad7e556c119fb" 502 | ], 503 | "index": "pypi", 504 | "version": "==2.1.1" 505 | }, 506 | "requests": { 507 | "hashes": [ 508 | "sha256:63b52e3c866428a224f97cab011de738c36aec0185aa91cfacd418b5d58911d1", 509 | "sha256:ec22d826a36ed72a7358ff3fe56cbd4ba69dd7a6718ffd450ff0e9df7a47ce6a" 510 | ], 511 | "index": "pypi", 512 | "version": "==2.20.0" 513 | }, 514 | "responses": { 515 | "hashes": [ 516 | "sha256:c6082710f4abfb60793899ca5f21e7ceb25aabf321560cc0726f8b59006811c9", 517 | "sha256:f23a29dca18b815d9d64a516b4a0abb1fbdccff6141d988ad8100facb81cf7b3" 518 | ], 519 | "index": "pypi", 520 | "version": "==0.9.0" 521 | }, 522 | "six": { 523 | "hashes": [ 524 | "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", 525 | "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb" 526 | ], 527 | "version": "==1.11.0" 528 | }, 529 | "toml": { 530 | "hashes": [ 531 | "sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c", 532 | "sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e", 533 | "sha256:f1db651f9657708513243e61e6cc67d101a39bad662eaa9b5546f789338e07a3" 534 | ], 535 | "version": "==0.10.0" 536 | }, 537 | "typed-ast": { 538 | "hashes": [ 539 | "sha256:0948004fa228ae071054f5208840a1e88747a357ec1101c17217bfe99b299d58", 540 | "sha256:10703d3cec8dcd9eef5a630a04056bbc898abc19bac5691612acba7d1325b66d", 541 | "sha256:1f6c4bd0bdc0f14246fd41262df7dfc018d65bb05f6e16390b7ea26ca454a291", 542 | "sha256:25d8feefe27eb0303b73545416b13d108c6067b846b543738a25ff304824ed9a", 543 | "sha256:29464a177d56e4e055b5f7b629935af7f49c196be47528cc94e0a7bf83fbc2b9", 544 | "sha256:2e214b72168ea0275efd6c884b114ab42e316de3ffa125b267e732ed2abda892", 545 | "sha256:3e0d5e48e3a23e9a4d1a9f698e32a542a4a288c871d33ed8df1b092a40f3a0f9", 546 | "sha256:519425deca5c2b2bdac49f77b2c5625781abbaf9a809d727d3a5596b30bb4ded", 547 | "sha256:57fe287f0cdd9ceaf69e7b71a2e94a24b5d268b35df251a88fef5cc241bf73aa", 548 | "sha256:668d0cec391d9aed1c6a388b0d5b97cd22e6073eaa5fbaa6d2946603b4871efe", 549 | "sha256:68ba70684990f59497680ff90d18e756a47bf4863c604098f10de9716b2c0bdd", 550 | "sha256:6de012d2b166fe7a4cdf505eee3aaa12192f7ba365beeefaca4ec10e31241a85", 551 | "sha256:79b91ebe5a28d349b6d0d323023350133e927b4de5b651a8aa2db69c761420c6", 552 | "sha256:8550177fa5d4c1f09b5e5f524411c44633c80ec69b24e0e98906dd761941ca46", 553 | "sha256:898f818399cafcdb93cbbe15fc83a33d05f18e29fb498ddc09b0214cdfc7cd51", 554 | "sha256:94b091dc0f19291adcb279a108f5d38de2430411068b219f41b343c03b28fb1f", 555 | "sha256:a26863198902cda15ab4503991e8cf1ca874219e0118cbf07c126bce7c4db129", 556 | "sha256:a8034021801bc0440f2e027c354b4eafd95891b573e12ff0418dec385c76785c", 557 | "sha256:bc978ac17468fe868ee589c795d06777f75496b1ed576d308002c8a5756fb9ea", 558 | "sha256:c05b41bc1deade9f90ddc5d988fe506208019ebba9f2578c622516fd201f5863", 559 | "sha256:c9b060bd1e5a26ab6e8267fd46fc9e02b54eb15fffb16d112d4c7b1c12987559", 560 | "sha256:edb04bdd45bfd76c8292c4d9654568efaedf76fe78eb246dde69bdb13b2dad87", 561 | "sha256:f19f2a4f547505fe9072e15f6f4ae714af51b5a681a97f187971f50c283193b6" 562 | ], 563 | "markers": "python_version < '3.7' and implementation_name == 'cpython'", 564 | "version": "==1.1.0" 565 | }, 566 | "urllib3": { 567 | "hashes": [ 568 | "sha256:a68ac5e15e76e7e5dd2b8f94007233e01effe3e50e8daddf69acfd81cb686baf", 569 | "sha256:b5725a0bd4ba422ab0e66e89e030c806576753ea3ee08554382c14e685d117b5" 570 | ], 571 | "version": "==1.24.2" 572 | }, 573 | "wrapt": { 574 | "hashes": [ 575 | "sha256:d4d560d479f2c21e1b5443bbd15fe7ec4b37fe7e53d335d3b9b0a7b1226fe3c6" 576 | ], 577 | "version": "==1.10.11" 578 | } 579 | } 580 | } 581 | --------------------------------------------------------------------------------