├── .circleci └── config.yml ├── .gitignore ├── LICENSE ├── README.rst ├── RELEASING.md ├── badge_server ├── .gcloudignore ├── Dockerfile ├── README.rst ├── app.yaml ├── datastore_cache.py ├── deployment │ └── app-with-secret.yaml ├── fake_cache.py ├── main.py ├── redis_cache.py ├── requirements.txt ├── static │ └── css │ │ └── theme.css ├── templates │ ├── all-badges.html │ ├── dependency-result.html │ ├── google-compatibility.html │ ├── one-badge.html │ └── self-compatibility.html ├── test_all_badges.py ├── test_badge_server.py ├── test_badges.py └── utils.py ├── best_practices └── README.rst ├── cloud_build ├── Dockerfile ├── README.rst ├── app.yaml └── cloudbuild.yaml ├── cloud_sql_proxy ├── compatibility_lib ├── README.rst ├── compatibility_lib │ ├── __init__.py │ ├── compatibility_checker.py │ ├── compatibility_store.py │ ├── configs.py │ ├── dependency_highlighter.py │ ├── dependency_highlighter_stub.py │ ├── deprecated_dep_finder.py │ ├── deprecated_dep_finder_stub.py │ ├── fake_compatibility_store.py │ ├── get_compatibility_data.py │ ├── package.py │ ├── package_crawler_static.py │ ├── semver_checker.py │ ├── test_compatibility_checker.py │ ├── test_compatibility_store.py │ ├── test_dependency_highlighter.py │ ├── test_deprecated_dep_finder.py │ ├── test_fake_compatibility_store.py │ ├── test_get_compatibility_data.py │ ├── test_semver_checker.py │ ├── test_utils.py │ ├── testdata │ │ ├── __init__.py │ │ └── mock_depinfo_data.py │ ├── testpkgs │ │ ├── added_args │ │ │ ├── 0.1.0 │ │ │ │ └── main.py │ │ │ └── 0.2.0 │ │ │ │ └── main.py │ │ ├── added_func │ │ │ ├── 0.1.0 │ │ │ │ └── main.py │ │ │ └── 0.2.0 │ │ │ │ └── main.py │ │ ├── removed_args │ │ │ ├── 0.1.0 │ │ │ │ └── main.py │ │ │ └── 0.2.0 │ │ │ │ └── main.py │ │ ├── removed_func │ │ │ ├── 0.1.0 │ │ │ │ └── main.py │ │ │ └── 0.2.0 │ │ │ │ └── main.py │ │ └── simple_function │ │ │ └── simple_function.py │ └── utils.py ├── requirements.txt ├── setup.cfg └── setup.py ├── compatibility_server ├── Dockerfile ├── README.rst ├── compatibility_checker_server.py ├── configs.py ├── deployment │ ├── deploy.yaml │ └── service.yaml ├── fake_pip.py ├── loadtest │ └── locustfile.py ├── pip_checker.py ├── requirements.txt ├── run-in-docker.sh ├── test_configs.py ├── test_pip_checker.py ├── test_sanitize_packages.py └── views.py ├── credentials.json.enc ├── dashboard ├── css │ └── theme.css ├── dashboard_builder.py ├── grid-template.html ├── index.html ├── js │ └── main.js ├── main-template.html ├── monitoring.html └── test_dashboard_builder.py ├── nox.py ├── python-compatibility-tools.json ├── python-compatibility-tools.json.enc ├── requirements-test.txt ├── scripts ├── twine_upload.sh └── update_dashboard.sh ├── sql_schema ├── pairwise_compatibility_status_schema.json ├── release_time_for_dependencies.json └── self_compatibility_status_schema.json ├── system_test ├── test_badge_server.py └── test_compatibility_checker_server.py └── tox.ini /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | # Python CircleCI 2.0 configuration file 2 | # 3 | # Check https://circleci.com/docs/2.0/language-python/ for more details 4 | # 5 | version: 2 6 | 7 | jobs: 8 | build: 9 | docker: 10 | - image: googleapis/nox:0.17.0 11 | 12 | working_directory: ~/repo 13 | 14 | steps: 15 | - checkout 16 | - run: 17 | name: Decrypt credentials 18 | command: | 19 | if [ -n "$GOOGLE_APPLICATION_CREDENTIALS" ]; then 20 | openssl aes-256-cbc -d -a -k "$GOOGLE_CREDENTIALS_PASSPHRASE" \ 21 | -in credentials.json.enc \ 22 | -out $GOOGLE_APPLICATION_CREDENTIALS 23 | else 24 | echo "No credentials. System tests will not run." 25 | fi 26 | - run: 27 | name: Run tests - compatibility_checker 28 | command: | 29 | nox -f nox.py 30 | - deploy: 31 | name: Push to PyPI (if this is a release tag). 32 | command: scripts/twine_upload.sh 33 | 34 | deployment: 35 | tag_build_for_cci2: 36 | tag: /(([a-z]+)_)*([0-9]+)\.([0-9]+)\.([0-9]+)/ 37 | commands: 38 | - true 39 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | *.sw[op] 3 | 4 | # C extensions 5 | *.so 6 | 7 | # Packages 8 | *.egg 9 | *.egg-info 10 | dist 11 | build 12 | eggs 13 | parts 14 | bin 15 | var 16 | sdist 17 | develop-eggs 18 | .installed.cfg 19 | lib 20 | lib64 21 | __pycache__ 22 | venv/ 23 | 24 | # Installer logs 25 | pip-log.txt 26 | 27 | # Unit test / coverage reports 28 | .coverage 29 | .nox 30 | .tox 31 | .cache 32 | htmlcov 33 | 34 | # Translations 35 | *.mo 36 | 37 | # Mac 38 | .DS_Store 39 | 40 | # Mr Developer 41 | .mr.developer.cfg 42 | .project 43 | .pydevproject 44 | 45 | # JetBrains 46 | .idea 47 | 48 | # VSCode 49 | .vscode 50 | 51 | # Built documentation 52 | docs/_build 53 | docs/_build_doc2dash 54 | 55 | # Virtual environment 56 | env/ 57 | venv/ 58 | coverage.xml 59 | 60 | # Make sure a generated file isn't accidentally committed. 61 | pylintrc 62 | pylintrc.test 63 | .pytest_cache/ 64 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | **NOTE: this repository is no longer being maintained** 2 | 3 | Dependency Management Toolkit for Google Cloud Python Projects 4 | ============================================================== 5 | 6 | Version conflicts between dependencies has been a big issue for GCP Python 7 | users. The issue typically happens when a user depends on two libraries A 8 | and B, both of which depend on incompatible versions of library C. This 9 | can lead to non-deterministic behavior, since only one version of C 10 | actually gets loaded into the library. 11 | 12 | This repository is providing a toolkit for GCP Python open source projects 13 | to bootstrap their development infrastructure, enforcing centralized 14 | dependency management, CI, and release process, thus ensuring compatibility 15 | across all of our GCP Python open source libraries for our end-users. 16 | 17 | |circleci| |pypi| |package_details| |github_head| 18 | 19 | .. |circleci| image:: https://circleci.com/gh/GoogleCloudPlatform/cloud-opensource-python/tree/master.svg?style=svg&circle-token=edd37af38ff6d303b11cd0620890537168144137 20 | :target: https://circleci.com/gh/GoogleCloudPlatform/cloud-opensource-python/tree/master 21 | .. |pypi| image:: https://img.shields.io/pypi/v/compatibility-lib.svg 22 | :target: https://pypi.org/project/compatibility-lib/ 23 | .. |package_details| image:: https://python-compatibility-tools.appspot.com/one_badge_image?package=compatibility-lib 24 | :target: https://python-compatibility-tools.appspot.com/one_badge_target?package=compatibility-lib 25 | .. |github_head| image:: https://python-compatibility-tools.appspot.com/one_badge_image?package=git%2Bgit://github.com/GoogleCloudPlatform/cloud-opensource-python.git%23subdirectory=compatibility_lib&force_run_check=1 26 | :target: https://python-compatibility-tools.appspot.com/one_badge_target?package=git%2Bgit://github.com/GoogleCloudPlatform/cloud-opensource-python.git%23subdirectory=compatibility_lib 27 | 28 | ----------------- 29 | Compatibility Lib 30 | ----------------- 31 | 32 | `Compatibility Lib`_ is a library to get compatibility status and dependency information of Python packages. 33 | It contains three tools: compatibility checker, outdated dependency highlighter and deprecated dependency finder. 34 | And it also provides utilities to query data from the BigQuery tables (external user will need to set up tables 35 | with the same schema that this library is using). 36 | 37 | .. _Compatibility Lib: https://pypi.org/project/compatibility-lib/ 38 | 39 | Installation: 40 | 41 | .. code-block:: bash 42 | 43 | pip install compatibility-lib 44 | 45 | 46 | Compatibility Checker 47 | --------------------- 48 | 49 | Compatibility checker gets the compatibility data by sending requests to the Compatibility Server endpoint, 50 | or by querying the BigQuery table (if the given package is listed in our configs, which are pre-computed). 51 | 52 | Usage like below, 53 | 54 | .. code-block:: python 55 | 56 | import itertools 57 | from compatibility_lib import compatibility_checker 58 | 59 | packages = ['package1', 'package2', 'package3'] 60 | package_pairs = itertools.combinations(packages, 2) 61 | checker = compatibility_checker.CompatibilityChecker() 62 | 63 | # Get self compatibility data 64 | checker.get_self_compatibility(python_version='3', packages=packages) 65 | 66 | # Get pairwise compatibility data 67 | checker.get_pairwise_compatibility( 68 | python_version='3', pkg_sets=package_pairs) 69 | 70 | Outdated Dependency Highlighter 71 | ------------------------------- 72 | 73 | Outdated Dependency Highlighter finds out the outdated dependencies of a Python package, and determines 74 | the priority of updating the dependency version based on a set of criteria below: 75 | 76 | - Mark “High Priority” if dependencies have widely adopted major release. (e.g 1.0.0 -> 2.0.0) 77 | - Mark “High Priority” if a new version has been available for more than 6 months. 78 | - Mark “High Priority” if dependencies are 3 or more sub-versions behind the newest one. (e.g 1.0.0 -> 1.3.0) 79 | - Mark “Low Priority” for other dependency updates. 80 | 81 | Usage: 82 | 83 | .. code-block:: python 84 | 85 | from compatibility_lib import dependency_highlighter 86 | 87 | packages = ['package1', 'package2', 'package3'] 88 | highlighter = dependency_highlighter.DependencyHighlighter() 89 | highlighter.check_packages(packages) 90 | 91 | Deprecated Dependency Finder 92 | ---------------------------- 93 | 94 | Deprecated Dependency Finder can find out the deprecated dependencies that a Python package 95 | depends on. 96 | 97 | Usage: 98 | 99 | .. code-block:: python 100 | 101 | from compatibility_lib import deprecated_dep_finder 102 | 103 | packages = ['package1', 'package2', 'package3'] 104 | finder = deprecated_dep_finder.DeprecatedDepFinder() 105 | for res in finder.get_deprecated_deps(packages): 106 | print(res) 107 | 108 | ------------ 109 | Badge Server 110 | ------------ 111 | 112 | Displaying the compatibility status for your package as a Github Badge. 113 | 114 | Types of badges 115 | --------------- 116 | 117 | 1. Self Compatibility 118 | 2. Compatibility with Google OSS Python packages 119 | 3. Dependency version status 120 | 121 | Usage 122 | ----- 123 | 124 | See the usage `here`_. 125 | 126 | .. _here: https://github.com/GoogleCloudPlatform/cloud-opensource-python/blob/master/badge_server/README.rst 127 | 128 | ------------ 129 | Contributing 130 | ------------ 131 | 132 | Set up environment 133 | ------------------ 134 | 135 | - Set Up Python Environment 136 | 137 | https://cloud.google.com/python/setup 138 | 139 | 140 | - Install py 3.6 (may not be included in previous step) 141 | 142 | .. code-block:: bash 143 | 144 | sudo apt install python3.6 145 | 146 | 147 | - Clone the cloud-opensource-python project and cd to project 148 | 149 | .. code-block:: bash 150 | 151 | git clone git@github.com:GoogleCloudPlatform/cloud-opensource-python.git 152 | cd cloud-opensource-python 153 | 154 | 155 | - Fork project and configure git remote settings 156 | 157 | .. code-block:: bash 158 | 159 | git remote add upstream git@github.com:GoogleCloudPlatform/cloud-opensource-python.git 160 | git config --global user.email "email@example.com" 161 | 162 | 163 | - Create a virtualenv, and source 164 | 165 | .. code-block:: bash 166 | 167 | tox -e py36 168 | source .tox/py36/bin/activate 169 | 170 | - Install gcloud SDK and initialize 171 | 172 | .. code-block:: bash 173 | 174 | curl https://sdk.cloud.google.com | bash 175 | gcloud init 176 | 177 | Set up credentials 178 | ------------------ 179 | 180 | - Create new service account key 181 | 182 | 1. In your browser, navigate to Cloud Console 183 | 184 | 2. menu > IAM & admin > Service accounts 185 | 186 | 3. under bigquery-admin, actions > create new key 187 | 188 | - Set GOOGLE_APPLICATION_CREDENTIALS 189 | 190 | .. code-block:: bash 191 | 192 | export GOOGLE_APPLICATION_CREDENTIALS=”path/to/service/key.json” 193 | 194 | Contributing to compatibility_lib 195 | --------------------------------- 196 | 197 | - Build compatibility_lib library from source and install 198 | 199 | .. code-block:: bash 200 | 201 | python compatibility_lib/setup.py bdist_wheel 202 | pip install compatibility_lib/dist/* 203 | 204 | ------- 205 | Testing 206 | ------- 207 | 208 | We use nox test suite for running tests. 209 | 210 | - Install Nox for testing 211 | 212 | .. code-block:: bash 213 | 214 | pip install nox-automation 215 | 216 | - Run the tests 217 | 218 | .. code-block:: bash 219 | 220 | nox -s unit # unit tests 221 | nox -s lint # linter 222 | nox -s system # system tests 223 | nox -l # see available options 224 | nox # run everything 225 | 226 | ------- 227 | License 228 | ------- 229 | 230 | Apache 2.0 - See `LICENSE `__ for more information. 231 | 232 | ---------- 233 | Disclaimer 234 | ---------- 235 | 236 | This is not an official Google product. 237 | -------------------------------------------------------------------------------- /RELEASING.md: -------------------------------------------------------------------------------- 1 | # Release Compatibility_Lib 2 | 3 | ## Steps 4 | 5 | ### Update the version number in `setup.py` 6 | 7 | ### Build the Python wheel 8 | 9 | ``` 10 | python setup.py bdist_wheel 11 | ``` 12 | 13 | ### Upload the package to PyPI using twine 14 | 15 | ``` 16 | # If not installed 17 | pip install twine 18 | 19 | # Running the command below needs a PyPI user account which owns this library 20 | twine upload dist/* 21 | ``` 22 | 23 | ### Create a Github release 24 | -------------------------------------------------------------------------------- /badge_server/.gcloudignore: -------------------------------------------------------------------------------- 1 | # This file specifies files that are *not* uploaded to Google Cloud Platform 2 | # using gcloud. It follows the same syntax as .gitignore, with the addition of 3 | # "#!include" directives (which insert the entries of the given .gitignore-style 4 | # file at that point). 5 | # 6 | # For more information, run: 7 | # $ gcloud topic gcloudignore 8 | # 9 | .gcloudignore 10 | # If you would like to upload your .git directory, .gitignore file or files 11 | # from your .gitignore file, remove the corresponding line 12 | # below: 13 | .git 14 | .gitignore 15 | 16 | # Python pycache: 17 | __pycache__/ 18 | 19 | -------------------------------------------------------------------------------- /badge_server/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # An image used to run a Python webserver that does compatibility checking 16 | # between pip-installable packages. 17 | 18 | # [START docker] 19 | FROM gcr.io/google_appengine/python 20 | ADD requirements.txt /app/requirements.txt 21 | RUN pip3 install -r /app/requirements.txt 22 | ADD . /app 23 | ENTRYPOINT ["python3"] 24 | CMD ["main.py"] 25 | # [END docker] 26 | -------------------------------------------------------------------------------- /badge_server/README.rst: -------------------------------------------------------------------------------- 1 | Badge Server 2 | ============ 3 | 4 | Displaying the compatibility status for your package as a Github Badge. 5 | 6 | Types of badges 7 | --------------- 8 | 9 | 1. Self Compatibility 10 | 2. Compatibility with Google OSS Python packages 11 | 3. Dependency version status 12 | 13 | Usage 14 | ----- 15 | 16 | What to Check 17 | ~~~~~~~~~~~~~ 18 | 19 | - Latest released version on PyPI 20 | 21 | 22 | .. code-block:: 23 | 24 | package=[name_on_pypi] 25 | 26 | - Github head version 27 | 28 | .. code-block:: 29 | 30 | package=git%2Bgit://github.com/[your_repo_name].git%23subdirectory=[subdirectory_containing_setup_py_file] 31 | 32 | How to Check 33 | ~~~~~~~~~~~~ 34 | 35 | - One badge 36 | 37 | Add this line to your README file on Github: 38 | 39 | .. code-block:: 40 | 41 | .. |package_details_example| image:: https://python-compatibility-tools.appspot.com/one_badge_image?package=compatibility_lib 42 | :target: https://python-compatibility-tools.appspot.com/one_badge_target?package=compatibility_lib 43 | 44 | And the badge for package details will show up like below: 45 | 46 | .. image:: https://user-images.githubusercontent.com/12888824/46687056-c1255f00-cbae-11e8-9066-91d62cb120e0.png 47 | 48 | - Multiple badges 49 | 50 | Adding the link of the badge image and badge target to your README file on 51 | Github: 52 | 53 | .. code-block:: 54 | 55 | .. csv-table:: 56 | :header: "CHECK_TYPE", "RESULT" 57 | :widths: 20, 30 58 | 59 | "Self Compatibility", |self_compatibility| 60 | "Google Compatibility", |google_compatibility| 61 | "Dependency Version Status", |dependency_version_status| 62 | 63 | .. |self_compatibility| image:: https://python-compatibility-tools.appspot.com/self_compatibility_badge_image?package=compatibility_lib 64 | :target: https://python-compatibility-tools.appspot.com/self_compatibility_badge_target?package=compatibility_lib 65 | .. |google_compatibility| image:: https://python-compatibility-tools.appspot.com/google_compatibility_badge_image?package=compatibility_lib 66 | :target: https://python-compatibility-tools.appspot.com/google_compatibility_badge_target?package=compatibility_lib 67 | .. |dependency_version_status| image:: https://python-compatibility-tools.appspot.com/self_dependency_badge_image?package=compatibility_lib 68 | :target: https://python-compatibility-tools.appspot.com/self_dependency_badge_target?package=compatibility_lib 69 | 70 | And the badges will show up like below: 71 | 72 | .. image:: https://user-images.githubusercontent.com/12888824/46686958-8a4f4900-cbae-11e8-80dc-017bfd9ea437.png 73 | 74 | For maintainers 75 | --------------- 76 | 77 | Steps for building the docker image and deploying to GKE: 78 | 79 | - Update the dependency version in `requirements.txt` if there are any. 80 | 81 | - Deploy! 82 | 83 | .. code-block:: 84 | 85 | gcloud app deploy 86 | -------------------------------------------------------------------------------- /badge_server/app.yaml: -------------------------------------------------------------------------------- 1 | runtime: python37 2 | instance_class: F4 3 | 4 | handlers: 5 | - url: /static 6 | static_dir: static/ 7 | - url: /.* 8 | script: auto 9 | secure: always 10 | 11 | automatic_scaling: 12 | min_instances: 3 13 | 14 | env_variables: 15 | MYSQL_USER: 'root' 16 | MYSQL_PASSWORD: '19931228' 17 | -------------------------------------------------------------------------------- /badge_server/datastore_cache.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """A key/value cache using Google Cloud Datastore.""" 16 | 17 | import json 18 | from typing import Any 19 | 20 | from google.cloud import datastore 21 | 22 | 23 | class DatastoreCache: 24 | def __init__(self): 25 | self._datastore_client = datastore.Client() 26 | 27 | def get(self, name: str) -> Any: 28 | """Returns a Python value given a key. None if not found.""" 29 | key = self._datastore_client.key('_Cache', name) 30 | e = self._datastore_client.get(key) 31 | if e is None: 32 | return None 33 | else: 34 | return json.loads(e['value']) 35 | 36 | def set(self, name: str, value: Any): 37 | """Sets a key name to any Python object.""" 38 | key = self._datastore_client.key('_Cache', name) 39 | e = datastore.Entity(key, exclude_from_indexes=['value']) 40 | e.update(value=json.dumps(value)) 41 | self._datastore_client.put(e) 42 | -------------------------------------------------------------------------------- /badge_server/deployment/app-with-secret.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: badge-server-cluster 5 | spec: 6 | replicas: 10 7 | template: 8 | metadata: 9 | creationTimestamp: null 10 | labels: 11 | run: badge-server-cluster 12 | spec: 13 | volumes: 14 | - name: google-cloud-key 15 | secret: 16 | secretName: bigquery-key 17 | containers: 18 | - image: gcr.io/python-compatibility-tools/badge_server:ver11 19 | imagePullPolicy: IfNotPresent 20 | name: badge-server-cluster 21 | ports: 22 | - containerPort: 8080 23 | protocol: TCP 24 | resources: {} 25 | terminationMessagePath: /dev/termination-log 26 | terminationMessagePolicy: File 27 | volumeMounts: 28 | - name: google-cloud-key 29 | mountPath: /var/secrets/google 30 | env: 31 | - name: GOOGLE_APPLICATION_CREDENTIALS 32 | value: /var/secrets/google/key.json 33 | -------------------------------------------------------------------------------- /badge_server/fake_cache.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """An in-memory key/value cache.""" 16 | 17 | from typing import Any 18 | 19 | 20 | class FakeCache: 21 | def __init__(self): 22 | self._cache = {} 23 | 24 | def get(self, name: str) -> Any: 25 | """Returns a Python value given a key. None if not found.""" 26 | return self._cache.get(name) 27 | 28 | def set(self, name: str, value: Any): 29 | """Sets a key name to any Python object.""" 30 | self._cache[name] = value 31 | -------------------------------------------------------------------------------- /badge_server/redis_cache.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """A key/value cache using Redis.""" 16 | 17 | import os 18 | import json 19 | from typing import Any 20 | 21 | import redis 22 | 23 | 24 | class RedisCache: 25 | def __init__(self): 26 | redis_host = os.environ.get('REDISHOST', '10.0.0.3') 27 | redis_port = int(os.environ.get('REDISPORT', 6379)) 28 | self._redis_client = redis.StrictRedis( 29 | host=redis_host, port=redis_port) 30 | 31 | def get(self, name: str) -> Any: 32 | """Returns a Python value given a key. None if not found.""" 33 | return json.loads(self._redis_client.get(name)) 34 | 35 | def set(self, name: str, value: Any): 36 | """Sets a key name to any Python object.""" 37 | return self._redis_client.set(name, json.loads(value)) 38 | -------------------------------------------------------------------------------- /badge_server/requirements.txt: -------------------------------------------------------------------------------- 1 | compatibility_lib>=0.1.0 2 | Flask==1.0.2 3 | google-cloud-datastore==1.7.0 4 | grpcio==1.15.0 5 | pexpect==4.6.0 6 | pybadges==1.0.2 7 | pymysql==0.9.3 8 | redis==2.10.5 9 | requests==2.20.0 10 | retrying==1.3.3 11 | wrapt==1.10.11 12 | -------------------------------------------------------------------------------- /badge_server/static/css/theme.css: -------------------------------------------------------------------------------- 1 | .m-b-30 { 2 | margin-bottom: 30px; 3 | } 4 | 5 | .center { 6 | display: flex; 7 | justify-content: center; 8 | } 9 | 10 | .title--sbold { 11 | font-weight: 600; 12 | } 13 | 14 | .title-1 { 15 | text-transform: capitalize; 16 | font-weight: 400; 17 | font-size: 30px; 18 | } 19 | 20 | .title-2 { 21 | text-transform: capitalize; 22 | font-weight: 400; 23 | font-size: 24px; 24 | line-height: 1; 25 | } 26 | 27 | .title-3 { 28 | text-transform: capitalize; 29 | font-weight: 400; 30 | font-size: 24px; 31 | color: #333; 32 | } 33 | 34 | .title-3 i { 35 | margin-right: 13px; 36 | vertical-align: baseline; 37 | } 38 | 39 | .title-4 { 40 | margin-top: 30px; 41 | margin-bottom: 30px; 42 | font-weight: 500; 43 | font-size: 30px; 44 | color: #393939; 45 | } 46 | 47 | .title-5 { 48 | text-transform: capitalize; 49 | font-size: 22px; 50 | font-weight: 500; 51 | color: #393939; 52 | } 53 | 54 | .title-6 { 55 | font-size: 24px; 56 | font-weight: 500; 57 | color: #fff; 58 | } 59 | 60 | .user-data { 61 | border: 1px solid #e5e5e5; 62 | background: #fff; 63 | padding-top: 44px; 64 | } 65 | 66 | .user-data .title-3 { 67 | padding-left: 40px; 68 | padding-right: 55px; 69 | } 70 | 71 | .user-data .filters { 72 | padding-left: 40px; 73 | padding-right: 55px; 74 | } 75 | 76 | .user-data__footer { 77 | padding: 29px 0; 78 | text-align: center; 79 | } 80 | 81 | .table { 82 | margin: 0; 83 | } 84 | 85 | .table-responsive { 86 | padding-right: 1px; 87 | } 88 | 89 | .table-data thead tr td { 90 | font-size: 12px; 91 | font-weight: 600; 92 | color: #808080; 93 | text-transform: uppercase; 94 | } 95 | 96 | .table-data .table td { 97 | border-top: none; 98 | border-bottom: 1px solid #f2f2f2; 99 | padding-top: 23px; 100 | padding-bottom: 33px; 101 | padding-left: 40px; 102 | padding-right: 10px; 103 | } 104 | 105 | .table-data .table tr td:last-child { 106 | padding-right: 24px; 107 | } 108 | 109 | .table-data tbody tr:hover td .more { 110 | -webkit-transform: scale(1); 111 | -moz-transform: scale(1); 112 | -ms-transform: scale(1); 113 | -o-transform: scale(1); 114 | transform: scale(1); 115 | } 116 | 117 | .table-data__info h6 { 118 | font-size: 14px; 119 | color: #808080; 120 | text-transform: capitalize; 121 | font-weight: 400; 122 | } 123 | 124 | .table-data__info span a { 125 | font-size: 12px; 126 | color: #999; 127 | } 128 | 129 | .table-data__info span a:hover { 130 | color: #666; 131 | } 132 | 133 | .card { 134 | -webkit-border-radius: 3px; 135 | -moz-border-radius: 3px; 136 | border-radius: 3px; 137 | } 138 | 139 | .more { 140 | display: inline-block; 141 | cursor: pointer; 142 | width: 30px; 143 | height: 30px; 144 | background: #e5e5e5; 145 | -webkit-border-radius: 100%; 146 | -moz-border-radius: 100%; 147 | border-radius: 100%; 148 | position: relative; 149 | -webkit-transition: all 0.4s ease; 150 | -o-transition: all 0.4s ease; 151 | -moz-transition: all 0.4s ease; 152 | transition: all 0.4s ease; 153 | -webkit-transform: scale(0); 154 | -moz-transform: scale(0); 155 | -ms-transform: scale(0); 156 | -o-transform: scale(0); 157 | transform: scale(0); 158 | } 159 | 160 | .more i { 161 | font-size: 20px; 162 | color: #808080; 163 | position: absolute; 164 | top: 50%; 165 | left: 50%; 166 | -webkit-transform: translate(-50%, -50%); 167 | -moz-transform: translate(-50%, -50%); 168 | -ms-transform: translate(-50%, -50%); 169 | -o-transform: translate(-50%, -50%); 170 | transform: translate(-50%, -50%); 171 | } 172 | 173 | .line-seprate { 174 | height: 1px; 175 | width: 100%; 176 | background: #e5e5e5; 177 | border: none; 178 | margin-top: 20px; 179 | margin-bottom: 0; 180 | } 181 | 182 | .status { 183 | display: inline-block; 184 | line-height: 30px; 185 | font-size: 14px; 186 | color: #fff; 187 | padding: 0 15px; 188 | -webkit-border-radius: 3px; 189 | -moz-border-radius: 3px; 190 | border-radius: 3px; 191 | text-transform: capitalize; 192 | white-space: nowrap; 193 | } 194 | -------------------------------------------------------------------------------- /badge_server/templates/all-badges.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | All Compatibility Badges 4 | 5 | 6 |

PyPI packages

7 | 8 | {% for package in pypi_packages | sort(attribute='name') %} 9 | 10 | 13 | 18 | {% endfor %} 19 |
11 | {{ package.name }} 12 | 14 | 15 | 16 | 17 |
20 |

Github packages

21 | 22 | {% for package in github_packages | sort(attribute='name') %} 23 | 24 | 27 | 32 | {% endfor %} 33 |
25 | {{ package.name }} 26 | 28 | 29 | 30 | 31 |
34 | 35 | 36 | -------------------------------------------------------------------------------- /badge_server/templates/dependency-result.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | Dependency Check Result 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 |
25 |
26 | 27 |
28 |
29 |
30 |
31 |

Package name: {{ package_name }} 32 |

33 |
34 |
35 |
36 |
37 |
38 | 39 | 40 | 41 |
42 |
43 |
44 |

45 | Dependency Check Result: {{ result.status }} 46 |

47 |
48 | {% if result.status in ['LOW_PRIORITY', 'HIGH_PRIORITY'] %} 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | {% for dep, detail in result.details.items() %} 61 | 62 | 67 | 70 | 75 | 80 | 83 | 84 | {% endfor %} 85 | 86 |
Dependency NamePriority to UpdateInstalled VersionLatest VersionDetails
63 |
64 |
{{ dep }}
65 |
66 | 		68 | {{ detail.priority }} 69 | 71 |
72 |
{{ detail.installed_version }}
73 |
74 | 		76 |
77 |
{{ detail.latest_version }}
78 |
79 | 		81 | {{ detail.detail }} 82 |
87 | {% elif result.status == 'UP_TO_DATE' %} 88 |

All dependencies are up to date!

89 | {% endif %} 90 |
91 |
92 | {% if result.deprecated_deps | length > 0 %} 93 |
94 |
95 |
96 | Deprecated Dependencies 97 |
98 |
99 |

{{ result.deprecated_deps }}

100 |
101 |
102 |
103 | {% endif %} 104 |
105 |
106 | 107 |
108 |
109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | -------------------------------------------------------------------------------- /badge_server/templates/google-compatibility.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | Google Compatibility Result 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 |
25 |
26 | 27 |
28 |
29 |
30 |
31 |

Package name: {{ package_name }} 32 |

33 |
34 |
35 |
36 |
37 |
38 | 39 | 40 |
41 |
42 |
43 |
44 |

Google Compatibility Check Results

45 |
46 |
47 |

Click on the tabs to see results for different Python versions.

48 | 49 | 65 |
66 |
67 |
68 |

Check Result: {{ result.py2.status }}

69 |
70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | {% for dep_name, detail in result.py2.details.items() %} 79 | 80 | 85 | 88 | 89 | {% endfor %} 90 | 91 |
Package NameDetails
81 |
82 |
{{ dep_name }}
83 |
84 | 		86 | {{ detail }} 87 |
92 |
93 |
94 |
95 |
96 |
97 |

Check Result: {{ result.py3.status }}

98 |
99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | {% for dep_name, detail in result.py3.details.items() %} 108 | 109 | 114 | 117 | 118 | {% endfor %} 119 | 120 |
Dependency NameDetails
110 |
111 |
{{ dep_name }}
112 |
113 | 		115 | {{ detail }} 116 |
121 |
122 |
123 |
124 |
125 |
126 |
127 |
128 |
129 |
130 |
131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | -------------------------------------------------------------------------------- /badge_server/templates/self-compatibility.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | Self Compatibility Result 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 |
25 |
26 | 27 |
28 |
29 |
30 |
31 |

Package name: {{ package_name }} 32 |

33 |
34 |
35 |
36 |
37 |
38 | 39 | 40 | 41 |
42 |
43 |
44 |

45 | Self Compatibility Check Results

46 |
47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 62 | 65 | 68 | 69 | 70 | 75 | 78 | 81 | 82 | 83 |
Python VersionResultDetails
58 |
59 |
Py2
60 |
61 | 		63 | {{ result.py2.status }} 64 | 66 | {{ result.py2.details }} 67 |
71 |
72 |
Py3
73 |
74 | 		76 | {{ result.py3.status }} 77 | 79 | {{ result.py3.details }} 80 |
84 |
85 |
86 |
87 |
88 | 89 |
90 |
91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | -------------------------------------------------------------------------------- /badge_server/test_all_badges.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Add tests for the badge server all badges page.""" 15 | 16 | import unittest 17 | import unittest.mock 18 | 19 | import main 20 | import utils 21 | 22 | 23 | class AllBadgesTestCase(unittest.TestCase): 24 | """Test case for the badge server all badges endpoint.""" 25 | 26 | def setUp(self): 27 | main.app.config['TESTING'] = True 28 | self.client = main.app.test_client() 29 | 30 | self._pkg_list_patch = unittest.mock.patch( 31 | 'compatibility_lib.configs.PKG_LIST', [ 32 | 'apache-beam[gcp]', 33 | 'google-api-core', 34 | 'google-api-python-client', 35 | 'tensorflow', 36 | ]) 37 | self._whitelist_urls_patch = unittest.mock.patch( 38 | 'compatibility_lib.configs.WHITELIST_URLS', { 39 | 'git+git://github.com/google/apache-beam.git': 40 | 'apache-beam[gcp]', 41 | 'git+git://github.com/google/api-core.git': 'google-api-core', 42 | 'git+git://github.com/google/api-python-client.git': 43 | 'google-api-python-client', 44 | 'git+git://github.com/google/tensorflow.git': 'tensorflow', 45 | }) 46 | 47 | self._pkg_list_patch.start() 48 | self.addCleanup(self._pkg_list_patch.stop) 49 | self._whitelist_urls_patch.start() 50 | self.addCleanup(self._whitelist_urls_patch.stop) 51 | 52 | def test_all(self): 53 | html = self.client.get('/all').get_data(as_text=True) 54 | 55 | self.assertIn('apache-beam[gcp]', html) 56 | self.assertIn('https://pypi.org/project/apache-beam', html) 57 | self.assertIn('https://github.com/google/apache-beam.git', html) 58 | self.assertIn( 59 | '/one_badge_target?package=apache-beam%5Bgcp%5D', 60 | html) 61 | self.assertIn( 62 | '/one_badge_image?package=apache-beam%5Bgcp%5D', 63 | html) 64 | self.assertIn( 65 | '/one_badge_target?package=git%2Bgit%3A//github.com/google/apache-beam.git', 66 | html) 67 | self.assertIn( 68 | '/one_badge_image?package=git%2Bgit%3A//github.com/google/apache-beam.git', 69 | html) 70 | -------------------------------------------------------------------------------- /badge_server/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Common utils methods for badge server.""" 16 | 17 | import enum 18 | import json 19 | import logging 20 | from urllib.parse import urlparse 21 | from urllib.request import urlopen 22 | 23 | from typing import Optional 24 | 25 | import pybadges 26 | 27 | from compatibility_lib import compatibility_checker 28 | from compatibility_lib import compatibility_store 29 | from compatibility_lib import configs 30 | from compatibility_lib import dependency_highlighter 31 | from compatibility_lib import deprecated_dep_finder 32 | 33 | # Initializations 34 | DB_CONNECTION_NAME = 'python-compatibility-tools:us-central1:' \ 35 | 'compatibility-data' 36 | UNIX_SOCKET = '/cloudsql/{}'.format(DB_CONNECTION_NAME) 37 | 38 | checker = compatibility_checker.CompatibilityChecker() 39 | store = compatibility_store.CompatibilityStore(mysql_unix_socket=UNIX_SOCKET) 40 | highlighter = dependency_highlighter.DependencyHighlighter( 41 | checker=checker, store=store) 42 | finder = deprecated_dep_finder.DeprecatedDepFinder( 43 | checker=checker, store=store) 44 | priority_level = dependency_highlighter.PriorityLevel 45 | 46 | TIMESTAMP_FORMAT = "%Y-%m-%dT%H:%M:%S" 47 | 48 | GITHUB_HEAD_NAME = 'github head' 49 | GITHUB_API = 'https://api.github.com/repos' 50 | SVG_CONTENT_TYPE = 'image/svg+xml' 51 | EMPTY_DETAILS = 'NO DETAILS' 52 | 53 | PY_VER_MAPPING = { 54 | 2: 'py2', 55 | 3: 'py3', 56 | } 57 | 58 | STATUS_COLOR_MAPPING = { 59 | 'SUCCESS': 'green', 60 | 'UNKNOWN': 'purple', 61 | 'INSTALL_ERROR': 'yellow', 62 | 'CHECK_WARNING': 'red', 63 | 'CALCULATING': 'blue', 64 | 'CONVERSION_ERROR': 'orange', 65 | 'UP_TO_DATE': 'green', 66 | 'LOW_PRIORITY': 'yellow', 67 | 'HIGH_PRIORITY': 'red', 68 | } 69 | 70 | 71 | class BadgeType(enum.Enum): 72 | """Enum class for badge types.""" 73 | DEP_BADGE = 'dependency_badge' 74 | SELF_COMP_BADGE = 'self_comp_badge' 75 | GOOGLE_COMP_BADGE = 'google_comp_badge' 76 | 77 | 78 | def _build_default_result( 79 | status: str, 80 | include_pyversion: bool = True, 81 | details: Optional = EMPTY_DETAILS) -> dict: 82 | """Build the default result for different conditions.""" 83 | # Dependency badge 84 | if not include_pyversion: 85 | result = { 86 | 'status': status, 87 | 'details': details, 88 | } 89 | # Compatibility badge 90 | else: 91 | result = { 92 | 'py2': { 93 | 'status': status, 94 | 'details': details, 95 | }, 96 | 'py3': { 97 | 'status': status, 98 | 'details': details, 99 | } 100 | } 101 | return result 102 | 103 | 104 | def _get_badge(res: dict, badge_name: str) -> str: 105 | """Generate badge using the check result.""" 106 | if 'github.com' in badge_name: 107 | badge_name = GITHUB_HEAD_NAME 108 | 109 | status = res.get('status') 110 | if status is not None: 111 | # Dependency badge 112 | color = STATUS_COLOR_MAPPING[status] 113 | else: 114 | # Compatibility badge 115 | status = res['py3']['status'] 116 | if status == 'SUCCESS' and \ 117 | badge_name not in \ 118 | configs.PKG_PY_VERSION_NOT_SUPPORTED.get(2): 119 | status = res['py2']['status'] 120 | 121 | color = STATUS_COLOR_MAPPING[status] 122 | 123 | status = status.replace('_', ' ') 124 | return pybadges.badge( 125 | left_text=badge_name, 126 | right_text=status, 127 | right_color=color) 128 | 129 | 130 | def _calculate_commit_number(package: str) -> Optional[str]: 131 | """Calculate the github head version commit number.""" 132 | url_parsed = urlparse(package) 133 | if url_parsed.scheme and url_parsed.netloc == 'github.com': 134 | try: 135 | owner, repo, *_ = url_parsed.path[1:].split('/') 136 | repo = repo.split('.git')[0] 137 | except ValueError: 138 | return None 139 | else: 140 | url = '{0}/{1}/{2}/commits'.format(GITHUB_API, owner, repo) 141 | try: 142 | with urlopen(url) as f: 143 | commits = json.loads(f.read()) 144 | return commits[0]['sha'] 145 | except Exception as e: 146 | logging.warning( 147 | 'Unable to generate caching key for "%s": %s', package, e) 148 | return None 149 | 150 | return None 151 | -------------------------------------------------------------------------------- /best_practices/README.rst: -------------------------------------------------------------------------------- 1 | Python Package Compatibility Guidelines 2 | ======================================= 3 | 4 | This document uses terminology (MUST, SHOULD, etc) from `RFC 2119`_. 5 | 6 | .. _RFC 2119: https://www.ietf.org/rfc/rfc2119.txt 7 | 8 | ---------- 9 | Background 10 | ---------- 11 | 12 | Incompatibilities between packages published on the `Python Package Index (PyPI)`_ 13 | have been a long standing issue. Diamond dependencies are a common problem where 14 | a package or application has dependencies that are, themselves, dependant on 15 | incompatible versions of shared packages. 16 | 17 | .. _Python Package Index (PyPI): https://pypi.org/ 18 | 19 | Incompatibilities between packages can occur when: 20 | 21 | - A package makes breaking API changes and doesn't follow `Semantic Versioning`_ 22 | - A package has a pinned dependency version which conflicts with other dependencies. 23 | - A package depends on outdated dependencies. 24 | - A package is dependent on deprecated dependencies. 25 | 26 | .. _Semantic Versioning: https://semver.org/ 27 | 28 | This guide is a list of best practices that Python package authors can follow 29 | to help reduce future incompatibilities. Google-sponsored projects are expected 30 | to follow these guidelines but other projects may benefit from them as well. 31 | 32 | ----- 33 | Tools 34 | ----- 35 | 36 | To detect and prevent version incompatibilities between Google Open Source Python 37 | packages, we provide a set of tooling to ensure we are compatible with 38 | ourselves. Our tools are as follows: 39 | 40 | Badge Server 41 | ------------ 42 | 43 | The badge server runs checks and generates an svg format badge image to show the 44 | status of a given package. Supported usage includes: 45 | 46 | - Self compatibility (the package has internally consistent dependencies) 47 | - Google-wise compatibility (the package is compatible with other Google packages) 48 | - Dependency version status (the package does not rely on obsolete dependencies) 49 | - One badge for all of the checks above 50 | 51 | For more details please refer to `here`_. 52 | 53 | .. _here: https://github.com/GoogleCloudPlatform/cloud-opensource-python/tree/master/badge_server 54 | 55 | ---------------------- 56 | Python Packaging Rules 57 | ---------------------- 58 | 59 | Package Versioning 60 | ------------------ 61 | 62 | We should use semantic versioning for all Google OSS Python distribution 63 | packages. Semantic versioning requires that given a version number 64 | `MAJOR.MINOR.PATCH`, increment the: 65 | 66 | * MAJOR version when you make incompatible API changes. 67 | * MINOR version when you add functionality in a backwards-compatible manner. 68 | * PATCH version when you make backwards-compatible bug fixes. 69 | 70 | Requirements: 71 | 72 | - `GA(Generally Available)`_ libraries must conform to semantic versioning. 73 | - Non-GA libraries should use major version 0, and be promoted to 1.0 when reaching GA. 74 | - Non-GA libraries could be excluded from semver stability guarantees. 75 | - Dropping support for a Python major version(e.g. Python 2) should result in a major version increment 76 | 77 | .. _GA(Generally Available): https://cloud.google.com/terms/launch-stages 78 | 79 | Dependencies 80 | ------------ 81 | 82 | **1. Specify dependency version using closed ranges** 83 | 84 | - Minor or patch versions shouldn’t be used as an upper bound for 1st party dependencies unless the dependency is not GA. 85 | - Specific versions can be excluded if they are known to be incompatible. e.g. google-cloud-pubsub >= 0.1.1 !=2.0.0 !=2.0.1 86 | - Specific versions may be specified if a package exists as a wrapper around another. 87 | 88 | **2. Avoid depending on unstable release version dependencies** 89 | 90 | - It’s not recommended to depend on non-GA packages. 91 | - Avoid depending on pre-release, post-release and development release versions. 92 | - GA packages must not depend on non-GA packages. 93 | 94 | **3. Version range upper bound should be updated when there is a newer version available as soon as possible.** 95 | 96 | - We allow a 30 day grace period for package owners to migrate to support new major version bump of the dependencies. 97 | 98 | **4. Minimize dependencies** 99 | 100 | - Packages should use Python built-in modules if possible. e.g. logging, unittest 101 | 102 | **5. Never vendor dependencies** 103 | 104 | Vendoring means having a copy of a specific version of the dependencies and ship it together with the library code. 105 | 106 | Release and Support 107 | ------------------- 108 | 109 | - Major version bumps should be rare 110 | - Minimize the cost for users to go from one major version to another. 111 | - Support every semver-major HEAD of every package that is 1.0+ for at least one year. 112 | - Dropping support for any older version should be semver-major. 113 | 114 | --------------- 115 | GA Requirements 116 | --------------- 117 | 118 | The GA requirements are validated using the `github badge`_ service, so the badge 119 | should be green before any GA launch. 120 | 121 | - Packages must be self compatible 122 | If package A has dependencies B and C, and they require different versions 123 | of dependency D, package A is not self compatible. Packages that are not internally 124 | compatible will have conflicts with all the rest of the packages in the world. 125 | 126 | - Packages must be google-wise compatible 127 | It’s required for any new package owned by Google to be compatible with all the other Google Python packages. So that using any combination of Google Python packages will not cause any conflicts during installation or failures during runtime. 128 | 129 | - Packages must support latest version of its dependencies 130 | 131 | .. _github badge: https://github.com/GoogleCloudPlatform/cloud-opensource-python/blob/master/badge_server/README.rst 132 | -------------------------------------------------------------------------------- /cloud_build/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.6 2 | 3 | RUN mkdir /compatibility_dashboard 4 | RUN mkdir /compatibility_dashboard/dashboard 5 | RUN mkdir /static 6 | ADD static/ /static/ 7 | ADD main-template.html /compatibility_dashboard/dashboard/ 8 | ADD grid-template.html /compatibility_dashboard/dashboard/ 9 | ADD dashboard_builder.py /compatibility_dashboard/dashboard/ 10 | ADD requirements-test.txt /compatibility_dashboard 11 | ADD python-compatibility-tools.json /compatibility_dashboard 12 | ENV GOOGLE_APPLICATION_CREDENTIALS=/compatibility_dashboard/python-compatibility-tools.json 13 | 14 | RUN pip3 install -r /compatibility_dashboard/requirements-test.txt 15 | 16 | RUN cd /compatibility_dashboard && python dashboard/dashboard_builder.py && cp dashboard/index.html /static/ && cp dashboard/grid.html /static/ 17 | -------------------------------------------------------------------------------- /cloud_build/README.rst: -------------------------------------------------------------------------------- 1 | Cloud Build for Dashboard 2 | ========================= 3 | 4 | Generating the dashboard static files 5 | ------------------------------------- 6 | 7 | This step is defined in the Dockerfile, which is basically the same as running it locally. 8 | It is using the dashboard_builder.py script and the jinja templates to generate the HTML files, 9 | and keep the files in the static/ folder. 10 | 11 | As this step is done in a docker container, the generated files will not located in host by themselves. 12 | We will need to mount the volume to the docker container and copy the files from docker to host. 13 | This will be done by this command: 14 | 15 | :: 16 | 17 | docker run --name dashboard_builder -h dashboard_container -v /workspace/static:/export dashboard_builder /bin/bash -c "cp /compatibility_dashboard/dashboard/*.html /export/" 18 | 19 | Deploy to App Engine 20 | -------------------- 21 | 22 | This deploys the generated static files to app engine using: 23 | 24 | :: 25 | 26 | gcloud app deploy [PROJECT_NAME] 27 | 28 | Note that the credential used in this step is different than in step #1, as the data for generating the files 29 | is stored in a different project. But because the two steps are not running in the same environment (step #1 runs 30 | in its docker container), the two credentials won't interfere with each other. 31 | 32 | 33 | Build and deploy periodically 34 | ----------------------------- 35 | 36 | In order to have a way to automatically build and deploy the dashboard to App Engine, we use the Cloud Build here. 37 | The cloudbuild.yaml is the config file defining the steps described above. And running the command below will start the 38 | build defined in the yaml file: 39 | 40 | :: 41 | 42 | gcloud builds submit --config=cloudbuild.yaml . 43 | 44 | There is a cron job configured which can grab the credentials from a secret key store, 45 | and then runs the gcloud command for starting the build periodically. 46 | -------------------------------------------------------------------------------- /cloud_build/app.yaml: -------------------------------------------------------------------------------- 1 | runtime: python37 2 | 3 | handlers: 4 | - url: /(.*\.(appcache|manifest)) 5 | mime_type: text/cache-manifest 6 | static_files: static/\1 7 | upload: static/(.*\.(appcache|manifest)) 8 | expiration: "0m" 9 | 10 | - url: /(.*\.css) 11 | mime_type: text/css 12 | static_files: static/\1 13 | upload: static/(.*\.css) 14 | 15 | - url: /(.*\.eot) 16 | mime_type: application/vnd.ms-fontobject 17 | static_files: static/\1 18 | upload: static/(.*\.eot) 19 | 20 | - url: /(.*\.html) 21 | mime_type: text/html 22 | static_files: static/\1 23 | upload: static/(.*\.html) 24 | expiration: "1h" 25 | 26 | - url: /(.*\.ico) 27 | mime_type: image/x-icon 28 | static_files: static/\1 29 | upload: static/(.*\.ico) 30 | expiration: "7d" 31 | 32 | - url: /(.*\.js) 33 | mime_type: text/javascript 34 | static_files: static/\1 35 | upload: static/(.*\.js) 36 | 37 | - url: /(.*\.json) 38 | mime_type: application/json 39 | static_files: static/\1 40 | upload: static/(.*\.json) 41 | expiration: "1h" 42 | 43 | - url: /(.*\.m4v) 44 | mime_type: video/m4v 45 | static_files: static/\1 46 | upload: static/(.*\.m4v) 47 | 48 | - url: /(.*\.mp4) 49 | mime_type: video/mp4 50 | static_files: static/\1 51 | upload: static/(.*\.mp4) 52 | 53 | - url: /(.*\.(ogg|oga)) 54 | mime_type: audio/ogg 55 | static_files: static/\1 56 | upload: static/(.*\.(ogg|oga)) 57 | 58 | - url: /(.*\.ogv) 59 | mime_type: video/ogg 60 | static_files: static/\1 61 | upload: static/(.*\.ogv) 62 | 63 | - url: /(.*\.otf) 64 | mime_type: font/opentype 65 | static_files: static/\1 66 | upload: static/(.*\.otf) 67 | 68 | - url: /(.*\.(svg|svgz)) 69 | mime_type: images/svg+xml 70 | static_files: static/\1 71 | upload: static/(.*\.(svg|svgz)) 72 | 73 | - url: /(.*\.swf) 74 | mime_type: application/x-shockwave-flash 75 | static_files: static/\1 76 | upload: static/(.*\.swf) 77 | 78 | - url: /(.*\.ttf) 79 | mime_type: font/truetype 80 | static_files: static/\1 81 | upload: static/(.*\.ttf) 82 | 83 | - url: /(.*\.txt) 84 | mime_type: text/plain 85 | static_files: static/\1 86 | upload: static/(.*\.txt) 87 | 88 | - url: /(.*\.webm) 89 | mime_type: video/webm 90 | static_files: static/\1 91 | upload: static/(.*\.webm) 92 | 93 | - url: /(.*\.webp) 94 | mime_type: image/webp 95 | static_files: static/\1 96 | upload: static/(.*\.webp) 97 | 98 | - url: /(.*\.woff) 99 | mime_type: application/x-font-woff 100 | static_files: static/\1 101 | upload: static/(.*\.woff) 102 | 103 | - url: /(.*\.xml) 104 | mime_type: application/xml 105 | static_files: static/\1 106 | upload: static/(.*\.xml) 107 | expiration: "1h" 108 | 109 | - url: /(.*\.pdf) 110 | mime_type: application/pdf 111 | static_files: static/\1 112 | upload: static/(.*\.pdf) 113 | 114 | - url: /(.*\.jpg) 115 | mime_type: image/jpeg 116 | static_files: static/\1 117 | upload: static/(.*\.jpg) 118 | 119 | - url: /(.*\.jpeg) 120 | mime_type: image/jpeg 121 | static_files: static/\1 122 | upload: static/(.*\.jpeg) 123 | 124 | - url: /(.*\.tif) 125 | mime_type: image/tiff 126 | static_files: static/\1 127 | upload: static/(.*\.tif) 128 | 129 | - url: /(.*\.tiff) 130 | mime_type: image/tiff 131 | static_files: static/\1 132 | upload: static/(.*\.tiff) 133 | 134 | - url: /(.*\.png) 135 | mime_type: image/png 136 | static_files: static/\1 137 | upload: static/(.*\.png) 138 | 139 | - url: /(.*\.gif) 140 | mime_type: image/gif 141 | static_files: static/\1 142 | upload: static/(.*\.gif) 143 | 144 | - url: / 145 | static_files: static/index.html 146 | upload: static/index.html 147 | 148 | - url: /(.*) 149 | static_files: static/\1 150 | upload: static/(.*) 151 | 152 | 153 | -------------------------------------------------------------------------------- /cloud_build/cloudbuild.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: 'gcr.io/cloud-builders/docker' 3 | args: ['build', '-t', 'dashboard_builder', '.'] 4 | id: build_dashboard 5 | - name: 'gcr.io/cloud-builders/docker' 6 | args: ['run', '--name', 'dashboard_builder', '-h', 'dashboard_container', '-v', '/workspace/static:/export', 'dashboard_builder', '/bin/bash', '-c', 'cp /compatibility_dashboard/dashboard/*.html /export/'] 7 | id: gen_htmls 8 | - name: "gcr.io/cloud-builders/gcloud" 9 | args: ["app", "deploy", "--project", "google.com:python-compatibility-dashboard"] 10 | timeout: "1600s" 11 | -------------------------------------------------------------------------------- /cloud_sql_proxy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/cloud-opensource-python/d1197e9d4e9ab48fa3ddb0db0b658ac611769eca/cloud_sql_proxy -------------------------------------------------------------------------------- /compatibility_lib/README.rst: -------------------------------------------------------------------------------- 1 | Compatibility Lib 2 | ================= 3 | 4 | A library that calls the compatibility server to get compatibility 5 | information about Python packages, and provides utilities to store the 6 | results into BigQuery. 7 | 8 | Running the server locally 9 | -------------------------- 10 | 11 | 1. Install Docker_ 12 | 13 | .. _Docker: https://www.docker.com/community-edition 14 | 15 | 2. Download the code: 16 | 17 | :: 18 | 19 | git clone git@github.com:GoogleCloudPlatform/cloud-opensource-python.git 20 | 21 | 3. Build and run the Docker image 22 | 23 | :: 24 | 25 | cd cloud-opensource-python/compatbility_server 26 | ./run-in-docker.sh 27 | 28 | Testing the server out 29 | ---------------------- 30 | 31 | :: 32 | 33 | curl 'http://0.0.0.0:8888/?package=six&package=Django&python-version=3' | python3 -m json.tool 34 | { 35 | "result": "SUCCESS", 36 | "packages": [ 37 | "six", 38 | "Django" 39 | ], 40 | "description": null, 41 | "requirements": "absl-py==0.2.2\napparmor==2.11.1\nasn1crypto==0.24.0\nastor==0.6.2\natomicwrites==1.1.5\nattrs==18.1.0\nbleach==1.5.0\nblinker==1.3\nBrlapi==0.6.6\ncachetools==2.1.0\ncertifi==2018.4.16\nchardet==3.0.4\ncheckbox-ng==0.23\ncheckbox-support==0.22\ncolorlog==2.10.0\ncryptography==2.1.4\ncupshelpers==1.0\ndecorator==4.3.0\ndefer==1.0.6\nDjango==2.0.6\nfeedparser==5.2.1\ngast==0.2.0\nglinux-rebootd==0.1\ngoobuntu-config-tools==0.1\ngoogle-api-core==1.2.0\ngoogle-auth==1.5.0\ngoogleapis-common-protos==1.5.3\ngpg==1.10.0\ngrpcio==1.12.1\nguacamole==0.9.2\nhtml5lib==0.9999999\nhttplib2==0.9.2\nidna==2.6\nimportlab==0.1.1\nIPy==0.83\nJinja2==2.9.6\nkeyring==10.5.1\nkeyrings.alt==2.2\nLibAppArmor==2.11.1\nlouis==3.3.0\nlxml==4.0.0\nMako==1.0.7\nMarkdown==2.6.11\nMarkupSafe==1.0\nmore-itertools==4.2.0\nnetworkx==2.1\nnox-automation==0.19.0\nnumpy==1.14.5\noauthlib==2.0.4\nobno==29\nolefile==0.44\nonboard==1.4.1\nopencensus==0.1.5\npadme==1.1.1\npexpect==4.2.1\nPillow==4.3.0\nplainbox==0.25\npluggy==0.6.0\nprotobuf==3.5.2.post1\npsutil==5.4.2\npy==1.5.3\npyasn1==0.4.3\npyasn1-modules==0.2.1\npycairo==1.15.4\npycrypto==2.6.1\npycups==1.9.73\npycurl==7.43.0\npygobject==3.26.1\npyinotify==0.9.6\nPyJWT==1.5.3\npyOpenSSL==17.5.0\npyparsing==2.1.10\npysmbc==1.0.15.6\npytest==3.6.1\npython-apt==1.4.0b3\npython-debian==0.1.31\npython-xapp==1.0.0\npython-xlib==0.20\npytype==2018.5.22.1\npytz==2018.4\npyxdg==0.25\nPyYAML==3.12\nreportlab==3.3.0\nrequests==2.18.4\nretrying==1.3.3\nrsa==3.4.2\nSecretStorage==2.3.1\nsetproctitle==1.1.10\nsix==1.11.0\ntensorboard==1.8.0\ntensorflow==1.8.0\ntermcolor==1.1.0\nufw==0.35\nunattended-upgrades==0.1\nurllib3==1.22\nvirtualenv==16.0.0\nWerkzeug==0.14.1\nXlsxWriter==0.9.6\nyoutube-dl==2017.11.6\n" 42 | } 43 | 44 | Get Compatibility Data using the library 45 | ---------------------------------------- 46 | 47 | The library is sending requests to the compatibility server running GKE. 48 | 49 | :: 50 | 51 | from compatibility_lib import compatibility_checker 52 | 53 | checker = compatibility_checker.CompatibilityChecker() 54 | 55 | # Get self compatibility data 56 | checker.get_self_compatibility(python_version='2') 57 | 58 | # Get pairwise compatibility data 59 | checker.get_pairwise_compatibility(python_version='2') 60 | 61 | # Get the data and store to BigQuery 62 | from compatibility_lib import get_compatibility_data 63 | 64 | get_compatibility_data.write_to_status_table() 65 | 66 | Disclaimer 67 | ---------- 68 | 69 | This is not an official Google product. 70 | -------------------------------------------------------------------------------- /compatibility_lib/compatibility_lib/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | try: 16 | import pkg_resources 17 | pkg_resources.declare_namespace(__name__) 18 | except ImportError: 19 | import pkgutil 20 | __path__ = pkgutil.extend_path(__path__, __name__) 21 | -------------------------------------------------------------------------------- /compatibility_lib/compatibility_lib/compatibility_checker.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Send request to server to get self and pairwise compatibility data.""" 16 | 17 | import itertools 18 | import logging 19 | import concurrent.futures 20 | import json 21 | import requests 22 | import retrying 23 | import time 24 | 25 | from compatibility_lib import configs 26 | from compatibility_lib import utils 27 | 28 | SERVER_URL = 'http://34.68.40.69/' 29 | 30 | UNKNOWN_STATUS_RESULT = { 31 | 'result': 'UNKNOWN', 32 | } 33 | 34 | 35 | class CompatibilityChecker(object): 36 | 37 | def __init__(self, max_workers=20): 38 | self.max_workers = max_workers 39 | 40 | def check(self, packages, python_version): 41 | """Call the checker server to get back status results.""" 42 | if not utils._is_package_in_whitelist(packages): 43 | 44 | UNKNOWN_STATUS_RESULT['packages'] = packages 45 | UNKNOWN_STATUS_RESULT['description'] = 'Package is not supported' \ 46 | ' by our checker server.' 47 | return UNKNOWN_STATUS_RESULT 48 | 49 | start_time = time.time() 50 | data = { 51 | 'python-version': python_version, 52 | 'package': packages 53 | } 54 | # Set the timeout to 299 seconds, which should be less than the 55 | # docker timeout (300 seconds). 56 | try: 57 | result = requests.get(SERVER_URL, params=data, timeout=299) 58 | content = result.content.decode('utf-8') 59 | except Exception as e: 60 | check_time = time.time() - start_time 61 | logging.getLogger("compatibility_lib").error( 62 | 'Checked {} in {:.1f} seconds: {}'.format( 63 | packages, check_time, e)) 64 | raise 65 | check_time = time.time() - start_time 66 | if result.ok: 67 | logging.getLogger("compatibility_lib").debug( 68 | 'Checked {} in {:.1f} seconds (success!)'.format( 69 | packages, check_time)) 70 | else: 71 | logging.getLogger("compatibility_lib").debug( 72 | 'Checked {} in {:.1f} seconds: {}'.format( 73 | packages, check_time, content)) 74 | result.raise_for_status() 75 | 76 | return json.loads(content), python_version 77 | 78 | def filter_packages(self, packages, python_version): 79 | """Filter out the packages not supported by the given py version.""" 80 | filtered_packages = [] 81 | for pkg in packages: 82 | if 'github.com' in pkg: 83 | pkg_name = configs.WHITELIST_URLS[pkg] 84 | else: 85 | pkg_name = pkg 86 | if pkg_name not in configs.PKG_PY_VERSION_NOT_SUPPORTED[ 87 | int(python_version)]: 88 | filtered_packages.append(pkg) 89 | return filtered_packages 90 | 91 | @retrying.retry(wait_random_min=1000, 92 | wait_random_max=2000) 93 | def retrying_check(self, args): 94 | """Retrying logic for sending requests to checker server.""" 95 | packages = args[0] 96 | python_version = args[1] 97 | return self.check(packages, python_version) 98 | 99 | def collect_check_packages( 100 | self, python_version=None, packages=None, pkg_sets=None): 101 | # Generating single packages 102 | if packages is None: 103 | packages = configs.PKG_LIST 104 | 105 | check_singles = [] 106 | if python_version is None: 107 | for py_ver in ['2', '3']: 108 | # Remove the package not supported in the python_version 109 | filtered_single = self.filter_packages(packages, py_ver) 110 | for pkg in filtered_single: 111 | check_singles.append(([pkg], py_ver)) 112 | else: 113 | filtered_single = self.filter_packages(packages, python_version) 114 | check_singles = [ 115 | ([pkg], python_version) for pkg in filtered_single] 116 | 117 | # Generating pairs 118 | if pkg_sets is None: 119 | pkg_sets = list(itertools.combinations(configs.PKG_LIST, 2)) 120 | 121 | check_pairs = [] 122 | if python_version is None: 123 | for py_ver in ['2', '3']: 124 | filtered_pkgs = [] 125 | for pkgs in pkg_sets: 126 | if list(pkgs) != self.filter_packages(pkgs, 127 | py_ver): 128 | continue 129 | filtered_pkgs.append(pkgs) 130 | for pkg_set in filtered_pkgs: 131 | check_pairs.append((list(pkg_set), py_ver)) 132 | else: 133 | filtered_pkgs = [] 134 | for pkgs in pkg_sets: 135 | if list(pkgs) != self.filter_packages(pkgs, 136 | python_version): 137 | continue 138 | filtered_pkgs.append(pkgs) 139 | check_pairs = [(list(pkg_set), python_version) 140 | for pkg_set in pkg_sets] 141 | 142 | res = tuple(check_singles) + tuple(check_pairs) 143 | return res 144 | 145 | def get_compatibility( 146 | self, python_version=None, packages=None, pkg_sets=None): 147 | """Get the compatibility data for each package and package pairs.""" 148 | check_packages = self.collect_check_packages( 149 | python_version, packages, pkg_sets) 150 | 151 | start_time = time.time() 152 | with concurrent.futures.ThreadPoolExecutor( 153 | max_workers=self.max_workers) as p: 154 | pkg_set_results = p.map(self.retrying_check, tuple(check_packages)) 155 | 156 | for count, result in enumerate(zip(pkg_set_results)): 157 | if (count and count % self.max_workers == 0 or 158 | count == len(check_packages) - 1): 159 | logging.getLogger("compatibility_lib").info( 160 | "Successfully checked {}/{} in {:.1f}s".format( 161 | count, len(check_packages), 162 | time.time() - start_time)) 163 | yield result 164 | -------------------------------------------------------------------------------- /compatibility_lib/compatibility_lib/dependency_highlighter_stub.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """A stub implementation of dependency_highlighter.DependencyHighlighter.""" 15 | 16 | from typing import List 17 | 18 | from compatibility_lib import dependency_highlighter 19 | 20 | 21 | class DependencyHighlighterStub: 22 | def __init__(self): 23 | self._outdated_dependencies = {} 24 | 25 | def check_package(self, package_name: str 26 | ) -> List[dependency_highlighter.OutdatedDependency]: 27 | return self._outdated_dependencies.get(package_name, []) 28 | 29 | def set_outdated_dependencies( 30 | self, package_name: str, outdated_dependencies: List[ 31 | dependency_highlighter.OutdatedDependency]): 32 | self._outdated_dependencies[package_name] = outdated_dependencies 33 | -------------------------------------------------------------------------------- /compatibility_lib/compatibility_lib/deprecated_dep_finder.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import concurrent.futures 16 | import logging 17 | 18 | from compatibility_lib import compatibility_checker 19 | from compatibility_lib import configs 20 | from compatibility_lib import utils 21 | 22 | DEPRECATED_STATUS = "Development Status :: 7 - Inactive" 23 | 24 | 25 | class DeprecatedDepFinder(object): 26 | """A tool for finding if there are deprecated pacakges in the deps. 27 | 28 | This tool looks at the development status field in the package info from 29 | PyPI JSON API, and if the status if 'Development Status :: 7 - Inactive', 30 | the package is deprecated. 31 | """ 32 | 33 | def __init__(self, 34 | py_version=None, 35 | checker=None, 36 | store=None, 37 | max_workers=50): 38 | if py_version is None: 39 | py_version = '3' 40 | 41 | if checker is None: 42 | checker = compatibility_checker.CompatibilityChecker() 43 | 44 | self.max_workers = max_workers 45 | self.py_version = py_version 46 | self._checker = checker 47 | self._store = store 48 | self._dependency_info_getter = utils.DependencyInfo( 49 | py_version, self._checker, self._store) 50 | 51 | # Share a common pool for PyPI requests to avoid creating too many 52 | # threads. 53 | self._pypi_thread_pool = concurrent.futures.ThreadPoolExecutor( 54 | max_workers=self.max_workers) 55 | 56 | def _get_development_status_from_pypi(self, package_name): 57 | """Get the development status for a package. 58 | 59 | All kinds of development statuses: 60 | 61 | Development Status :: 1 - Planning 62 | Development Status :: 2 - Pre-Alpha 63 | Development Status :: 3 - Alpha 64 | Development Status :: 4 - Beta 65 | Development Status :: 5 - Production/Stable 66 | Development Status :: 6 - Mature 67 | Development Status :: 7 - Inactive 68 | 69 | Args: 70 | package_name: the package needs to be checked. 71 | 72 | Returns: 73 | The development status of the package. 74 | """ 75 | pkg_info = utils.call_pypi_json_api(package_name=package_name) 76 | 77 | try: 78 | development_status = pkg_info['info']['classifiers'][0] 79 | except (KeyError, IndexError, TypeError): 80 | logging.warning("No development status available.") 81 | development_status = None 82 | 83 | return development_status 84 | 85 | def get_deprecated_dep(self, package_name): 86 | """Get deprecated dep for a single package.""" 87 | dependency_info = self._dependency_info_getter.get_dependency_info( 88 | package_name) 89 | deprecated_deps = [] 90 | for dep_name, development_status in zip( 91 | dependency_info, 92 | self._pypi_thread_pool.map( 93 | self._get_development_status_from_pypi, 94 | dependency_info)): 95 | if development_status == DEPRECATED_STATUS: 96 | deprecated_deps.append(dep_name) 97 | 98 | return package_name, deprecated_deps 99 | 100 | def get_deprecated_deps(self, packages=None): 101 | """Get deprecated deps for all the Google OSS packages.""" 102 | if packages is None: 103 | packages = configs.PKG_LIST 104 | 105 | # Create a small number of threads since get_deprecated_dep also 106 | # uses a thread pool. 107 | with concurrent.futures.ThreadPoolExecutor(max_workers=5) as p: 108 | results = p.map( 109 | self.get_deprecated_dep, 110 | ((pkg) for pkg in packages)) 111 | 112 | for result in zip(results): 113 | yield result 114 | -------------------------------------------------------------------------------- /compatibility_lib/compatibility_lib/deprecated_dep_finder_stub.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """A stub implementation of deprecated_dep_finder.DeprecatedDepFinder.""" 15 | 16 | 17 | class DeprecatedDepFinderStub: 18 | def get_deprecated_dep(self, package_name): 19 | return package_name, [] 20 | -------------------------------------------------------------------------------- /compatibility_lib/compatibility_lib/fake_compatibility_store.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """In memory storage for package compatibility information.""" 16 | 17 | import collections 18 | import itertools 19 | from typing import Iterable, FrozenSet, List, Mapping 20 | 21 | from compatibility_lib import package 22 | from compatibility_lib import compatibility_store 23 | from compatibility_lib import configs 24 | 25 | 26 | class CompatibilityStore: 27 | """Storage for package compatibility information.""" 28 | 29 | def __init__(self): 30 | self._packages_to_compatibility_result = {} 31 | self._package_to_dependency_info = {} 32 | 33 | def get_packages(self) -> Iterable[package.Package]: 34 | """Returns all packages tracked by the system.""" 35 | 36 | return [list(p)[0] 37 | for p in self._packages_to_compatibility_result.keys() 38 | if len(p) == 1] 39 | 40 | @staticmethod 41 | def _filter_older_versions( 42 | crs: Iterable[compatibility_store.CompatibilityResult]) \ 43 | -> Iterable[compatibility_store.CompatibilityResult]: 44 | """Remove old versions of CompatibilityResults from the given list.""" 45 | 46 | def key_func(cr): 47 | return frozenset(cr.packages), cr.python_major_version 48 | 49 | filtered_results = [] 50 | crs = sorted(crs, key=key_func) 51 | for _, results in itertools.groupby(crs, key_func): 52 | filtered_results.append(max(results, key=lambda cr: cr.timestamp)) 53 | return filtered_results 54 | 55 | def get_self_compatibility(self, 56 | p: package.Package) -> \ 57 | Iterable[compatibility_store.CompatibilityResult]: 58 | """Returns CompatibilityStatuses for internal package compatibility. 59 | 60 | Args: 61 | p: The package to check internal compatibility for. 62 | 63 | Yields: 64 | One CompatibilityResult per Python version. 65 | """ 66 | return self._filter_older_versions( 67 | self._packages_to_compatibility_result.get( 68 | frozenset([p]), [])) 69 | 70 | def get_self_compatibilities(self, 71 | packages: Iterable[package.Package]) -> \ 72 | Mapping[package.Package, List[ 73 | compatibility_store.CompatibilityResult]]: 74 | """Returns CompatibilityStatuses for internal package compatibility. 75 | 76 | Args: 77 | packages: The packages to check internal compatibility for. 78 | 79 | Returns: 80 | A mapping between the given packages and a (possibly empty) 81 | list of CompatibilityResults for each one. 82 | """ 83 | 84 | return {p: list(self.get_self_compatibility(p)) for p in packages} 85 | 86 | def get_pair_compatibility(self, packages: List[package.Package]) -> \ 87 | Iterable[compatibility_store.CompatibilityResult]: 88 | """Returns CompatibilityStatuses for internal package compatibility. 89 | 90 | Args: 91 | packages: The packages to check compatibility for. 92 | 93 | Yields: 94 | One CompatibilityResult per Python version. 95 | """ 96 | return self._filter_older_versions( 97 | self._packages_to_compatibility_result.get( 98 | frozenset(packages), 99 | [])) 100 | 101 | def get_pairwise_compatibility_for_package(self, package_name: str) -> \ 102 | Mapping[FrozenSet[package.Package], 103 | List[compatibility_store.CompatibilityResult]]: 104 | """Returns a mapping between package pairs and CompatibilityResults. 105 | 106 | Args: 107 | package_name: The package to check compatibility for. 108 | 109 | Returns: 110 | A mapping between every pairing between the given package with 111 | each google cloud python package (found in configs.PKG_LIST) and 112 | their pairwise CompatibilityResults. For example: 113 | Given package_name = 'p1', configs.PKG_LIST = [p2, p3, p4] => 114 | { 115 | frozenset([p1, p2]): [CompatibilityResult...], 116 | frozenset([p1, p3]): [CompatibilityResult...], 117 | frozenset([p1, p4]): [CompatibilityResult...], 118 | }. 119 | """ 120 | package_pairs = [frozenset([package.Package(package_name), 121 | package.Package(name)]) 122 | for name in configs.PKG_LIST 123 | if package_name != name] 124 | results = {pair: self.get_pair_compatibility(pair) 125 | for pair in package_pairs 126 | if self.get_pair_compatibility(pair)} 127 | return results 128 | 129 | def get_compatibility_combinations(self, 130 | packages: List[package.Package]) -> \ 131 | Mapping[FrozenSet[package.Package], List[ 132 | compatibility_store.CompatibilityResult]]: 133 | """Returns a mapping between package pairs and CompatibilityResults. 134 | 135 | Args: 136 | packages: The packages to check compatibility for. 137 | 138 | Returns: 139 | A mapping between every combination of input packages and their 140 | CompatibilityResults. For example: 141 | get_compatibility_combinations(packages = [p1, p2, p3]) => 142 | { 143 | frozenset([p1, p2]): [CompatibilityResult...], 144 | frozenset([p1, p3]): [CompatibilityResult...], 145 | frozenset([p2, p3]): [CompatibilityResult...], 146 | }. 147 | """ 148 | return {frozenset([p1, p2]): self.get_pair_compatibility([p1, p2]) 149 | for (p1, p2) in itertools.combinations(packages, r=2)} 150 | 151 | def save_compatibility_statuses( 152 | self, 153 | compatibility_statuses: Iterable[ 154 | compatibility_store.CompatibilityResult]): 155 | """Save the given CompatibilityStatuses""" 156 | 157 | name_to_compatibility_results = collections.defaultdict(list) 158 | for cr in compatibility_statuses: 159 | self._packages_to_compatibility_result.setdefault( 160 | frozenset(cr.packages), []).append(cr) 161 | 162 | if len(cr.packages) == 1: 163 | install_name = cr.packages[0].install_name 164 | name_to_compatibility_results[install_name].append(cr) 165 | 166 | for install_name, compatibility_results in ( 167 | name_to_compatibility_results.items()): 168 | compatibility_result = ( 169 | compatibility_store.get_latest_compatibility_result_by_version( 170 | compatibility_results)) 171 | if compatibility_result.dependency_info: 172 | self._package_to_dependency_info[ 173 | install_name] = compatibility_result.dependency_info 174 | 175 | def get_dependency_info(self, package_name): 176 | return self._package_to_dependency_info.get(package_name, {}) 177 | -------------------------------------------------------------------------------- /compatibility_lib/compatibility_lib/get_compatibility_data.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Get self and pairwise compatibility data and write to bigquery.""" 16 | 17 | import argparse 18 | import contextlib 19 | import datetime 20 | import itertools 21 | import logging 22 | import signal 23 | 24 | import pexpect 25 | from pexpect import popen_spawn 26 | 27 | from compatibility_lib import compatibility_checker 28 | from compatibility_lib import compatibility_store 29 | from compatibility_lib import configs 30 | from compatibility_lib import package 31 | 32 | # The production compatibility server configuration currently runs 33 | # 200 pods and each pod has two workers. 34 | checker = compatibility_checker.CompatibilityChecker(max_workers=400) 35 | store = compatibility_store.CompatibilityStore() 36 | 37 | PY2 = '2' 38 | PY3 = '3' 39 | 40 | INSTANCE_CONNECTION_NAME = 'python-compatibility-tools:us-central1:' \ 41 | 'compatibility-data' 42 | 43 | PORT = '3306' 44 | 45 | LOG_LEVEL_TO_RUNTIME_CONSTANT = { 46 | 'debug': logging.DEBUG, 47 | 'info': logging.INFO, 48 | 'warning': logging.WARNING, 49 | 'error': logging.ERROR, 50 | 'critical': logging.CRITICAL, 51 | } 52 | 53 | 54 | class ConnectionError(Exception): 55 | pass 56 | 57 | 58 | def _result_dict_to_compatibility_result(results): 59 | res_list = [] 60 | 61 | for item in results: 62 | res_dict = item[0] 63 | result_content, python_version = res_dict 64 | check_result = result_content.get('result') 65 | packages_list = [package.Package(pkg) 66 | for pkg in result_content.get('packages')] 67 | details = result_content.get('description') 68 | timestamp = datetime.datetime.now().isoformat() 69 | dependency_info = result_content.get('dependency_info') 70 | 71 | compatibility_result = compatibility_store.CompatibilityResult( 72 | packages=packages_list, 73 | python_major_version=python_version, 74 | status=compatibility_store.Status(check_result), 75 | details=details, 76 | timestamp=timestamp, 77 | dependency_info=dependency_info 78 | ) 79 | res_list.append(compatibility_result) 80 | 81 | return res_list 82 | 83 | 84 | @contextlib.contextmanager 85 | def run_cloud_sql_proxy(cloud_sql_proxy_path): 86 | instance_flag = '-instances={}=tcp:{}'.format( 87 | INSTANCE_CONNECTION_NAME, PORT) 88 | if cloud_sql_proxy_path is None: 89 | assert cloud_sql_proxy_path, 'Could not find cloud_sql_proxy path' 90 | process = popen_spawn.PopenSpawn([cloud_sql_proxy_path, instance_flag]) 91 | 92 | try: 93 | process.expect('Ready for new connection', timeout=5) 94 | yield 95 | except pexpect.exceptions.TIMEOUT: 96 | raise ConnectionError( 97 | ('Cloud SQL Proxy was unable to start after 5 seconds. Output ' 98 | 'of cloud_sql_proxy: \n{}').format(process.before)) 99 | except pexpect.exceptions.EOF: 100 | raise ConnectionError( 101 | ('Cloud SQL Proxy exited unexpectedly. Output of ' 102 | 'cloud_sql_proxy: \n{}').format(process.before)) 103 | finally: 104 | process.kill(signal.SIGTERM) 105 | 106 | 107 | def get_package_pairs(check_pypi=False, check_github=False): 108 | """Get package pairs for pypi and github head.""" 109 | self_packages = [] 110 | pair_packages = [] 111 | if check_pypi: 112 | # Get pypi packages for single checks 113 | self_packages.extend(configs.PKG_LIST) 114 | # Get pypi packages for pairwise checks 115 | pypi_pairs = list(itertools.combinations(configs.PKG_LIST, 2)) 116 | pair_packages.extend(pypi_pairs) 117 | if check_github: 118 | # Get github head packages for single checks 119 | self_packages.extend(list(configs.WHITELIST_URLS.keys())) 120 | # Get github head packages for pairwise checks 121 | for gh_url in configs.WHITELIST_URLS: 122 | pairs = [] 123 | gh_name = configs.WHITELIST_URLS[gh_url] 124 | for pypi_pkg in configs.PKG_LIST: 125 | if pypi_pkg != gh_name: 126 | pairs.append((gh_url, pypi_pkg)) 127 | pair_packages.extend(pairs) 128 | 129 | return self_packages, pair_packages 130 | 131 | 132 | def write_to_status_table( 133 | check_pypi=False, check_github=False, cloud_sql_proxy_path=None): 134 | """Get the compatibility status for PyPI versions.""" 135 | # Write self compatibility status to BigQuery 136 | self_packages, pair_packages = get_package_pairs(check_pypi, check_github) 137 | results = checker.get_compatibility( 138 | packages=self_packages, pkg_sets=pair_packages) 139 | res_list = _result_dict_to_compatibility_result(results) 140 | 141 | with run_cloud_sql_proxy(cloud_sql_proxy_path): 142 | store.save_compatibility_statuses(res_list) 143 | 144 | 145 | if __name__ == '__main__': 146 | parser = argparse.ArgumentParser(description='Determine what to check.') 147 | parser.add_argument( 148 | '--pypi', 149 | type=bool, 150 | default=False, 151 | help='Check PyPI released packages or not.') 152 | parser.add_argument( 153 | '--github', 154 | type=bool, 155 | default=False, 156 | help='Check GitHub head packages or not.') 157 | parser.add_argument( 158 | '--cloud_sql_proxy_path', 159 | type=str, 160 | default='cloud_sql_proxy', 161 | help='Path to cloud_sql_proxy.') 162 | parser.add_argument( 163 | '--log_level', 164 | choices=LOG_LEVEL_TO_RUNTIME_CONSTANT.keys(), 165 | default='info', 166 | help=('the log level above which logging messages are written to ' + 167 | 'stderr')) 168 | 169 | args = parser.parse_args() 170 | 171 | log_level = LOG_LEVEL_TO_RUNTIME_CONSTANT[args.log_level] 172 | logger = logging.getLogger("compatibility_lib") 173 | logger.setLevel(log_level) 174 | ch = logging.StreamHandler() 175 | logger.addHandler(ch) 176 | 177 | check_pypi = args.pypi 178 | check_github = args.github 179 | cloud_sql_proxy_path = args.cloud_sql_proxy_path 180 | write_to_status_table(check_pypi, check_github, cloud_sql_proxy_path) 181 | -------------------------------------------------------------------------------- /compatibility_lib/compatibility_lib/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Represents a pip-installable Python package.""" 16 | 17 | from typing import Optional 18 | 19 | 20 | class Package: 21 | """A pip-installable Python package. 22 | 23 | Attributes: 24 | installed_name: The name that would be used in the "pip install" 25 | command e.g. "tensorflow" or 26 | "git+git://github.com/apache/beam#subdirectory=sdks/python". 27 | friendly_name: The friendly name of the package e.g. "tensorflow" 28 | or "apache_beam (git HEAD)" 29 | """ 30 | 31 | def __init__(self, install_name: str, friendly_name: Optional[str] = None): 32 | """Initializer for Package. 33 | 34 | Args: 35 | install_name: The name that would be used in the "pip install" 36 | command e.g. "tensorflow" or 37 | "git+git://github.com/apache/beam#subdirectory=sdks/python". 38 | friendly_name: The friendly name of the package e.g. "tensorflow" 39 | or "apache_beam (git HEAD)" 40 | """ 41 | self._install_name = install_name 42 | self._friendly_name = friendly_name or install_name 43 | 44 | def __repr__(self): 45 | return 'Package({})'.format(self.install_name) 46 | 47 | def __hash__(self): 48 | return hash(self.install_name) 49 | 50 | def __eq__(self, o): 51 | if isinstance(o, Package): 52 | return self.install_name == o.install_name 53 | return NotImplemented 54 | 55 | @property 56 | def install_name(self) -> str: 57 | return self._install_name 58 | 59 | @property 60 | def friendly_name(self) -> str: 61 | return self._friendly_name 62 | -------------------------------------------------------------------------------- /compatibility_lib/compatibility_lib/package_crawler_static.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """A set of functions that aid in crawling a package 16 | and creating a corresponding data model""" 17 | 18 | import ast 19 | import importlib 20 | import os 21 | 22 | 23 | class PackageNotFound(Exception): 24 | pass 25 | 26 | 27 | def get_package_location(pkgname): 28 | """gets the package location 29 | 30 | Args: 31 | pkgname: the package name as a string 32 | 33 | Returns: 34 | A string containing the path of the give package 35 | 36 | Raises: 37 | PackageNotFound: An error that occurs when the package is not found 38 | """ 39 | spec = importlib.util.find_spec(pkgname) 40 | if spec is None: 41 | errmsg = ('Could not find "%s". Please make sure that ' 42 | 'this package is installed.' % pkgname) 43 | raise PackageNotFound(errmsg) 44 | locations = [l for l in spec.submodule_search_locations] 45 | root_dir = locations[0] 46 | return root_dir 47 | 48 | 49 | def get_package_info(root_dir): 50 | """gets package info 51 | 52 | Crawls the package directory with filesystem tooling and 53 | creates a data model containing relevant info about 54 | subpackages, modules, classes, functions, and args 55 | 56 | Args: 57 | root_dir: the location of the package 58 | 59 | Returns: 60 | A dict mapping keys to the corresponding data derived 61 | For example: 62 | { 63 | module_name: { 64 | 'classes': { 65 | class_name: { 66 | 'args': [arg1, arg2, ...], 67 | 'functions': { 68 | function_name: {'args': [...]}, 69 | } 70 | }, 71 | class_name: {...}, 72 | }, 73 | 'functions': {...}, 74 | 'subclasses': {...}, 75 | }, 76 | 'subpackages': {...} 77 | } 78 | """ 79 | 80 | info = {} 81 | info['modules'] = {} 82 | subpackages = [] 83 | for name in os.listdir(root_dir): 84 | path = os.path.join(root_dir, name) 85 | if name.startswith('_'): 86 | continue 87 | elif name.endswith('.py'): 88 | if (name.startswith('test_') or name.endswith('_test.py')): 89 | continue 90 | with open(path) as f: 91 | node = ast.parse(f.read(), path) 92 | modname = os.path.splitext(name)[0] 93 | info['modules'][modname] = get_module_info(node) 94 | elif os.path.isdir(path) and not name.startswith('.'): 95 | subpackages.append(name) 96 | 97 | info['subpackages'] = {} 98 | for name in subpackages: 99 | path = os.path.join(root_dir, name) 100 | info['subpackages'][name] = get_package_info(path) 101 | 102 | return info 103 | 104 | 105 | def get_module_info(node): 106 | """gets module info 107 | 108 | Recursively crawls the ast node and creates a data model 109 | containing relevant info about the module's classes, 110 | subclasses, functions, and args 111 | 112 | Args: 113 | node: the module 114 | 115 | Returns: 116 | A dict mapping keys to the corresponding data derived 117 | For example: 118 | { 119 | 'classes': { 120 | class_name: { 121 | 'args': [arg1, arg2, ...], 122 | 'functions': { 123 | function_name: {'args': [...]}, 124 | } 125 | }, 126 | class_name: {...}, 127 | }, 128 | 'functions': {...}, 129 | 'subclasses': {...}, 130 | } 131 | """ 132 | classes = [n for n in node.body if isinstance(n, ast.ClassDef)] 133 | functions = [n for n in node.body if isinstance(n, ast.FunctionDef)] 134 | 135 | res = {} 136 | res['classes'] = _get_class_info(classes) 137 | res['functions'] = _get_function_info(functions) 138 | 139 | return res 140 | 141 | 142 | def _get_class_info(classes): 143 | """returns a dict containing info on args, subclasses and functions""" 144 | res = {} 145 | for node in classes: 146 | # in classes with multiple base classes, there may be subclasses or 147 | # functions that overlap (share the same name), to mirror the actual 148 | # pythonic behavior, clashes are ignored 149 | if node.name.startswith('_') or res.get(node.name) is not None: 150 | continue 151 | 152 | # assumption is that bases are user-defined within the same module 153 | init_func, subclasses, functions = _get_class_attrs(node, classes) 154 | args = [] 155 | if init_func is not None: 156 | args = _get_args(init_func.args) 157 | 158 | res[node.name] = {} 159 | res[node.name]['args'] = args 160 | res[node.name]['subclasses'] = _get_class_info(subclasses) 161 | res[node.name]['functions'] = _get_function_info(functions) 162 | 163 | return res 164 | 165 | 166 | def _get_class_attrs(node, classes): 167 | """returns operational init func, subclasses, and functions of a class 168 | including those of any base classes defined within the same module by 169 | crawling through class nodes 170 | """ 171 | init_func, subclasses, functions = None, [], [] 172 | for n in node.body: 173 | if hasattr(n, 'name') and n.name == '__init__': 174 | init_func = n 175 | if isinstance(n, ast.ClassDef): 176 | subclasses.append(n) 177 | elif isinstance(n, ast.FunctionDef): 178 | functions.append(n) 179 | 180 | # inheritance priority is preorder 181 | basenames = _get_basenames(node.bases) 182 | bases = {n.name: n for n in classes if n.name in basenames} 183 | for bname in basenames: 184 | if bases.get(bname) is None: 185 | continue 186 | n = bases[bname] 187 | _init_func, _subclasses, _functions = _get_class_attrs(n, classes) 188 | if init_func is None: 189 | init_func = _init_func 190 | subclasses.extend(_subclasses) 191 | functions.extend(_functions) 192 | return (init_func, subclasses, functions) 193 | 194 | 195 | def _get_basenames(bases): 196 | res = [] 197 | for n in bases: 198 | name = [] 199 | if isinstance(n, ast.Attribute): 200 | while isinstance(n, ast.Attribute): 201 | name.append(n.attr) 202 | n = n.value 203 | if isinstance(n, ast.Name): 204 | name.append(n.id) 205 | res.append('.'.join(name[::-1])) 206 | return res 207 | 208 | 209 | def _get_function_info(functions): 210 | """returns a dict mapping function name to function args""" 211 | res = {} 212 | for node in functions: 213 | fname = node.name 214 | if fname.startswith('_') or res.get(fname) is not None: 215 | continue 216 | res[fname] = {} 217 | res[fname]['args'] = _get_args(node.args) 218 | return res 219 | 220 | 221 | def _get_args(node): 222 | """returns a list of args""" 223 | args = [] 224 | for arg in node.args: 225 | if isinstance(arg, ast.arg): 226 | args.append(arg.arg) 227 | elif isinstance(arg, ast.Name): 228 | args.append(arg.id) 229 | return args 230 | -------------------------------------------------------------------------------- /compatibility_lib/compatibility_lib/semver_checker.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """checks two packages for semver breakage""" 16 | 17 | from compatibility_lib import package_crawler_static as crawler 18 | 19 | 20 | # TODO: This needs more sophisticated logic 21 | # - needs to look at args 22 | def check(old_dir, new_dir): 23 | """checks for semver breakage for two local directories 24 | it looks at all the attributes found by get_package_info 25 | (module, class, function names) for old_dir and making sure they are also 26 | in new_dir in a BFS 27 | 28 | Args: 29 | old_dir: directory containing old files 30 | new_dir: directory containing new files 31 | 32 | Returns: 33 | False if changes breaks semver, True if semver is preserved 34 | """ 35 | old_pkg_info = crawler.get_package_info(old_dir) 36 | new_pkg_info = crawler.get_package_info(new_dir) 37 | 38 | unseen = [(old_pkg_info, new_pkg_info)] 39 | errors = [] 40 | missing = 'missing attribute "%s" from new version' 41 | bad_args = 'args do not match; expecting: "%s", got: "%s"' 42 | 43 | i = 0 44 | while i < len(unseen): 45 | old, new = unseen[i] 46 | for key in old.keys(): 47 | if new.get(key) is None: 48 | errors.append(missing % key) 49 | continue 50 | if key != 'args': 51 | unseen.append((old[key], new[key])) 52 | elif old[key] != new[key]: 53 | old_args = ', '.join(old[key]) 54 | new_args = ', '.join(new[key]) 55 | errors.append(bad_args % (old_args, new_args)) 56 | i += 1 57 | 58 | return errors 59 | -------------------------------------------------------------------------------- /compatibility_lib/compatibility_lib/test_compatibility_checker.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import unittest 16 | 17 | import mock 18 | 19 | from compatibility_lib import compatibility_checker 20 | 21 | 22 | class TestCompatibilityChecker(unittest.TestCase): 23 | 24 | def test_check(self): 25 | checker = compatibility_checker.CompatibilityChecker() 26 | 27 | packages = ['opencensus'] 28 | python_version = 3 29 | expected_server_url = 'http://34.68.40.69/' 30 | 31 | data = { 32 | 'python-version': python_version, 33 | 'package': packages, 34 | } 35 | 36 | mock_requests = mock.Mock() 37 | mock_response = mock.Mock(content=b'{}') 38 | mock_requests.get.return_value = mock_response 39 | 40 | patch_request = mock.patch( 41 | 'compatibility_lib.compatibility_checker.requests', 42 | mock_requests) 43 | 44 | with patch_request: 45 | checker.check(packages, python_version) 46 | 47 | mock_requests.get.assert_called_with( 48 | compatibility_checker.SERVER_URL, params=data, timeout=299) 49 | self.assertEqual(compatibility_checker.SERVER_URL, 50 | expected_server_url) 51 | 52 | def _mock_retrying_check(self, *args): 53 | packages = args[0][0] 54 | python_version = args[0][1] 55 | return (packages, python_version, 'SUCCESS') 56 | 57 | def test_get_compatibility(self): 58 | checker = compatibility_checker.CompatibilityChecker() 59 | 60 | pkg_list = ['pkg1', 'pkg2', 'pkg3'] 61 | pkg_py_version_not_supported = { 62 | 2: ['tensorflow', ], 63 | 3: ['apache-beam[gcp]', 'gsutil', ], 64 | } 65 | 66 | mock_config = mock.Mock() 67 | mock_config.PKG_LIST = pkg_list 68 | mock_config.PKG_PY_VERSION_NOT_SUPPORTED = pkg_py_version_not_supported 69 | patch_config = mock.patch( 70 | 'compatibility_lib.compatibility_checker.configs', mock_config) 71 | 72 | patch_executor = mock.patch( 73 | 'compatibility_lib.compatibility_checker.concurrent.futures.ThreadPoolExecutor', 74 | FakeExecutor) 75 | patch_retrying_check = mock.patch.object( 76 | compatibility_checker.CompatibilityChecker, 77 | 'retrying_check', 78 | self._mock_retrying_check) 79 | 80 | res = [] 81 | with patch_config, patch_executor, patch_retrying_check: 82 | result = checker.get_compatibility() 83 | 84 | for item in result: 85 | res.append(item) 86 | 87 | expected = sorted([ 88 | ((['pkg1'], '2', 'SUCCESS'),), 89 | ((['pkg2'], '2', 'SUCCESS'),), 90 | ((['pkg3'], '2', 'SUCCESS'),), 91 | ((['pkg1'], '3', 'SUCCESS'),), 92 | ((['pkg2'], '3', 'SUCCESS'),), 93 | ((['pkg3'], '3', 'SUCCESS'),), 94 | ((['pkg1', 'pkg2'], '2', 'SUCCESS'),), 95 | ((['pkg1', 'pkg3'], '2', 'SUCCESS'),), 96 | ((['pkg2', 'pkg3'], '2', 'SUCCESS'),), 97 | ((['pkg1', 'pkg2'], '3', 'SUCCESS'),), 98 | ((['pkg1', 'pkg3'], '3', 'SUCCESS'),), 99 | ((['pkg2', 'pkg3'], '3', 'SUCCESS'),)]) 100 | 101 | self.assertEqual(sorted(res), expected) 102 | 103 | 104 | class FakeExecutor(object): 105 | def __init__(self, max_workers=10): 106 | self.max_workers = max_workers 107 | 108 | def map(self, check_func, pkgs): 109 | results = [] 110 | 111 | for pkg in pkgs: 112 | results.append(check_func(pkg)) 113 | 114 | return results 115 | 116 | def __enter__(self): 117 | return self 118 | 119 | def __exit__(self, exception_type, exception_value, traceback): 120 | return None 121 | -------------------------------------------------------------------------------- /compatibility_lib/compatibility_lib/test_deprecated_dep_finder.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import unittest 16 | 17 | import mock 18 | 19 | from compatibility_lib import deprecated_dep_finder 20 | from compatibility_lib import compatibility_store 21 | from compatibility_lib import fake_compatibility_store 22 | from compatibility_lib import package 23 | 24 | 25 | class TestDeprecatedDepFinder(unittest.TestCase): 26 | PKG_INFO = { 27 | 'info': { 28 | 'classifiers': [ 29 | "Development Status :: 7 - Inactive", 30 | "Intended Audience :: Developers", 31 | "License :: OSI Approved:: Apache Software License", 32 | "Operating System :: POSIX", 33 | "Programming Language:: Python :: 2", 34 | "Programming Language:: Python :: 2.7", 35 | "Programming Language:: Python :: 3", 36 | "Programming Language:: Python :: 3.4", 37 | "Programming Language:: Python :: 3.5", 38 | "Topic :: Internet :: WWW / HTTP", 39 | ] 40 | } 41 | } 42 | 43 | def setUp(self): 44 | self.DEP_INFO = { 45 | 'dep1': { 46 | 'installed_version': '2.1.0', 47 | 'installed_version_time': '2018-05-12T16:26:31', 48 | 'latest_version': '2.1.0', 49 | 'current_time': '2018-08-27T17:04:57.260105', 50 | 'latest_version_time': '2018-05-12T16:26:31', 51 | 'is_latest': True, 52 | }, 53 | 'dep2': { 54 | 'installed_version': '3.6.1', 55 | 'installed_version_time': '2018-08-13T22:47:09', 56 | 'latest_version': '3.6.1', 57 | 'current_time': '2018-08-27T17:04:57.934866', 58 | 'latest_version_time': '2018-08-13T22:47:09', 59 | 'is_latest': True, 60 | }, 61 | } 62 | 63 | self.SELF_COMP_RES = (( 64 | { 65 | 'result': 'SUCCESS', 66 | 'packages': ['package1'], 67 | 'description': None, 68 | 'dependency_info': self.DEP_INFO, 69 | }, 70 | ),) 71 | self.mock_checker = mock.Mock(autospec=True) 72 | self.fake_store = fake_compatibility_store.CompatibilityStore() 73 | 74 | self.mock_checker.get_compatibility.return_value = \ 75 | self.SELF_COMP_RES 76 | 77 | def test_constructor_default(self): 78 | from compatibility_lib import utils 79 | 80 | finder = deprecated_dep_finder.DeprecatedDepFinder( 81 | checker=self.mock_checker, store=self.fake_store) 82 | 83 | self.assertEqual(finder.py_version, '3') 84 | self.assertTrue( 85 | isinstance(finder._dependency_info_getter, utils.DependencyInfo)) 86 | 87 | def test_constructor_explicit(self): 88 | from compatibility_lib import utils 89 | 90 | finder = deprecated_dep_finder.DeprecatedDepFinder( 91 | py_version='2', checker=self.mock_checker, store=self.fake_store) 92 | 93 | self.assertEqual(finder.py_version, '2') 94 | self.assertIsInstance( 95 | finder._dependency_info_getter, utils.DependencyInfo) 96 | 97 | def test__get_development_status_from_pypi_error(self): 98 | PKG_INFO = { 99 | 'test': { 100 | 'test_key_error': [], 101 | } 102 | } 103 | 104 | mock_call_pypi_json_api = mock.Mock(autospec=True) 105 | mock_call_pypi_json_api.return_value = PKG_INFO 106 | 107 | patch_utils = mock.patch( 108 | 'compatibility_lib.deprecated_dep_finder.utils.call_pypi_json_api', 109 | mock_call_pypi_json_api) 110 | 111 | with patch_utils: 112 | finder = deprecated_dep_finder.DeprecatedDepFinder(checker=self.mock_checker, store=self.fake_store) 113 | development_status = finder._get_development_status_from_pypi( 114 | 'package1') 115 | 116 | self.assertIsNone(development_status) 117 | 118 | def test__get_development_status_from_pypi(self): 119 | mock_call_pypi_json_api = mock.Mock(autospec=True) 120 | mock_call_pypi_json_api.return_value = self.PKG_INFO 121 | 122 | patch_utils = mock.patch( 123 | 'compatibility_lib.deprecated_dep_finder.utils.call_pypi_json_api', 124 | mock_call_pypi_json_api) 125 | 126 | with patch_utils: 127 | finder = deprecated_dep_finder.DeprecatedDepFinder( 128 | checker=self.mock_checker, store=self.fake_store) 129 | development_status = finder._get_development_status_from_pypi( 130 | 'package1') 131 | 132 | expected_development_status = "Development Status :: 7 - Inactive" 133 | self.assertEqual(development_status, expected_development_status) 134 | 135 | def test_get_deprecated_dep(self): 136 | # TODO: use a more convincing mock that doesn't say that *all* packages 137 | # are deprecated. 138 | mock_call_pypi_json_api = mock.Mock(autospec=True) 139 | mock_call_pypi_json_api.return_value = self.PKG_INFO 140 | 141 | self.fake_store.save_compatibility_statuses([ 142 | compatibility_store.CompatibilityResult( 143 | packages=[package.Package('opencencus')], 144 | python_major_version='3', 145 | status=compatibility_store.Status.SUCCESS, 146 | details=None, 147 | dependency_info=self.DEP_INFO), 148 | ]) 149 | patch_utils = mock.patch( 150 | 'compatibility_lib.deprecated_dep_finder.utils.call_pypi_json_api', 151 | mock_call_pypi_json_api) 152 | 153 | with patch_utils: 154 | finder = deprecated_dep_finder.DeprecatedDepFinder( 155 | checker=self.mock_checker, store=self.fake_store) 156 | deprecated_deps = finder.get_deprecated_dep('opencencus') 157 | 158 | expected_deprecated_deps = set(['dep1', 'dep2']) 159 | self.assertEqual(set(deprecated_deps[1]), expected_deprecated_deps) 160 | -------------------------------------------------------------------------------- /compatibility_lib/compatibility_lib/test_get_compatibility_data.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import mock 16 | import unittest 17 | 18 | from compatibility_lib import fake_compatibility_store 19 | from compatibility_lib import package 20 | from compatibility_lib import compatibility_store 21 | 22 | 23 | class TestGetCompatibilityData(unittest.TestCase): 24 | dependency_info = { 25 | 'cachetools': { 26 | 'installed_version': '2.1.0', 27 | 'latest_version': '2.1.0', 28 | 'current_time': '2018-07-10T11:02:40.481246', 29 | 'latest_version_time': '2018-05-12T16:26:31', 30 | 'is_latest': True 31 | }, 32 | 'certifi': { 33 | 'installed_version': '2018.4.16', 34 | 'latest_version': '2018.4.16', 35 | 'current_time': '2018-07-10T11:02:40.544879', 36 | 'latest_version_time': '2018-04-16T18:50:10', 37 | 'is_latest': True 38 | } 39 | } 40 | results = ( 41 | (( 42 | { 43 | 'result': 'SUCCESS', 44 | 'packages': ['google-api-core'], 45 | 'description': None, 46 | 'dependency_info': dependency_info, 47 | }, 48 | '3', 49 | ),), 50 | ) 51 | 52 | packages = [package.Package('google-api-core')] 53 | status = compatibility_store.Status.SUCCESS 54 | 55 | def setUp(self): 56 | self.mock_checker = mock.Mock() 57 | self.mock_checker.get_compatibility.return_value = self.results 58 | 59 | self.fake_store = fake_compatibility_store.CompatibilityStore() 60 | 61 | def mock_init(): 62 | return None 63 | 64 | self.patch_constructor = mock.patch.object( 65 | compatibility_store.CompatibilityStore, 66 | '__init__', 67 | side_effect=mock_init) 68 | self.patch_checker = mock.patch( 69 | 'compatibility_lib.get_compatibility_data.checker', 70 | self.mock_checker) 71 | self.patch_store = mock.patch( 72 | 'compatibility_lib.get_compatibility_data.store', 73 | self.fake_store) 74 | 75 | def test_get_package_pairs_pypi(self): 76 | mock_config = mock.Mock() 77 | PKG_LIST = ['package1', 'package2', 'package3'] 78 | mock_config.PKG_LIST = PKG_LIST 79 | WHITELIST_URLS = { 80 | 'github.com/pkg1.git': 'package1', 81 | 'github.com/pkg2.git': 'package2', 82 | 'github.com/pkg3.git': 'package3' 83 | } 84 | mock_config.WHITELIST_URLS = WHITELIST_URLS 85 | patch_config = mock.patch( 86 | 'compatibility_lib.get_compatibility_data.configs', 87 | mock_config) 88 | 89 | with patch_config, self.patch_constructor, self.patch_checker, self.patch_store: 90 | from compatibility_lib import get_compatibility_data 91 | 92 | self_packages, pair_packages = get_compatibility_data.get_package_pairs( 93 | check_pypi=True, check_github=False) 94 | 95 | expected_self_packages = sorted(['package1', 'package2', 'package3']) 96 | self.assertEqual(sorted(self_packages), expected_self_packages) 97 | 98 | expected_pair_packages = sorted( 99 | [('package1', 'package2'), 100 | ('package1', 'package3'), 101 | ('package2', 'package3')]) 102 | self.assertEqual( 103 | sorted(pair_packages), 104 | expected_pair_packages) 105 | 106 | def test_get_package_pairs_github(self): 107 | mock_config = mock.Mock() 108 | PKG_LIST = ['package1', 'package2', 'package3'] 109 | mock_config.PKG_LIST = PKG_LIST 110 | WHITELIST_URLS = { 111 | 'github.com/pkg1.git': 'package1', 112 | 'github.com/pkg2.git': 'package2', 113 | 'github.com/pkg3.git': 'package3' 114 | } 115 | mock_config.WHITELIST_URLS = WHITELIST_URLS 116 | patch_config = mock.patch( 117 | 'compatibility_lib.get_compatibility_data.configs', 118 | mock_config) 119 | 120 | with patch_config, self.patch_constructor, self.patch_checker, self.patch_store: 121 | from compatibility_lib import get_compatibility_data 122 | 123 | self_packages, pair_packages = get_compatibility_data.get_package_pairs( 124 | check_pypi=False, check_github=True) 125 | 126 | expected_self_packages = sorted( 127 | ['github.com/pkg1.git', 128 | 'github.com/pkg2.git', 129 | 'github.com/pkg3.git']) 130 | self.assertEqual( 131 | sorted(self_packages), expected_self_packages) 132 | 133 | expected_pair_packages = sorted( 134 | [('github.com/pkg1.git', 'package2'), 135 | ('github.com/pkg1.git', 'package3'), 136 | ('github.com/pkg2.git', 'package1'), 137 | ('github.com/pkg2.git', 'package3'), 138 | ('github.com/pkg3.git', 'package1'), 139 | ('github.com/pkg3.git', 'package2')]) 140 | self.assertEqual( 141 | sorted(pair_packages), expected_pair_packages) 142 | 143 | def test__result_dict_to_compatibility_result(self): 144 | with self.patch_constructor, self.patch_checker, self.patch_store: 145 | from compatibility_lib import compatibility_store 146 | from compatibility_lib import get_compatibility_data 147 | 148 | res_list = get_compatibility_data._result_dict_to_compatibility_result( 149 | self.results) 150 | 151 | self.assertTrue(isinstance( 152 | res_list[0], compatibility_store.CompatibilityResult)) 153 | self.assertEqual(res_list[0].dependency_info, self.dependency_info) 154 | self.assertEqual(res_list[0].packages, self.packages) 155 | self.assertEqual(res_list[0].status, self.status) 156 | 157 | def test_write_to_status_table(self): 158 | patch_cloud_sql_proxy = mock.patch( 159 | 'compatibility_lib.get_compatibility_data.run_cloud_sql_proxy', 160 | return_value=MockProxy()) 161 | 162 | with self.patch_checker, self.patch_store, patch_cloud_sql_proxy: 163 | from compatibility_lib import get_compatibility_data 164 | 165 | get_compatibility_data.write_to_status_table() 166 | 167 | self.assertTrue(self.mock_checker.get_compatibility.called) 168 | saved_results = self.fake_store._packages_to_compatibility_result.get( 169 | frozenset({self.packages[0]})) 170 | self.assertIsNotNone(saved_results) 171 | self.assertEqual(len(saved_results), 1) 172 | saved_item = saved_results[0] 173 | self.assertEqual(saved_item.packages, self.packages) 174 | self.assertEqual(saved_item.dependency_info, self.dependency_info) 175 | self.assertEqual(saved_item.status, self.status) 176 | 177 | class MockProxy(object): 178 | 179 | def __enter__(self): 180 | return mock.Mock() 181 | 182 | def __exit__(self, exc_type, exc_val, exc_tb): 183 | pass 184 | -------------------------------------------------------------------------------- /compatibility_lib/compatibility_lib/test_semver_checker.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import ast 16 | import os 17 | import unittest 18 | 19 | from compatibility_lib.semver_checker import check 20 | from compatibility_lib.package_crawler_static import get_package_info, get_module_info 21 | 22 | 23 | CWD = os.path.dirname(os.path.realpath(__file__)) 24 | TEST_DIR = os.path.join(CWD, 'testpkgs') 25 | 26 | 27 | class TestSimplePackages(unittest.TestCase): 28 | 29 | def test_get_package_info(self): 30 | expected = { 31 | 'modules': { 32 | 'simple_function': { 33 | 'classes': {}, 34 | 'functions': { 35 | 'hello': { 36 | 'args': [] 37 | } 38 | } 39 | }, 40 | }, 41 | 'subpackages': {} 42 | } 43 | location = os.path.join(TEST_DIR, 'simple_function') 44 | info = get_package_info(location) 45 | self.assertEqual(expected, info) 46 | 47 | def test_semver_check_on_added_func(self): 48 | old_dir = os.path.join(TEST_DIR, 'added_func/0.1.0') 49 | new_dir = os.path.join(TEST_DIR, 'added_func/0.2.0') 50 | 51 | res = check(old_dir, new_dir) 52 | self.assertEqual([], res) 53 | 54 | def test_semver_check_on_removed_func(self): 55 | old_dir = os.path.join(TEST_DIR, 'removed_func/0.1.0') 56 | new_dir = os.path.join(TEST_DIR, 'removed_func/0.2.0') 57 | 58 | res = check(old_dir, new_dir) 59 | expected = ['missing attribute "bar" from new version'] 60 | self.assertEqual(expected, res) 61 | 62 | def test_semver_check_on_added_args(self): 63 | old_dir = os.path.join(TEST_DIR, 'added_args/0.1.0') 64 | new_dir = os.path.join(TEST_DIR, 'added_args/0.2.0') 65 | 66 | res = check(old_dir, new_dir) 67 | expected = ['args do not match; expecting: "self, x", got: "self, x, y"'] 68 | self.assertEqual(expected, res) 69 | 70 | def test_semver_check_on_removed_args(self): 71 | old_dir = os.path.join(TEST_DIR, 'removed_args/0.1.0') 72 | new_dir = os.path.join(TEST_DIR, 'removed_args/0.2.0') 73 | 74 | res = check(old_dir, new_dir) 75 | expected = ['args do not match; expecting: "self, x", got: "self"'] 76 | self.assertEqual(expected, res) 77 | -------------------------------------------------------------------------------- /compatibility_lib/compatibility_lib/test_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the 'License'); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an 'AS IS' BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import mock 16 | import unittest 17 | 18 | from compatibility_lib import fake_compatibility_store 19 | from compatibility_lib import utils 20 | 21 | UNKNOWN_STATUS_RESULT = { 22 | 'result': 'UNKNOWN', 23 | } 24 | 25 | DEP_INFO = { 26 | 'dep1': { 27 | 'installed_version': '2.1.0', 28 | 'installed_version_time': '2018-05-12T16:26:31', 29 | 'latest_version': '2.1.0', 30 | 'current_time': '2018-08-27T17:04:57.260105', 31 | 'latest_version_time': '2018-05-12T16:26:31', 32 | 'is_latest': True, 33 | }, 34 | 'dep2': { 35 | 'installed_version': '3.6.1', 36 | 'installed_version_time': '2018-08-13T22:47:09', 37 | 'latest_version': '3.6.1', 38 | 'current_time': '2018-08-27T17:04:57.934866', 39 | 'latest_version_time': '2018-08-13T22:47:09', 40 | 'is_latest': True, 41 | }, 42 | } 43 | 44 | 45 | class MockChecker(object): 46 | def check(self, packages, python_version): 47 | if not utils._is_package_in_whitelist(packages): 48 | UNKNOWN_STATUS_RESULT['packages'] = packages 49 | UNKNOWN_STATUS_RESULT['description'] = 'Package is not supported' \ 50 | ' by our checker server.' 51 | return UNKNOWN_STATUS_RESULT 52 | 53 | return { 54 | 'result': 'SUCCESS', 55 | 'packages': packages, 56 | 'description': None, 57 | 'dependency_info': DEP_INFO, 58 | } 59 | 60 | def get_compatibility(self, python_version, packages=None): 61 | return [[self.check( 62 | packages=packages, python_version=python_version)]] 63 | 64 | 65 | class TestDependencyInfo(unittest.TestCase): 66 | 67 | def setUp(self): 68 | self.mock_checker = MockChecker() 69 | self.fake_store = fake_compatibility_store.CompatibilityStore() 70 | 71 | def test_constructor_default(self): 72 | dep_info_getter = utils.DependencyInfo( 73 | checker=self.mock_checker, store=self.fake_store) 74 | 75 | self.assertEqual(dep_info_getter.py_version, '3') 76 | 77 | def test_constructor_explicit(self): 78 | dep_info_getter = utils.DependencyInfo( 79 | py_version='2', checker=self.mock_checker, store=self.fake_store) 80 | 81 | self.assertEqual(dep_info_getter.py_version, '2') 82 | 83 | def test__get_from_cloud_sql_exists(self): 84 | dep_info_getter = utils.DependencyInfo( 85 | checker=self.mock_checker, store=self.fake_store) 86 | dep_info = dep_info_getter._get_from_cloud_sql('opencensus') 87 | 88 | self.assertIsNotNone(dep_info) 89 | 90 | def test__get_from_cloud_sql_not_exists(self): 91 | dep_info_getter = utils.DependencyInfo( 92 | checker=self.mock_checker, store=self.fake_store) 93 | dep_info = dep_info_getter._get_from_cloud_sql('pkg_not_in_config') 94 | 95 | self.assertIsNone(dep_info) 96 | 97 | def test__get_from_endpoint(self): 98 | dep_info_getter = utils.DependencyInfo( 99 | checker=self.mock_checker, store=self.fake_store) 100 | dep_info = dep_info_getter._get_from_endpoint('opencensus') 101 | 102 | self.assertEqual(dep_info, DEP_INFO) 103 | 104 | def test__get_from_endpoint_raise_exception(self): 105 | dep_info_getter = utils.DependencyInfo( 106 | checker=self.mock_checker, store=self.fake_store) 107 | 108 | with self.assertRaises(utils.PackageNotSupportedError): 109 | dep_info_getter._get_from_endpoint('pkg_not_in_config') 110 | 111 | def test_get_dependency_info_compatibility_store(self): 112 | dep_info_getter = utils.DependencyInfo( 113 | checker=self.mock_checker, store=self.fake_store) 114 | dep_info = dep_info_getter.get_dependency_info('opencensus') 115 | 116 | self.assertIsNotNone(dep_info) 117 | 118 | 119 | class Test__parse_datetime(unittest.TestCase): 120 | 121 | def test__parse_datetime(self): 122 | date_string = '2018-08-16T15:42:04.351677' 123 | expected = '2018-08-16 00:00:00' 124 | res = utils._parse_datetime(date_string) 125 | self.assertEqual(str(res), expected) 126 | 127 | def test__parse_datetime_empty(self): 128 | res = utils._parse_datetime(None) 129 | self.assertIsNone(res) 130 | -------------------------------------------------------------------------------- /compatibility_lib/compatibility_lib/testdata/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/cloud-opensource-python/d1197e9d4e9ab48fa3ddb0db0b658ac611769eca/compatibility_lib/compatibility_lib/testdata/__init__.py -------------------------------------------------------------------------------- /compatibility_lib/compatibility_lib/testpkgs/added_args/0.1.0/main.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | class Foo(object): 17 | 18 | def __init__(self, x): 19 | pass 20 | -------------------------------------------------------------------------------- /compatibility_lib/compatibility_lib/testpkgs/added_args/0.2.0/main.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | class Foo(object): 17 | 18 | def __init__(self, x, y): 19 | pass 20 | -------------------------------------------------------------------------------- /compatibility_lib/compatibility_lib/testpkgs/added_func/0.1.0/main.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | def foo(x): 17 | pass 18 | -------------------------------------------------------------------------------- /compatibility_lib/compatibility_lib/testpkgs/added_func/0.2.0/main.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | def foo(x): 17 | pass 18 | 19 | 20 | def bar(x): 21 | pass 22 | -------------------------------------------------------------------------------- /compatibility_lib/compatibility_lib/testpkgs/removed_args/0.1.0/main.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | class Foo(object): 17 | 18 | def __init__(self, x): 19 | pass 20 | -------------------------------------------------------------------------------- /compatibility_lib/compatibility_lib/testpkgs/removed_args/0.2.0/main.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | class Foo(object): 17 | 18 | def __init__(self): 19 | pass 20 | -------------------------------------------------------------------------------- /compatibility_lib/compatibility_lib/testpkgs/removed_func/0.1.0/main.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | def foo(x): 17 | pass 18 | 19 | 20 | def bar(x): 21 | pass 22 | -------------------------------------------------------------------------------- /compatibility_lib/compatibility_lib/testpkgs/removed_func/0.2.0/main.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | def foo(x): 17 | pass 18 | -------------------------------------------------------------------------------- /compatibility_lib/compatibility_lib/testpkgs/simple_function/simple_function.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | def hello(): 17 | return 'hello' 18 | -------------------------------------------------------------------------------- /compatibility_lib/compatibility_lib/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Common utils for compatibility_lib.""" 16 | 17 | from datetime import datetime 18 | import json 19 | import logging 20 | import urllib.request 21 | 22 | from compatibility_lib import compatibility_checker 23 | from compatibility_lib import configs 24 | 25 | DATETIME_FORMAT = "%Y-%m-%d" 26 | 27 | PYPI_URL = 'https://pypi.org/pypi/' 28 | 29 | 30 | class PackageNotSupportedError(Exception): 31 | """Package is not supported by our checker server.""" 32 | 33 | def __init__(self, package_name): 34 | super(PackageNotSupportedError, self).__init__( 35 | 'Package {} is not supported by our checker server.'.format( 36 | package_name)) 37 | self.package_name = package_name 38 | 39 | 40 | def call_pypi_json_api(package_name, pkg_version=None): 41 | if pkg_version is not None: 42 | pypi_pkg_url = PYPI_URL + '{}/{}/json'.format( 43 | package_name, pkg_version) 44 | else: 45 | pypi_pkg_url = PYPI_URL + '{}/json'.format(package_name) 46 | 47 | try: 48 | r = urllib.request.Request(pypi_pkg_url) 49 | 50 | with urllib.request.urlopen(r) as f: 51 | result = json.loads(f.read().decode('utf-8')) 52 | except urllib.error.HTTPError: 53 | logging.error('Package {} with version {} not found in Pypi'. 54 | format(package_name, pkg_version)) 55 | return None 56 | return result 57 | 58 | 59 | def _is_package_in_whitelist(packages): 60 | """Return True if all the given packages are in whitelist. 61 | 62 | Args: 63 | packages: A list of package names. 64 | 65 | Returns: 66 | True if all packages are in whitelist, else False. 67 | """ 68 | for pkg in packages: 69 | if pkg not in configs.PKG_LIST and pkg not in configs.WHITELIST_URLS: 70 | return False 71 | 72 | return True 73 | 74 | 75 | class DependencyInfo(object): 76 | """Common utils of getting dependency info for a package.""" 77 | 78 | def __init__(self, py_version=None, checker=None, store=None): 79 | if py_version is None: 80 | py_version = '3' 81 | 82 | if checker is None: 83 | checker = compatibility_checker.CompatibilityChecker() 84 | 85 | self.py_version = py_version 86 | self.checker = checker 87 | self.store = store 88 | 89 | def _get_from_cloud_sql(self, package_name): 90 | """Gets the package dependency info from compatibility store 91 | which gets its dependnecy info from cloud sql. 92 | 93 | Args: 94 | package_name: the name of the package to query 95 | Returns: 96 | a dict mapping from dependency package name (string) to 97 | the info (dict) 98 | """ 99 | if self.store is not None: 100 | if (package_name in configs.PKG_LIST or 101 | package_name in configs.WHITELIST_URLS): 102 | depinfo = self.store.get_dependency_info(package_name) 103 | return depinfo 104 | else: 105 | return None 106 | 107 | def _get_from_endpoint(self, package_name): 108 | """Gets the package dependency info from the compatibility checker 109 | endpoint. 110 | 111 | Args: 112 | package_name: the name of the package to query (string) 113 | Returns: 114 | a dict mapping from dependency package name (string) to 115 | the info (dict) 116 | """ 117 | _result = self.checker.get_compatibility( 118 | python_version=self.py_version, packages=[package_name]) 119 | result = [item for item in _result] 120 | depinfo = result[0][0].get('dependency_info') 121 | 122 | # depinfo can be None if there is an exception during pip install or 123 | # the package is not supported by checker server (not in whitelist). 124 | if depinfo is None: 125 | logging.warning( 126 | "Could not get the dependency info of package {} from server." 127 | .format(package_name)) 128 | raise PackageNotSupportedError(package_name) 129 | 130 | fields = ('installed_version_time', 131 | 'current_time', 'latest_version_time') 132 | for pkgname in depinfo.keys(): 133 | for field in fields: 134 | depinfo[pkgname][field] = _parse_datetime( 135 | depinfo[pkgname][field]) 136 | 137 | return depinfo 138 | 139 | def get_dependency_info(self, package_name): 140 | """Gets the package dependency info 141 | 142 | Args: 143 | package_name: the name of the package to query (string) 144 | Returns: 145 | a dict mapping from dependency package name (string) to 146 | the info (dict) 147 | """ 148 | depinfo = self._get_from_cloud_sql(package_name) 149 | 150 | if depinfo is None: 151 | depinfo = self._get_from_endpoint(package_name) 152 | return depinfo 153 | 154 | 155 | def _parse_datetime(date_string): 156 | """Converts a date string into a datetime obj 157 | 158 | Args: 159 | date_string: a date as a string 160 | Returns: 161 | the date as a datetime obj 162 | """ 163 | if date_string is None: 164 | return None 165 | 166 | date_string = date_string.replace('T', ' ') 167 | short_date = date_string.split(' ')[0] 168 | return datetime.strptime(short_date, DATETIME_FORMAT) 169 | -------------------------------------------------------------------------------- /compatibility_lib/requirements.txt: -------------------------------------------------------------------------------- 1 | retrying==1.3.3 2 | -------------------------------------------------------------------------------- /compatibility_lib/setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | universal = 1 3 | -------------------------------------------------------------------------------- /compatibility_lib/setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import setuptools 16 | 17 | with open("README.rst", "r") as fh: 18 | long_description = fh.read() 19 | 20 | 21 | namespaces = ['compatibility_lib'] 22 | 23 | 24 | setuptools.setup( 25 | name="compatibility_lib", 26 | version="0.1.11", 27 | author="Cloud Python", 28 | description="A library to get and store the dependency compatibility " 29 | "status data to BigQuery.", 30 | long_description=long_description, 31 | license="Apache-2.0", 32 | include_package_data=True, 33 | url="https://github.com/GoogleCloudPlatform/cloud-opensource-python/tree/" 34 | "master/compatibility_lib", 35 | packages=setuptools.find_packages(), 36 | namespace_packages=namespaces, 37 | classifiers=( 38 | "Intended Audience :: Developers", 39 | "Development Status :: 3 - Alpha", 40 | "Programming Language :: Python :: 3", 41 | "License :: OSI Approved :: Apache Software License", 42 | ), 43 | ) 44 | -------------------------------------------------------------------------------- /compatibility_server/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # An image used to run a Python webserver that does compatibility checking 16 | # between pip-installable packages. 17 | 18 | FROM python:3.6 19 | 20 | RUN mkdir /compatibility_checker 21 | ADD compatibility_checker_server.py /compatibility_checker 22 | ADD pip_checker.py /compatibility_checker 23 | ADD configs.py /compatibility_checker 24 | ADD requirements.txt /compatibility_checker 25 | ADD views.py /compatibility_checker 26 | 27 | RUN pip3 install -r /compatibility_checker/requirements.txt 28 | RUN pip3 install gunicorn[gevent] 29 | 30 | EXPOSE 8888 31 | 32 | # Restart each worker periodically so that `auto_remove` 33 | # (https://docker-py.readthedocs.io/en/stable/containers.html) has a chance to work. 34 | # 5 replicas per vm, each replica has 10 workers which allows 50 docker images 35 | # created at most in the vm. 36 | CMD cd compatibility_checker && \ 37 | gunicorn -b 0.0.0.0:8888 -w 2 --worker-class sync --max-requests 20 --max-requests-jitter 10 --timeout 300 \ 38 | -e EXPORT_METRICS=1 compatibility_checker_server:app 39 | -------------------------------------------------------------------------------- /compatibility_server/README.rst: -------------------------------------------------------------------------------- 1 | Compatibility Server 2 | ==================== 3 | 4 | A web application that returns compatibility information about Python packages. 5 | 6 | Running the server 7 | ------------------ 8 | 9 | 1. Install Docker_ 10 | 11 | .. _Docker: https://www.docker.com/community-edition 12 | 13 | 2. Download the code: 14 | 15 | :: 16 | 17 | git clone git@github.com:GoogleCloudPlatform/cloud-opensource-python.git 18 | 19 | 3. Build and run the Docker image 20 | 21 | :: 22 | 23 | cd cloud-opensource-python/compatbility_server 24 | ./run-in-docker.sh 25 | 26 | Testing it out 27 | -------------- 28 | 29 | :: 30 | 31 | curl 'http://0.0.0.0:8888/?package=six&package=Django&python-version=3' | python3 -m json.tool 32 | { 33 | "result": "SUCCESS", 34 | "packages": [ 35 | "six", 36 | "Django" 37 | ], 38 | "description": null, 39 | "requirements": "absl-py==0.2.2\napparmor==2.11.1\nasn1crypto==0.24.0\nastor==0.6.2\natomicwrites==1.1.5\nattrs==18.1.0\nbleach==1.5.0\nblinker==1.3\nBrlapi==0.6.6\ncachetools==2.1.0\ncertifi==2018.4.16\nchardet==3.0.4\ncheckbox-ng==0.23\ncheckbox-support==0.22\ncolorlog==2.10.0\ncryptography==2.1.4\ncupshelpers==1.0\ndecorator==4.3.0\ndefer==1.0.6\nDjango==2.0.6\nfeedparser==5.2.1\ngast==0.2.0\nglinux-rebootd==0.1\ngoobuntu-config-tools==0.1\ngoogle-api-core==1.2.0\ngoogle-auth==1.5.0\ngoogleapis-common-protos==1.5.3\ngpg==1.10.0\ngrpcio==1.12.1\nguacamole==0.9.2\nhtml5lib==0.9999999\nhttplib2==0.9.2\nidna==2.6\nimportlab==0.1.1\nIPy==0.83\nJinja2==2.9.6\nkeyring==10.5.1\nkeyrings.alt==2.2\nLibAppArmor==2.11.1\nlouis==3.3.0\nlxml==4.0.0\nMako==1.0.7\nMarkdown==2.6.11\nMarkupSafe==1.0\nmore-itertools==4.2.0\nnetworkx==2.1\nnox-automation==0.19.0\nnumpy==1.14.5\noauthlib==2.0.4\nobno==29\nolefile==0.44\nonboard==1.4.1\nopencensus==0.1.5\npadme==1.1.1\npexpect==4.2.1\nPillow==4.3.0\nplainbox==0.25\npluggy==0.6.0\nprotobuf==3.5.2.post1\npsutil==5.4.2\npy==1.5.3\npyasn1==0.4.3\npyasn1-modules==0.2.1\npycairo==1.15.4\npycrypto==2.6.1\npycups==1.9.73\npycurl==7.43.0\npygobject==3.26.1\npyinotify==0.9.6\nPyJWT==1.5.3\npyOpenSSL==17.5.0\npyparsing==2.1.10\npysmbc==1.0.15.6\npytest==3.6.1\npython-apt==1.4.0b3\npython-debian==0.1.31\npython-xapp==1.0.0\npython-xlib==0.20\npytype==2018.5.22.1\npytz==2018.4\npyxdg==0.25\nPyYAML==3.12\nreportlab==3.3.0\nrequests==2.18.4\nretrying==1.3.3\nrsa==3.4.2\nSecretStorage==2.3.1\nsetproctitle==1.1.10\nsix==1.11.0\ntensorboard==1.8.0\ntensorflow==1.8.0\ntermcolor==1.1.0\nufw==0.35\nunattended-upgrades==0.1\nurllib3==1.22\nvirtualenv==16.0.0\nWerkzeug==0.14.1\nXlsxWriter==0.9.6\nyoutube-dl==2017.11.6\n" 40 | } 41 | 42 | Commands for deployment 43 | ----------------------- 44 | 45 | 1. Set the current context to compatibility-server 46 | 47 | :: 48 | 49 | gcloud container clusters get-credentials compatibility-checker-cluster --zone us-central1-b --project python-compatibility-tools 50 | 51 | 2. Build, test, push, and deploy a new image 52 | 53 | TAG_NAME is in the format `v[NUMBER_OF_REVISION]`. Incrementing the number by 1 will be the new tag name. 54 | IMAGE_NAME is the gcr.io/python-compatibility-tools/compatibility-image:[TAG_NAME]. 55 | 56 | :: 57 | 58 | docker build -t gcr.io/python-compatibility-tools/compatibility-image:[TAG_NAME] . 59 | sudo docker run -v /var/run/docker.sock:/var/run/docker.sock -p 8888:8888 [IMAGE_NAME] 60 | gcloud docker -- push gcr.io/python-compatibility-tools/compatibility-image:[TAG_NAME] 61 | kubectl set image deployment/compatibility-server compatibility-server=gcr.io/python-compatibility-tools/compatibility-image:[TAG_NAME] 62 | 63 | Alternatively, we can use the deploy.yaml to push a new deployment. 64 | 65 | :: 66 | 67 | kubectl apply -f deployment/deploy.yaml 68 | 69 | If asked to sync up the current deploy.yaml with the latest one, 70 | 71 | :: 72 | 73 | kubectl get deployment compatibility-server -o yaml > deployment/deploy.yaml 74 | 75 | 3. After the deployment, be sure to send a new PR to update the TAG_NAME in the deploy.yaml. 76 | 77 | Commands for Maintenance 78 | ------------------------ 79 | 80 | 1. Resizing the cluster (adding more instances) 81 | 82 | Currently we have 24 instances in our cluster, this number is from our load 83 | test and requirement for completing each round of checks within 15 minutes. 84 | Potentially we will have 75 packages, and checking for both py2 and py3 pairwisely with one order 85 | will need 75 + 75 * 74 * 2 * 2 / 2 = 11175 checks, and the average time spent for each check is 60 seconds, 86 | which means we will need to run 745 checks at the same time. 87 | 88 | :: 89 | 90 | gcloud container clusters resize compatibility-server --size [SIZE] 91 | 92 | 2. Change the number of replicas 93 | 94 | The current number of replicas is 120, which means each instance has 5 replicas. 95 | We have 10 workers per replica, so that in total we have 120 * 10 = 1200 workers, which is 96 | much more than the number 745 as we need. 97 | 98 | The number of workers a single vm can tolerant is around 60, and we use 50 workers here. 99 | Be sure to run the load test before making any changes to the number of replicas. 100 | 101 | :: 102 | 103 | kubectl scale deployment/compatibility-server --replicas [NUMBER_OF_REPLICAS] 104 | 105 | Disclaimer 106 | ---------- 107 | 108 | This is not an official Google product. 109 | -------------------------------------------------------------------------------- /compatibility_server/compatibility_checker_server.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """A HTTP server that wraps pip_checker. 16 | 17 | Requires Python 3.6 or later. 18 | 19 | Url format for package check: 20 | http://:/?python-version=&package= 21 | 22 | Note that you can check multi package installation compatibility by adding 23 | extra "&package=" statements: 24 | http://:/?python-version=&package=&package=&...&package= 25 | 26 | 27 | Example usage: 28 | 29 | $ python3 compatibility_checker_server.py 30 | $ curl "http://127.0.0.1:8888/?python-version=3&package=compatibility-lib" \ 31 | | python3 -m json.tool 32 | { 33 | "result": "SUCCESS", 34 | "packages": [ 35 | "compatibility-lib" 36 | ], 37 | "description": null, 38 | "dependency_info": { 39 | "compatibility-lib": { 40 | "installed_version": "0.0.18", 41 | "installed_version_time": "2019-01-16T18:35:09", 42 | "latest_version": "0.0.18", 43 | "current_time": "2019-01-18T21:09:15.172255", 44 | "latest_version_time": "2019-01-16T18:35:09", 45 | "is_latest": true 46 | }, 47 | "pip": { 48 | "installed_version": "18.1", 49 | "installed_version_time": "2018-10-05T11:20:31", 50 | "latest_version": "18.1", 51 | "current_time": "2019-01-18T21:09:15.166074", 52 | "latest_version_time": "2018-10-05T11:20:31", 53 | "is_latest": true 54 | }, 55 | "setuptools": { 56 | "installed_version": "40.6.3", 57 | "installed_version_time": "2018-12-11T19:51:02", 58 | "latest_version": "40.6.3", 59 | "current_time": "2019-01-18T21:09:15.187775", 60 | "latest_version_time": "2018-12-11T19:51:02", 61 | "is_latest": true 62 | }, 63 | "wheel": { 64 | "installed_version": "0.32.3", 65 | "installed_version_time": "2018-11-19T00:25:58", 66 | "latest_version": "0.32.3", 67 | "current_time": "2019-01-18T21:09:15.170863", 68 | "latest_version_time": "2018-11-19T00:25:58", 69 | "is_latest": true 70 | } 71 | } 72 | } 73 | 74 | For complete usage information: 75 | $ python3 compatibility_checker_server.py --help 76 | 77 | For production uses, this module exports a WSGI application called `app`. 78 | For example: 79 | 80 | $ gunicorn -w 4 --timeout 120 compatibility_checker_server:app 81 | 82 | """ 83 | 84 | import argparse 85 | import configs 86 | import flask 87 | import logging 88 | import os 89 | import pprint 90 | import sys 91 | import wsgiref.simple_server 92 | 93 | import pip_checker 94 | import views 95 | 96 | from google import auth as google_auth 97 | from opencensus.stats import stats as stats_module 98 | from opencensus.stats.exporters import stackdriver_exporter 99 | 100 | PYTHON_VERSION_TO_COMMAND = { 101 | '2': ['python2', '-m', 'pip'], 102 | '3': ['python3', '-m', 'pip'], 103 | } 104 | 105 | STATS = stats_module.Stats() 106 | app = flask.Flask(__name__) 107 | 108 | 109 | def _get_project_id(): 110 | # get project id from default setting 111 | try: 112 | _, project_id = google_auth.default() 113 | except google_auth.exceptions.DefaultCredentialsError: 114 | raise ValueError("Couldn't find Google Cloud credentials, set the " 115 | "project ID with 'gcloud set project'") 116 | return project_id 117 | 118 | 119 | def _enable_exporter(): 120 | """Create and register the stackdriver exporter. 121 | 122 | For any data to be exported to stackdriver, an exporter needs to be created 123 | and registered with the view manager. Collected data will be reported via 124 | all the registered exporters. By not creating and registering an exporter, 125 | all collected data will stay local and will not appear on stackdriver. 126 | """ 127 | project_id = _get_project_id() 128 | exporter = stackdriver_exporter.new_stats_exporter( 129 | stackdriver_exporter.Options(project_id=project_id)) 130 | STATS.view_manager.register_exporter(exporter) 131 | 132 | 133 | def _sanitize_packages(packages): 134 | """Checks if packages are whitelisted 135 | 136 | Args: 137 | packages: a list of packages 138 | Returns: 139 | a subset of packages that are whitelisted 140 | """ 141 | sanitized_packages = [] 142 | for pkg in packages: 143 | if pkg in configs.WHITELIST_PKGS or pkg in configs.WHITELIST_URLS: 144 | sanitized_packages.append(pkg) 145 | return sanitized_packages 146 | 147 | 148 | @app.route('/health_check') 149 | def health_check(): 150 | return 'hello world' 151 | 152 | 153 | @app.route('/') 154 | def check(): 155 | packages = flask.request.args.getlist('package') 156 | if not packages: 157 | return flask.make_response( 158 | "Request must specify at least one 'package' parameter", 400) 159 | 160 | sanitized_packages = _sanitize_packages(packages) 161 | unsupported_packages = frozenset(packages) - frozenset(sanitized_packages) 162 | if unsupported_packages: 163 | return flask.make_response( 164 | 'Request contains unrecognized packages: {}'.format( 165 | ', '.join(unsupported_packages)), 166 | 400) 167 | 168 | python_version = flask.request.args.get('python-version') 169 | if not python_version: 170 | return flask.make_response( 171 | "Request must specify 'python-version' parameter", 400) 172 | if python_version not in PYTHON_VERSION_TO_COMMAND: 173 | return flask.make_response( 174 | 'Invalid Python version specified. Must be one of: {}'.format( 175 | ', '.join(PYTHON_VERSION_TO_COMMAND), 400)) 176 | 177 | python_command = PYTHON_VERSION_TO_COMMAND[python_version] 178 | 179 | try: 180 | pip_result = pip_checker.check( 181 | python_command, packages, STATS) 182 | except pip_checker.PipCheckerError as pip_error: 183 | return flask.make_response(pip_error.error_msg, 500) 184 | 185 | return flask.jsonify( 186 | result=pip_result.result_type.name, 187 | packages=pip_result.packages, 188 | description=pip_result.result_text, 189 | dependency_info=pip_result.dependency_info) 190 | 191 | 192 | def main(): 193 | 194 | class Handler(wsgiref.simple_server.WSGIRequestHandler): 195 | def log_message(self, format, *args): 196 | # Override the default log_message method to avoid logging 197 | # remote addresses. 198 | sys.stderr.write("[%s] %s\n" % (self.log_date_time_string(), 199 | format % args)) 200 | 201 | parser = argparse.ArgumentParser(description='Process some integers.') 202 | parser.add_argument( 203 | '--host', 204 | default='0.0.0.0', 205 | help='host name to which the server should bind') 206 | parser.add_argument( 207 | '--port', 208 | type=int, 209 | default=8888, 210 | help='port to which the server should bind') 211 | export_metrics = os.environ.get('EXPORT_METRICS') is not None 212 | 213 | args = parser.parse_args() 214 | argsdict = vars(args) 215 | argsdict['export_metrics'] = export_metrics 216 | logging.info('Running server with:\n%s', pprint.pformat(argsdict)) 217 | 218 | if export_metrics: 219 | _enable_exporter() 220 | 221 | # The views need to be registered with the view manager for data to be 222 | # collected. Once a view is registered, it reports data to any registered 223 | # exporters. 224 | for view in views.ALL_VIEWS: 225 | STATS.view_manager.register_view(view) 226 | 227 | logging.basicConfig( 228 | level=logging.INFO, 229 | format='%(levelname)-8s %(asctime)s ' + 230 | '%(filename)s:%(lineno)s] %(message)s') 231 | 232 | with wsgiref.simple_server.make_server( 233 | args.host, 234 | args.port, 235 | app, 236 | handler_class=Handler) as httpd: 237 | httpd.serve_forever() 238 | 239 | 240 | if __name__ == '__main__': 241 | main() 242 | -------------------------------------------------------------------------------- /compatibility_server/deployment/deploy.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | labels: 5 | run: compatibility-server 6 | name: compatibility-server-deployment 7 | namespace: default 8 | spec: 9 | progressDeadlineSeconds: 600 10 | replicas: 200 11 | revisionHistoryLimit: 10 12 | selector: 13 | matchLabels: 14 | run: compatibility-server 15 | strategy: 16 | type: Recreate 17 | template: 18 | metadata: 19 | creationTimestamp: null 20 | labels: 21 | run: compatibility-server 22 | spec: 23 | affinity: 24 | podAntiAffinity: 25 | requiredDuringSchedulingIgnoredDuringExecution: 26 | - labelSelector: 27 | matchExpressions: 28 | - key: run 29 | operator: In 30 | values: 31 | - compatibility-server 32 | topologyKey: kubernetes.io/hostname 33 | containers: 34 | - image: gcr.io/python-compatibility-tools/compatibility-image:v1 35 | imagePullPolicy: IfNotPresent 36 | name: compatibility-server 37 | ports: 38 | - containerPort: 8888 39 | protocol: TCP 40 | resources: {} 41 | terminationMessagePath: /dev/termination-log 42 | terminationMessagePolicy: File 43 | volumeMounts: 44 | - mountPath: /var/run/docker.sock 45 | name: docker-sock 46 | dnsPolicy: ClusterFirst 47 | restartPolicy: Always 48 | schedulerName: default-scheduler 49 | securityContext: {} 50 | terminationGracePeriodSeconds: 310 51 | volumes: 52 | - hostPath: 53 | path: /var/run/docker.sock 54 | type: File 55 | name: docker-sock 56 | -------------------------------------------------------------------------------- /compatibility_server/deployment/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: compatibility-server-service 5 | labels: 6 | run: compatibility-server 7 | spec: 8 | ports: 9 | - port: 80 10 | targetPort: 8888 11 | selector: 12 | run: compatibility-server 13 | type: LoadBalancer 14 | loadBalancerIP: $PROJECT_IPADDRESS 15 | -------------------------------------------------------------------------------- /compatibility_server/fake_pip.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright 2018 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | """A fake implementation of pip whose behavior can be modified using flags.""" 18 | 19 | import argparse 20 | import sys 21 | 22 | 23 | def assert_args(expected, actual): 24 | if expected and expected != actual: 25 | print('{!r} != {!r}'.format(expected, actual), end='', file=sys.stderr) 26 | sys.exit(1) 27 | 28 | 29 | def main(): 30 | parser = argparse.ArgumentParser(description='Process some integers.') 31 | parser.add_argument( 32 | '--install-returncode', 33 | type=int, 34 | default=0, 35 | help='the return code for "pip install"') 36 | parser.add_argument( 37 | '--install-output', 38 | default='install', 39 | help='the stderr output for "pip install"') 40 | 41 | parser.add_argument( 42 | '--check-returncode', 43 | type=int, 44 | default=0, 45 | help='the return code for "pip check"') 46 | parser.add_argument( 47 | '--check-output', 48 | default='check', 49 | help='the stdout output for "pip check"') 50 | 51 | parser.add_argument( 52 | '--freeze-returncode', 53 | type=int, 54 | default=0, 55 | help='the return code for "pip freeze"') 56 | parser.add_argument( 57 | '--freeze-output', 58 | default='freeze', 59 | help='the stdout output for "pip freeze"') 60 | 61 | parser.add_argument( 62 | '--list-returncode', 63 | type=int, 64 | default=0, 65 | help='the return code for "pip list"') 66 | parser.add_argument( 67 | '--list-output', 68 | default='list', 69 | help='the stdout output for "pip list"') 70 | 71 | parser.add_argument( 72 | '--uninstall-returncode', 73 | type=int, 74 | default=1, 75 | help='the return code for "pip uninstall"') 76 | 77 | parser.add_argument( 78 | '--expected-install-args', 79 | type=lambda s: s.split(','), 80 | default=[], 81 | help='the expected arguments for "pip install "') 82 | 83 | known, unknown_args = parser.parse_known_args() 84 | command, *command_args = unknown_args 85 | 86 | if command == 'install': 87 | assert_args(known.expected_install_args, command_args) 88 | print(known.install_output, end='', file=sys.stderr) 89 | sys.exit(known.install_returncode) 90 | elif command == 'check': 91 | assert_args([], command_args) 92 | print(known.check_output, end='') 93 | sys.exit(known.check_returncode) 94 | elif command == 'freeze': 95 | assert_args([], command_args) 96 | print(known.freeze_output, end='') 97 | sys.exit(known.freeze_returncode) 98 | elif command == 'list': 99 | assert_args([], command_args) 100 | print(known.list_output, end='') 101 | sys.exit(known.list_returncode) 102 | elif command == 'uninstall': 103 | sys.exit(known.uninstall_returncode) 104 | else: 105 | print('unexpected command: {}'.format(command), 106 | end='', file=sys.stderr) 107 | sys.exit(1) 108 | 109 | 110 | if __name__ == '__main__': 111 | main() 112 | -------------------------------------------------------------------------------- /compatibility_server/loadtest/locustfile.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Perform a load test on the compatibility server. Usage: 16 | 17 | $ pip install locustio 18 | $ locust --host=http://104.197.8.72 19 | """ 20 | 21 | import random 22 | import urllib.parse 23 | 24 | import locust 25 | 26 | 27 | PYTHON2_PACKAGES = [ 28 | 'apache-beam[gcp]', 29 | 'google-cloud-bigtable', 30 | 'google-cloud-dns', 31 | 'google-cloud-vision', 32 | 'tensorboard', 33 | 'tensorflow', 34 | ] 35 | 36 | PYTHON3_PACKAGES = [ 37 | 'google-cloud-bigtable', 38 | 'google-cloud-dns', 39 | 'google-cloud-vision', 40 | 'tensorboard', 41 | 'tensorflow', 42 | ] 43 | 44 | 45 | class CompatibilityCheck(locust.TaskSet): 46 | @locust.task 47 | def single_python2(self): 48 | query = urllib.parse.urlencode( 49 | {'python-version': '2', 50 | 'package': random.choice(PYTHON2_PACKAGES)}) 51 | self.client.get('/?%s' % query) 52 | 53 | @locust.task 54 | def single_python3(self): 55 | query = urllib.parse.urlencode( 56 | {'python-version': '3', 57 | 'package': random.choice(PYTHON3_PACKAGES)}) 58 | self.client.get('/?%s' % query) 59 | 60 | @locust.task 61 | def double_python2(self): 62 | package1 = random.choice(PYTHON2_PACKAGES) 63 | package2 = random.choice(list(set(PYTHON2_PACKAGES) - {package1})) 64 | 65 | query = urllib.parse.urlencode([('python-version', '2'), 66 | ('package', package1), 67 | ('package', package2)]) 68 | self.client.get('/?%s' % query) 69 | 70 | @locust.task 71 | def double_python3(self): 72 | package1 = random.choice(PYTHON3_PACKAGES) 73 | package2 = random.choice(list(set(PYTHON3_PACKAGES) - {package1})) 74 | 75 | query = urllib.parse.urlencode([('python-version', '3'), 76 | ('package', package1), 77 | ('package', package2)]) 78 | self.client.get('/?%s' % query) 79 | 80 | 81 | class CompatibilityChecker(locust.HttpLocust): 82 | task_set = CompatibilityCheck 83 | min_wait = 0 84 | max_wait = 0 85 | -------------------------------------------------------------------------------- /compatibility_server/requirements.txt: -------------------------------------------------------------------------------- 1 | certifi 2 | docker 3 | flask 4 | google-cloud-monitoring==0.31.1 5 | grpcio==1.18.0 6 | opencensus==0.2.0 7 | requests 8 | -------------------------------------------------------------------------------- /compatibility_server/run-in-docker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Copyright 2018 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | # Run the compatibility_checker_server locally. 18 | 19 | set -e 20 | 21 | docker build -t compatibility-image . 22 | docker run -p 8888:8888 compatibility-image 23 | -------------------------------------------------------------------------------- /compatibility_server/test_configs.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Tests for configs.""" 16 | 17 | import os 18 | import unittest 19 | 20 | 21 | class TestConfigs(unittest.TestCase): 22 | 23 | def test_config_files_match(self): 24 | cwd = os.path.dirname(os.path.realpath(__file__)) 25 | root, _ = os.path.split(cwd) 26 | first_path = os.path.join(cwd, 'configs.py') 27 | second_path = os.path.join( 28 | root, 'compatibility_lib/compatibility_lib/configs.py') 29 | with open(first_path) as fd: 30 | first_file = fd.read() 31 | with open(second_path) as fd: 32 | second_file = fd.read() 33 | self.assertEqual(first_file, second_file) 34 | -------------------------------------------------------------------------------- /compatibility_server/test_sanitize_packages.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the 'License'); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an 'AS IS' BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from compatibility_checker_server import _sanitize_packages 16 | import configs 17 | import logging 18 | import pip_checker 19 | import unittest 20 | 21 | class Test__sanitize_packages(unittest.TestCase): 22 | 23 | def test__sanitize_packages(self): 24 | packages = configs.WHITELIST_PKGS 25 | res = _sanitize_packages(packages) 26 | self.assertEqual(res, packages) 27 | 28 | urls = [url for url in configs.WHITELIST_URLS.keys()] 29 | res = _sanitize_packages(urls) 30 | self.assertEqual(res, urls) 31 | 32 | @unittest.skip("local testing only") 33 | def test_all_whitelists_pip_install(self): 34 | py2_pkgs, py3_pkgs = [], [] 35 | for pkg in configs.WHITELIST_PKGS: 36 | if pkg not in configs.PKG_PY_VERSION_NOT_SUPPORTED[2]: 37 | py2_pkgs.append(pkg) 38 | if pkg not in configs.PKG_PY_VERSION_NOT_SUPPORTED[3]: 39 | py3_pkgs.append(pkg) 40 | 41 | for url, pkg in configs.WHITELIST_URLS.items(): 42 | if pkg not in configs.PKG_PY_VERSION_NOT_SUPPORTED[2]: 43 | py2_pkgs.append(url) 44 | if pkg not in configs.PKG_PY_VERSION_NOT_SUPPORTED[3]: 45 | py3_pkgs.append(url) 46 | 47 | args = [ 48 | # ('python', py2_pkgs), 49 | ('python3', py3_pkgs), 50 | ] 51 | for command, packages in args: 52 | pip_result = pip_checker.check( 53 | [command, '-m', 'pip'], packages, clean=True) 54 | self.assertIsNotNone(pip_result) 55 | 56 | results = dict( 57 | result=pip_result.result_type.name, 58 | packages=pip_result.packages, 59 | description=pip_result.result_text, 60 | dependency_info=pip_result.dependency_info) 61 | if results['result'] == 'INSTALL_ERROR': 62 | logging.warning(command) 63 | logging.warning(results['description']) 64 | self.assertFalse(results['result'] == 'INSTALL_ERROR') 65 | 66 | def test_nonwhitelisted_packages(self): 67 | packages = [ 68 | 'requests', 69 | 'Scrapy', 70 | 'wxPython', 71 | 'Pillow', 72 | 'numpy', 73 | 'Pygame', 74 | ] 75 | res = _sanitize_packages(packages) 76 | self.assertEqual(res, []) 77 | -------------------------------------------------------------------------------- /compatibility_server/views.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from opencensus.stats import aggregation as aggregation_module 16 | from opencensus.stats import measure as measure_module 17 | from opencensus.stats import view as view_module 18 | 19 | # docker error 20 | DOCKER_ERROR_MEASURE = measure_module.MeasureInt( 21 | 'docker_error', 'The number of docker errors.', 'Errors') 22 | DOCKER_ERROR_VIEW = view_module.View( 23 | "docker_error_count", 24 | "The number of the docker errors", 25 | [], 26 | DOCKER_ERROR_MEASURE, 27 | aggregation_module.CountAggregation()) 28 | 29 | ALL_VIEWS = [ 30 | DOCKER_ERROR_VIEW, 31 | ] 32 | -------------------------------------------------------------------------------- /credentials.json.enc: -------------------------------------------------------------------------------- 1 | U2FsdGVkX1845FAGaZItJg3NDaNX9MU1mV2U/oEeF6gZ9E5oIgbPsZoRKSwgYToA 2 | toGDEC+dL/fenLlVBFuk/JOYdqW9BkSMA4SISUUozHceRieqqR/SVnxiaS4n5scw 3 | OvSkoDZv4hxt8pR4DUphdC4kGOO6QwsE90rk02EzeFpMsjWOESJtuDIWR9L/unVH 4 | mEUoVnymusexsTEikLJO0QtAjImnV7WIRq4kosvLFv0HBGYbdsqmEAXHS5lHmV88 5 | X+nNh3YQrdRldkNuTt3ElL4VMd6JLrenW8HxJsDOnSpS01yI5A+WFetb3/TiwGcg 6 | 7jylbv/cdJZI1SIr3RYp/RdBI1x80FjlB+JyPXWkQ2s4z5AVE8bj+sIXGzVXM5O9 7 | Eear+5gzxzpz667avyKUotwLa2y9lzru7U+pWhMLGc7t1+mVv1rMQ47EFnfTkrvg 8 | vWBdiSW/ledhGqM5eJOwlyf9V02uMRJW//bDO9dYUVcpmA9N1wKoXG79dQjm8XIx 9 | cV/uzXB2g3F58qAxlXjF6OVI/pGyo2FW/awnnogzTo0M4/n8fsUZtXYTt2m4VyWs 10 | pKWfzLAsZLJpxNYpWf/HM0LuI2stJvAEtuJBCE0OsyJ1KJvUc/w34qGZP7Btt4bp 11 | Uh324XJkgLMsiZbq9tO26GPFm9EPnkbGRo+Aty/t64yZNacpyVTc2R6tzASlLZ0O 12 | VeqS4oK22HSUd4tgiouFBZB1kZ6FzE/bS6B1waXrmIqUsG4evLqePnsfOQRxbkCS 13 | 4yKGvxKOyJdLBlh0Ek3Kg2FDiNewnHNHB/vAD9W2p3uWis5JaxwNKWjYBHIYpXcU 14 | mDKNnZLilzQ5zoxnO+j38/ZJBqWgD6Nh0pR6g0+4uyBSoB/81GUA74odKu45T6Ej 15 | tg31mTxA7i6tXsTM4Gr4jQE1Ag84mkfej2dVcgvuALmjLqwvTUIhdaJ6yonfMTEf 16 | cWTfUu9npy9JXPYsKXnHkmJnJtBBvmOInDpc/4pWFwZJ9HrTSGb1Ey/xruhaUbfS 17 | xlfReYAiqfhOdoZZxjlO8PA6cGbJaGrfqtT1V5Y/wO6gP1rTA3cOJANe7Bimui0U 18 | 6NjeGtJV0h7b6fblFmanhhSeoxtGMhPr2DE01mKxKJx5h/MJyZz7TEgfA0SeZOzY 19 | ATGjBP3x9V71H8ytbCdpyhu4TJa45GslEnYCu2O+QvZl7f2CAncRUPODQCtdpfUP 20 | vStw5tr3dRqBVIsMosvYWUmbAaLTACCFVJY7Nw8jgGf9XV8OJigsGq3CtnJdqH0q 21 | 5ln76RW/dlCPL1uSFOg83algdgG4J/mr5e4JvhsyvfIRIRJNXkN/yH7Cy5CMpebb 22 | Xe9aj4Xwm6tQzLHrC+qbWKJwVuhLChYUU/Bb2jJ+VmXowqLLarjOGve9caUIXtWv 23 | YW61Hx6qPA+bwERvzQwk/R8qBVnLiUmBlIdKhNjEX2krc9xnMbeDEWwShpTOlhFD 24 | 34SItFGG/v9km8U7cp+VyxvYDfiuPCXX5KEMWkdcZtZcspuQcxVH7OSM7B4LWzL+ 25 | Ai7jW7C0I5drGC1zo0Y1CH1WF+pXnpL5icqlBAlfmMtt4JEQCDacexCYfPnguXcT 26 | Iu+Z8opAtHSt83Pkm0Rd1JQZGbn/GU1Wi0tBQYdLKLZZxrMYrLEdbJ57HR0h2Hda 27 | K0vJVNE3HAvgPlVBOkYRxmWpq1/8Yv7HEvfuRsbKK1sVuDA6O7LxvVQwL6kaceqr 28 | VVuALyfjR8qv1PiJYVkZM/Q4KMJt+Ic0kCw19rUGXFoVZ0936WIe+DQw7a0UZbnV 29 | 8E/ORPJPaLvx+ZD2mZ4hS9tZ15jhF7WkU6jbiTb57FlwObk7pGwTh/HZk1J75VLI 30 | EyUjSxcPtnSk7gZZjM1OzrqMY00qYriZHnuTI+ZUQsdQ8oCbiD6M+SYE1qOZr1TN 31 | Wwcn4IGzDI2N/H/+8zW8R4hbgNXgKH8w9nlQZq7Jkpooq0YPVzdYsAkXHXIGtFuU 32 | 5HMti7RGUqy5JPClAfQD5zVMcazvu6prySAqbn9tMLX3JVKR67MwtbryQTwC2g6A 33 | DDu5at2Us8Vi2VgruWV1Gp2/N86bSuAQo8f7zxPr+88Wt0oBlP/nGArtBNbmppbA 34 | zVyoUiCsZtT9DkNANp5A5W7gk+bljA799bk85wlEnc5RMM9yLA4GfL2arRKzRtzr 35 | voXnPLEMN2Mpk9W8sUm9AjmS3Yq2i2Zjy6A5HkgSf5uyjrYz3lSpvt2j3jRvYjIY 36 | 5hY0dPj4QuixcP9YW3+3mXKoLTv8JrtNZr3dNGqiac5RPP6onw/QtIGASmhvt496 37 | FvzCynGaWtX+xAvREwPEylFdhfloYyr1TLgFNjOUNck4+TVd0xeUC+cM/4/7xBd6 38 | nAZD49JIrDJFBc7iQR8fpVhHcGewHvamaA+Arln/VbiC3eLmZVQUkL2zEASuVOtk 39 | lKmeHpvLGkYfSVxeXyHSC+jmVfPSJKfocfml4o8bcHRt6ySur1z1D1FP8jKvDeDS 40 | XkEooEQlnGRxKsaWYq4TqZMHu5BFP0zMPtfdwhgI4T9vQV6P75y9SNzmM0o/p53c 41 | 3MrkUpegjVgUUL7Xpj8R/ftfKQV2Ekapdqr9UIAy0ZwlwoQEzGpfrSVc8elUWnG+ 42 | 2qRAnbz8lsaw0WgkmjoHgxQqeBCjFcBp/gUJMYrM5Un2+/SR4Adif4HHUVVuelVu 43 | ZM6+0H4Wc4txMQwDO0IThInlY89PhhI4V1nxXOZ+joBDYkRmaBfQblMkr+4lt5hm 44 | BJl8uQVJOvLHaPdsjXNqkVNuyCiJYeKkPfSih0r34DMiyoRW5Z/w3R7i15ZE0BBo 45 | SPKAGNLAb160fkOpDjtnu0/SXtWurCcEPcl5nJnzRo63QgdahkWYCu77EV40dthE 46 | R0QDyJAYSQTFsLy2GwJ0xsvLANywmoY41BCAAWsNnPL8o0Fd3n7RH26Rg7c5Mw3F 47 | kZDoGVlMSvlsWrj9BkZGrurl5rsVoRJi/OQT+htsWGSVV7257awWCynRwenfeiVk 48 | YA6JSIxiVn5FbV+/wjqQrp1z90Gk28RxL518IRT+fcKrNQU+KVvFFP+rn/TzMsGR 49 | gikIDnpw8RGpyIwFLYRj6Jco4jD/yl9Rx1x5gOWYAvhkT2lJag11doHzfRprRZuI 50 | voHQNf9jlu4yebPjfWVBF7OTHHiM75rYvUgGyJ9dobk3tlau2/dm2bIHlHEo6tXT 51 | -------------------------------------------------------------------------------- /dashboard/grid-template.html: -------------------------------------------------------------------------------- 1 | {# Copyright 2018 Google LLC 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. #} 14 | 15 | 16 | 17 | 18 | 19 | 20 | Compatibility Grid 21 | 95 | 96 | 97 | 98 | Dashboard updated on: {{ current_timestamp }} 99 |
100 | Color legend: 101 |
102 |
Self-Unknown
103 |
Self-Success
104 |
Self-InstallError
105 |
Self-CheckWarning
106 |
107 |
108 |
Pairwise-Unknown
109 |
Pairwise-Success
110 |
Pairwise-InstallError
111 |
Pairwise-CheckWarning
112 |
113 |
114 | 115 | 116 | 117 | {% for col_package in packages %} 118 | 119 | {% endfor %} 120 | 121 | {% for row_package in packages %} 122 | {% set row = loop.index0 %} 123 | 124 | 131 | {% for col_package in packages %} 132 | {% set column = loop.index0 %} 133 | {% if column <= row %} 134 | {% set result = results.get_result(row_package, col_package) %} 135 | {% set cellName = row | string + '-' + column | string %} 136 | {% set bgColorName = result.status_type | lower %} 137 | {% if result.pairwise_compatibility_check | length > 0 %} 138 | {% set pairwise_status = result.pairwise_compatibility_check[0].status %} 139 | {% set pairwise_details = result.pairwise_compatibility_check[0].details %} 140 | {% endif %} 141 | {% if result.self_compatibility_check | length > 0 %} 142 | {% set self_1_status = result.self_compatibility_check[0].status %} 143 | {% set self_1_details = result.self_compatibility_check[0].details %} 144 | {% if result.self_compatibility_check | length > 1 %} 145 | {% set self_2_status = result.self_compatibility_check[1].status %} 146 | {% set self_2_details = result.self_compatibility_check[1].details %} 147 | {% endif %} 148 | {% endif %} 149 | 153 | {% else %} 154 | 155 | {% endif %} 156 | {% endfor %} 157 | 158 | {% endfor %} 159 |
{{ col_package.friendly_name }}
{{ row_package.friendly_name }} 125 | {% if results.has_issues(row_package) %} 126 | {% set issues = results.deprecated_deps[row_package.friendly_name][0] %} 127 | 128 |
129 | {% endif %} 130 | 		150 |
151 |
152 | 

160 | 
161 | 
162 | 
163 | 
184 | 
185 | 
186 | 


--------------------------------------------------------------------------------
/dashboard/index.html:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/GoogleCloudPlatform/cloud-opensource-python/d1197e9d4e9ab48fa3ddb0db0b658ac611769eca/dashboard/index.html


--------------------------------------------------------------------------------
/dashboard/js/main.js:
--------------------------------------------------------------------------------
  1 | (function ($) {
  2 |   // USE STRICT
  3 |   "use strict";
  4 | 
  5 |   try {
  6 |     // Percent Chart
  7 |     var doughnutChart = document.getElementById("percent-chart-compatibility");
  8 |     var num_success = doughnutChart.getAttribute("data-success");
  9 |     var num_conflict = doughnutChart.getAttribute("data-conflict");
 10 |     doughnutChart.height = 280;
 11 |     new Chart(doughnutChart, {
 12 |         type: 'doughnut',
 13 |         data: {
 14 |           datasets: [
 15 |             {
 16 |               label: "Compatibility Status",
 17 |               data: [num_success, num_conflict],
 18 |               backgroundColor: [
 19 |                 '#00b5e9',
 20 |                 '#fa4251'
 21 |               ],
 22 |               hoverBackgroundColor: [
 23 |                 '#00b5e9',
 24 |                 '#fa4251'
 25 |               ],
 26 |               borderWidth: [
 27 |                 0, 0
 28 |               ],
 29 |               hoverBorderColor: [
 30 |                 'transparent',
 31 |                 'transparent'
 32 |               ]
 33 |             }
 34 |           ],
 35 |           labels: [
 36 |             'Success',
 37 |             'Having Conflicts'
 38 |           ]
 39 |         },
 40 |         options: {
 41 |           maintainAspectRatio: false,
 42 |           responsive: true,
 43 |           cutoutPercentage: 55,
 44 |           animation: {
 45 |             animateScale: true,
 46 |             animateRotate: true
 47 |           },
 48 |           legend: {
 49 |             display: false
 50 |           },
 51 |           tooltips: {
 52 |             titleFontFamily: "Poppins",
 53 |             xPadding: 15,
 54 |             yPadding: 10,
 55 |             caretPadding: 0,
 56 |             bodyFontSize: 16
 57 |           }
 58 |         }
 59 |     });
 60 | 
 61 |   } catch (error) {
 62 |     console.log(error);
 63 |   }
 64 | 
 65 | })(jQuery);
 66 | 
 67 | (function ($) {
 68 |   // USE STRICT
 69 |   "use strict";
 70 | 
 71 |   try {
 72 |     // Percent Chart
 73 |     var doughnutChart = document.getElementById("percent-chart-dependency");
 74 |     var num_success = doughnutChart.getAttribute("data-success");
 75 |     var num_deprecated = doughnutChart.getAttribute("data-deprecated");
 76 |     var num_outdated = doughnutChart.getAttribute("data-outdated");
 77 |     doughnutChart.height = 280;
 78 |     new Chart(doughnutChart, {
 79 |         type: 'doughnut',
 80 |         data: {
 81 |           datasets: [
 82 |             {
 83 |               label: "Compatibility Status",
 84 |               data: [num_success, num_deprecated, num_outdated],
 85 |               backgroundColor: [
 86 |                 '#00b5e9',
 87 |                 '#fa4251',
 88 |                 '#f1c40f'
 89 |               ],
 90 |               hoverBackgroundColor: [
 91 |                 '#00b5e9',
 92 |                 '#fa4251',
 93 |                 '#f1c40f'
 94 |               ],
 95 |               borderWidth: [
 96 |                 0, 0, 0
 97 |               ],
 98 |               hoverBorderColor: [
 99 |                 'transparent',
100 |                 'transparent',
101 |                 'transparent'
102 |               ]
103 |             }
104 |           ],
105 |           labels: [
106 |             'Success',
107 |             'Deprecated Dependency',
108 |             'Outdated Dependency'
109 |           ]
110 |         },
111 |         options: {
112 |           maintainAspectRatio: false,
113 |           responsive: true,
114 |           cutoutPercentage: 55,
115 |           animation: {
116 |             animateScale: true,
117 |             animateRotate: true
118 |           },
119 |           legend: {
120 |             display: false
121 |           },
122 |           tooltips: {
123 |             titleFontFamily: "Poppins",
124 |             xPadding: 15,
125 |             yPadding: 10,
126 |             caretPadding: 0,
127 |             bodyFontSize: 16
128 |           }
129 |         }
130 |     });
131 | 
132 |   } catch (error) {
133 |     console.log(error);
134 |   }
135 | 
136 | })(jQuery);
137 | 
138 | // Sort the table rows by the details column to display the packages having
139 | // issues first.
140 | (function ($) {
141 |     var table = document.getElementById("package-details");
142 |     switching = true;
143 |     while (switching) {
144 |         switching = false;
145 |         rows = table.rows;
146 |         for (i = 1; i < (rows.length - 1); i++) {
147 |           shouldSwitch = false;
148 |           x = rows[i].getElementsByTagName("TD")[3];
149 |           y = rows[i + 1].getElementsByTagName("TD")[3];
150 |           if (x.innerHTML.toLowerCase() < y.innerHTML.toLowerCase()) {
151 |             shouldSwitch = true;
152 |             break;
153 |           }
154 |         }
155 |         if (shouldSwitch) {
156 |           rows[i].parentNode.insertBefore(rows[i + 1], rows[i]);
157 |           switching = true;
158 |         }
159 |     }
160 | })(jQuery);
161 | 
162 | (function ($) {
163 |     // USE STRICT
164 |     "use strict";
165 |     $(".animsition").animsition({
166 |       inClass: 'fade-in',
167 |       outClass: 'fade-out',
168 |       inDuration: 900,
169 |       outDuration: 900,
170 |       linkElement: 'a:not([target="_blank"]):not([href^="#"]):not([class^="chosen-single"])',
171 |       loading: true,
172 |       loadingParentElement: 'html',
173 |       loadingClass: 'page-loader',
174 |       loadingInner: '
',
175 |       timeout: false,
176 |       timeoutCountdown: 5000,
177 |       onLoadEvent: true,
178 |       browser: ['animation-duration', '-webkit-animation-duration'],
179 |       overlay: false,
180 |       overlayClass: 'animsition-overlay-slide',
181 |       overlayParentElement: 'html',
182 |       transition: function (url) {
183 |         window.location.href = url;
184 |       }
185 |     });
186 | 
187 | })(jQuery);
188 | 


--------------------------------------------------------------------------------
/dashboard/monitoring.html:
--------------------------------------------------------------------------------
 1 | 
 2 | 
 3 | 
 4 | 
 5 | 
 6 | 
 7 | 
 8 | 
 9 | 
10 | 


--------------------------------------------------------------------------------
/nox.py:
--------------------------------------------------------------------------------
  1 | # Copyright 2018 Google LLC. All Rights Reserved.
  2 | #
  3 | # Licensed under the Apache License, Version 2.0 (the "License");
  4 | # you may not use this file except in compliance with the License.
  5 | # You may obtain a copy of the License at
  6 | #
  7 | #      http://www.apache.org/licenses/LICENSE-2.0
  8 | #
  9 | # Unless required by applicable law or agreed to in writing, software
 10 | # distributed under the License is distributed on an "AS IS" BASIS,
 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | # See the License for the specific language governing permissions and
 13 | # limitations under the License.
 14 | 
 15 | """Nox config for running lint and unit tests."""
 16 | 
 17 | from __future__ import absolute_import
 18 | 
 19 | import os
 20 | 
 21 | import nox
 22 | 
 23 | LOCAL_DEPS = ['compatibility_lib']
 24 | 
 25 | 
 26 | @nox.session
 27 | def lint(session):
 28 |     """Run flake8.
 29 | 
 30 |     Returns a failure if flake8 finds linting errors or sufficiently
 31 |     serious code quality issues.
 32 |     """
 33 |     session.interpreter = 'python3.6'
 34 |     session.install('flake8')
 35 |     session.run('flake8',
 36 |                 '--exclude=--exclude=__pycache__,dist,.git,'
 37 |                 'build,.tox,.nox,.idea,mock_*,test_*,*_test')
 38 | 
 39 | 
 40 | @nox.session
 41 | @nox.parametrize('py', ['3.5', '3.6'])
 42 | def unit(session, py):
 43 |     """Run the unit test suite.
 44 | 
 45 |     Unit test files should be named like test_*.py and in the same directory
 46 |     as the file being tested.
 47 |     """
 48 |     session.interpreter = 'python{}'.format(py)
 49 | 
 50 |     # Install all test dependencies, then install this package in-place.
 51 |     session.install('-e', ','.join(LOCAL_DEPS))
 52 |     session.install('-r', 'requirements-test.txt')
 53 | 
 54 |     # Run py.test against the unit tests.
 55 |     session.run(
 56 |         'py.test',
 57 |         '--quiet',
 58 |         '.',
 59 |         '--ignore=system_test',
 60 |         *session.posargs
 61 |     )
 62 | 
 63 | 
 64 | @nox.session
 65 | @nox.parametrize('py', ['3.6'])
 66 | def system(session, py):
 67 |     """Run the system test suite."""
 68 | 
 69 |     # Sanity check: Only run system tests if the environment variable is set.
 70 |     if not os.environ.get('GOOGLE_APPLICATION_CREDENTIALS', ''):
 71 |         session.skip('Credentials must be set via environment variable.')
 72 | 
 73 |     # Run the system tests against latest Python 2 and Python 3 only.
 74 |     session.interpreter = 'python{}'.format(py)
 75 | 
 76 |     # Set the virtualenv dirname.
 77 |     session.virtualenv_dirname = 'sys-' + py
 78 | 
 79 |     # Build and install compatibility_lib
 80 |     session.install('-e', ','.join(LOCAL_DEPS))
 81 | 
 82 |     # Install all test dependencies.
 83 |     session.install('-r', 'requirements-test.txt')
 84 | 
 85 |     # Run py.test against the system tests.
 86 |     session.run(
 87 |         'py.test',
 88 |         '-s',
 89 |         'system_test/',
 90 |         # Skip the system test for compatibility server as circle ci does not
 91 |         # support running docker in docker.
 92 |         '--ignore=system_test/test_compatibility_checker_server.py',
 93 |         *session.posargs
 94 |     )
 95 | 
 96 | 
 97 | @nox.session
 98 | def update_dashboard(session):
 99 |     """Build the dashboard."""
100 | 
101 |     session.interpreter = 'python3.6'
102 |     session.install('compatibility-lib')
103 |     session.install('-r', 'requirements-test.txt')
104 |     session.install('-e', ','.join(LOCAL_DEPS))
105 | 
106 |     # Set the virtualenv dirname.
107 |     session.virtualenv_dirname = 'dashboard'
108 | 
109 |     session.chdir(os.path.realpath(os.path.dirname(__file__)))
110 | 
111 |     # Build the dashboard!
112 |     session.run(
113 |         'bash', os.path.join('.', 'scripts', 'update_dashboard.sh'))
114 | 


--------------------------------------------------------------------------------
/python-compatibility-tools.json:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/GoogleCloudPlatform/cloud-opensource-python/d1197e9d4e9ab48fa3ddb0db0b658ac611769eca/python-compatibility-tools.json


--------------------------------------------------------------------------------
/python-compatibility-tools.json.enc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/GoogleCloudPlatform/cloud-opensource-python/d1197e9d4e9ab48fa3ddb0db0b658ac611769eca/python-compatibility-tools.json.enc


--------------------------------------------------------------------------------
/requirements-test.txt:
--------------------------------------------------------------------------------
 1 | docker==3.6.0
 2 | Flask==1.0.2
 3 | google-cloud-bigquery==1.3.0
 4 | google-cloud-datastore==1.7.0
 5 | google-cloud-monitoring==0.31.1
 6 | grpcio==1.15.0
 7 | Jinja2==2.10.1
 8 | mock==2.0.0
 9 | opencensus==0.2.0
10 | pexpect==4.6.0
11 | pybadges==1.0.2
12 | pymysql==0.9.3
13 | pytest==3.6.1
14 | redis==2.10.6
15 | requests==2.20.0
16 | retrying==1.3.3
17 | wrapt==1.10.11
18 | 


--------------------------------------------------------------------------------
/scripts/twine_upload.sh:
--------------------------------------------------------------------------------
 1 | # Copyright 2017, OpenCensus Authors
 2 | #
 3 | # Licensed under the Apache License, Version 2.0 (the "License");
 4 | # you may not use this file except in compliance with the License.
 5 | # You may obtain a copy of the License at
 6 | #
 7 | #     http://www.apache.org/licenses/LICENSE-2.0
 8 | #
 9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | 
15 | set -ev
16 | 
17 | # If this is not a CircleCI tag, no-op.
18 | if [[ -z "$CIRCLE_TAG" ]]; then
19 |   echo "This is not a release tag. Doing nothing."
20 |   exit 0
21 | fi
22 | 
23 | echo -e "[pypi]" >> ~/.pypirc
24 | echo -e "username = $PYPI_USERNAME" >> ~/.pypirc
25 | echo -e "password = $PYPI_PASSWORD" >> ~/.pypirc
26 | 
27 | # Ensure that we have the latest versions of Twine, Wheel, and Setuptools.
28 | python3 -m pip install --upgrade twine wheel setuptools
29 | 
30 | # Build the distribution and upload.
31 | cd compatibility_lib
32 | python3 setup.py bdist_wheel
33 | twine upload dist/*
34 | 


--------------------------------------------------------------------------------
/scripts/update_dashboard.sh:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env bash
 2 | # Copyright 2018, Google LLC
 3 | #
 4 | # Licensed under the Apache License, Version 2.0 (the "License");
 5 | # you may not use this file except in compliance with the License.
 6 | # You may obtain a copy of the License at
 7 | #
 8 | #     http://www.apache.org/licenses/LICENSE-2.0
 9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 | 
16 | set -ev
17 | 
18 | # Build dashboard
19 | function build_dashboard {
20 |     if [ -n "$GOOGLE_APPLICATION_CREDENTIALS" ]; then
21 |       wget https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64 -O cloud_sql_proxy
22 |       chmod +x cloud_sql_proxy
23 |       python dashboard/dashboard_builder.py
24 |     else
25 |       echo "No credentials. Dashboard will not build."
26 |     fi
27 |     return $?
28 | }
29 | 
30 | build_dashboard
31 | 


--------------------------------------------------------------------------------
/sql_schema/pairwise_compatibility_status_schema.json:
--------------------------------------------------------------------------------
 1 | [
 2 |   {
 3 |     "mode": "REQUIRED", 
 4 |     "name": "install_name_lower", 
 5 |     "type": "STRING"
 6 |   }, 
 7 |   {
 8 |     "mode": "REQUIRED", 
 9 |     "name": "install_name_higher", 
10 |     "type": "STRING"
11 |   }, 
12 |   {
13 |     "mode": "REQUIRED", 
14 |     "name": "status", 
15 |     "type": "STRING"
16 |   }, 
17 |   {
18 |     "mode": "REQUIRED", 
19 |     "name": "py_version", 
20 |     "type": "STRING"
21 |   }, 
22 |   {
23 |     "mode": "REQUIRED", 
24 |     "name": "timestamp", 
25 |     "type": "TIMESTAMP"
26 |   }, 
27 |   {
28 |     "name": "details", 
29 |     "type": "STRING"
30 |   }
31 | ]
32 | 


--------------------------------------------------------------------------------
/sql_schema/release_time_for_dependencies.json:
--------------------------------------------------------------------------------
 1 | [
 2 |   {
 3 |     "mode": "REQUIRED",
 4 |     "name": "install_name",
 5 |     "type": "STRING"
 6 |   },
 7 |   {
 8 |     "mode": "REQUIRED",
 9 |     "name": "dep_name",
10 |     "type": "STRING"
11 |   },
12 |   {
13 |     "mode": "REQUIRED",
14 |     "name": "installed_version",
15 |     "type": "STRING"
16 |   },
17 |   {
18 |     "mode": "NULLABLE",
19 |     "name": "installed_version_time",
20 |     "type": "TIMESTAMP"
21 |   },
22 |   {
23 |     "mode": "NULLABLE",
24 |     "name": "latest_version",
25 |     "type": "STRING"
26 |   },
27 |   {
28 |     "mode": "NULLABLE",
29 |     "name": "latest_version_time",
30 |     "type": "TIMESTAMP"
31 |   },
32 |   {
33 |     "mode": "REQUIRED",
34 |     "name": "is_latest",
35 |     "type": "BOOLEAN"
36 |   },
37 |   {
38 |     "mode": "REQUIRED",
39 |     "name": "timestamp",
40 |     "type": "TIMESTAMP"
41 |   }
42 | ]
43 | 


--------------------------------------------------------------------------------
/sql_schema/self_compatibility_status_schema.json:
--------------------------------------------------------------------------------
 1 | [
 2 |   {
 3 |     "mode": "REQUIRED", 
 4 |     "name": "install_name", 
 5 |     "type": "STRING"
 6 |   }, 
 7 |   {
 8 |     "mode": "REQUIRED", 
 9 |     "name": "status", 
10 |     "type": "STRING"
11 |   }, 
12 |   {
13 |     "mode": "REQUIRED", 
14 |     "name": "py_version", 
15 |     "type": "STRING"
16 |   }, 
17 |   {
18 |     "mode": "REQUIRED", 
19 |     "name": "timestamp", 
20 |     "type": "TIMESTAMP"
21 |   }, 
22 |   {
23 |     "name": "details", 
24 |     "type": "STRING"
25 |   }
26 | ]
27 | 


--------------------------------------------------------------------------------
/system_test/test_badge_server.py:
--------------------------------------------------------------------------------
 1 | # Copyright 2018 Google LLC
 2 | #
 3 | # Licensed under the Apache License, Version 2.0 (the "License");
 4 | # you may not use this file except in compliance with the License.
 5 | # You may obtain a copy of the License at
 6 | #
 7 | #     http://www.apache.org/licenses/LICENSE-2.0
 8 | #
 9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | 
15 | import ast
16 | import os
17 | import requests
18 | import signal
19 | import subprocess
20 | 
21 | from retrying import retry
22 | 
23 | import unittest
24 | 
25 | HOST_PORT = '0.0.0.0:8080'
26 | BASE_URL = 'http://0.0.0.0:8080/'
27 | PACKAGE_FOR_TEST = 'opencensus'
28 | 
29 | RETRY_WAIT_PERIOD = 8000 # Wait 8 seconds between each retry
30 | RETRY_MAX_ATTEMPT = 10 # Retry 10 times
31 | 
32 | 
33 | def wait_app_to_start():
34 |     """Wait the application to start running."""
35 |     cmd = 'wget --retry-connrefused --tries=5 \'{}\''.format(BASE_URL)
36 |     subprocess.check_call(cmd, shell=True)
37 | 
38 | 
39 | def run_application():
40 |     """Start running the compatibility checker server."""
41 |     cmd = 'python badge_server/main.py '
42 |     process = subprocess.Popen(
43 |         cmd,
44 |         stdout=subprocess.PIPE,
45 |         shell=True,
46 |         preexec_fn=os.setsid)
47 |     return process
48 | 
49 | 
50 | class TestBadgeServer(unittest.TestCase):
51 | 
52 |     def setUp(self):
53 |         # Run application
54 |         self.process = run_application()
55 | 
56 |         # Wait app to start
57 |         wait_app_to_start()
58 | 
59 |     def tearDown(self):
60 |         # Kill the application process
61 |         os.killpg(os.getpgid(self.process.pid), signal.SIGTERM)
62 | 
63 |     @retry(wait_fixed=RETRY_WAIT_PERIOD,
64 |            stop_max_attempt_number=RETRY_MAX_ATTEMPT)
65 |     def test_one_badge(self):
66 |         response = requests.get('{}/'.format(BASE_URL))
67 |         status_code = response.status_code
68 |         content = response.content
69 | 
70 |         self.assertEqual(status_code, 200)
71 |         self.assertIn(b"Hello World!", content)
72 | 


--------------------------------------------------------------------------------
/system_test/test_compatibility_checker_server.py:
--------------------------------------------------------------------------------
 1 | # Copyright 2018 Google LLC
 2 | #
 3 | # Licensed under the Apache License, Version 2.0 (the "License");
 4 | # you may not use this file except in compliance with the License.
 5 | # You may obtain a copy of the License at
 6 | #
 7 | #     http://www.apache.org/licenses/LICENSE-2.0
 8 | #
 9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | 
15 | import ast
16 | import os
17 | import requests
18 | import signal
19 | import subprocess
20 | 
21 | from retrying import retry
22 | 
23 | import unittest
24 | 
25 | HOST_PORT = '0.0.0.0:8888'
26 | BASE_URL = 'http://0.0.0.0:8888/'
27 | PACKAGE_FOR_TEST = 'opencensus'
28 | 
29 | RETRY_WAIT_PERIOD = 8000 # Wait 8 seconds between each retry
30 | RETRY_MAX_ATTEMPT = 10 # Retry 10 times
31 | 
32 | 
33 | def wait_app_to_start():
34 |     """Wait the application to start running."""
35 |     url = '{}?package={}&python-version={}'.format(
36 |         BASE_URL, PACKAGE_FOR_TEST, 3)
37 |     cmd = 'wget --retry-connrefused --tries=5 \'{}\''.format(url)
38 |     subprocess.check_call(cmd, shell=True)
39 | 
40 | 
41 | def run_application():
42 |     """Start running the compatibility checker server."""
43 |     cmd = 'python compatibility_server/compatibility_checker_server.py ' \
44 |           '--host=\'0.0.0.0\' --port=8888'
45 |     process = subprocess.Popen(
46 |         cmd,
47 |         stdout=subprocess.PIPE,
48 |         shell=True,
49 |         preexec_fn=os.setsid)
50 |     return process
51 | 
52 | 
53 | class TestCompatibilityCheckerServer(unittest.TestCase):
54 | 
55 |     def setUp(self):
56 |         # Run application
57 |         self.process = run_application()
58 | 
59 |         # Wait app to start
60 |         wait_app_to_start()
61 | 
62 |     def tearDown(self):
63 |         # Kill the flask application process
64 |         os.killpg(os.getpgid(self.process.pid), signal.SIGTERM)
65 | 
66 |     @retry(wait_fixed=RETRY_WAIT_PERIOD,
67 |            stop_max_attempt_number=RETRY_MAX_ATTEMPT)
68 |     def test_check_compatibility(self):
69 |         response = requests.get('{}?package={}&python-version={}'.format(
70 |             BASE_URL, PACKAGE_FOR_TEST, 3))
71 |         status_code = response.status_code
72 |         content = response.content.decode('utf-8')
73 | 
74 |         content_dict = ast.literal_eval(content.replace(
75 |             'true', '"true"').replace(
76 |             'false', '"false"').replace('null', '"null"'))
77 | 
78 |         self.assertEqual(status_code, 200)
79 |         self.assertEqual(content_dict['packages'], [PACKAGE_FOR_TEST])
80 |         self.assertEqual(content_dict['result'], "SUCCESS")
81 |         self.assertIsNotNone(content_dict['dependency_info'])
82 | 


--------------------------------------------------------------------------------
/tox.ini:
--------------------------------------------------------------------------------
1 | [tox]
2 | skipsdist=True
3 | envlist = py27,py36
4 | 


--------------------------------------------------------------------------------