├── .github └── workflows │ └── basic.yml ├── .gitignore ├── LICENSE-2.0.txt ├── README.rst ├── hiplot_mlflow.py ├── images ├── notebook_name.png ├── server_multi.png └── server_name.png ├── pyproject.toml ├── setup.cfg ├── setup.py ├── tests └── test_basic.py └── tox.ini /.github/workflows/basic.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | # Trigger the workflow on push or pull request, 5 | # but only for the master branch 6 | push: 7 | branches: 8 | - master 9 | pull_request: 10 | branches: 11 | - master 12 | 13 | jobs: 14 | lint: 15 | runs-on: ubuntu-latest 16 | strategy: 17 | matrix: 18 | tox-env: ["license", "black", "flake8", "mypy"] 19 | steps: 20 | - uses: actions/checkout@v2 21 | - name: Set up Python 3.8 22 | uses: actions/setup-python@v1 23 | with: 24 | python-version: 3.8 25 | - name: Install dependencies 26 | run: | 27 | python -m pip install --upgrade pip 28 | python -m pip install tox 29 | - name: Lint with ${{ matrix.tox-env }} 30 | run: tox -e ${{ matrix.tox-env }} 31 | 32 | test: 33 | runs-on: ubuntu-latest 34 | strategy: 35 | matrix: 36 | python-version: [3.6, 3.7, 3.8] 37 | steps: 38 | - uses: actions/checkout@v2 39 | - name: Set up Python ${{ matrix.python-version }} 40 | uses: actions/setup-python@v1 41 | with: 42 | python-version: ${{ matrix.python-version }} 43 | - name: Install dependencies 44 | run: | 45 | python -m pip install --upgrade pip 46 | python -m pip install tox 47 | - name: Test 48 | run: tox -e py 49 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | build/ 2 | dist/ 3 | *.egg-info/ 4 | .eggs/ 5 | .mypy_cache/ 6 | pytest_cache/ 7 | __pycache__/ 8 | *.pyc 9 | .tox/ 10 | pip-wheel-metadata/ 11 | -------------------------------------------------------------------------------- /LICENSE-2.0.txt: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | hiplot-mlflow 2 | ============= 3 | 4 | A `HiPlot `_ experiment 5 | fetcher plugin for `MLflow `_, to help visualise your 6 | tracked experiments. 7 | 8 | Installation 9 | ------------ 10 | 11 | Install this library with ``pip`` as: 12 | 13 | .. code-block:: sh 14 | 15 | pip install hiplot_mlflow 16 | 17 | Usage 18 | ----- 19 | 20 | You can visualise experiments either in a Jupyter notebook or using HiPlot's 21 | built in server. 22 | 23 | Notebook 24 | ~~~~~~~~ 25 | 26 | In a Jupyter notebook, use ``hiplot_mlflow.fetch`` to retrieve an MLflow 27 | experiment by name, and display it with HiPlot: 28 | 29 | .. code-block:: python 30 | 31 | import hiplot_mlflow 32 | experiments = hiplot_mlflow.fetch("my-lovely-experiment") 33 | experiments.display(force_full_width=True) 34 | 35 | You can also retrieve experiments by their MLflow experiment ID: 36 | 37 | .. code-block:: python 38 | 39 | experiment = hiplot_mlflow.fetch_by_id(0) 40 | 41 | By default, MLflow tags are not shown (only MLflow metrics and parameters are 42 | shown). To display them, pass ``include_tag=True`` to either of the fetch 43 | functions, for example: 44 | 45 | .. code-block:: python 46 | 47 | experiment = hiplot_mlflow.fetch("my-lovely-experiment", include_tags=True) 48 | 49 | .. image:: images/notebook_name.png 50 | :alt: Loading HiPlot in a notebook 51 | 52 | See more about what you can do with the returned ``hiplot.Experiment`` values 53 | in the `HiPlot documentation 54 | `_. 55 | 56 | HiPlot Server 57 | ~~~~~~~~~~~~~ 58 | 59 | To use `HiPlot's built in webserver 60 | `_ with 61 | ``hiplot-mlflow``, you can start it up with the custom `experiment fetcher 62 | `_ 63 | implemented by this package: 64 | 65 | .. code-block:: sh 66 | 67 | hiplot hiplot_mlflow.fetch_by_uri 68 | 69 | You can then use the ``mlflow://`` schema to access MLflow experiments in 70 | HiPlot by either experiment or name, for example:: 71 | 72 | mlflow://name/experiment-name 73 | mlflow://id/0 74 | 75 | .. image:: images/server_name.png 76 | :alt: Loading HiPlot server with experiment name 77 | 78 | You can also add ``tags=yes`` as a query string parameter to include tags in 79 | the output, for example:: 80 | 81 | mlflow://name/experiment-name?tags=yes 82 | 83 | You can also use the `multiple experiments 84 | `_ 85 | loading syntax. Either the dictionary format (to define your own labels):: 86 | 87 | multi://{ 88 | "first-experiment": "mlflow://id/1", 89 | "another-experiment": "mlflow://name/another-experiment?tags=yes" 90 | } 91 | 92 | or list format:: 93 | 94 | multi://[ 95 | "mlflow://id/1", 96 | "mlflow://name/another-experiment?tags=yes" 97 | ] 98 | 99 | .. image:: images/server_multi.png 100 | :alt: Multiple experiments in HiPlot server 101 | -------------------------------------------------------------------------------- /hiplot_mlflow.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Faculty Science Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import uuid 16 | from typing import Any 17 | from urllib.parse import parse_qs, urlparse 18 | 19 | import hiplot 20 | from mlflow import search_runs, tracking 21 | from mlflow.exceptions import MlflowException 22 | from numpy import isfinite 23 | from pandas import DataFrame 24 | 25 | 26 | def fetch(name: str, include_tags: bool = False) -> hiplot.Experiment: 27 | """Fetch MLflow experiment by name. 28 | 29 | Parameters 30 | ---------- 31 | name: str 32 | The name of the tracked experiment to look up 33 | include_tags: bool, optional 34 | Whether to include tags in the resulting HiPlot experiment (False) 35 | 36 | Returns 37 | ------- 38 | hiplot.Experiment 39 | The resulting experiment 40 | """ 41 | return fetch_by_id(_get_experiment_id_from_name(name), include_tags) 42 | 43 | 44 | def fetch_by_id(id: Any, include_tags: bool = False) -> hiplot.Experiment: 45 | """Fetch MLFlow experiment by ID. 46 | 47 | Parameters 48 | ---------- 49 | id: int or str 50 | The ID if the experiment to look up 51 | include_tags: bool, optional 52 | Whether to include tags in the resulting HiPlot experiment (False) 53 | 54 | Returns 55 | ------- 56 | hiplot.Experiment 57 | The resulting experiment 58 | """ 59 | try: 60 | df = search_runs([id]) 61 | except MlflowException as e: 62 | raise hiplot.ExperimentValidationError(str(e)) 63 | return _create_experiment_from_dataframe(df, include_tags) 64 | 65 | 66 | def fetch_by_uri(uri: str) -> hiplot.Experiment: 67 | """Fetch MLflow experiment using URI. 68 | 69 | The URI can take these forms: 70 | * ``mlflow://id/`` to look up experiments by ID 71 | * ``mlflow://name/`` to look up experiments by name 72 | 73 | You can also add a query string to modify the results: 74 | * Setting ``tags`` to any value will return results with 75 | tags included. For example: ``mlflow://id/?tags=yes`` 76 | 77 | Parameters 78 | ---------- 79 | uri: str 80 | The fetcher URI to use 81 | 82 | Returns 83 | ------- 84 | hiplot.Experiment 85 | The resulting experiment 86 | """ 87 | # Only apply this fetcher if the URI matches our schema 88 | parsed_uri = urlparse(uri) 89 | if parsed_uri.scheme != "mlflow": 90 | raise hiplot.ExperimentFetcherDoesntApply() 91 | 92 | reference_type = parsed_uri.netloc 93 | reference = parsed_uri.path.strip("/") 94 | include_tags = parse_qs(parsed_uri.query).get("tags") is not None 95 | if reference_type == "name": 96 | return fetch(reference, include_tags) 97 | elif reference_type == "id": 98 | return fetch_by_id(reference, include_tags) 99 | else: 100 | raise hiplot.ExperimentValidationError() 101 | 102 | 103 | def _get_experiment_id_from_name(name: str) -> Any: 104 | """Get an MLflow experiment's ID from its name 105 | """ 106 | client = tracking.MlflowClient() 107 | experiment = client.get_experiment_by_name(name) 108 | if experiment is None: 109 | raise hiplot.ExperimentValidationError() 110 | return experiment.experiment_id 111 | 112 | 113 | def _create_experiment_from_dataframe( 114 | df: DataFrame, include_tags: bool 115 | ) -> hiplot.Experiment: 116 | """Generate HiPlot experiment from MLFlow runs. 117 | 118 | Parameters 119 | ---------- 120 | df: pandas.DataFrame 121 | A dataframe (returned by ``mlflow.search_runs`` normally) 122 | to turn process 123 | include_tags: bool 124 | Whether or not to include tags in the results (False) 125 | 126 | Returns 127 | ------- 128 | hiplot.Experiment 129 | The processed experiment 130 | """ 131 | exp = hiplot.Experiment() 132 | params = [p for p in df.columns if p.startswith("params.")] 133 | metrics = [m for m in df.columns if m.startswith("metrics.")] 134 | if include_tags: 135 | tags = [t for t in df.columns if t.startswith("tags.")] 136 | for _, row in df.iterrows(): 137 | values = {} 138 | for p in params: 139 | values[p] = row[p] 140 | 141 | for m in metrics: 142 | if isfinite(row[m]): 143 | values[m] = row[m] 144 | 145 | if include_tags: 146 | for t in tags: 147 | values[t] = row[t] 148 | 149 | dp = hiplot.Datapoint( 150 | uid=str(uuid.UUID(row["run_id"])), values=values, 151 | ) 152 | exp.datapoints.append(dp) 153 | return exp 154 | -------------------------------------------------------------------------------- /images/notebook_name.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facultyai/hiplot-mlflow/bd3bccc9bbcd0e2c6fe39e6b60f66bf4453f6497/images/notebook_name.png -------------------------------------------------------------------------------- /images/server_multi.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facultyai/hiplot-mlflow/bd3bccc9bbcd0e2c6fe39e6b60f66bf4453f6497/images/server_multi.png -------------------------------------------------------------------------------- /images/server_name.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facultyai/hiplot-mlflow/bd3bccc9bbcd0e2c6fe39e6b60f66bf4453f6497/images/server_name.png -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.black] 2 | line-length = 79 3 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | ignore = E203 W503 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Faculty Science Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | from setuptools import setup 17 | from pathlib import Path 18 | 19 | README = Path(__file__).parent / "README.rst" 20 | 21 | setup( 22 | name="hiplot-mlflow", 23 | description="HiPlot fetcher plugin for MLflow experiment tracking.", 24 | long_description=README.read_text(), 25 | url="https://github.com/facultyai/hiplot-mlflow", 26 | author="Faculty", 27 | author_email="opensource@faculty.ai", 28 | license="Apache Software License", 29 | py_modules=["hiplot_mlflow"], 30 | use_scm_version={"version_scheme": "post-release"}, 31 | setup_requires=["setuptools_scm"], 32 | install_requires=["hiplot", "mlflow", "numpy"], 33 | extras_require={ 34 | "dev": ["black", "flake8", "flake8-black", "mypy", "pytest", "tox"] 35 | }, 36 | ) 37 | -------------------------------------------------------------------------------- /tests/test_basic.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Faculty Science Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import uuid 16 | 17 | import hiplot 18 | from pandas import DataFrame 19 | 20 | import hiplot_mlflow 21 | import pytest 22 | 23 | RUN_ID_1 = str(uuid.uuid4()) 24 | RUN_ID_2 = str(uuid.uuid4()) 25 | 26 | SEARCH_RUNS_RESULT = DataFrame( 27 | { 28 | "run_id": [RUN_ID_1, RUN_ID_2], 29 | "params.numeric": [0, 1], 30 | "params.category": ["value1", "value2"], 31 | "metrics.third": [2, 4], 32 | "tags.test": ["yes", "no"], 33 | } 34 | ) 35 | EXPERIMENT = hiplot.Experiment( 36 | [ 37 | hiplot.Datapoint( 38 | uid=RUN_ID_1, 39 | values={ 40 | "params.numeric": 0, 41 | "params.category": "value1", 42 | "metrics.third": 2, 43 | }, 44 | ), 45 | hiplot.Datapoint( 46 | uid=RUN_ID_2, 47 | values={ 48 | "params.numeric": 1, 49 | "params.category": "value2", 50 | "metrics.third": 4, 51 | }, 52 | ), 53 | ] 54 | ) 55 | EXPERIMENT_WITH_TAGS = hiplot.Experiment( 56 | [ 57 | hiplot.Datapoint( 58 | uid=RUN_ID_1, 59 | values={ 60 | "params.numeric": 0, 61 | "params.category": "value1", 62 | "metrics.third": 2, 63 | "tags.test": "yes", 64 | }, 65 | ), 66 | hiplot.Datapoint( 67 | uid=RUN_ID_2, 68 | values={ 69 | "params.numeric": 1, 70 | "params.category": "value2", 71 | "metrics.third": 4, 72 | "tags.test": "no", 73 | }, 74 | ), 75 | ] 76 | ) 77 | 78 | 79 | def test_wrong_schema(): 80 | """Test passing unsupported schema. 81 | """ 82 | with pytest.raises(hiplot.ExperimentFetcherDoesntApply): 83 | hiplot_mlflow.fetch_by_uri("something://") 84 | 85 | 86 | def test_unsupported_reference_type(): 87 | """Test passing correct schema and unsupported reference type. 88 | """ 89 | with pytest.raises(hiplot.ExperimentValidationError): 90 | hiplot_mlflow.fetch_by_uri(f"mlflow://something/else") 91 | 92 | 93 | @pytest.mark.parametrize( 94 | "include_tags, expected_result", 95 | [(False, EXPERIMENT), (True, EXPERIMENT_WITH_TAGS)], 96 | ) 97 | def test_dataframe_to_experiment(include_tags, expected_result): 98 | """Test the conversion from dataframe to HiPlot experiment, 99 | with or without tags. 100 | """ 101 | experiment = hiplot_mlflow._create_experiment_from_dataframe( 102 | SEARCH_RUNS_RESULT, include_tags 103 | ) 104 | for test_dp, exp_dp in zip( 105 | experiment.datapoints, expected_result.datapoints 106 | ): 107 | assert test_dp.uid == exp_dp.uid 108 | assert test_dp.values == exp_dp.values 109 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py{36,37,38}, flake8, black, mypy, license 3 | 4 | [testenv] 5 | sitepackages = False 6 | deps = 7 | pytest 8 | commands = pytest {posargs} 9 | 10 | [testenv:flake8] 11 | skip_install = True 12 | deps = 13 | flake8 14 | commands = 15 | flake8 {posargs:--count --show-source --statistics} 16 | 17 | [testenv:black] 18 | skip_install = True 19 | deps = 20 | black 21 | commands = 22 | black {posargs:--check setup.py hiplot_mlflow.py tests} 23 | 24 | [testenv:mypy] 25 | skip_install = True 26 | deps = 27 | mypy 28 | commands = 29 | mypy {posargs:--ignore-missing-imports hiplot_mlflow.py} 30 | 31 | [testenv:license] 32 | skip_install = True 33 | deps = 34 | apache-license-check 35 | commands = 36 | apache-license-check setup.py hiplot_mlflow.py tests --copyright "Faculty Science Limited" 37 | --------------------------------------------------------------------------------