73 | #
79 |
80 | # %%
81 | # Get sklearn extension to run sklearn models easily on OpenML tasks.
82 | from openml_sklearn import SklearnExtension
83 |
84 | extension = SklearnExtension()
85 |
86 | # Model to be used
87 | clf = RandomForestClassifier()
88 |
89 | # We'll create a study with one run on 3 datasets present in the suite
90 | tasks = [115, 259, 307]
91 |
92 | # To verify
93 | # https://test.openml.org/api/v1/study/1
94 | suite = openml.study.get_suite("OpenML100")
95 | print(all(t_id in suite.tasks for t_id in tasks))
96 |
97 | run_ids = []
98 | for task_id in tasks:
99 | task = openml.tasks.get_task(task_id)
100 | run = openml.runs.run_model_on_task(clf, task)
101 | run.publish()
102 | run_ids.append(run.run_id)
103 |
104 | # The study needs a machine-readable and unique alias. To obtain this,
105 | # we simply generate a random uuid.
106 | alias = uuid.uuid4().hex
107 |
108 | new_study = openml.study.create_study(
109 | name="Test-Study",
110 | description="Test study for the Python tutorial on studies",
111 | run_ids=run_ids,
112 | alias=alias,
113 | benchmark_suite=suite.study_id,
114 | )
115 | new_study.publish()
116 | print(new_study)
117 |
118 |
119 | # %%
120 | openml.config.stop_using_configuration_for_example()
121 |
--------------------------------------------------------------------------------
/docs/index.md:
--------------------------------------------------------------------------------
1 | # OpenML
2 |
3 | **The Python API for a World of Data and More**
4 |
5 | Welcome to the documentation of the OpenML Python API, a connector to
6 | the collaborative machine learning platform
7 | [OpenML.org](https://www.openml.org).
8 | OpenML-Python can download or upload data from OpenML, such as datasets
9 | and machine learning experiment results.
10 |
11 | If you are new to OpenML, we recommend checking out the [OpenML documentation](https://docs.openml.org/)
12 | to get familiar with the concepts and features of OpenML. In particular, we recommend
13 | reading more about the [OpenML concepts](https://docs.openml.org/concepts/).
14 |
15 | ## :joystick: Minimal Examples
16 |
17 | Use the following code to get the [credit-g](https://www.openml.org/search?type=data&sort=runs&status=active&id=31) [dataset](https://docs.openml.org/concepts/data/):
18 |
19 | ```python
20 | import openml
21 |
22 | dataset = openml.datasets.get_dataset("credit-g") # or by ID get_dataset(31)
23 | X, y, categorical_indicator, attribute_names = dataset.get_data(target="class")
24 | ```
25 |
26 | Get a [task](https://docs.openml.org/concepts/tasks/) for [supervised classification on credit-g](https://www.openml.org/search?type=task&id=31&source_data.data_id=31):
27 |
28 | ```python
29 | import openml
30 |
31 | task = openml.tasks.get_task(31)
32 | dataset = task.get_dataset()
33 | X, y, categorical_indicator, attribute_names = dataset.get_data(target=task.target_name)
34 | # get splits for the first fold of 10-fold cross-validation
35 | train_indices, test_indices = task.get_train_test_split_indices(fold=0)
36 | ```
37 |
38 | Use an [OpenML benchmarking suite](https://docs.openml.org/concepts/benchmarking/) to get a curated list of machine-learning tasks:
39 | ```python
40 | import openml
41 |
42 | suite = openml.study.get_suite("amlb-classification-all") # Get a curated list of tasks for classification
43 | for task_id in suite.tasks:
44 | task = openml.tasks.get_task(task_id)
45 | ```
46 | Find more examples in the navbar at the top.
47 |
48 | ## :magic_wand: Installation
49 |
50 | OpenML-Python is available on Linux, MacOS, and Windows.
51 |
52 | You can install OpenML-Python with:
53 |
54 | ```bash
55 | pip install openml
56 | ```
57 |
58 | For more advanced installation information, please see the
59 | ["Introduction"](../examples/Basics/introduction_tutorial) example.
60 |
61 |
62 | ## Further information
63 |
64 | - [OpenML documentation](https://docs.openml.org/)
65 | - [OpenML client APIs](https://docs.openml.org/APIs/)
66 | - [OpenML developer guide](https://docs.openml.org/contributing/)
67 | - [Contact information](https://www.openml.org/contact)
68 | - [Citation request](https://www.openml.org/cite)
69 | - [OpenML blog](https://medium.com/open-machine-learning)
70 | - [OpenML twitter account](https://twitter.com/open_ml)
71 |
72 |
73 | ## Contributing
74 |
75 | Contributing to the OpenML package is highly appreciated. Please see the
76 | ["Contributing"](contributing.md) page for more information.
77 |
78 | ## Citing OpenML-Python
79 |
80 | If you use OpenML-Python in a scientific publication, we would
81 | appreciate a reference to our JMLR-MLOSS paper
82 | ["OpenML-Python: an extensible Python API for OpenML"](https://www.jmlr.org/papers/v22/19-920.html):
83 |
84 | === "Bibtex"
85 |
86 | ```bibtex
87 | @article{JMLR:v22:19-920,
88 | author = {Matthias Feurer and Jan N. van Rijn and Arlind Kadra and Pieter Gijsbers and Neeratyoy Mallik and Sahithya Ravi and Andreas Müller and Joaquin Vanschoren and Frank Hutter},
89 | title = {OpenML-Python: an extensible Python API for OpenML},
90 | journal = {Journal of Machine Learning Research},
91 | year = {2021},
92 | volume = {22},
93 | number = {100},
94 | pages = {1--5},
95 | url = {http://jmlr.org/papers/v22/19-920.html}
96 | }
97 | ```
98 |
99 | === "MLA"
100 |
101 | Feurer, Matthias, et al.
102 | "OpenML-Python: an extensible Python API for OpenML."
103 | _Journal of Machine Learning Research_ 22.100 (2021):1−5.
104 |
--------------------------------------------------------------------------------
/examples/Basics/simple_flows_and_runs_tutorial.py:
--------------------------------------------------------------------------------
1 | # %% [markdown]
2 | # A simple tutorial on how to upload results from a machine learning experiment to OpenML.
3 |
4 | # %%
5 | import sklearn
6 | from sklearn.neighbors import KNeighborsClassifier
7 |
8 | import openml
9 |
10 | # %% [markdown]
11 | # Warning
74 | #75 | # For the rest of this tutorial, we will require the `openml-sklearn` package. 76 | # Install it with `pip install openml-sklearn`. 77 | #
78 | #
12 | #
23 |
24 | # %%
25 | openml.config.start_using_configuration_for_example()
26 |
27 | # %% [markdown]
28 | # ## Train a machine learning model and evaluate it
29 | # NOTE: We are using task 119 from the test server: https://test.openml.org/d/20
30 |
31 | # %%
32 | task = openml.tasks.get_task(119)
33 |
34 | # Get the data
35 | dataset = task.get_dataset()
36 | X, y, categorical_indicator, attribute_names = dataset.get_data(
37 | target=dataset.default_target_attribute
38 | )
39 |
40 | # Get the holdout split from the task
41 | train_indices, test_indices = task.get_train_test_split_indices(fold=0, repeat=0)
42 | X_train, X_test = X.iloc[train_indices], X.iloc[test_indices]
43 | y_train, y_test = y.iloc[train_indices], y.iloc[test_indices]
44 |
45 | knn_parameters = {
46 | "n_neighbors": 3,
47 | }
48 | clf = KNeighborsClassifier(**knn_parameters)
49 | clf.fit(X_train, y_train)
50 |
51 | # Get experiment results
52 | y_pred = clf.predict(X_test)
53 | y_pred_proba = clf.predict_proba(X_test)
54 |
55 | # %% [markdown]
56 | # ## Upload the machine learning experiments to OpenML
57 | # First, create a fow and fill it with metadata about the machine learning model.
58 |
59 | # %%
60 | knn_flow = openml.flows.OpenMLFlow(
61 | # Metadata
62 | model=clf, # or None, if you do not want to upload the model object.
63 | name="CustomKNeighborsClassifier",
64 | description="A custom KNeighborsClassifier flow for OpenML.",
65 | external_version=f"{sklearn.__version__}",
66 | language="English",
67 | tags=["openml_tutorial_knn"],
68 | dependencies=f"{sklearn.__version__}",
69 | # Hyperparameters
70 | parameters={k: str(v) for k, v in knn_parameters.items()},
71 | parameters_meta_info={
72 | "n_neighbors": {"description": "number of neighbors to use", "data_type": "int"}
73 | },
74 | # If you have a pipeline with subcomponents, such as preprocessing, add them here.
75 | components={},
76 | )
77 | knn_flow.publish()
78 | print(f"knn_flow was published with the ID {knn_flow.flow_id}")
79 |
80 | # %% [markdown]
81 | # Second, we create a run to store the results associated with the flow.
82 |
83 | # %%
84 |
85 | # Format the predictions for OpenML
86 | predictions = []
87 | for test_index, y_true_i, y_pred_i, y_pred_proba_i in zip(
88 | test_indices, y_test, y_pred, y_pred_proba
89 | ):
90 | predictions.append(
91 | openml.runs.functions.format_prediction(
92 | task=task,
93 | repeat=0,
94 | fold=0,
95 | index=test_index,
96 | prediction=y_pred_i,
97 | truth=y_true_i,
98 | proba=dict(zip(task.class_labels, y_pred_proba_i)),
99 | )
100 | )
101 |
102 | # Format the parameters for OpenML
103 | oml_knn_parameters = [
104 | {"oml:name": k, "oml:value": v, "oml:component": knn_flow.flow_id}
105 | for k, v in knn_parameters.items()
106 | ]
107 |
108 | knn_run = openml.runs.OpenMLRun(
109 | task_id=task.task_id,
110 | flow_id=knn_flow.flow_id,
111 | dataset_id=dataset.dataset_id,
112 | parameter_settings=oml_knn_parameters,
113 | data_content=predictions,
114 | tags=["openml_tutorial_knn"],
115 | description_text="Run generated by the tutorial.",
116 | )
117 | knn_run = knn_run.publish()
118 | print(f"Run was uploaded to {knn_run.openml_url}")
119 | print(f"The flow can be found at {knn_run.flow.openml_url}")
120 |
121 | # %%
122 | openml.config.stop_using_configuration_for_example()
123 |
--------------------------------------------------------------------------------
/openml/extensions/functions.py:
--------------------------------------------------------------------------------
1 | # License: BSD 3-Clause
2 | from __future__ import annotations
3 |
4 | from typing import TYPE_CHECKING, Any
5 |
6 | # Need to implement the following by its full path because otherwise it won't be possible to
7 | # access openml.extensions.extensions
8 | import openml.extensions
9 |
10 | # Avoid import cycles: https://mypy.readthedocs.io/en/latest/common_issues.html#import-cycles
11 | if TYPE_CHECKING:
12 | from openml.flows import OpenMLFlow
13 |
14 | from . import Extension
15 |
16 | SKLEARN_HINT = (
17 | "But it looks related to scikit-learn. "
18 | "Please install the OpenML scikit-learn extension (openml-sklearn) and try again. "
19 | "For more information, see "
20 | "https://github.com/openml/openml-sklearn?tab=readme-ov-file#installation"
21 | )
22 |
23 |
24 | def register_extension(extension: type[Extension]) -> None:
25 | """Register an extension.
26 |
27 | Registered extensions are considered by ``get_extension_by_flow`` and
28 | ``get_extension_by_model``, which are used by ``openml.flow`` and ``openml.runs``.
29 |
30 | Parameters
31 | ----------
32 | extension : Type[Extension]
33 |
34 | Returns
35 | -------
36 | None
37 | """
38 | openml.extensions.extensions.append(extension)
39 |
40 |
41 | def get_extension_by_flow(
42 | flow: OpenMLFlow,
43 | raise_if_no_extension: bool = False, # noqa: FBT001, FBT002
44 | ) -> Extension | None:
45 | """Get an extension which can handle the given flow.
46 |
47 | Iterates all registered extensions and checks whether they can handle the presented flow.
48 | Raises an exception if two extensions can handle a flow.
49 |
50 | Parameters
51 | ----------
52 | flow : OpenMLFlow
53 |
54 | raise_if_no_extension : bool (optional, default=False)
55 | Raise an exception if no registered extension can handle the presented flow.
56 |
57 | Returns
58 | -------
59 | Extension or None
60 | """
61 | candidates = []
62 | for extension_class in openml.extensions.extensions:
63 | if extension_class.can_handle_flow(flow):
64 | candidates.append(extension_class())
65 | if len(candidates) == 0:
66 | if raise_if_no_extension:
67 | install_instruction = ""
68 | if flow.name.startswith("sklearn"):
69 | install_instruction = SKLEARN_HINT
70 | raise ValueError(
71 | f"No extension registered which can handle flow: {flow.flow_id} ({flow.name}). "
72 | f"{install_instruction}"
73 | )
74 |
75 | return None
76 |
77 | if len(candidates) == 1:
78 | return candidates[0]
79 |
80 | raise ValueError(
81 | f"Multiple extensions registered which can handle flow: {flow}, but only one "
82 | f"is allowed ({candidates}).",
83 | )
84 |
85 |
86 | def get_extension_by_model(
87 | model: Any,
88 | raise_if_no_extension: bool = False, # noqa: FBT001, FBT002
89 | ) -> Extension | None:
90 | """Get an extension which can handle the given flow.
91 |
92 | Iterates all registered extensions and checks whether they can handle the presented model.
93 | Raises an exception if two extensions can handle a model.
94 |
95 | Parameters
96 | ----------
97 | model : Any
98 |
99 | raise_if_no_extension : bool (optional, default=False)
100 | Raise an exception if no registered extension can handle the presented model.
101 |
102 | Returns
103 | -------
104 | Extension or None
105 | """
106 | candidates = []
107 | for extension_class in openml.extensions.extensions:
108 | if extension_class.can_handle_model(model):
109 | candidates.append(extension_class())
110 | if len(candidates) == 0:
111 | if raise_if_no_extension:
112 | install_instruction = ""
113 | if type(model).__module__.startswith("sklearn"):
114 | install_instruction = SKLEARN_HINT
115 | raise ValueError(
116 | f"No extension registered which can handle model: {model}. {install_instruction}"
117 | )
118 |
119 | return None
120 |
121 | if len(candidates) == 1:
122 | return candidates[0]
123 |
124 | raise ValueError(
125 | f"Multiple extensions registered which can handle model: {model}, but only one "
126 | f"is allowed ({candidates}).",
127 | )
128 |
--------------------------------------------------------------------------------
/examples/_external_or_deprecated/2015_neurips_feurer_example.py:
--------------------------------------------------------------------------------
1 | """
2 | Feurer et al. (2015)
3 | ====================
4 |
5 | A tutorial on how to get the datasets used in the paper introducing *Auto-sklearn* by Feurer et al..
6 |
7 | Auto-sklearn website: https://automl.github.io/auto-sklearn/
8 |
9 | Publication
10 | ~~~~~~~~~~~
11 |
12 | | Efficient and Robust Automated Machine Learning
13 | | Matthias Feurer, Aaron Klein, Katharina Eggensperger, Jost Springenberg, Manuel Blum and Frank Hutter
14 | | In *Advances in Neural Information Processing Systems 28*, 2015
15 | | Available at https://papers.nips.cc/paper/5872-efficient-and-robust-automated-machine-learning.pdf
16 | """ # noqa F401
17 |
18 | # License: BSD 3-Clause
19 |
20 | import pandas as pd
21 |
22 | import openml
23 |
24 | ####################################################################################################
25 | # List of dataset IDs given in the supplementary material of Feurer et al.:
26 | # https://papers.nips.cc/paper/5872-efficient-and-robust-automated-machine-learning-supplemental.zip
27 | # fmt: off
28 | dataset_ids = [
29 | 3, 6, 12, 14, 16, 18, 21, 22, 23, 24, 26, 28, 30, 31, 32, 36, 38, 44, 46,
30 | 57, 60, 179, 180, 181, 182, 184, 185, 273, 293, 300, 351, 354, 357, 389,
31 | 390, 391, 392, 393, 395, 396, 398, 399, 401, 554, 679, 715, 718, 720, 722,
32 | 723, 727, 728, 734, 735, 737, 740, 741, 743, 751, 752, 761, 772, 797, 799,
33 | 803, 806, 807, 813, 816, 819, 821, 822, 823, 833, 837, 843, 845, 846, 847,
34 | 849, 866, 871, 881, 897, 901, 903, 904, 910, 912, 913, 914, 917, 923, 930,
35 | 934, 953, 958, 959, 962, 966, 971, 976, 977, 978, 979, 980, 991, 993, 995,
36 | 1000, 1002, 1018, 1019, 1020, 1021, 1036, 1040, 1041, 1049, 1050, 1053,
37 | 1056, 1067, 1068, 1069, 1111, 1112, 1114, 1116, 1119, 1120, 1128, 1130,
38 | 1134, 1138, 1139, 1142, 1146, 1161, 1166,
39 | ]
40 | # fmt: on
41 |
42 | ####################################################################################################
43 | # The dataset IDs could be used directly to load the dataset and split the data into a training set
44 | # and a test set. However, to be reproducible, we will first obtain the respective tasks from
45 | # OpenML, which define both the target feature and the train/test split.
46 | #
47 | # .. note::
48 | # It is discouraged to work directly on datasets and only provide dataset IDs in a paper as
49 | # this does not allow reproducibility (unclear splitting). Please do not use datasets but the
50 | # respective tasks as basis for a paper and publish task IDS. This example is only given to
51 | # showcase the use of OpenML-Python for a published paper and as a warning on how not to do it.
52 | # Please check the `OpenML documentation of tasks Warning
13 | #
14 | # This example uploads data. For that reason, this example connects to the
15 | # test server at test.openml.org.
17 | # This prevents the main server from becoming overloaded with example datasets, tasks,
18 | # runs, and other submissions.
19 | # Using this test server may affect the behavior and performance of the
20 | # OpenML-Python API.
21 | #
4 |
5 |
25 |
26 | OpenML-Python provides an easy-to-use and straightforward Python interface for [OpenML](http://openml.org), an online platform for open science collaboration in machine learning.
27 | It can download or upload data from OpenML, such as datasets and machine learning experiment results.
28 |
29 | ## :joystick: Minimal Example
30 |
31 | Use the following code to get the [credit-g](https://www.openml.org/search?type=data&sort=runs&status=active&id=31) [dataset](https://docs.openml.org/concepts/data/):
32 |
33 | ```python
34 | import openml
35 |
36 | dataset = openml.datasets.get_dataset("credit-g") # or by ID get_dataset(31)
37 | X, y, categorical_indicator, attribute_names = dataset.get_data(target="class")
38 | ```
39 |
40 | Get a [task](https://docs.openml.org/concepts/tasks/) for [supervised classification on credit-g](https://www.openml.org/search?type=task&id=31&source_data.data_id=31):
41 |
42 | ```python
43 | import openml
44 |
45 | task = openml.tasks.get_task(31)
46 | dataset = task.get_dataset()
47 | X, y, categorical_indicator, attribute_names = dataset.get_data(target=task.target_name)
48 | # get splits for the first fold of 10-fold cross-validation
49 | train_indices, test_indices = task.get_train_test_split_indices(fold=0)
50 | ```
51 |
52 | Use an [OpenML benchmarking suite](https://docs.openml.org/concepts/benchmarking/) to get a curated list of machine-learning tasks:
53 | ```python
54 | import openml
55 |
56 | suite = openml.study.get_suite("amlb-classification-all") # Get a curated list of tasks for classification
57 | for task_id in suite.tasks:
58 | task = openml.tasks.get_task(task_id)
59 | ```
60 |
61 | ## :magic_wand: Installation
62 |
63 | OpenML-Python is supported on Python 3.8 - 3.13 and is available on Linux, MacOS, and Windows.
64 |
65 | You can install OpenML-Python with:
66 |
67 | ```bash
68 | pip install openml
69 | ```
70 |
71 | ## :page_facing_up: Citing OpenML-Python
72 |
73 | If you use OpenML-Python in a scientific publication, we would appreciate a reference to the following paper:
74 |
75 | [Matthias Feurer, Jan N. van Rijn, Arlind Kadra, Pieter Gijsbers, Neeratyoy Mallik, Sahithya Ravi, Andreas Müller, Joaquin Vanschoren, Frank Hutter
6 |
14 |
15 | ## The Python API for a World of Data and More :dizzy:
16 |
17 | [](https://github.com/openml/openml-python/releases)
18 | [](https://pypi.org/project/openml/)
19 | [](https://pepy.tech/project/openml)
20 | [](https://opensource.org/licenses/BSD-3-Clause)
21 |
22 |
23 | [Installation](https://openml.github.io/openml-python/main/#how-to-get-openml-for-python) | [Documentation](https://openml.github.io/openml-python) | [Contribution guidelines](https://github.com/openml/openml-python/blob/develop/CONTRIBUTING.md)
24 | -
7 |
9 | OpenML-Python
10 |
11 | 76 | **OpenML-Python: an extensible Python API for OpenML**
77 | Journal of Machine Learning Research, 22(100):1−5, 2021](https://www.jmlr.org/papers/v22/19-920.html) 78 | 79 | Bibtex entry: 80 | ```bibtex 81 | @article{JMLR:v22:19-920, 82 | author = {Matthias Feurer and Jan N. van Rijn and Arlind Kadra and Pieter Gijsbers and Neeratyoy Mallik and Sahithya Ravi and Andreas Müller and Joaquin Vanschoren and Frank Hutter}, 83 | title = {OpenML-Python: an extensible Python API for OpenML}, 84 | journal = {Journal of Machine Learning Research}, 85 | year = {2021}, 86 | volume = {22}, 87 | number = {100}, 88 | pages = {1--5}, 89 | url = {http://jmlr.org/papers/v22/19-920.html} 90 | } 91 | ``` 92 | ## :handshake: Contributing 93 | 94 | We welcome contributions from both new and experienced developers! 95 | 96 | If you would like to contribute to OpenML-Python, please read our 97 | [Contribution Guidelines](https://github.com/openml/openml-python/blob/develop/CONTRIBUTING.md). 98 | 99 | If you are new to open-source development, a great way to get started is by 100 | looking at issues labeled **"good first issue"** in our GitHub issue tracker. 101 | These tasks are beginner-friendly and help you understand the project structure, 102 | development workflow, and how to submit a pull request. 103 | -------------------------------------------------------------------------------- /openml/evaluations/evaluation.py: -------------------------------------------------------------------------------- 1 | # License: BSD 3-Clause 2 | from __future__ import annotations 3 | 4 | import openml.config 5 | import openml.datasets 6 | import openml.flows 7 | import openml.runs 8 | import openml.tasks 9 | 10 | 11 | # TODO(eddiebergman): A lot of this class is automatically 12 | # handled by a dataclass 13 | class OpenMLEvaluation: 14 | """ 15 | Contains all meta-information about a run / evaluation combination, 16 | according to the evaluation/list function 17 | 18 | Parameters 19 | ---------- 20 | run_id : int 21 | Refers to the run. 22 | task_id : int 23 | Refers to the task. 24 | setup_id : int 25 | Refers to the setup. 26 | flow_id : int 27 | Refers to the flow. 28 | flow_name : str 29 | Name of the referred flow. 30 | data_id : int 31 | Refers to the dataset. 32 | data_name : str 33 | The name of the dataset. 34 | function : str 35 | The evaluation metric of this item (e.g., accuracy). 36 | upload_time : str 37 | The time of evaluation. 38 | uploader: int 39 | Uploader ID (user ID) 40 | upload_name : str 41 | Name of the uploader of this evaluation 42 | value : float 43 | The value (score) of this evaluation. 44 | values : List[float] 45 | The values (scores) per repeat and fold (if requested) 46 | array_data : str 47 | list of information per class. 48 | (e.g., in case of precision, auroc, recall) 49 | """ 50 | 51 | def __init__( # noqa: PLR0913 52 | self, 53 | run_id: int, 54 | task_id: int, 55 | setup_id: int, 56 | flow_id: int, 57 | flow_name: str, 58 | data_id: int, 59 | data_name: str, 60 | function: str, 61 | upload_time: str, 62 | uploader: int, 63 | uploader_name: str, 64 | value: float | None, 65 | values: list[float] | None, 66 | array_data: str | None = None, 67 | ): 68 | self.run_id = run_id 69 | self.task_id = task_id 70 | self.setup_id = setup_id 71 | self.flow_id = flow_id 72 | self.flow_name = flow_name 73 | self.data_id = data_id 74 | self.data_name = data_name 75 | self.function = function 76 | self.upload_time = upload_time 77 | self.uploader = uploader 78 | self.uploader_name = uploader_name 79 | self.value = value 80 | self.values = values 81 | self.array_data = array_data 82 | 83 | def _to_dict(self) -> dict: 84 | return { 85 | "run_id": self.run_id, 86 | "task_id": self.task_id, 87 | "setup_id": self.setup_id, 88 | "flow_id": self.flow_id, 89 | "flow_name": self.flow_name, 90 | "data_id": self.data_id, 91 | "data_name": self.data_name, 92 | "function": self.function, 93 | "upload_time": self.upload_time, 94 | "uploader": self.uploader, 95 | "uploader_name": self.uploader_name, 96 | "value": self.value, 97 | "values": self.values, 98 | "array_data": self.array_data, 99 | } 100 | 101 | def __repr__(self) -> str: 102 | header = "OpenML Evaluation" 103 | header = f"{header}\n{'=' * len(header)}\n" 104 | 105 | fields = { 106 | "Upload Date": self.upload_time, 107 | "Run ID": self.run_id, 108 | "OpenML Run URL": openml.runs.OpenMLRun.url_for_id(self.run_id), 109 | "Task ID": self.task_id, 110 | "OpenML Task URL": openml.tasks.OpenMLTask.url_for_id(self.task_id), 111 | "Flow ID": self.flow_id, 112 | "OpenML Flow URL": openml.flows.OpenMLFlow.url_for_id(self.flow_id), 113 | "Setup ID": self.setup_id, 114 | "Data ID": self.data_id, 115 | "Data Name": self.data_name, 116 | "OpenML Data URL": openml.datasets.OpenMLDataset.url_for_id(self.data_id), 117 | "Metric Used": self.function, 118 | "Result": self.value, 119 | } 120 | 121 | order = [ 122 | "Uploader Date", 123 | "Run ID", 124 | "OpenML Run URL", 125 | "Task ID", 126 | "OpenML Task URL" "Flow ID", 127 | "OpenML Flow URL", 128 | "Setup ID", 129 | "Data ID", 130 | "Data Name", 131 | "OpenML Data URL", 132 | "Metric Used", 133 | "Result", 134 | ] 135 | _fields = [(key, fields[key]) for key in order if key in fields] 136 | 137 | longest_field_name_length = max(len(name) for name, _ in _fields) 138 | field_line_format = f"{{:.<{longest_field_name_length}}}: {{}}" 139 | body = "\n".join(field_line_format.format(name, value) for name, value in _fields) 140 | return header + body 141 | -------------------------------------------------------------------------------- /tests/test_openml/test_api_calls.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import unittest.mock 4 | from pathlib import Path 5 | from typing import NamedTuple, Iterable, Iterator 6 | from unittest import mock 7 | 8 | import minio 9 | import pytest 10 | 11 | import openml 12 | from openml.config import ConfigurationForExamples 13 | import openml.testing 14 | from openml._api_calls import _download_minio_bucket, API_TOKEN_HELP_LINK 15 | 16 | 17 | class TestConfig(openml.testing.TestBase): 18 | def test_too_long_uri(self): 19 | with pytest.raises(openml.exceptions.OpenMLServerError, match="URI too long!"): 20 | openml.datasets.list_datasets(data_id=list(range(10000))) 21 | 22 | @unittest.mock.patch("time.sleep") 23 | @unittest.mock.patch("requests.Session") 24 | def test_retry_on_database_error(self, Session_class_mock, _): 25 | response_mock = unittest.mock.Mock() 26 | response_mock.text = ( 27 | "
29 | #
35 |
36 | # %%
37 | # Get sklearn extension to run sklearn models easily on OpenML tasks.
38 | from openml_sklearn import SklearnExtension, cat, cont
39 |
40 | extension = SklearnExtension()
41 |
42 | # Set your openml api key if you want to upload your results to OpenML (eg:
43 | # https://openml.org/search?type=run&sort=date) . To get one, simply make an
44 | # account (you don't need one for anything else, just to upload your results),
45 | # go to your profile and select the API-KEY.
46 | # Or log in, and navigate to https://www.openml.org/auth/api-key
47 | openml.config.apikey = ""
48 | ############################################################################
49 | # Prepare for preprocessors and an OpenML task
50 | # ============================================
51 |
52 | # OpenML contains several key concepts which it needs to make machine learning research shareable.
53 | # A machine learning experiment consists of one or several runs, which describe the performance of
54 | # an algorithm (called a flow in OpenML), its hyperparameter settings (called a setup) on a task.
55 | # A Task is the combination of a dataset, a split and an evaluation metric We choose a dataset from
56 | # OpenML, (https://www.openml.org/d/1464) and a subsequent task (https://www.openml.org/t/10101) To
57 | # make your own dataset and task, please refer to
58 | # https://openml.github.io/openml-python/main/examples/30_extended/create_upload_tutorial.html
59 |
60 | # https://www.openml.org/search?type=study&study_type=task&id=218
61 | task_id = 10101
62 | seed = 42
63 | categorical_preproc = (
64 | "categorical",
65 | OneHotEncoder(sparse_output=False, handle_unknown="ignore"),
66 | cat,
67 | )
68 | numerical_preproc = ("numerical", SimpleImputer(strategy="median"), cont)
69 | preproc = ColumnTransformer([categorical_preproc, numerical_preproc])
70 |
71 | ############################################################################
72 | # Define a pipeline for the hyperparameter optimization (this is standark for Optuna)
73 | # =====================================================
74 |
75 | # Optuna explanation
76 | # we follow the `Optuna Warning
30 | #31 | # For the rest of this tutorial, we will require the `openml-sklearn` package. 32 | # Install it with `pip install openml-sklearn`. 33 | #
34 | #