diff --git a/openml/_api/clients/minio.py b/openml/_api/clients/minio.py
index 920b485e04..852b678cd0 100644
--- a/openml/_api/clients/minio.py
+++ b/openml/_api/clients/minio.py
@@ -1,8 +1,17 @@
from __future__ import annotations
+import contextlib
+import shutil
+import urllib
+import zipfile
from pathlib import Path
+import minio
+import requests
+from urllib3 import ProxyManager
+
import openml
+from openml.utils import ProgressBar
class MinIOClient:
@@ -16,13 +25,125 @@ class MinIOClient:
Attributes
----------
- path : pathlib.Path or None
+ path : pathlib.Path
Configured base path for storage operations.
headers : dict of str to str
Default HTTP headers, including a user-agent identifying the
OpenML Python client version.
"""
+ @property
+ def headers(self) -> dict[str, str]:
+ return openml.config._HEADERS
+
@property
def path(self) -> Path:
return Path(openml.config.get_cache_directory())
+
+ def download_minio_file(
+ self,
+ source: str,
+ exists_ok: bool = True, # noqa: FBT002
+ proxy: str | None = "auto",
+ ) -> Path:
+ """Download file ``source`` from a MinIO Bucket and store it at ``destination``.
+
+ Parameters
+ ----------
+ source : str
+ URL to a file in a MinIO bucket.
+ exists_ok : bool, optional (default=True)
+ If False, raise FileExists if a file already exists in ``destination``.
+ proxy: str, optional (default = "auto")
+ The proxy server to use. By default it's "auto" which uses ``requests`` to
+ automatically find the proxy to use. Pass None or the environment variable
+ ``no_proxy="*"`` to disable proxies.
+ """
+ destination = Path(openml.config.get_minio_download_path(source))
+ parsed_url = urllib.parse.urlparse(source)
+
+ # expect path format: /BUCKET/path/to/file.ext
+ bucket, object_name = parsed_url.path[1:].split("/", maxsplit=1)
+ if destination.is_dir():
+ destination = Path(destination, object_name)
+ if destination.is_file() and not exists_ok:
+ raise FileExistsError(f"File already exists in {destination}.")
+
+ destination = destination.expanduser()
+ destination.parent.mkdir(parents=True, exist_ok=True)
+
+ if proxy == "auto":
+ resolved_proxies = requests.utils.get_environ_proxies(parsed_url.geturl())
+ proxy = requests.utils.select_proxy(parsed_url.geturl(), resolved_proxies) # type: ignore
+
+ proxy_client = ProxyManager(proxy) if proxy else None
+
+ client = minio.Minio(endpoint=parsed_url.netloc, secure=False, http_client=proxy_client)
+ try:
+ client.fget_object(
+ bucket_name=bucket,
+ object_name=object_name,
+ file_path=str(destination),
+ progress=ProgressBar() if openml.config.show_progress else None,
+ request_headers=self.headers,
+ )
+ if destination.is_file() and destination.suffix == ".zip":
+ with zipfile.ZipFile(destination, "r") as zip_ref:
+ zip_ref.extractall(destination.parent)
+
+ except minio.error.S3Error as e:
+ if e.message is not None and e.message.startswith("Object does not exist"):
+ raise FileNotFoundError(f"Object at '{source}' does not exist.") from e
+ # e.g. permission error, or a bucket does not exist (which is also interpreted as a
+ # permission error on minio level).
+ raise FileNotFoundError("Bucket does not exist or is private.") from e
+
+ return destination
+
+ def download_minio_bucket(self, source: str) -> None:
+ """Download file ``source`` from a MinIO Bucket and store it at ``destination``.
+
+ Does not redownload files which already exist.
+
+ Parameters
+ ----------
+ source : str
+ URL to a MinIO bucket.
+ """
+ destination = Path(openml.config.get_minio_download_path(source))
+ parsed_url = urllib.parse.urlparse(source)
+ if destination.suffix:
+ destination = destination.parent
+ # expect path format: /BUCKET/path/to/file.ext
+ _, bucket, *prefixes, _ = parsed_url.path.split("/")
+ prefix = "/".join(prefixes)
+
+ client = minio.Minio(endpoint=parsed_url.netloc, secure=False)
+
+ for file_object in client.list_objects(bucket, prefix=prefix, recursive=True):
+ if file_object.object_name is None:
+ raise ValueError(f"Object name is None for object {file_object!r}")
+ if file_object.etag is None:
+ raise ValueError(f"Object etag is None for object {file_object!r}")
+
+ marker = destination / file_object.etag
+ if marker.exists():
+ continue
+
+ file_destination = destination / file_object.object_name.rsplit("/", 1)[1]
+ if (file_destination.parent / file_destination.stem).exists():
+ # Marker is missing but archive exists means the server archive changed
+ # force a refresh
+ shutil.rmtree(file_destination.parent / file_destination.stem)
+
+ with contextlib.suppress(FileExistsError):
+ self.download_minio_file(
+ source=source.rsplit("/", 1)[0]
+ + "/"
+ + file_object.object_name.rsplit("/", 1)[1],
+ exists_ok=False,
+ )
+
+ if file_destination.is_file() and file_destination.suffix == ".zip":
+ file_destination.unlink()
+ marker.touch()
diff --git a/openml/_api/resources/base/resources.py b/openml/_api/resources/base/resources.py
index 61b576e278..45b3f910b5 100644
--- a/openml/_api/resources/base/resources.py
+++ b/openml/_api/resources/base/resources.py
@@ -1,7 +1,7 @@
from __future__ import annotations
from abc import abstractmethod
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, Literal
from openml.enums import ResourceType
@@ -10,9 +10,12 @@
if TYPE_CHECKING:
import builtins
from collections.abc import Iterable
+ from pathlib import Path
import pandas as pd
+ from openml.datasets.data_feature import OpenMLDataFeature
+ from openml.datasets.dataset import OpenMLDataset
from openml.estimation_procedures import OpenMLEstimationProcedure
from openml.evaluations import OpenMLEvaluation
from openml.flows.flow import OpenMLFlow
@@ -26,6 +29,117 @@ class DatasetAPI(ResourceAPI):
resource_type: ResourceType = ResourceType.DATASET
+ @abstractmethod
+ def get( # noqa: PLR0913
+ self,
+ dataset_id: int,
+ *,
+ download_data: bool = False,
+ cache_format: Literal["pickle", "feather"] = "pickle",
+ download_qualities: bool = False,
+ download_features_meta_data: bool = False,
+ download_all_files: bool = False,
+ force_refresh_cache: bool = False,
+ ) -> OpenMLDataset: ...
+
+ @abstractmethod
+ def list(
+ self,
+ limit: int,
+ offset: int,
+ *,
+ data_id: list[int] | None = None, # type: ignore
+ **kwargs: Any,
+ ) -> pd.DataFrame: ...
+
+ @abstractmethod
+ def edit( # noqa: PLR0913
+ self,
+ dataset_id: int,
+ *,
+ description: str | None = None,
+ creator: str | None = None,
+ contributor: str | None = None,
+ collection_date: str | None = None,
+ language: str | None = None,
+ default_target_attribute: str | None = None,
+ ignore_attribute: str | list[str] | None = None, # type: ignore
+ citation: str | None = None,
+ row_id_attribute: str | None = None,
+ original_data_url: str | None = None,
+ paper_url: str | None = None,
+ ) -> int: ...
+
+ @abstractmethod
+ def fork(self, dataset_id: int) -> int: ...
+
+ @abstractmethod
+ def status_update(self, dataset_id: int, status: Literal["active", "deactivated"]) -> None: ...
+
+ @abstractmethod
+ def list_qualities(self) -> builtins.list[str]: ...
+
+ @abstractmethod
+ def feature_add_ontology(self, dataset_id: int, index: int, ontology: str) -> bool: ...
+
+ @abstractmethod
+ def feature_remove_ontology(self, dataset_id: int, index: int, ontology: str) -> bool: ...
+
+ @abstractmethod
+ def get_features(
+ self,
+ dataset_id: int,
+ force_refresh_cache: bool = False, # noqa: FBT002
+ ) -> dict[int, OpenMLDataFeature]: ...
+
+ @abstractmethod
+ def get_qualities(
+ self,
+ dataset_id: int,
+ force_refresh_cache: bool = False, # noqa: FBT002
+ ) -> dict[str, float] | None: ...
+
+ @abstractmethod
+ def parse_features_file(
+ self, features_file: Path, features_pickle_file: Path | None = None
+ ) -> dict[int, OpenMLDataFeature]: ...
+
+ @abstractmethod
+ def parse_qualities_file(
+ self, qualities_file: Path, qualities_pickle_file: Path | None = None
+ ) -> dict[str, float]: ...
+
+ @abstractmethod
+ def _download_file(self, url_ext: str) -> Path: ...
+
+ @abstractmethod
+ def download_features_file(self, dataset_id: int) -> Path: ...
+
+ @abstractmethod
+ def download_qualities_file(self, dataset_id: int) -> Path | None: ...
+
+ @abstractmethod
+ def download_dataset_parquet(
+ self,
+ description: dict | OpenMLDataset,
+ download_all_files: bool = False, # noqa: FBT002
+ ) -> Path | None: ...
+
+ @abstractmethod
+ def download_dataset_arff(
+ self,
+ description: dict | OpenMLDataset,
+ ) -> Path: ...
+
+ @abstractmethod
+ def add_topic(self, dataset_id: int, topic: str) -> int: ...
+
+ @abstractmethod
+ def delete_topic(self, dataset_id: int, topic: str) -> int: ...
+
+ @abstractmethod
+ def get_online_dataset_format(self, dataset_id: int) -> str: ...
+
class TaskAPI(ResourceAPI):
"""Abstract API interface for task resources."""
diff --git a/openml/_api/resources/dataset.py b/openml/_api/resources/dataset.py
index 520594df9d..c69f3fbdc8 100644
--- a/openml/_api/resources/dataset.py
+++ b/openml/_api/resources/dataset.py
@@ -1,11 +1,1584 @@
+# ruff: noqa: PLR0913
from __future__ import annotations
+import json
+import logging
+import os
+import pickle
+from collections import OrderedDict
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Literal
+
+import minio
+import pandas as pd
+import urllib3
+import xmltodict
+
+import openml
+from openml.datasets.data_feature import OpenMLDataFeature
+from openml.datasets.dataset import OpenMLDataset
+from openml.exceptions import (
+ OpenMLHashException,
+ OpenMLPrivateDatasetError,
+ OpenMLServerException,
+)
+
from .base import DatasetAPI, ResourceV1API, ResourceV2API
+if TYPE_CHECKING:
+ import builtins
+
+logger = logging.getLogger(__name__)
+
+
+NO_ACCESS_GRANTED_ERRCODE = 112
+
class DatasetV1API(ResourceV1API, DatasetAPI):
"""Version 1 API implementation for dataset resources."""
+ def get(
+ self,
+ dataset_id: int,
+ *,
+ download_data: bool = False,
+ cache_format: Literal["pickle", "feather"] = "pickle",
+ download_qualities: bool = False,
+ download_features_meta_data: bool = False,
+ download_all_files: bool = False,
+ force_refresh_cache: bool = False,
+ ) -> OpenMLDataset:
+ """Download the OpenML dataset representation, optionally also download actual data file.
+
+ Parameters
+ ----------
+ dataset_id : int or str
+ Dataset ID (integer) or dataset name (string) of the dataset to download.
+ download_data : bool (default=False)
+ If True, download the data file.
+ cache_format : str (default='pickle') in {'pickle', 'feather'}
+ Format for caching the dataset - may be feather or pickle
+ Note that the default 'pickle' option may load slower than feather when
+ no.of.rows is very high.
+ download_qualities : bool (default=False)
+ Option to download 'qualities' meta-data with the minimal dataset description.
+ download_features_meta_data : bool (default=False)
+ Option to download 'features' meta-data with the minimal dataset description.
+ download_all_files: bool (default=False)
+ EXPERIMENTAL. Download all files related to the dataset that reside on the server.
+ force_refresh_cache : bool (default=False)
+ Force the cache to delete the cache directory and re-download the data.
+
+ Returns
+ -------
+ dataset : :class:`openml.OpenMLDataset`
+ The downloaded dataset.
+ """
+ path = f"data/{dataset_id}"
+ try:
+ response = self._http.get(path, enable_cache=True, refresh_cache=force_refresh_cache)
+ xml_content = response.text
+ description = xmltodict.parse(xml_content)["oml:data_set_description"]
+
+ features_file = None
+ qualities_file = None
+
+ if download_features_meta_data:
+ features_file = self.download_features_file(dataset_id)
+ if download_qualities:
+ qualities_file = self.download_qualities_file(dataset_id)
+
+ parquet_file = None
+ skip_parquet = (
+ os.environ.get(openml.config.OPENML_SKIP_PARQUET_ENV_VAR, "false").casefold()
+ == "true"
+ )
+ download_parquet = "oml:parquet_url" in description and not skip_parquet
+ if download_parquet and (download_data or download_all_files):
+ try:
+ parquet_file = self.download_dataset_parquet(
+ description,
+ download_all_files=download_all_files,
+ )
+ except urllib3.exceptions.MaxRetryError:
+ parquet_file = None
+
+ arff_file = None
+ if parquet_file is None and download_data:
+ if download_parquet:
+ logger.warning("Failed to download parquet, fallback on ARFF.")
+ arff_file = self.download_dataset_arff(description)
+ except OpenMLServerException as e:
+ # if there was an exception
+ # check if the user had access to the dataset
+ if e.code == NO_ACCESS_GRANTED_ERRCODE:
+ raise OpenMLPrivateDatasetError(e.message) from None
+
+ raise e
+
+ return self._create_dataset_from_xml(
+ description,
+ features_file=features_file,
+ qualities_file=qualities_file,
+ arff_file=arff_file,
+ parquet_file=parquet_file,
+ cache_format=cache_format,
+ )
+
+ def list(
+ self,
+ limit: int,
+ offset: int,
+ *,
+ data_id: builtins.list[int] | None = None,
+ **kwargs: Any,
+ ) -> pd.DataFrame:
+ """
+ Perform api call to return a list of all datasets.
+
+ Parameters
+ ----------
+ The arguments that are lists are separated from the single value
+ ones which are put into the kwargs.
+ display_errors is also separated from the kwargs since it has a
+ default value.
+
+ limit : int
+ The maximum number of datasets to show.
+ offset : int
+ The number of datasets to skip, starting from the first.
+ data_id : list, optional
+
+ kwargs : dict, optional
+ Legal filter operators (keys in the dict):
+ tag, status, limit, offset, data_name, data_version, number_instances,
+ number_features, number_classes, number_missing_values.
+
+ Returns
+ -------
+ datasets : dataframe
+ """
+ api_call = "data/list"
+
+ if limit is not None:
+ api_call += f"/limit/{limit}"
+ if offset is not None:
+ api_call += f"/offset/{offset}"
+
+ if kwargs is not None:
+ for operator, value in kwargs.items():
+ if value is not None:
+ api_call += f"/{operator}/{value}"
+ if data_id is not None:
+ api_call += f"/data_id/{','.join([str(int(i)) for i in data_id])}"
+ xml_string = self._http.get(api_call).text
+ return self._parse_list_xml(xml_string)
+
+ def edit(
+ self,
+ dataset_id: int,
+ *,
+ description: str | None = None,
+ creator: str | None = None,
+ contributor: str | None = None,
+ collection_date: str | None = None,
+ language: str | None = None,
+ default_target_attribute: str | None = None,
+ ignore_attribute: str | builtins.list[str] | None = None,
+ citation: str | None = None,
+ row_id_attribute: str | None = None,
+ original_data_url: str | None = None,
+ paper_url: str | None = None,
+ ) -> int:
+ """Edits an OpenMLDataset.
+
+ Parameters
+ ----------
+ dataset_id : int
+ ID of the dataset.
+ description : str, optional
+ Description of the dataset.
+ creator : str, optional
+ The person who created the dataset.
+ contributor : str, optional
+ People who contributed to the current version of the dataset.
+ collection_date : str, optional
+ The date the data was originally collected, given by the uploader.
+ language : str, optional
+ Language in which the data is represented.
+ Starts with 1 upper case letter, rest lower case, e.g. 'English'.
+ default_target_attribute : str, optional
+ The default target attribute, if it exists.
+ Can have multiple values, comma separated.
+ ignore_attribute : str | list, optional
+ Attributes that should be excluded in modelling,
+ such as identifiers and indexes.
+ citation : str, optional
+ Reference(s) that should be cited when building on this data.
+ row_id_attribute : str, optional
+ The attribute that represents the row-id column, if present in the
+ dataset. If ``data`` is a dataframe and ``row_id_attribute`` is not
+ specified, the index of the dataframe will be used as the
+ ``row_id_attribute``. If the name of the index is ``None``, it will
+ be discarded.
+
+ .. versionadded: 0.8
+ Inference of ``row_id_attribute`` from a dataframe.
+ original_data_url : str, optional
+ For derived data, the url to the original dataset.
+ paper_url : str, optional
+ Link to a paper describing the dataset.
+
+ Returns
+ -------
+ Dataset id
+ """
+ # compose data edit parameters as xml
+ form_data = {"data_id": dataset_id} # type: dict[str, str | int]
+ xml = OrderedDict() # type: 'OrderedDict[str, OrderedDict]'
+ xml["oml:data_edit_parameters"] = OrderedDict()
+ xml["oml:data_edit_parameters"]["@xmlns:oml"] = "http://openml.org/openml"
+ xml["oml:data_edit_parameters"]["oml:description"] = description
+ xml["oml:data_edit_parameters"]["oml:creator"] = creator
+ xml["oml:data_edit_parameters"]["oml:contributor"] = contributor
+ xml["oml:data_edit_parameters"]["oml:collection_date"] = collection_date
+ xml["oml:data_edit_parameters"]["oml:language"] = language
+ xml["oml:data_edit_parameters"]["oml:default_target_attribute"] = default_target_attribute
+ xml["oml:data_edit_parameters"]["oml:row_id_attribute"] = row_id_attribute
+ xml["oml:data_edit_parameters"]["oml:ignore_attribute"] = ignore_attribute
+ xml["oml:data_edit_parameters"]["oml:citation"] = citation
+ xml["oml:data_edit_parameters"]["oml:original_data_url"] = original_data_url
+ xml["oml:data_edit_parameters"]["oml:paper_url"] = paper_url
+
+ # delete None inputs
+ for k in list(xml["oml:data_edit_parameters"]):
+ if not xml["oml:data_edit_parameters"][k]:
+ del xml["oml:data_edit_parameters"][k]
+
+ file_elements = {
+ "edit_parameters": ("description.xml", xmltodict.unparse(xml)),
+ } # type: dict[str, str | tuple[str, str]]
+ result_xml = self._http.post("data/edit", data=form_data, files=file_elements).text
+ result = xmltodict.parse(result_xml)
+ dataset_id = result["oml:data_edit"]["oml:id"]
+ return int(dataset_id)
+
+ def fork(self, dataset_id: int) -> int:
+ """
+ Creates a new dataset version, with the authenticated user as the new owner.
+ The forked dataset can have distinct dataset meta-data,
+ but the actual data itself is shared with the original version.
+
+ Parameters
+ ----------
+ dataset_id : int
+ id of the dataset to be forked
+
+ Returns
+ -------
+ Dataset id of the forked dataset
+
+ """
+ # compose data fork parameters
+ form_data = {"data_id": dataset_id}
+ result_xml = self._http.post("data/fork", data=form_data).text
+ result = xmltodict.parse(result_xml)
+ dataset_id = result["oml:data_fork"]["oml:id"]
+ return int(dataset_id)
+
+ def status_update(self, dataset_id: int, status: Literal["active", "deactivated"]) -> None:
+ """
+ Updates the status of a dataset to either 'active' or 'deactivated'.
+ Please see the OpenML API documentation for a description of the status
+ and all legal status transitions:
+ https://docs.openml.org/concepts/data/#dataset-status
+
+ Parameters
+ ----------
+ dataset_id : int
+ The data id of the dataset
+ status : str,
+ 'active' or 'deactivated'
+ """
+ legal_status = {"active", "deactivated"}
+ if status not in legal_status:
+ raise ValueError(f"Illegal status value. Legal values: {legal_status}")
+
+ data: dict[str, str | int] = {"data_id": dataset_id, "status": status}
+ result_xml = self._http.post("data/status/update", data=data).text
+ result = xmltodict.parse(result_xml)
+ server_data_id = result["oml:data_status_update"]["oml:id"]
+ server_status = result["oml:data_status_update"]["oml:status"]
+ if status != server_status or int(dataset_id) != int(server_data_id):
+ # This should never happen
+ raise ValueError("Data id/status does not match")
+
+ def list_qualities(self) -> builtins.list[str]:
+ """Return list of data qualities available.
+
+ The function performs an API call to retrieve the entire list of
+ data qualities that are computed on the datasets uploaded.
+
+ Returns
+ -------
+ list
+ """
+ api_call = "data/qualities/list"
+ xml_string = self._http.get(api_call).text
+ qualities = xmltodict.parse(xml_string, force_list=("oml:quality"))
+ # Minimalistic check if the XML is useful
+ if "oml:data_qualities_list" not in qualities:
+ raise ValueError('Error in return XML, does not contain "oml:data_qualities_list"')
+
+ if not isinstance(qualities["oml:data_qualities_list"]["oml:quality"], list):
+ raise TypeError('Error in return XML, does not contain "oml:quality" as a list')
+
+ return qualities["oml:data_qualities_list"]["oml:quality"]
+
+ def _create_dataset_from_xml(
+ self,
+ description: dict,
+ *,
+ features_file: Path | None = None,
+ qualities_file: Path | None = None,
+ arff_file: Path | None = None,
+ parquet_file: Path | None = None,
+ cache_format: Literal["pickle", "feather"] = "pickle",
+ ) -> OpenMLDataset:
+ """Create a dataset given a parsed xml dict.
+
+ Parameters
+ ----------
+ description : dict
+ Parsed xml dict representing the dataset description.
+ features_file : Path, optional
+ Path to features file.
+ qualities_file : Path, optional
+ Path to qualities file.
+ arff_file : Path, optional
+ Path to arff file.
+ parquet_file : Path, optional
+ Path to parquet file.
+ cache_format : str (default='pickle') in {'pickle', 'feather'}
+ Format for caching the dataset - may be feather or pickle
+
+ Returns
+ -------
+ OpenMLDataset
+ """
+ return OpenMLDataset(
+ description["oml:name"],
+ description.get("oml:description"),
+ data_format=description["oml:format"],
+ dataset_id=int(description["oml:id"]),
+ version=int(description["oml:version"]),
+ creator=description.get("oml:creator"),
+ contributor=description.get("oml:contributor"),
+ collection_date=description.get("oml:collection_date"),
+ upload_date=description.get("oml:upload_date"),
+ language=description.get("oml:language"),
+ licence=description.get("oml:licence"),
+ url=description["oml:url"],
+ default_target_attribute=description.get("oml:default_target_attribute"),
+ row_id_attribute=description.get("oml:row_id_attribute"),
+ ignore_attribute=description.get("oml:ignore_attribute"),
+ version_label=description.get("oml:version_label"),
+ citation=description.get("oml:citation"),
+ tag=description.get("oml:tag"),
+ cache_format=cache_format,
+ visibility=description.get("oml:visibility"),
+ original_data_url=description.get("oml:original_data_url"),
+ paper_url=description.get("oml:paper_url"),
+ update_comment=description.get("oml:update_comment"),
+ md5_checksum=description.get("oml:md5_checksum"),
+ data_file=str(arff_file) if arff_file is not None else None,
+ features_file=str(features_file) if features_file is not None else None,
+ qualities_file=str(qualities_file) if qualities_file is not None else None,
+ parquet_url=description.get("oml:parquet_url"),
+ parquet_file=str(parquet_file) if parquet_file is not None else None,
+ )
+
+ def feature_add_ontology(self, dataset_id: int, index: int, ontology: str) -> bool:
+ """
+ An ontology describes the concept that are described in a feature. An
+ ontology is defined by an URL where the information is provided. Adds
+ an ontology (URL) to a given dataset feature (defined by a dataset id
+ and index). The dataset has to exists on OpenML and needs to have been
+ processed by the evaluation engine.
+
+ Parameters
+ ----------
+ dataset_id : int
+ id of the dataset to which the feature belongs
+ index : int
+ index of the feature in dataset (0-based)
+ ontology : str
+ URL to ontology (max. 256 characters)
+
+ Returns
+ -------
+ True or throws an OpenML server exception
+ """
+ upload_data: dict[str, int | str] = {
+ "data_id": dataset_id,
+ "index": index,
+ "ontology": ontology,
+ }
+ self._http.post("data/feature/ontology/add", data=upload_data)
+ # an error will be thrown in case the request was unsuccessful
+ return True
+
+ def feature_remove_ontology(self, dataset_id: int, index: int, ontology: str) -> bool:
+ """
+ Removes an existing ontology (URL) from a given dataset feature (defined
+ by a dataset id and index). The dataset has to exists on OpenML and needs
+ to have been processed by the evaluation engine. Ontology needs to be
+ attached to the specific fearure.
+
+ Parameters
+ ----------
+ dataset_id : int
+ id of the dataset to which the feature belongs
+ index : int
+ index of the feature in dataset (0-based)
+ ontology : str
+ URL to ontology (max. 256 characters)
+
+ Returns
+ -------
+ True or throws an OpenML server exception
+ """
+ upload_data: dict[str, int | str] = {
+ "data_id": dataset_id,
+ "index": index,
+ "ontology": ontology,
+ }
+ self._http.post("data/feature/ontology/remove", data=upload_data)
+ # an error will be thrown in case the request was unsuccessful
+ return True
+
+ def get_features(
+ self,
+ dataset_id: int,
+ force_refresh_cache: bool = False, # noqa: FBT002
+ ) -> dict[int, OpenMLDataFeature]:
+ """Get features of a dataset from server.
+
+ Parameters
+ ----------
+ dataset_id : int
+ ID of the dataset.
+ force_refresh_cache : bool (default=False)
+ Refresh the cache and re-download the features file.
+
+ Returns
+ -------
+ dict[int, OpenMLDataFeature]
+ """
+ path = f"data/features/{dataset_id}"
+ xml = self._http.get(path, enable_cache=True, refresh_cache=force_refresh_cache).text
+ return self._parse_features_xml(xml)
+
+ def get_qualities(
+ self,
+ dataset_id: int,
+ force_refresh_cache: bool = False, # noqa: FBT002
+ ) -> dict[str, float] | None:
+ """Get qualities of a dataset from server.
+
+ Parameters
+ ----------
+ dataset_id : int
+ ID of the dataset.
+ force_refresh_cache : bool (default=False)
+ Refresh the cache and re-download the qualities file.
+
+ Returns
+ -------
+ dict[str, float] | None
+ """
+ path = f"data/qualities/{dataset_id!s}"
+ try:
+ xml = self._http.get(path, enable_cache=True, refresh_cache=force_refresh_cache).text
+ except OpenMLServerException as e:
+ if e.code == 362 and str(e) == "No qualities found - None":
+ # quality file stays as None
+ logger.warning(f"No qualities found for dataset {dataset_id}")
+ return None
+
+ raise e
+ return self._parse_qualities_xml(xml)
+
+ def parse_features_file(
+ self, features_file: Path, features_pickle_file: Path | None = None
+ ) -> dict[int, OpenMLDataFeature]:
+ """
+ Parse features file (xml) and store it as a pickle file.
+
+ Parameters
+ ----------
+ features_file : Path
+ Path to features file.
+ features_pickle_file : Path, optional
+ Path to pickle file for features.
+
+ Returns
+ -------
+ features : dict[int, OpenMLDataFeature]
+ """
+ if features_pickle_file is None:
+ features_pickle_file = features_file.with_suffix(features_file.suffix + ".pkl")
+ assert features_file.suffix == ".xml"
+
+ with Path(features_file).open("r", encoding="utf8") as fh:
+ features_xml = fh.read()
+
+ features = self._parse_features_xml(features_xml)
+
+ with features_pickle_file.open("wb") as fh_binary:
+ pickle.dump(features, fh_binary)
+
+ return features
+
+ def parse_qualities_file(
+ self, qualities_file: Path, qualities_pickle_file: Path | None = None
+ ) -> dict[str, float]:
+ """Parse qualities file (xml) and store it as a pickle file.
+
+ Parameters
+ ----------
+ qualities_file : Path
+ Path to qualities file.
+ qualities_pickle_file : Path, optional
+ Path to pickle file for qualities.
+
+ Returns
+ -------
+ qualities : dict[str, float]
+ """
+ if qualities_pickle_file is None:
+ qualities_pickle_file = qualities_file.with_suffix(qualities_file.suffix + ".pkl")
+ assert qualities_file.suffix == ".xml"
+
+ with Path(qualities_file).open("r", encoding="utf8") as fh:
+ qualities_xml = fh.read()
+
+ qualities = self._parse_qualities_xml(qualities_xml)
+
+ with qualities_pickle_file.open("wb") as fh_binary:
+ pickle.dump(qualities, fh_binary)
+
+ return qualities
+
+ def _parse_features_xml(self, features_xml_string: str) -> dict[int, OpenMLDataFeature]:
+ """Parse features xml string.
+
+ Parameters
+ ----------
+ features_xml_string : str
+ Features xml string.
+
+ Returns
+ -------
+ features : dict[int, OpenMLDataFeature]
+ """
+ xml_dict = xmltodict.parse(
+ features_xml_string,
+ force_list=("oml:feature", "oml:nominal_value"),
+ strip_whitespace=False,
+ )
+ features_xml = xml_dict["oml:data_features"]
+
+ features: dict[int, OpenMLDataFeature] = {}
+ for idx, xmlfeature in enumerate(features_xml["oml:feature"]):
+ nr_missing = xmlfeature.get("oml:number_of_missing_values", 0)
+ feature = OpenMLDataFeature(
+ int(xmlfeature["oml:index"]),
+ xmlfeature["oml:name"],
+ xmlfeature["oml:data_type"],
+ xmlfeature.get("oml:nominal_value"),
+ int(nr_missing),
+ xmlfeature.get("oml:ontology"),
+ )
+ if idx != feature.index:
+ raise ValueError("Data features not provided in right order")
+ features[feature.index] = feature
+
+ return features
+
+ def _parse_qualities_xml(self, qualities_xml: str) -> dict[str, float]:
+ """Parse qualities xml string.
+
+ Parameters
+ ----------
+ qualities_xml : str
+ Qualities xml string.
+
+ Returns
+ -------
+ qualities : dict[str, float]
+ """
+ xml_as_dict = xmltodict.parse(qualities_xml, force_list=("oml:quality",))
+ qualities = xml_as_dict["oml:data_qualities"]["oml:quality"]
+ qualities_ = {}
+ for xmlquality in qualities:
+ name = xmlquality["oml:name"]
+ if xmlquality.get("oml:value", None) is None or xmlquality["oml:value"] == "null":
+ value = float("NaN")
+ else:
+ value = float(xmlquality["oml:value"])
+ qualities_[name] = value
+ return qualities_
+
+ def _parse_list_xml(self, xml_string: str) -> pd.DataFrame:
+ """Parse list response xml string.
+
+ Parameters
+ ----------
+ xml_string : str
+ List response xml string.
+
+ Returns
+ -------
+ pd.DataFrame
+ """
+ datasets_dict = xmltodict.parse(xml_string, force_list=("oml:dataset",))
+ # Minimalistic check if the XML is useful
+ assert isinstance(datasets_dict["oml:data"]["oml:dataset"], list), type(
+ datasets_dict["oml:data"],
+ )
+ assert datasets_dict["oml:data"]["@xmlns:oml"] == "http://openml.org/openml", datasets_dict[
+ "oml:data"
+ ]["@xmlns:oml"]
+
+ datasets = {}
+ for dataset_ in datasets_dict["oml:data"]["oml:dataset"]:
+ ignore_attribute = ["oml:file_id", "oml:quality"]
+ dataset = {
+ k.replace("oml:", ""): v for (k, v) in dataset_.items() if k not in ignore_attribute
+ }
+ dataset["did"] = int(dataset["did"])
+ dataset["version"] = int(dataset["version"])
+
+ # The number of qualities can range from 0 to infinity
+ for quality in dataset_.get("oml:quality", []):
+ try:
+ dataset[quality["@name"]] = int(quality["#text"])
+ except ValueError:
+ dataset[quality["@name"]] = float(quality["#text"])
+ datasets[dataset["did"]] = dataset
+
+ return pd.DataFrame.from_dict(datasets, orient="index").astype(
+ {
+ "did": int,
+ "version": int,
+ "status": pd.CategoricalDtype(["active", "deactivated", "in_preparation"]),
+ }
+ )
+
+ def _download_file(self, url_ext: str) -> Path:
+ """Helper method to pass respective handler to downloader.
+
+ Parameters
+ ----------
+ url_ext : str
+ URL extension to download from.
+
+ Returns
+ -------
+ Path
+ """
+ self._http.get(url_ext, enable_cache=True)
+ return self._http.cache_path_from_url(url_ext)
+
+ def download_features_file(self, dataset_id: int) -> Path:
+ """Download features file.
+
+ Parameters
+ ----------
+ dataset_id : int
+ ID of the dataset.
+
+ Returns
+ -------
+ Path
+ """
+ path = f"data/features/{dataset_id}"
+ file = self._download_file(path)
+ self.parse_features_file(file)
+ return file
+
+ def download_qualities_file(self, dataset_id: int) -> Path | None:
+ """Download qualities file.
+
+ Parameters
+ ----------
+ dataset_id : int
+ ID of the dataset.
+
+ Returns
+ -------
+ Path
+ """
+ path = f"data/qualities/{dataset_id}"
+ try:
+ file = self._download_file(path)
+ except OpenMLServerException as e:
+ if e.code == 362 and str(e) == "No qualities found - None":
+ # quality file stays as None
+ logger.warning(f"No qualities found for dataset {dataset_id}")
+ return None
+ raise e
+
+ self.parse_qualities_file(file)
+ return file
+
+ def download_dataset_parquet(
+ self,
+ description: dict | OpenMLDataset,
+ download_all_files: bool = False, # noqa: FBT002
+ ) -> Path | None:
+ """Download dataset parquet file.
+
+ Parameters
+ ----------
+ description : dictionary or OpenMLDataset
+ Either a dataset description as dict or OpenMLDataset.
+ download_all_files: bool, optional (default=False)
+ If `True`, download all data found in the bucket to which the description's
+ ``parquet_url`` points, only download the parquet file otherwise.
+
+ Returns
+ -------
+ Path | None
+ """
+ if isinstance(description, dict):
+ url = str(description.get("oml:parquet_url"))
+ elif isinstance(description, OpenMLDataset):
+ url = str(description._parquet_url)
+ assert description.dataset_id is not None
+ else:
+ raise TypeError("`description` should be either OpenMLDataset or Dict.")
+
+ output_file_path = Path(openml.config.get_minio_download_path(url))
+ if download_all_files:
+ self._minio.download_minio_bucket(source=url)
+
+ if not output_file_path.is_file():
+ try:
+ output_file_path = self._minio.download_minio_file(
+ source=url,
+ )
+ except (
+ FileNotFoundError,
+ urllib3.exceptions.MaxRetryError,
+ minio.error.ServerError,
+ ) as e:
+ logger.warning(f"Could not download file from {url}: {e}")
+ return None
+ return output_file_path
+
+ def download_dataset_arff(
+ self,
+ description: dict | OpenMLDataset,
+ ) -> Path:
+ """Download dataset arff file.
+
+ Parameters
+ ----------
+ description : dictionary or OpenMLDataset
+ Either a dataset description as dict or OpenMLDataset.
+
+ Returns
+ -------
+ output_filename : Path
+ Location of ARFF file.
+ """
+ if isinstance(description, dict):
+ md5_checksum_fixture = description.get("oml:md5_checksum")
+ url = str(description["oml:url"])
+ did = int(description.get("oml:id")) # type: ignore
+ elif isinstance(description, OpenMLDataset):
+ md5_checksum_fixture = description.md5_checksum
+ assert description.url is not None
+ assert description.dataset_id is not None
+
+ url = description.url
+ did = int(description.dataset_id)
+ else:
+ raise TypeError("`description` should be either OpenMLDataset or Dict.")
+
+ try:
+ # save the file in cache and get it's path
+ self._http.get(url, enable_cache=True, md5_checksum=md5_checksum_fixture)
+ output_file_path = self._http.cache_path_from_url(url)
+ except OpenMLHashException as e:
+ additional_info = f" Raised when downloading dataset {did}."
+ e.args = (e.args[0] + additional_info,)
+ raise e
+
+ return output_file_path
+
+ def add_topic(self, dataset_id: int, topic: str) -> int:
+ """
+ Adds a topic to a dataset.
+ This API is not available for all OpenML users and is accessible only by admins.
+
+ Parameters
+ ----------
+ dataset_id : int
+ id of the dataset to be forked
+ topic : str
+ Topic to be added
+
+ Returns
+ -------
+ Dataset id
+ """
+ form_data = {"data_id": dataset_id, "topic": topic} # type: dict[str, str | int]
+ result_xml = self._http.post("data/topicadd", data=form_data).text
+ result = xmltodict.parse(result_xml)
+ dataset_id = result["oml:data_topic"]["oml:id"]
+ return int(dataset_id)
+
+ def delete_topic(self, dataset_id: int, topic: str) -> int:
+ """
+ Removes a topic from a dataset.
+ This API is not available for all OpenML users and is accessible only by admins.
+
+ Parameters
+ ----------
+ dataset_id : int
+ id of the dataset to be forked
+ topic : str
+ Topic to be deleted
+
+ Returns
+ -------
+ Dataset id
+ """
+ form_data = {"data_id": dataset_id, "topic": topic} # type: dict[str, str | int]
+ result_xml = self._http.post("data/topicdelete", data=form_data).text
+ result = xmltodict.parse(result_xml)
+ dataset_id = result["oml:data_topic"]["oml:id"]
+ return int(dataset_id)
+
+ def get_online_dataset_format(self, dataset_id: int) -> str:
+ """Get the dataset format for a given dataset id from the OpenML website.
+
+ Parameters
+ ----------
+ dataset_id : int
+ A dataset id.
+
+ Returns
+ -------
+ str
+ Dataset format.
+ """
+ dataset_xml = self._http.get(f"data/{dataset_id}").text
+ # build a dict from the xml and get the format from the dataset description
+ return xmltodict.parse(dataset_xml)["oml:data_set_description"]["oml:format"].lower() # type: ignore
+
class DatasetV2API(ResourceV2API, DatasetAPI):
"""Version 2 API implementation for dataset resources."""
+
+ def get(
+ self,
+ dataset_id: int,
+ *,
+ download_data: bool = False,
+ cache_format: Literal["pickle", "feather"] = "pickle",
+ download_qualities: bool = False,
+ download_features_meta_data: bool = False,
+ download_all_files: bool = False,
+ force_refresh_cache: bool = False,
+ ) -> OpenMLDataset:
+ """Download the OpenML dataset representation, optionally also download actual data file.
+
+ Parameters
+ ----------
+ dataset_id : int or str
+ Dataset ID (integer) or dataset name (string) of the dataset to download.
+ download_data : bool (default=False)
+ If True, download the data file.
+ cache_format : str (default='pickle') in {'pickle', 'feather'}
+ Format for caching the dataset - may be feather or pickle
+ Note that the default 'pickle' option may load slower than feather when
+ no.of.rows is very high.
+ download_qualities : bool (default=False)
+ Option to download 'qualities' meta-data with the minimal dataset description.
+ download_features_meta_data : bool (default=False)
+ Option to download 'features' meta-data with the minimal dataset description.
+ download_all_files: bool (default=False)
+ EXPERIMENTAL. Download all files related to the dataset that reside on the server.
+ force_refresh_cache : bool (default=False)
+ Force the cache to delete the cache directory and re-download the data.
+
+ Returns
+ -------
+ dataset : :class:`openml.OpenMLDataset`
+ The downloaded dataset.
+ """
+ path = f"datasets/{dataset_id}"
+ try:
+ response = self._http.get(path, enable_cache=True, refresh_cache=force_refresh_cache)
+ json_content = response.json()
+ features_file = None
+ qualities_file = None
+
+ if download_features_meta_data:
+ features_file = self.download_features_file(dataset_id)
+ if download_qualities:
+ qualities_file = self.download_qualities_file(dataset_id)
+
+ parquet_file = None
+ skip_parquet = (
+ os.environ.get(openml.config.OPENML_SKIP_PARQUET_ENV_VAR, "false").casefold()
+ == "true"
+ )
+ download_parquet = "parquet_url" in json_content and not skip_parquet
+ if download_parquet and (download_data or download_all_files):
+ try:
+ parquet_file = self.download_dataset_parquet(
+ json_content,
+ download_all_files=download_all_files,
+ )
+ except urllib3.exceptions.MaxRetryError:
+ parquet_file = None
+
+ arff_file = None
+ if parquet_file is None and download_data:
+ if download_parquet:
+ logger.warning("Failed to download parquet, fallback on ARFF.")
+ arff_file = self.download_dataset_arff(json_content)
+ except OpenMLServerException as e:
+ # if there was an exception
+ # check if the user had access to the dataset
+ if e.code == NO_ACCESS_GRANTED_ERRCODE:
+ raise OpenMLPrivateDatasetError(e.message) from None
+
+ raise e
+
+ return self._create_dataset_from_json(
+ json_content,
+ features_file=features_file,
+ qualities_file=qualities_file,
+ arff_file=arff_file,
+ parquet_file=parquet_file,
+ cache_format=cache_format,
+ )
+
+ def list(
+ self,
+ limit: int,
+ offset: int,
+ *,
+ data_id: builtins.list[int] | None = None,
+ **kwargs: Any,
+ ) -> pd.DataFrame:
+ """
+ Perform api call to return a list of all datasets.
+
+ Parameters
+ ----------
+ The arguments that are lists are separated from the single value
+ ones which are put into the kwargs.
+ display_errors is also separated from the kwargs since it has a
+ default value.
+
+ limit : int
+ The maximum number of datasets to show.
+ offset : int
+ The number of datasets to skip, starting from the first.
+ data_id: list[int], optional
+
+ kwargs : dict, optional
+ Legal filter operators (keys in the dict):
+ tag, status, limit, offset, data_name, data_version, number_instances,
+ number_features, number_classes, number_missing_values.
+
+ Returns
+ -------
+ datasets : dataframe
+ """
+ json: dict[str, Any] = {"pagination": {}}
+
+ if limit is not None:
+ json["pagination"]["limit"] = limit
+ if offset is not None:
+ json["pagination"]["offset"] = offset
+ if data_id is not None:
+ json["data_id"] = data_id
+ if kwargs is not None:
+ for operator, value in kwargs.items():
+ if value is not None:
+ json[operator] = value
+
+ api_call = "datasets/list"
+ datasets_list = self._http.post(path=api_call, json=json, use_api_key=False).json()
+ # Minimalistic check if the JSON is useful
+ assert isinstance(datasets_list, list), type(datasets_list)
+
+ return self._parse_list_json(datasets_list)
+
+ def edit(
+ self,
+ dataset_id: int,
+ *,
+ description: str | None = None,
+ creator: str | None = None,
+ contributor: str | None = None,
+ collection_date: str | None = None,
+ language: str | None = None,
+ default_target_attribute: str | None = None,
+ ignore_attribute: str | builtins.list[str] | None = None,
+ citation: str | None = None,
+ row_id_attribute: str | None = None,
+ original_data_url: str | None = None,
+ paper_url: str | None = None,
+ ) -> int:
+ _ = (
+ dataset_id,
+ description,
+ creator,
+ contributor,
+ collection_date,
+ language,
+ default_target_attribute,
+ ignore_attribute,
+ citation,
+ row_id_attribute,
+ original_data_url,
+ paper_url,
+ ) # unused method arg mypy error
+ raise self._not_supported(method="edit")
+
+ def fork(self, dataset_id: int) -> int:
+ _ = dataset_id # unused method arg mypy error
+ raise self._not_supported(method="fork")
+
+ def status_update(self, dataset_id: int, status: Literal["active", "deactivated"]) -> None:
+ """
+ Updates the status of a dataset to either 'active' or 'deactivated'.
+ Please see the OpenML API documentation for a description of the status
+ and all legal status transitions:
+ https://docs.openml.org/concepts/data/#dataset-status
+
+ Parameters
+ ----------
+ dataset_id : int
+ The data id of the dataset
+ status : str,
+ 'active' or 'deactivated'
+ """
+ legal_status = {"active", "deactivated"}
+ if status not in legal_status:
+ raise ValueError(f"Illegal status value. Legal values: {legal_status}")
+
+ data: dict[str, str | int] = {"dataset_id": dataset_id, "status": status}
+ result = self._http.post(
+ f"datasets/status/update/?api_key={self._http.api_key}", json=data, use_api_key=False
+ ).json()
+ server_data_id = result["dataset_id"]
+ server_status = result["status"]
+ if status != server_status or int(dataset_id) != int(server_data_id):
+ # This should never happen
+ raise ValueError("Data id/status does not match")
+
+ def list_qualities(self) -> builtins.list[str]:
+ """Return list of data qualities available.
+
+ The function performs an API call to retrieve the entire list of
+ data qualities that are computed on the datasets uploaded.
+
+ Returns
+ -------
+ list
+ """
+ api_call = "datasets/qualities/list"
+ qualities = self._http.get(api_call).json()
+ # Minimalistic check if the JSON is useful
+ if "data_qualities_list" not in qualities:
+ raise ValueError('Error in return JSON, does not contain "data_qualities_list"')
+
+ if not isinstance(qualities["data_qualities_list"]["quality"], list):
+ raise TypeError('Error in return JSON, does not contain "quality" as a list')
+
+ return qualities["data_qualities_list"]["quality"]
+
+ def _create_dataset_from_json(
+ self,
+ json_content: dict,
+ *,
+ features_file: Path | None = None,
+ qualities_file: Path | None = None,
+ arff_file: Path | None = None,
+ parquet_file: Path | None = None,
+ cache_format: Literal["pickle", "feather"] = "pickle",
+ ) -> OpenMLDataset:
+ """Create a dataset given a json.
+
+ Parameters
+ ----------
+ json_content : dict
+ Dataset dict/json representation.
+ features_file : Path, optional
+ Path to features file.
+ qualities_file : Path, optional
+ Path to qualities file.
+ arff_file : Path, optional
+ Path to arff file.
+ parquet_file : Path, optional
+ Path to parquet file.
+ cache_format : str (default='pickle') in {'pickle', 'feather'}
+ Format for caching the dataset - may be feather or pickle
+
+ Returns
+ -------
+ OpenMLDataset
+ """
+ return OpenMLDataset(
+ json_content["name"],
+ json_content.get("description"),
+ data_format=json_content["format"],
+ dataset_id=int(json_content["id"]),
+ version=int(json_content["version"]),
+ creator=json_content.get("creator"),
+ contributor=json_content.get("contributor"),
+ collection_date=json_content.get("collection_date"),
+ upload_date=json_content.get("upload_date"),
+ language=json_content.get("language"),
+ licence=json_content.get("licence"),
+ url=json_content["url"],
+ default_target_attribute=json_content.get("default_target_attribute"),
+ row_id_attribute=json_content.get("row_id_attribute"),
+ ignore_attribute=json_content.get("ignore_attribute"),
+ version_label=json_content.get("version_label"),
+ citation=json_content.get("citation"),
+ tag=json_content.get("tag"),
+ cache_format=cache_format,
+ visibility=json_content.get("visibility"),
+ original_data_url=json_content.get("original_data_url"),
+ paper_url=json_content.get("paper_url"),
+ update_comment=json_content.get("update_comment"),
+ md5_checksum=json_content.get("md5_checksum"),
+ data_file=str(arff_file) if arff_file is not None else None,
+ features_file=str(features_file) if features_file is not None else None,
+ qualities_file=str(qualities_file) if qualities_file is not None else None,
+ parquet_url=json_content.get("parquet_url"),
+ parquet_file=str(parquet_file) if parquet_file is not None else None,
+ )
+
+ def feature_add_ontology(self, dataset_id: int, index: int, ontology: str) -> bool:
+ _ = (dataset_id, index, ontology) # unused method arg mypy error
+ raise self._not_supported(method="feature_add_ontology")
+
+ def feature_remove_ontology(self, dataset_id: int, index: int, ontology: str) -> bool:
+ _ = (dataset_id, index, ontology) # unused method arg mypy error
+ raise self._not_supported(method="feature_remove_ontology")
+
+ def get_features(
+ self,
+ dataset_id: int,
+ force_refresh_cache: bool = False, # noqa: FBT002
+ ) -> dict[int, OpenMLDataFeature]:
+ """Get features of a dataset from server.
+
+ Parameters
+ ----------
+ dataset_id : int
+ ID of the dataset.
+ force_refresh_cache : bool (default=False)
+ Refresh the cache and re-download the features file.
+
+ Returns
+ -------
+ dict[int, OpenMLDataFeature]
+ Dictionary mapping feature index to OpenMLDataFeature.
+ """
+ path = f"datasets/features/{dataset_id}"
+ json = self._http.get(path, enable_cache=True, refresh_cache=force_refresh_cache).json()
+
+ return self._parse_features_json(json)
+
+ def get_qualities(
+ self,
+ dataset_id: int,
+ force_refresh_cache: bool = False, # noqa: FBT002
+ ) -> dict[str, float] | None:
+ """Get qualities of a dataset from server.
+
+ Parameters
+ ----------
+ dataset_id : int
+ ID of the dataset.
+ force_refresh_cache : bool (default=False)
+ Refresh the cache and re-download the qualities file.
+
+ Returns
+ -------
+ dict[str, float] | None
+ Dictionary mapping quality name to quality value.
+ """
+ path = f"datasets/qualities/{dataset_id!s}"
+ try:
+ qualities_json = self._http.get(
+ path, enable_cache=True, refresh_cache=force_refresh_cache
+ ).json()
+ except OpenMLServerException as e:
+ if e.code == 362 and str(e) == "No qualities found - None":
+ logger.warning(f"No qualities found for dataset {dataset_id}")
+ return None
+
+ raise e
+
+ qualities = self._parse_qualities_json(qualities_json)
+ if not qualities:
+ logger.warning(f"No qualities found for dataset {dataset_id}")
+ return None
+ return qualities
+
+ def parse_features_file(
+ self, features_file: Path, features_pickle_file: Path | None = None
+ ) -> dict[int, OpenMLDataFeature]:
+ """
+ Parse features file (json) and store it as a pickle file.
+
+ Parameters
+ ----------
+ features_file : Path
+ Path to features file.
+ features_pickle_file : Path, optional
+ Path to pickle file for features.
+
+ Returns
+ -------
+ dict[int, OpenMLDataFeature]
+ """
+ if features_pickle_file is None:
+ features_pickle_file = features_file.with_suffix(features_file.suffix + ".pkl")
+ if features_file.suffix == ".xml":
+ # can fallback to v1 if the file is .xml
+ raise NotImplementedError("Unable to Parse .xml from v1")
+
+ with Path(features_file).open("r", encoding="utf8") as fh:
+ features_json = json.load(fh)
+
+ features = self._parse_features_json(features_json)
+
+ with features_pickle_file.open("wb") as fh_binary:
+ pickle.dump(features, fh_binary)
+
+ return features
+
+ def parse_qualities_file(
+ self, qualities_file: Path, qualities_pickle_file: Path | None = None
+ ) -> dict[str, float]:
+ """Parse qualities file (json) and store it as a pickle file.
+
+ Parameters
+ ----------
+ qualities_file : Path
+ Path to qualities file.
+ qualities_pickle_file : Path, optional
+ Path to pickle file for qualities.
+
+ Returns
+ -------
+ qualities : dict[str, float]
+ """
+ if qualities_pickle_file is None:
+ qualities_pickle_file = qualities_file.with_suffix(qualities_file.suffix + ".pkl")
+ if qualities_file.suffix == ".xml":
+ # can fallback to v1 if the file is .xml
+ raise NotImplementedError("Unable to Parse .xml from v1")
+
+ with Path(qualities_file).open("r", encoding="utf8") as fh:
+ qualities_json = json.load(fh)
+
+ qualities = self._parse_qualities_json(qualities_json)
+
+ with qualities_pickle_file.open("wb") as fh_binary:
+ pickle.dump(qualities, fh_binary)
+
+ return qualities
+
+ def _parse_features_json(
+ self,
+ features_json: dict[str, Any] | builtins.list[dict[str, Any]],
+ ) -> dict[int, OpenMLDataFeature]:
+ """Parse features json.
+
+ Parameters
+ ----------
+ features_json : dict[str, Any] | list[dict[str, Any]]
+ Features JSON payload. Accepts either a list of feature objects or
+ an object containing a ``features`` list.
+
+ Returns
+ -------
+ dict[int, OpenMLDataFeature]
+ """
+ if isinstance(features_json, dict):
+ features_data = features_json.get("features")
+ if not isinstance(features_data, list):
+ raise TypeError('Error in return JSON, does not contain "features" as a list')
+ features_list = features_data
+ else:
+ features_list = features_json
+
+ features: dict[int, OpenMLDataFeature] = {}
+ for idx, jsonfeatures in enumerate(features_list):
+ nr_missing = jsonfeatures.get("number_of_missing_values", 0)
+ feature = OpenMLDataFeature(
+ int(jsonfeatures["index"]),
+ jsonfeatures["name"],
+ jsonfeatures["data_type"],
+ jsonfeatures.get("nominal_values"),
+ int(nr_missing),
+ jsonfeatures.get("ontology"),
+ )
+ if idx != feature.index:
+ raise ValueError("Data features not provided in right order")
+ features[feature.index] = feature
+
+ return features
+
+ def _parse_qualities_json(
+ self,
+ qualities_json: dict[str, Any] | builtins.list[dict[str, Any]],
+ ) -> dict[str, float]:
+ """Parse qualities json.
+
+ Parameters
+ ----------
+ qualities_json : dict[str, Any] | list[dict[str, Any]]
+ Qualities JSON payload. Accepts either a list of quality objects or
+ an object containing a ``qualities`` list.
+
+ Returns
+ -------
+ dict[str, float]
+ """
+ if isinstance(qualities_json, dict):
+ qualities_data = qualities_json.get("qualities")
+ if not isinstance(qualities_data, list):
+ raise TypeError('Error in return JSON, does not contain "qualities" as a list')
+ qualities_list = qualities_data
+ else:
+ qualities_list = qualities_json
+
+ qualities_ = {}
+ for quality in qualities_list:
+ name = quality["name"]
+ if quality.get("value", None) is None or quality["value"] == "null":
+ value = float("NaN")
+ else:
+ value = float(quality["value"])
+ qualities_[name] = value
+ return qualities_
+
+ def _parse_list_json(self, datasets_list: builtins.list) -> pd.DataFrame:
+ """Parse list response json.
+
+ Parameters
+ ----------
+ datasets_list : list
+ List of datasets in json format.
+
+ Returns
+ -------
+ pd.DataFrame
+ """
+ datasets = {}
+ for dataset_ in datasets_list:
+ ignore_attribute = ["file_id", "quality", "md5_checksum"]
+ dataset = {k: v for (k, v) in dataset_.items() if k not in ignore_attribute}
+ dataset["did"] = int(dataset["did"])
+ dataset["version"] = int(dataset["version"])
+
+ # The number of qualities can range from 0 to infinity
+ for quality in dataset_.get("quality", []):
+ try:
+ dataset[quality["name"]] = int(quality["value"])
+ except ValueError:
+ dataset[quality["name"]] = float(quality["value"])
+ datasets[dataset["did"]] = dataset
+
+ return pd.DataFrame.from_dict(datasets, orient="index").astype(
+ {
+ "did": int,
+ "version": int,
+ "status": pd.CategoricalDtype(["active", "deactivated", "in_preparation"]),
+ }
+ )
+
+ def _download_file(self, url_ext: str) -> Path:
+ """Helper method to pass respective handler to downloader.
+
+ Parameters
+ ----------
+ url_ext : str
+ URL extension to download from.
+
+ Returns
+ -------
+ Path
+ """
+ self._http.get(url_ext, enable_cache=True)
+ return self._http.cache_path_from_url(url_ext)
+
+ def download_features_file(self, dataset_id: int) -> Path:
+ """Download features file.
+
+ Parameters
+ ----------
+ dataset_id : int
+ ID of the dataset.
+
+ Returns
+ -------
+ Path
+ """
+ path = f"datasets/features/{dataset_id}"
+ file = self._download_file(path)
+ self.parse_features_file(file)
+ return file
+
+ def download_qualities_file(self, dataset_id: int) -> Path | None:
+ """Download qualities file.
+
+ Parameters
+ ----------
+ dataset_id : int
+ ID of the dataset.
+
+ Returns
+ -------
+ Path
+ """
+ path = f"datasets/qualities/{dataset_id}"
+ try:
+ file = self._download_file(path)
+ except OpenMLServerException as e:
+ if e.code == 362 and str(e) == "No qualities found - None":
+ # quality file stays as None
+ logger.warning(f"No qualities found for dataset {dataset_id}")
+ return None
+ raise e
+
+ if not self.parse_qualities_file(file):
+ logger.warning(f"No qualities found for dataset {dataset_id}")
+ return None
+ return file
+
+ def download_dataset_parquet(
+ self,
+ description: dict | OpenMLDataset,
+ download_all_files: bool = False, # noqa: FBT002
+ ) -> Path | None:
+ """Download dataset parquet file.
+
+ Parameters
+ ----------
+ description : dictionary or OpenMLDataset
+ Either a dataset description as dict or OpenMLDataset.
+ download_all_files: bool, optional (default=False)
+ If `True`, download all data found in the bucket to which the description's
+ ``parquet_url`` points, only download the parquet file otherwise.
+
+ Returns
+ -------
+ Path | None
+ """
+ if isinstance(description, dict):
+ url = str(description.get("parquet_url"))
+ elif isinstance(description, OpenMLDataset):
+ url = str(description._parquet_url)
+ assert description.dataset_id is not None
+ else:
+ raise TypeError("`description` should be either OpenMLDataset or Dict.")
+
+ output_file_path = Path(openml.config.get_minio_download_path(url))
+ if download_all_files:
+ self._minio.download_minio_bucket(source=url)
+
+ if not output_file_path.is_file():
+ try:
+ output_file_path = self._minio.download_minio_file(source=url)
+ except (
+ FileNotFoundError,
+ urllib3.exceptions.MaxRetryError,
+ minio.error.ServerError,
+ ) as e:
+ logger.warning(f"Could not download file from {url}: {e}")
+ return None
+ return output_file_path
+
+ def download_dataset_arff(
+ self,
+ description: dict | OpenMLDataset,
+ ) -> Path:
+ """Download dataset arff file.
+
+ Parameters
+ ----------
+ description : dictionary or OpenMLDataset
+ Either a dataset description as dict or OpenMLDataset.
+
+ Returns
+ -------
+ output_filename : Path
+ Location of ARFF file.
+ """
+ if isinstance(description, dict):
+ url = str(description["url"])
+ did = int(description.get("id")) # type: ignore
+ md5_checksum_fixture = description.get("md5_checksum")
+ elif isinstance(description, OpenMLDataset):
+ assert description.url is not None
+ assert description.dataset_id is not None
+ md5_checksum_fixture = description.md5_checksum
+
+ url = description.url
+ did = int(description.dataset_id)
+ else:
+ raise TypeError("`description` should be either OpenMLDataset or Dict.")
+
+ try:
+ # save the file in cache and get it's path
+ self._http.get(url, enable_cache=True, md5_checksum=md5_checksum_fixture)
+ output_file_path = self._http.cache_path_from_url(url)
+ except OpenMLHashException as e:
+ additional_info = f" Raised when downloading dataset {did}."
+ e.args = (e.args[0] + additional_info,)
+ raise e
+
+ return output_file_path
+
+ def add_topic(self, dataset_id: int, topic: str) -> int:
+ _ = (dataset_id, topic) # unused method arg mypy error
+ raise self._not_supported(method="add_topic")
+
+ def delete_topic(self, dataset_id: int, topic: str) -> int:
+ _ = (dataset_id, topic) # unused method arg mypy error
+ raise self._not_supported(method="delete_topic")
+
+ def get_online_dataset_format(self, dataset_id: int) -> str:
+ """Get the dataset format for a given dataset id from the OpenML website.
+
+ Parameters
+ ----------
+ dataset_id : int
+ A dataset id.
+
+ Returns
+ -------
+ str
+ Dataset format.
+ """
+ dataset_json = self._http.get(f"datasets/{dataset_id}").json()
+ # build a dict from the json and get the format from the dataset description
+ return dataset_json["format"].lower() # type: ignore
diff --git a/openml/_config.py b/openml/_config.py
index 73d154acc7..852cf4ed37 100644
--- a/openml/_config.py
+++ b/openml/_config.py
@@ -499,6 +499,10 @@ def get_cache_directory(self) -> str:
reversed_url_suffix = os.sep.join(url_parts) # noqa: PTH118
return os.path.join(self._root_cache_directory, reversed_url_suffix) # noqa: PTH118
+ def get_minio_download_path(self, url: str) -> str:
+ parsed_url = urlparse(url)
+ return os.path.join(self.get_cache_directory(), "minio", parsed_url.path.lstrip("/")) # noqa: PTH118
+
def set_root_cache_directory(self, root_cache_directory: str | Path) -> None:
"""Set the root cache directory."""
self._root_cache_directory = Path(root_cache_directory)
diff --git a/openml/datasets/dataset.py b/openml/datasets/dataset.py
index c3d4a71c68..b87e5cee0d 100644
--- a/openml/datasets/dataset.py
+++ b/openml/datasets/dataset.py
@@ -16,7 +16,6 @@
import numpy as np
import pandas as pd
import scipy.sparse
-import xmltodict
import openml
from openml.base import OpenMLBase
@@ -384,6 +383,7 @@ def _download_data(self) -> None:
if self._parquet_url is not None and not skip_parquet:
parquet_file = _get_dataset_parquet(self)
self.parquet_file = None if parquet_file is None else str(parquet_file)
+
if self.parquet_file is None:
self.data_file = str(_get_dataset_arff(self))
@@ -828,7 +828,6 @@ def _load_features(self) -> None:
"""Load the features metadata from the server and store it in the dataset object."""
# Delayed Import to avoid circular imports or having to import all of dataset.functions to
# import OpenMLDataset.
- from openml.datasets.functions import _get_dataset_features_file
if self.dataset_id is None:
raise ValueError(
@@ -836,13 +835,11 @@ def _load_features(self) -> None:
"metadata.",
)
- features_file = _get_dataset_features_file(None, self.dataset_id)
- self._features = _read_features(features_file)
+ self._features = openml._backend.dataset.get_features(self.dataset_id)
def _load_qualities(self) -> None:
"""Load qualities information from the server and store it in the dataset object."""
# same reason as above for _load_features
- from openml.datasets.functions import _get_dataset_qualities_file
if self.dataset_id is None:
raise ValueError(
@@ -850,12 +847,12 @@ def _load_qualities(self) -> None:
"metadata.",
)
- qualities_file = _get_dataset_qualities_file(None, self.dataset_id)
+ qualities = openml._backend.dataset.get_qualities(self.dataset_id)
- if qualities_file is None:
+ if qualities is None:
self._no_qualities_found = True
else:
- self._qualities = _read_qualities(qualities_file)
+ self._qualities = qualities
def retrieve_class_labels(self, target_name: str = "class") -> list[str] | None:
"""Reads the datasets arff to determine the class-labels.
@@ -973,6 +970,50 @@ def _parse_publish_response(self, xml_response: dict) -> None:
"""Parse the id from the xml_response and assign it to self."""
self.dataset_id = int(xml_response["oml:upload_data_set"]["oml:id"])
+ def publish(self) -> OpenMLDataset:
+ """Publish this dataset to the OpenML server.
+
+ Returns
+ -------
+ self : OpenMLDataset
+ """
+ file_elements = self._get_file_elements()
+ if "description" not in file_elements:
+ file_elements["description"] = self._to_xml()
+ dataset_id = openml._backend.dataset.publish(path="data", files=file_elements)
+ self.dataset_id = dataset_id
+ return self
+
+ def push_tag(self, tag: str) -> None:
+ """Annotates this dataset with a tag on the server.
+
+ Parameters
+ ----------
+ tag : str
+ Tag to attach to the dataset.
+ """
+ if self.dataset_id is None:
+ raise openml.exceptions.ObjectNotPublishedError(
+ "Cannot tag a dataset that has not been published yet."
+ "Please publish the object first before being able to tag it."
+ )
+ openml._backend.dataset.tag(self.dataset_id, tag)
+
+ def remove_tag(self, tag: str) -> None:
+ """Removes a tag from this dataset on the server.
+
+ Parameters
+ ----------
+ tag : str
+ Tag to remove from the dataset.
+ """
+ if self.dataset_id is None:
+ raise openml.exceptions.ObjectNotPublishedError(
+ "Cannot tag a dataset that has not been published yet."
+ "Please publish the object first before being able to tag it."
+ )
+ openml._backend.dataset.untag(self.dataset_id, tag)
+
def _to_dict(self) -> dict[str, dict]:
"""Creates a dictionary representation of self."""
props = [
@@ -1015,46 +1056,15 @@ def _to_dict(self) -> dict[str, dict]:
}
-def _read_features(features_file: Path) -> dict[int, OpenMLDataFeature]:
+def _read_features(features_file: str | Path) -> dict[int, OpenMLDataFeature]:
+ features_file = Path(features_file)
features_pickle_file = Path(_get_features_pickle_file(str(features_file)))
try:
with features_pickle_file.open("rb") as fh_binary:
return pickle.load(fh_binary) # type: ignore # noqa: S301
except: # noqa: E722
- with Path(features_file).open("r", encoding="utf8") as fh:
- features_xml_string = fh.read()
-
- features = _parse_features_xml(features_xml_string)
-
- with features_pickle_file.open("wb") as fh_binary:
- pickle.dump(features, fh_binary)
-
- return features
-
-
-def _parse_features_xml(features_xml_string: str) -> dict[int, OpenMLDataFeature]:
- xml_dict = xmltodict.parse(
- features_xml_string, force_list=("oml:feature", "oml:nominal_value"), strip_whitespace=False
- )
- features_xml = xml_dict["oml:data_features"]
-
- features: dict[int, OpenMLDataFeature] = {}
- for idx, xmlfeature in enumerate(features_xml["oml:feature"]):
- nr_missing = xmlfeature.get("oml:number_of_missing_values", 0)
- feature = OpenMLDataFeature(
- int(xmlfeature["oml:index"]),
- xmlfeature["oml:name"],
- xmlfeature["oml:data_type"],
- xmlfeature.get("oml:nominal_value"),
- int(nr_missing),
- xmlfeature.get("oml:ontology"),
- )
- if idx != feature.index:
- raise ValueError("Data features not provided in right order")
- features[feature.index] = feature
-
- return features
+ return openml._backend.dataset.parse_features_file(features_file, features_pickle_file)
# TODO(eddiebergman): Should this really exist?
@@ -1076,29 +1086,4 @@ def _read_qualities(qualities_file: str | Path) -> dict[str, float]:
with qualities_pickle_file.open("rb") as fh_binary:
return pickle.load(fh_binary) # type: ignore # noqa: S301
except: # noqa: E722
- with qualities_file.open(encoding="utf8") as fh:
- qualities_xml = fh.read()
-
- qualities = _parse_qualities_xml(qualities_xml)
- with qualities_pickle_file.open("wb") as fh_binary:
- pickle.dump(qualities, fh_binary)
-
- return qualities
-
-
-def _check_qualities(qualities: list[dict[str, str]]) -> dict[str, float]:
- qualities_ = {}
- for xmlquality in qualities:
- name = xmlquality["oml:name"]
- if xmlquality.get("oml:value", None) is None or xmlquality["oml:value"] == "null":
- value = float("NaN")
- else:
- value = float(xmlquality["oml:value"])
- qualities_[name] = value
- return qualities_
-
-
-def _parse_qualities_xml(qualities_xml: str) -> dict[str, float]:
- xml_as_dict = xmltodict.parse(qualities_xml, force_list=("oml:quality",))
- qualities = xml_as_dict["oml:data_qualities"]["oml:quality"]
- return _check_qualities(qualities)
+ return openml._backend.dataset.parse_qualities_file(qualities_file, qualities_pickle_file)
diff --git a/openml/datasets/functions.py b/openml/datasets/functions.py
index 12ad427841..b9ae7bf042 100644
--- a/openml/datasets/functions.py
+++ b/openml/datasets/functions.py
@@ -3,40 +3,26 @@
from __future__ import annotations
import logging
-import os
import warnings
-from collections import OrderedDict
from functools import partial
-from pathlib import Path
-from pyexpat import ExpatError
from typing import TYPE_CHECKING, Any, Literal
import arff
-import minio.error
import numpy as np
import pandas as pd
-import urllib3
-import xmltodict
from scipy.sparse import coo_matrix
import openml
-import openml._api_calls
import openml.utils
-from openml.exceptions import (
- OpenMLHashException,
- OpenMLPrivateDatasetError,
- OpenMLServerError,
- OpenMLServerException,
-)
from openml.utils import (
_create_cache_directory_for_id,
- _get_cache_dir_for_id,
- _remove_cache_dir_for_id,
)
from .dataset import OpenMLDataset
if TYPE_CHECKING:
+ from pathlib import Path
+
import scipy
DATASETS_CACHE_DIR_NAME = "datasets"
@@ -64,17 +50,7 @@ def list_qualities() -> list[str]:
-------
list
"""
- api_call = "data/qualities/list"
- xml_string = openml._api_calls._perform_api_call(api_call, "get")
- qualities = xmltodict.parse(xml_string, force_list=("oml:quality"))
- # Minimalistic check if the XML is useful
- if "oml:data_qualities_list" not in qualities:
- raise ValueError('Error in return XML, does not contain "oml:data_qualities_list"')
-
- if not isinstance(qualities["oml:data_qualities_list"]["oml:quality"], list):
- raise TypeError('Error in return XML, does not contain "oml:quality" as a list')
-
- return qualities["oml:data_qualities_list"]["oml:quality"]
+ return openml._backend.dataset.list_qualities()
def list_datasets(
@@ -128,7 +104,7 @@ def list_datasets(
these are also included as columns.
"""
listing_call = partial(
- _list_datasets,
+ openml._backend.dataset.list,
data_id=data_id,
status=status,
tag=tag,
@@ -146,92 +122,6 @@ def list_datasets(
return pd.concat(batches)
-def _list_datasets(
- limit: int,
- offset: int,
- *,
- data_id: list[int] | None = None,
- **kwargs: Any,
-) -> pd.DataFrame:
- """
- Perform api call to return a list of all datasets.
-
- Parameters
- ----------
- The arguments that are lists are separated from the single value
- ones which are put into the kwargs.
- display_errors is also separated from the kwargs since it has a
- default value.
-
- limit : int
- The maximum number of datasets to show.
- offset : int
- The number of datasets to skip, starting from the first.
- data_id : list, optional
-
- kwargs : dict, optional
- Legal filter operators (keys in the dict):
- tag, status, limit, offset, data_name, data_version, number_instances,
- number_features, number_classes, number_missing_values.
-
- Returns
- -------
- datasets : dataframe
- """
- api_call = "data/list"
-
- if limit is not None:
- api_call += f"/limit/{limit}"
- if offset is not None:
- api_call += f"/offset/{offset}"
-
- if kwargs is not None:
- for operator, value in kwargs.items():
- if value is not None:
- api_call += f"/{operator}/{value}"
- if data_id is not None:
- api_call += f"/data_id/{','.join([str(int(i)) for i in data_id])}"
- return __list_datasets(api_call=api_call)
-
-
-def __list_datasets(api_call: str) -> pd.DataFrame:
- xml_string = openml._api_calls._perform_api_call(api_call, "get")
- datasets_dict = xmltodict.parse(xml_string, force_list=("oml:dataset",))
-
- # Minimalistic check if the XML is useful
- assert isinstance(datasets_dict["oml:data"]["oml:dataset"], list), type(
- datasets_dict["oml:data"],
- )
- assert datasets_dict["oml:data"]["@xmlns:oml"] == "http://openml.org/openml", datasets_dict[
- "oml:data"
- ]["@xmlns:oml"]
-
- datasets = {}
- for dataset_ in datasets_dict["oml:data"]["oml:dataset"]:
- ignore_attribute = ["oml:file_id", "oml:quality"]
- dataset = {
- k.replace("oml:", ""): v for (k, v) in dataset_.items() if k not in ignore_attribute
- }
- dataset["did"] = int(dataset["did"])
- dataset["version"] = int(dataset["version"])
-
- # The number of qualities can range from 0 to infinity
- for quality in dataset_.get("oml:quality", []):
- try:
- dataset[quality["@name"]] = int(quality["#text"])
- except ValueError:
- dataset[quality["@name"]] = float(quality["#text"])
- datasets[dataset["did"]] = dataset
-
- return pd.DataFrame.from_dict(datasets, orient="index").astype(
- {
- "did": int,
- "version": int,
- "status": pd.CategoricalDtype(["active", "deactivated", "in_preparation"]),
- }
- )
-
-
def _expand_parameter(parameter: str | list[str] | None) -> list[str]:
expanded_parameter = []
if isinstance(parameter, str):
@@ -379,7 +269,7 @@ def get_datasets(
@openml.utils.thread_safe_if_oslo_installed
-def get_dataset( # noqa: C901, PLR0912
+def get_dataset(
dataset_id: int | str,
download_data: bool = False, # noqa: FBT002
version: int | None = None,
@@ -482,66 +372,14 @@ def get_dataset( # noqa: C901, PLR0912
f"`dataset_id` must be one of `str` or `int`, not {type(dataset_id)}.",
)
- if force_refresh_cache:
- did_cache_dir = _get_cache_dir_for_id(DATASETS_CACHE_DIR_NAME, dataset_id)
- if did_cache_dir.exists():
- _remove_cache_dir_for_id(DATASETS_CACHE_DIR_NAME, did_cache_dir)
-
- did_cache_dir = _create_cache_directory_for_id(
- DATASETS_CACHE_DIR_NAME,
+ return openml._backend.dataset.get(
dataset_id,
- )
-
- remove_dataset_cache = True
- try:
- description = _get_dataset_description(did_cache_dir, dataset_id)
- features_file = None
- qualities_file = None
-
- if download_features_meta_data:
- features_file = _get_dataset_features_file(did_cache_dir, dataset_id)
- if download_qualities:
- qualities_file = _get_dataset_qualities_file(did_cache_dir, dataset_id)
-
- parquet_file = None
- skip_parquet = (
- os.environ.get(openml.config.OPENML_SKIP_PARQUET_ENV_VAR, "false").casefold() == "true"
- )
- download_parquet = "oml:parquet_url" in description and not skip_parquet
- if download_parquet and (download_data or download_all_files):
- try:
- parquet_file = _get_dataset_parquet(
- description,
- download_all_files=download_all_files,
- )
- except urllib3.exceptions.MaxRetryError:
- parquet_file = None
-
- arff_file = None
- if parquet_file is None and download_data:
- if download_parquet:
- logger.warning("Failed to download parquet, fallback on ARFF.")
- arff_file = _get_dataset_arff(description)
-
- remove_dataset_cache = False
- except OpenMLServerException as e:
- # if there was an exception
- # check if the user had access to the dataset
- if e.code == NO_ACCESS_GRANTED_ERRCODE:
- raise OpenMLPrivateDatasetError(e.message) from None
-
- raise e
- finally:
- if remove_dataset_cache:
- _remove_cache_dir_for_id(DATASETS_CACHE_DIR_NAME, did_cache_dir)
-
- return _create_dataset_from_description(
- description,
- features_file,
- qualities_file,
- arff_file,
- parquet_file,
- cache_format,
+ download_data=download_data,
+ cache_format=cache_format,
+ download_qualities=download_qualities,
+ download_features_meta_data=download_features_meta_data,
+ download_all_files=download_all_files,
+ force_refresh_cache=force_refresh_cache,
)
@@ -821,14 +659,7 @@ def status_update(data_id: int, status: Literal["active", "deactivated"]) -> Non
if status not in legal_status:
raise ValueError(f"Illegal status value. Legal values: {legal_status}")
- data: openml._api_calls.DATA_TYPE = {"data_id": data_id, "status": status}
- result_xml = openml._api_calls._perform_api_call("data/status/update", "post", data=data)
- result = xmltodict.parse(result_xml)
- server_data_id = result["oml:data_status_update"]["oml:id"]
- server_status = result["oml:data_status_update"]["oml:status"]
- if status != server_status or int(data_id) != int(server_data_id):
- # This should never happen
- raise ValueError("Data id/status does not collide")
+ openml._backend.dataset.status_update(dataset_id=data_id, status=status)
def edit_dataset(
@@ -928,40 +759,20 @@ def edit_dataset(
"ignore_attribute or paper_url to edit.",
)
- # compose data edit parameters as xml
- form_data = {"data_id": data_id} # type: openml._api_calls.DATA_TYPE
- xml = OrderedDict() # type: 'OrderedDict[str, OrderedDict]'
- xml["oml:data_edit_parameters"] = OrderedDict()
- xml["oml:data_edit_parameters"]["@xmlns:oml"] = "http://openml.org/openml"
- xml["oml:data_edit_parameters"]["oml:description"] = description
- xml["oml:data_edit_parameters"]["oml:creator"] = creator
- xml["oml:data_edit_parameters"]["oml:contributor"] = contributor
- xml["oml:data_edit_parameters"]["oml:collection_date"] = collection_date
- xml["oml:data_edit_parameters"]["oml:language"] = language
- xml["oml:data_edit_parameters"]["oml:default_target_attribute"] = default_target_attribute
- xml["oml:data_edit_parameters"]["oml:row_id_attribute"] = row_id_attribute
- xml["oml:data_edit_parameters"]["oml:ignore_attribute"] = ignore_attribute
- xml["oml:data_edit_parameters"]["oml:citation"] = citation
- xml["oml:data_edit_parameters"]["oml:original_data_url"] = original_data_url
- xml["oml:data_edit_parameters"]["oml:paper_url"] = paper_url
-
- # delete None inputs
- for k in list(xml["oml:data_edit_parameters"]):
- if not xml["oml:data_edit_parameters"][k]:
- del xml["oml:data_edit_parameters"][k]
-
- file_elements = {
- "edit_parameters": ("description.xml", xmltodict.unparse(xml)),
- } # type: openml._api_calls.FILE_ELEMENTS_TYPE
- result_xml = openml._api_calls._perform_api_call(
- "data/edit",
- "post",
- data=form_data,
- file_elements=file_elements,
+ return openml._backend.dataset.edit(
+ data_id,
+ description=description,
+ creator=creator,
+ contributor=contributor,
+ collection_date=collection_date,
+ language=language,
+ default_target_attribute=default_target_attribute,
+ ignore_attribute=ignore_attribute,
+ citation=citation,
+ row_id_attribute=row_id_attribute,
+ original_data_url=original_data_url,
+ paper_url=paper_url,
)
- result = xmltodict.parse(result_xml)
- data_id = result["oml:data_edit"]["oml:id"]
- return int(data_id)
def fork_dataset(data_id: int) -> int:
@@ -995,12 +806,8 @@ def fork_dataset(data_id: int) -> int:
"""
if not isinstance(data_id, int):
raise TypeError(f"`data_id` must be of type `int`, not {type(data_id)}.")
- # compose data fork parameters
- form_data = {"data_id": data_id} # type: openml._api_calls.DATA_TYPE
- result_xml = openml._api_calls._perform_api_call("data/fork", "post", data=form_data)
- result = xmltodict.parse(result_xml)
- data_id = result["oml:data_fork"]["oml:id"]
- return int(data_id)
+
+ return openml._backend.dataset.fork(dataset_id=data_id)
def data_feature_add_ontology(data_id: int, index: int, ontology: str) -> bool:
@@ -1024,10 +831,7 @@ def data_feature_add_ontology(data_id: int, index: int, ontology: str) -> bool:
-------
True or throws an OpenML server exception
"""
- upload_data: dict[str, int | str] = {"data_id": data_id, "index": index, "ontology": ontology}
- openml._api_calls._perform_api_call("data/feature/ontology/add", "post", data=upload_data)
- # an error will be thrown in case the request was unsuccessful
- return True
+ return openml._backend.dataset.feature_add_ontology(data_id, index, ontology)
def data_feature_remove_ontology(data_id: int, index: int, ontology: str) -> bool:
@@ -1050,10 +854,7 @@ def data_feature_remove_ontology(data_id: int, index: int, ontology: str) -> boo
-------
True or throws an OpenML server exception
"""
- upload_data: dict[str, int | str] = {"data_id": data_id, "index": index, "ontology": ontology}
- openml._api_calls._perform_api_call("data/feature/ontology/remove", "post", data=upload_data)
- # an error will be thrown in case the request was unsuccessful
- return True
+ return openml._backend.dataset.feature_remove_ontology(data_id, index, ontology)
def _topic_add_dataset(data_id: int, topic: str) -> int:
@@ -1074,11 +875,8 @@ def _topic_add_dataset(data_id: int, topic: str) -> int:
"""
if not isinstance(data_id, int):
raise TypeError(f"`data_id` must be of type `int`, not {type(data_id)}.")
- form_data = {"data_id": data_id, "topic": topic} # type: openml._api_calls.DATA_TYPE
- result_xml = openml._api_calls._perform_api_call("data/topicadd", "post", data=form_data)
- result = xmltodict.parse(result_xml)
- data_id = result["oml:data_topic"]["oml:id"]
- return int(data_id)
+
+ return openml._backend.dataset.add_topic(data_id, topic)
def _topic_delete_dataset(data_id: int, topic: str) -> int:
@@ -1099,59 +897,12 @@ def _topic_delete_dataset(data_id: int, topic: str) -> int:
"""
if not isinstance(data_id, int):
raise TypeError(f"`data_id` must be of type `int`, not {type(data_id)}.")
- form_data = {"data_id": data_id, "topic": topic} # type: openml._api_calls.DATA_TYPE
- result_xml = openml._api_calls._perform_api_call("data/topicdelete", "post", data=form_data)
- result = xmltodict.parse(result_xml)
- data_id = result["oml:data_topic"]["oml:id"]
- return int(data_id)
-
-def _get_dataset_description(did_cache_dir: Path, dataset_id: int) -> dict[str, Any]:
- """Get the dataset description as xml dictionary.
-
- This function is NOT thread/multiprocessing safe.
-
- Parameters
- ----------
- did_cache_dir : Path
- Cache subdirectory for this dataset.
-
- dataset_id : int
- Dataset ID
-
- Returns
- -------
- dict
- XML Dataset description parsed to a dict.
-
- """
- # TODO implement a cache for this that invalidates itself after some time
- # This can be saved on disk, but cannot be cached properly, because
- # it contains the information on whether a dataset is active.
- description_file = did_cache_dir / "description.xml"
-
- try:
- with description_file.open(encoding="utf8") as fh:
- dataset_xml = fh.read()
- description = xmltodict.parse(dataset_xml)["oml:data_set_description"]
- except Exception: # noqa: BLE001
- url_extension = f"data/{dataset_id}"
- dataset_xml = openml._api_calls._perform_api_call(url_extension, "get")
- try:
- description = xmltodict.parse(dataset_xml)["oml:data_set_description"]
- except ExpatError as e:
- url = openml._api_calls._create_url_from_endpoint(url_extension)
- raise OpenMLServerError(f"Dataset description XML at '{url}' is malformed.") from e
-
- with description_file.open("w", encoding="utf8") as fh:
- fh.write(dataset_xml)
-
- return description # type: ignore
+ return openml._backend.dataset.delete_topic(data_id, topic)
def _get_dataset_parquet(
description: dict | OpenMLDataset,
- cache_directory: Path | None = None,
download_all_files: bool = False, # noqa: FBT002
) -> Path | None:
"""Return the path to the local parquet file of the dataset. If is not cached, it is downloaded.
@@ -1168,10 +919,6 @@ def _get_dataset_parquet(
description : dictionary or OpenMLDataset
Either a dataset description as dict or OpenMLDataset.
- cache_directory: Path, optional (default=None)
- Folder to store the parquet file in.
- If None, use the default cache directory for the dataset.
-
download_all_files: bool, optional (default=False)
If `True`, download all data found in the bucket to which the description's
``parquet_url`` points, only download the parquet file otherwise.
@@ -1181,47 +928,11 @@ def _get_dataset_parquet(
output_filename : Path, optional
Location of the Parquet file if successfully downloaded, None otherwise.
"""
- if isinstance(description, dict):
- url = str(description.get("oml:parquet_url"))
- did = int(description.get("oml:id")) # type: ignore
- elif isinstance(description, OpenMLDataset):
- url = str(description._parquet_url)
- assert description.dataset_id is not None
-
- did = int(description.dataset_id)
- else:
- raise TypeError("`description` should be either OpenMLDataset or Dict.")
-
- if cache_directory is None:
- cache_directory = _create_cache_directory_for_id(DATASETS_CACHE_DIR_NAME, did)
-
- output_file_path = cache_directory / f"dataset_{did}.pq"
-
- old_file_path = cache_directory / "dataset.pq"
- if old_file_path.is_file():
- old_file_path.rename(output_file_path)
-
- # The call below skips files already on disk, so avoids downloading the parquet file twice.
- # To force the old behavior of always downloading everything, use `force_refresh_cache`
- # of `get_dataset`
- if download_all_files:
- openml._api_calls._download_minio_bucket(source=url, destination=cache_directory)
-
- if not output_file_path.is_file():
- try:
- openml._api_calls._download_minio_file(
- source=url,
- destination=output_file_path,
- )
- except (FileNotFoundError, urllib3.exceptions.MaxRetryError, minio.error.ServerError) as e:
- logger.warning(f"Could not download file from {url}: {e}")
- return None
- return output_file_path
+ return openml._backend.dataset.download_dataset_parquet(description, download_all_files)
def _get_dataset_arff(
description: dict | OpenMLDataset,
- cache_directory: Path | None = None,
) -> Path:
"""Return the path to the local arff file of the dataset. If is not cached, it is downloaded.
@@ -1236,56 +947,17 @@ def _get_dataset_arff(
description : dictionary or OpenMLDataset
Either a dataset description as dict or OpenMLDataset.
- cache_directory: Path, optional (default=None)
- Folder to store the arff file in.
- If None, use the default cache directory for the dataset.
-
Returns
-------
output_filename : Path
Location of ARFF file.
"""
- if isinstance(description, dict):
- md5_checksum_fixture = description.get("oml:md5_checksum")
- url = str(description["oml:url"])
- did = int(description.get("oml:id")) # type: ignore
- elif isinstance(description, OpenMLDataset):
- md5_checksum_fixture = description.md5_checksum
- assert description.url is not None
- assert description.dataset_id is not None
-
- url = description.url
- did = int(description.dataset_id)
- else:
- raise TypeError("`description` should be either OpenMLDataset or Dict.")
-
- save_cache_directory = (
- _create_cache_directory_for_id(DATASETS_CACHE_DIR_NAME, did)
- if cache_directory is None
- else Path(cache_directory)
- )
- output_file_path = save_cache_directory / "dataset.arff"
-
- try:
- openml._api_calls._download_text_file(
- source=url,
- output_path=output_file_path,
- md5_checksum=md5_checksum_fixture,
- )
- except OpenMLHashException as e:
- additional_info = f" Raised when downloading dataset {did}."
- e.args = (e.args[0] + additional_info,)
- raise e
-
- return output_file_path
-
+ return openml._backend.dataset.download_dataset_arff(description)
-def _get_features_xml(dataset_id: int) -> str:
- url_extension = f"data/features/{dataset_id}"
- return openml._api_calls._perform_api_call(url_extension, "get")
-
-def _get_dataset_features_file(did_cache_dir: str | Path | None, dataset_id: int) -> Path:
+def _get_dataset_features_file(
+ dataset_id: int,
+) -> Path:
"""API call to load dataset features. Loads from cache or downloads them.
Features are feature descriptions for each column.
@@ -1295,9 +967,6 @@ def _get_dataset_features_file(did_cache_dir: str | Path | None, dataset_id: int
Parameters
----------
- did_cache_dir : str or None
- Cache subdirectory for this dataset
-
dataset_id : int
Dataset ID
@@ -1306,28 +975,10 @@ def _get_dataset_features_file(did_cache_dir: str | Path | None, dataset_id: int
Path
Path of the cached dataset feature file
"""
- did_cache_dir = Path(did_cache_dir) if did_cache_dir is not None else None
- if did_cache_dir is None:
- did_cache_dir = _create_cache_directory_for_id(DATASETS_CACHE_DIR_NAME, dataset_id)
-
- features_file = did_cache_dir / "features.xml"
-
- # Dataset features aren't subject to change...
- if not features_file.is_file():
- features_xml = _get_features_xml(dataset_id)
- with features_file.open("w", encoding="utf8") as fh:
- fh.write(features_xml)
-
- return features_file
-
-
-def _get_qualities_xml(dataset_id: int) -> str:
- url_extension = f"data/qualities/{dataset_id!s}"
- return openml._api_calls._perform_api_call(url_extension, "get")
+ return openml._backend.dataset.download_features_file(dataset_id)
def _get_dataset_qualities_file(
- did_cache_dir: str | Path | None,
dataset_id: int,
) -> Path | None:
"""Get the path for the dataset qualities file, or None if no qualities exist.
@@ -1339,9 +990,6 @@ def _get_dataset_qualities_file(
Parameters
----------
- did_cache_dir : str or None
- Cache subdirectory for this dataset
-
dataset_id : int
Dataset ID
@@ -1350,116 +998,7 @@ def _get_dataset_qualities_file(
str
Path of the cached qualities file
"""
- save_did_cache_dir = (
- _create_cache_directory_for_id(DATASETS_CACHE_DIR_NAME, dataset_id)
- if did_cache_dir is None
- else Path(did_cache_dir)
- )
-
- # Dataset qualities are subject to change and must be fetched every time
- qualities_file = save_did_cache_dir / "qualities.xml"
- try:
- with qualities_file.open(encoding="utf8") as fh:
- qualities_xml = fh.read()
- except OSError:
- try:
- qualities_xml = _get_qualities_xml(dataset_id)
- with qualities_file.open("w", encoding="utf8") as fh:
- fh.write(qualities_xml)
- except OpenMLServerException as e:
- if e.code == 362 and str(e) == "No qualities found - None":
- # quality file stays as None
- logger.warning(f"No qualities found for dataset {dataset_id}")
- return None
-
- raise e
-
- return qualities_file
-
-
-def _create_dataset_from_description(
- description: dict[str, str],
- features_file: Path | None = None,
- qualities_file: Path | None = None,
- arff_file: Path | None = None,
- parquet_file: Path | None = None,
- cache_format: Literal["pickle", "feather"] = "pickle",
-) -> OpenMLDataset:
- """Create a dataset object from a description dict.
-
- Parameters
- ----------
- description : dict
- Description of a dataset in xml dict.
- features_file : str
- Path of the dataset features as xml file.
- qualities_file : list
- Path of the dataset qualities as xml file.
- arff_file : string, optional
- Path of dataset ARFF file.
- parquet_file : string, optional
- Path of dataset Parquet file.
- cache_format: string, optional
- Caching option for datasets (feather/pickle)
-
- Returns
- -------
- dataset : dataset object
- Dataset object from dict and ARFF.
- """
- return OpenMLDataset(
- description["oml:name"],
- description.get("oml:description"),
- data_format=description["oml:format"], # type: ignore
- dataset_id=int(description["oml:id"]),
- version=int(description["oml:version"]),
- creator=description.get("oml:creator"),
- contributor=description.get("oml:contributor"),
- collection_date=description.get("oml:collection_date"),
- upload_date=description.get("oml:upload_date"),
- language=description.get("oml:language"),
- licence=description.get("oml:licence"),
- url=description["oml:url"],
- default_target_attribute=description.get("oml:default_target_attribute"),
- row_id_attribute=description.get("oml:row_id_attribute"),
- ignore_attribute=description.get("oml:ignore_attribute"),
- version_label=description.get("oml:version_label"),
- citation=description.get("oml:citation"),
- tag=description.get("oml:tag"),
- visibility=description.get("oml:visibility"),
- original_data_url=description.get("oml:original_data_url"),
- paper_url=description.get("oml:paper_url"),
- update_comment=description.get("oml:update_comment"),
- md5_checksum=description.get("oml:md5_checksum"),
- data_file=str(arff_file) if arff_file is not None else None,
- cache_format=cache_format,
- features_file=str(features_file) if features_file is not None else None,
- qualities_file=str(qualities_file) if qualities_file is not None else None,
- parquet_url=description.get("oml:parquet_url"),
- parquet_file=str(parquet_file) if parquet_file is not None else None,
- )
-
-
-def _get_online_dataset_arff(dataset_id: int) -> str | None:
- """Download the ARFF file for a given dataset id
- from the OpenML website.
-
- Parameters
- ----------
- dataset_id : int
- A dataset id.
-
- Returns
- -------
- str or None
- A string representation of an ARFF file. Or None if file already exists.
- """
- dataset_xml = openml._api_calls._perform_api_call(f"data/{dataset_id}", "get")
- # build a dict from the xml.
- # use the url from the dataset description and return the ARFF string
- return openml._api_calls._download_text_file(
- xmltodict.parse(dataset_xml)["oml:data_set_description"]["oml:url"],
- )
+ return openml._backend.dataset.download_qualities_file(dataset_id)
def _get_online_dataset_format(dataset_id: int) -> str:
@@ -1475,9 +1014,7 @@ def _get_online_dataset_format(dataset_id: int) -> str:
str
Dataset format.
"""
- dataset_xml = openml._api_calls._perform_api_call(f"data/{dataset_id}", "get")
- # build a dict from the xml and get the format from the dataset description
- return xmltodict.parse(dataset_xml)["oml:data_set_description"]["oml:format"].lower() # type: ignore
+ return openml._backend.dataset.get_online_dataset_format(dataset_id)
def delete_dataset(dataset_id: int) -> bool:
@@ -1496,4 +1033,4 @@ def delete_dataset(dataset_id: int) -> bool:
bool
True if the deletion was successful. False otherwise.
"""
- return openml.utils._delete_entity("data", dataset_id)
+ return openml._backend.dataset.delete(dataset_id)
diff --git a/openml/testing.py b/openml/testing.py
index 7f5230efee..0489509347 100644
--- a/openml/testing.py
+++ b/openml/testing.py
@@ -15,6 +15,8 @@
import requests
import openml
+from openml._api import HTTPClient, MinIOClient
+from openml.enums import APIVersion
from openml.exceptions import OpenMLServerException
from openml.tasks import TaskType
@@ -53,6 +55,9 @@ class TestBase(unittest.TestCase):
logger = logging.getLogger("unit_tests_published_entities")
logger.setLevel(logging.DEBUG)
+ http_client: HTTPClient = HTTPClient(api_version=APIVersion.V1)
+ minio_client = MinIOClient()
+
def setUp(self, n_levels: int = 1, tmpdir_suffix: str = "") -> None:
"""Setup variables and temporary directories.
diff --git a/pyproject.toml b/pyproject.toml
index c67af74e22..02b31ef2a9 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -350,6 +350,7 @@ module = [
"examples.Advanced.create_upload_tutorial",
# Files that need to be fixed after removing the entire tests directory from skipping mypy checks
+ "tests.test_api.test_datasets",
"tests.test_api.test_estimation_procedure",
"tests.test_api.test_evaluation",
"tests.test_api.test_evaluation_measure",
diff --git a/tests/conftest.py b/tests/conftest.py
index aabe8b2459..bf0941ec80 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -202,21 +202,7 @@ def _expected_static_cache_state(root_dir: Path) -> list[Path]:
_c_root_dir = root_dir / "org" / "openml" / "test"
res_paths = [root_dir, _c_root_dir]
- for _d in ["datasets", "tasks"]:
- res_paths.append(_c_root_dir / _d)
-
- for _id in ["-1", "2"]:
- tmp_p = _c_root_dir / "datasets" / _id
- res_paths.extend(
- [
- tmp_p / "dataset.arff",
- tmp_p / "features.xml",
- tmp_p / "qualities.xml",
- tmp_p / "description.xml",
- ]
- )
-
- res_paths.append(_c_root_dir / "datasets" / "30" / "dataset_30.pq")
+ res_paths.append(_c_root_dir / "tasks")
for _id in ["1", "3", "1882"]:
tmp_p = _c_root_dir / "tasks" / _id
@@ -238,6 +224,36 @@ def _expected_static_cache_state(root_dir: Path) -> list[Path]:
]
)
+ res_paths.extend(
+ [
+ _c_root_dir / "api" / "v1" / "xml" / "data",
+ _c_root_dir / "api" / "v1" / "xml" / "data" / "qualities",
+ _c_root_dir / "api" / "v1" / "xml" / "data" / "features",
+ _c_root_dir / "api" / "v1" / "xml" / "data" / "-1",
+ _c_root_dir / "api" / "v1" / "xml" / "data" / "-1" / "body.xml",
+ _c_root_dir / "api" / "v1" / "xml" / "data" / "qualities" / "-1",
+ _c_root_dir / "api" / "v1" / "xml" / "data" / "qualities" / "-1" / "body.xml",
+ _c_root_dir / "api" / "v1" / "xml" / "data" / "features" / "-1",
+ _c_root_dir / "api" / "v1" / "xml" / "data" / "features" / "-1" / "body.xml",
+ _c_root_dir
+ / "labs"
+ / "beta"
+ / "Projects"
+ / "autoweka"
+ / "datasets"
+ / "dexter.zip"
+ / "body.arff",
+ _c_root_dir / "api" / "v1" / "xml" / "data" / "2",
+ _c_root_dir / "api" / "v1" / "xml" / "data" / "2" / "body.xml",
+ _c_root_dir / "api" / "v1" / "xml" / "data" / "qualities" / "2",
+ _c_root_dir / "api" / "v1" / "xml" / "data" / "qualities" / "2" / "body.xml",
+ _c_root_dir / "api" / "v1" / "xml" / "data" / "features" / "2",
+ _c_root_dir / "api" / "v1" / "xml" / "data" / "features" / "2" / "body.xml",
+ _c_root_dir / "data" / "download" / "1666876" / "phpFsFYVN" / "body.arff",
+ _c_root_dir / "minio" / "dataset30" / "dataset_30.pq",
+ ]
+ )
+
return res_paths
diff --git a/tests/files/org/openml/test/datasets/-1/description.xml b/tests/files/org/openml/test/api/v1/xml/data/-1/body.xml
similarity index 100%
rename from tests/files/org/openml/test/datasets/-1/description.xml
rename to tests/files/org/openml/test/api/v1/xml/data/-1/body.xml
diff --git a/tests/files/org/openml/test/datasets/2/description.xml b/tests/files/org/openml/test/api/v1/xml/data/2/body.xml
similarity index 100%
rename from tests/files/org/openml/test/datasets/2/description.xml
rename to tests/files/org/openml/test/api/v1/xml/data/2/body.xml
diff --git a/tests/files/org/openml/test/datasets/-1/features.xml b/tests/files/org/openml/test/api/v1/xml/data/features/-1/body.xml
similarity index 100%
rename from tests/files/org/openml/test/datasets/-1/features.xml
rename to tests/files/org/openml/test/api/v1/xml/data/features/-1/body.xml
diff --git a/tests/files/org/openml/test/datasets/2/features.xml b/tests/files/org/openml/test/api/v1/xml/data/features/2/body.xml
similarity index 100%
rename from tests/files/org/openml/test/datasets/2/features.xml
rename to tests/files/org/openml/test/api/v1/xml/data/features/2/body.xml
diff --git a/tests/files/org/openml/test/datasets/-1/qualities.xml b/tests/files/org/openml/test/api/v1/xml/data/qualities/-1/body.xml
similarity index 100%
rename from tests/files/org/openml/test/datasets/-1/qualities.xml
rename to tests/files/org/openml/test/api/v1/xml/data/qualities/-1/body.xml
diff --git a/tests/files/org/openml/test/datasets/2/qualities.xml b/tests/files/org/openml/test/api/v1/xml/data/qualities/2/body.xml
similarity index 100%
rename from tests/files/org/openml/test/datasets/2/qualities.xml
rename to tests/files/org/openml/test/api/v1/xml/data/qualities/2/body.xml
diff --git a/tests/files/org/openml/test/datasets/2/dataset.arff b/tests/files/org/openml/test/data/download/1666876/phpFsFYVN/body.arff
similarity index 100%
rename from tests/files/org/openml/test/datasets/2/dataset.arff
rename to tests/files/org/openml/test/data/download/1666876/phpFsFYVN/body.arff
diff --git a/tests/files/org/openml/test/datasets/30/dataset_30.pq b/tests/files/org/openml/test/datasets/30/dataset_30.pq
deleted file mode 100644
index b355972819..0000000000
Binary files a/tests/files/org/openml/test/datasets/30/dataset_30.pq and /dev/null differ
diff --git a/tests/files/org/openml/test/datasets/-1/dataset.arff b/tests/files/org/openml/test/labs/beta/Projects/autoweka/datasets/dexter.zip/body.arff
similarity index 100%
rename from tests/files/org/openml/test/datasets/-1/dataset.arff
rename to tests/files/org/openml/test/labs/beta/Projects/autoweka/datasets/dexter.zip/body.arff
diff --git a/tests/files/org/openml/test/minio/dataset30/dataset_30.pq b/tests/files/org/openml/test/minio/dataset30/dataset_30.pq
new file mode 100644
index 0000000000..edcd02d7a5
Binary files /dev/null and b/tests/files/org/openml/test/minio/dataset30/dataset_30.pq differ
diff --git a/tests/test_api/test_datasets.py b/tests/test_api/test_datasets.py
new file mode 100644
index 0000000000..b809a734ca
--- /dev/null
+++ b/tests/test_api/test_datasets.py
@@ -0,0 +1,437 @@
+# License: BSD 3-Clause
+from __future__ import annotations
+
+import os
+import time
+import uuid
+from pathlib import Path
+from unittest.mock import patch
+
+import pandas as pd
+import pytest
+from requests import Response, Session
+
+import openml
+from openml import OpenMLDataset
+from openml._api import DatasetV1API, DatasetV2API
+from openml.enums import APIVersion
+from openml.exceptions import OpenMLNotSupportedError
+from openml.testing import TestBase
+
+
+@pytest.fixture
+def dataset_v1(http_client_v1, minio_client) -> DatasetV1API:
+ return DatasetV1API(http=http_client_v1, minio=minio_client)
+
+
+@pytest.fixture
+def dataset_v2(http_client_v2, minio_client) -> DatasetV2API:
+ return DatasetV2API(http=http_client_v2, minio=minio_client)
+
+
+def _wait_for_dataset_being_processed(dataset, did, status="active", n_tries=10, wait_time=10):
+ for attempt in range(n_tries):
+ try:
+ result = dataset.list(limit=1, offset=0, data_id=[did], status="all")
+ result = result.to_dict(orient="index")
+ TestBase.logger.warning(f"Dataset {did} status: {result[did]['status']}")
+ if result[did]["status"] == status:
+ return
+ except Exception as e: # noqa: BLE001
+ TestBase.logger.warning(f"Dataset {did} status check failed: {e}")
+
+ if attempt < n_tries - 1:
+ time.sleep(wait_time)
+ raise TimeoutError(f"Dataset did not become {status} within given time")
+
+
+def _status_update_check(dataset, dataset_id, status):
+ dataset.status_update(dataset_id, status)
+ _wait_for_dataset_being_processed(dataset, dataset_id, status)
+
+
+@pytest.mark.test_server
+def test_v1_get(dataset_v1):
+ dataset_id = 2
+ output = dataset_v1.get(dataset_id)
+ assert output.dataset_id == dataset_id
+
+
+@pytest.mark.test_server
+def test_v1_list(dataset_v1):
+ output = dataset_v1.list(limit=2, offset=0, status="active")
+ assert not output.empty
+ assert output.shape[0] == 2
+ assert (output["status"] == "active").all()
+
+
+@pytest.mark.test_server
+def test_v1_download_arff(dataset_v1):
+ from openml.datasets.functions import _get_dataset_arff
+
+ output = dataset_v1.get(2)
+ file = _get_dataset_arff(output)
+ assert file.exists()
+
+
+@pytest.mark.test_server
+def test_v1_download_parquet(dataset_v1):
+ from openml.datasets.functions import _get_dataset_parquet
+
+ output = dataset_v1.get(2)
+ file = _get_dataset_parquet(output)
+ assert file.exists()
+
+
+@pytest.mark.test_server
+def test_v1_download_arff_from_get(dataset_v1):
+ output = dataset_v1.get(2, download_data=True)
+ data = output.data_file is not None and Path(output.data_file).exists()
+ parquet = output.parquet_file is not None and Path(output.parquet_file).exists()
+ assert data or parquet
+
+
+@pytest.mark.test_server
+def test_v1_download_qualities_from_get(dataset_v1):
+ output = dataset_v1.get(2, download_qualities=True)
+
+ assert output._qualities is not None
+
+
+@pytest.mark.test_server
+def test_v1_download_features_from_get(dataset_v1):
+ output = dataset_v1.get(2, download_features_meta_data=True)
+
+ assert output._features is not None
+
+
+@pytest.mark.test_server
+def test_v1_get_features(dataset_v1):
+ output = dataset_v1.get_features(2)
+
+ assert isinstance(output, dict)
+ assert len(output.keys()) >= 35
+
+
+@pytest.mark.test_server
+def test_v1_get_qualities(dataset_v1):
+ output = dataset_v1.get_qualities(2)
+
+ assert isinstance(output, dict)
+ assert len(output.keys()) >= 100
+
+
+@pytest.mark.skipif(
+ not os.environ.get(openml.config.OPENML_TEST_SERVER_ADMIN_KEY_ENV_VAR),
+ reason="Test requires admin key. Set OPENML_TEST_SERVER_ADMIN_KEY environment variable.",
+)
+@pytest.mark.test_server
+def test_v1_status_update(dataset_v1):
+ openml.config.apikey = TestBase.admin_key
+ new_dataset = OpenMLDataset(
+ f"TEST-{time.time()!s}-UploadTestWithURL",
+ "test",
+ "ARFF",
+ version=1,
+ url="https://www.openml.org/data/download/61/dataset_61_iris.arff",
+ )
+ new_dataset.publish()
+ _status_update_check(dataset_v1, new_dataset.dataset_id, "deactivated")
+ _status_update_check(dataset_v1, new_dataset.dataset_id, "active")
+ dataset_v1.delete(new_dataset.dataset_id)
+
+
+@pytest.mark.test_server
+def test_v1_edit(dataset_v1):
+ did = 2
+ result = dataset_v1.fork(did)
+ TestBase._mark_entity_for_removal("data", result)
+ _wait_for_dataset_being_processed(dataset_v1, result, "active")
+
+ edited_did = dataset_v1.edit(
+ result, description="Forked dataset", default_target_attribute="shape"
+ )
+ assert result == edited_did
+ n_tries = 10
+ # we need to wait for the edit to be reflected on the server
+ for i in range(n_tries):
+ edited_dataset = dataset_v1.get(result, force_refresh_cache=True)
+ try:
+ assert edited_dataset.default_target_attribute == "shape", edited_dataset
+ assert edited_dataset.description == "Forked dataset", edited_dataset
+ break
+ except AssertionError as e:
+ if i == n_tries - 1:
+ raise e
+ time.sleep(10)
+
+
+@pytest.mark.test_server
+def test_v1_fork(dataset_v1):
+ did = 2
+ result = dataset_v1.fork(did)
+ TestBase._mark_entity_for_removal("data", result)
+ assert did != result
+ _wait_for_dataset_being_processed(dataset_v1, result, "active", n_tries=30)
+
+ listing = dataset_v1.list(limit=2, offset=0, data_id=[did, result], status="all")
+
+ assert listing.iloc[0]["name"] == listing.iloc[1]["name"]
+
+
+@pytest.mark.test_server
+def test_v1_list_qualities(dataset_v1):
+ output = dataset_v1.list_qualities()
+ assert len(output) >= 100
+ assert isinstance(output[0], str)
+
+
+@pytest.mark.test_server
+def test_v1_feature_add_remove_ontology(dataset_v1):
+ did = 11
+ fid = 0
+ ontology = "https://www.openml.org/unittest/" + str(time.time())
+ output = dataset_v1.feature_add_ontology(did, fid, ontology)
+ assert output
+ assert ontology in dataset_v1.get_features(did, force_refresh_cache=True)[fid].ontologies
+ output = dataset_v1.feature_remove_ontology(did, fid, ontology)
+ assert output
+ ontologies = dataset_v1.get_features(did, force_refresh_cache=True)[fid].ontologies
+ assert ontologies is None or ontology not in ontologies
+
+
+@pytest.mark.parametrize(
+ ("method", "endpoint"),
+ [("add_topic", "data/topicadd"), ("delete_topic", "data/topicdelete")],
+)
+def test_v1_add_delete_topic_mocked(dataset_v1, test_apikey_v1, method, endpoint):
+ dataset_id = 31
+ topic = f"test_topic_{uuid.uuid4().hex}"
+
+ with patch.object(Session, "request") as mock_request:
+ mock_request.return_value = Response()
+ mock_request.return_value.status_code = 200
+ mock_request.return_value._content = (
+ b'\n'
+ b" 31\n"
+ b"\n"
+ )
+
+ result = getattr(dataset_v1, method)(dataset_id, topic)
+
+ assert result == dataset_id
+ mock_request.assert_called_once_with(
+ method="POST",
+ url=openml.config.server + endpoint,
+ params={},
+ data={"data_id": dataset_id, "topic": topic, "api_key": test_apikey_v1},
+ headers=openml.config._HEADERS,
+ files=None,
+ )
+
+
+def test_v1_delete_mocked(dataset_v1, test_apikey_v1):
+ dataset_id = 456
+
+ with patch.object(Session, "request") as mock_request:
+ mock_request.return_value = Response()
+ mock_request.return_value.status_code = 200
+ mock_request.return_value._content = (
+ b'\n'
+ b" 456\n"
+ b"\n"
+ )
+
+ result = dataset_v1.delete(dataset_id)
+
+ assert result is True
+ mock_request.assert_called_once_with(
+ method="DELETE",
+ url=openml.config.server + f"data/{dataset_id}",
+ params={"api_key": test_apikey_v1},
+ data={},
+ headers=openml.config._HEADERS,
+ files=None,
+ )
+
+
+@pytest.mark.test_server
+def test_v2_get(dataset_v2):
+ dataset_id = 2
+ output = dataset_v2.get(dataset_id)
+ assert output.dataset_id == dataset_id
+
+
+@pytest.mark.test_server
+def test_v2_list(dataset_v2):
+ output = dataset_v2.list(limit=2, offset=0, status="active")
+ assert not output.empty
+ assert output.shape[0] == 2
+ assert (output["status"] == "active").all()
+
+
+@pytest.mark.test_server
+def test_v2_download_arff(dataset_v2):
+ from openml.datasets.functions import _get_dataset_arff
+
+ output = dataset_v2.get(2)
+ file = _get_dataset_arff(output)
+ assert file.exists()
+
+
+@pytest.mark.test_server
+def test_v2_download_parquet(dataset_v2):
+ from openml.datasets.functions import _get_dataset_parquet
+
+ output = dataset_v2.get(2)
+ file = _get_dataset_parquet(output)
+ assert file.exists()
+
+
+@pytest.mark.test_server
+def test_v2_download_arff_from_get(dataset_v2):
+ output = dataset_v2.get(2, download_data=True)
+ data = output.data_file is not None and Path(output.data_file).exists()
+ parquet = output.parquet_file is not None and Path(output.parquet_file).exists()
+ assert data or parquet
+
+
+@pytest.mark.test_server
+def test_v2_download_qualities_from_get(dataset_v2):
+ output = dataset_v2.get(2, download_qualities=True)
+
+ assert output._qualities is not None
+
+
+@pytest.mark.test_server
+def test_v2_download_features_from_get(dataset_v2):
+ output = dataset_v2.get(2, download_features_meta_data=True)
+
+ assert output._features is not None
+
+
+@pytest.mark.test_server
+def test_v2_get_features(dataset_v2):
+ output = dataset_v2.get_features(2)
+
+ assert isinstance(output, dict)
+ assert len(output.keys()) >= 35
+
+
+@pytest.mark.test_server
+def test_v2_edit(dataset_v2):
+ with pytest.raises(OpenMLNotSupportedError):
+ dataset_v2.edit(2, description="Test")
+
+
+@pytest.mark.test_server
+def test_v2_fork(dataset_v2):
+ with pytest.raises(OpenMLNotSupportedError):
+ dataset_v2.fork(2)
+
+
+@pytest.mark.test_server
+def test_v2_feature_add_remove_ontology(dataset_v2):
+ with pytest.raises(OpenMLNotSupportedError):
+ dataset_v2.feature_add_ontology(2, 0, "https://www.openml.org/unittest/" + str(time.time()))
+
+
+@pytest.mark.test_server
+def test_v2_add_delete_topic(dataset_v2):
+ with pytest.raises(OpenMLNotSupportedError):
+ dataset_v2.add_topic(2, "test_topic_" + str(time.time()))
+ with pytest.raises(OpenMLNotSupportedError):
+ dataset_v2.delete_topic(2, "test_topic_" + str(time.time()))
+
+
+@pytest.mark.test_server
+def test_v2_get_qualities(dataset_v2):
+ output = dataset_v2.get_qualities(2)
+ assert isinstance(output, dict)
+ assert len(output.keys()) >= 100
+
+
+def test_v2_get_qualities_empty_returns_none(dataset_v2):
+ dataset_id = 2
+
+ with patch.object(Session, "request") as mock_request:
+ mock_request.return_value = Response()
+ mock_request.return_value.status_code = 200
+ mock_request.return_value.headers["Content-Type"] = "application/json"
+ mock_request.return_value._content = b"[]"
+
+ result = dataset_v2.get_qualities(dataset_id, force_refresh_cache=True)
+
+ assert result is None
+ mock_request.assert_called_once()
+ _, kwargs = mock_request.call_args
+ assert kwargs["method"] == "GET"
+ assert kwargs["url"] == (
+ openml.config.servers[APIVersion.V2]["server"] + f"datasets/qualities/{dataset_id}"
+ )
+
+
+@pytest.mark.test_server
+def test_v2_list_qualities(dataset_v2):
+ output = dataset_v2.list_qualities()
+ assert len(output) >= 100
+ assert isinstance(output[0], str)
+
+
+@pytest.mark.skip(reason="Needs valid v2 admin key required")
+@pytest.mark.test_server
+def test_v2_status_update(dataset_v2):
+ openml.config.apikey = TestBase.admin_key
+ # publish and fork is not supported in v2
+ _status_update_check(dataset_v2, 2, "deactivated")
+ _status_update_check(dataset_v2, 2, "active")
+
+
+@pytest.mark.test_server
+def test_get_matches(dataset_v1, dataset_v2):
+ output_v1 = dataset_v1.get(2)
+ output_v2 = dataset_v2.get(2)
+
+ assert output_v1.dataset_id == output_v2.dataset_id
+ assert output_v1.name == output_v2.name
+ assert output_v1.data_file is None
+ assert output_v1.data_file == output_v2.data_file
+
+
+@pytest.mark.test_server
+def test_get_features_matches(dataset_v1, dataset_v2):
+ output_v1 = dataset_v1.get_features(3)
+ output_v2 = dataset_v2.get_features(3)
+
+ assert output_v1.keys() == output_v2.keys()
+ # would not be same if v1 has ontology
+ assert output_v1 == output_v2
+
+
+@pytest.mark.test_server
+def test_list_matches(dataset_v1, dataset_v2):
+ output_v1 = dataset_v1.list(limit=2, offset=1)
+ output_v2 = dataset_v2.list(limit=2, offset=1)
+
+ pd.testing.assert_frame_equal(
+ output_v1[["did", "name", "version"]],
+ output_v2[["did", "name", "version"]],
+ check_like=True,
+ )
+
+
+@pytest.mark.test_server
+def test_get_qualities_matches(dataset_v1, dataset_v2):
+ output_v1 = dataset_v1.get_qualities(2)
+ output_v2 = dataset_v2.get_qualities(2)
+ assert set(output_v1.keys()) == set(output_v2.keys())
+ assert output_v1["AutoCorrelation"] == output_v2["AutoCorrelation"]
+ assert len(output_v1) == len(output_v2)
+
+
+@pytest.mark.test_server
+def test_list_qualities_matches(dataset_v1, dataset_v2):
+ output_v1 = dataset_v1.list_qualities()
+ output_v2 = dataset_v2.list_qualities()
+
+ assert output_v1 == output_v2
diff --git a/tests/test_datasets/test_dataset.py b/tests/test_datasets/test_dataset.py
index facc111d35..2c677354f2 100644
--- a/tests/test_datasets/test_dataset.py
+++ b/tests/test_datasets/test_dataset.py
@@ -2,7 +2,6 @@
from __future__ import annotations
import os
-from pathlib import Path
from time import time
import numpy as np
@@ -62,6 +61,9 @@ def iris(self):
self._iris = openml.datasets.get_dataset(61, download_data=False)
return self._iris
+ def _get_cache_filename(self, did):
+ return self.http_client.cache_path_from_url(f"data/{did}")
+
def test_repr(self):
# create a bare-bones dataset as would be returned by
# create_dataset
@@ -230,18 +232,14 @@ def test_get_data_corrupt_pickle(self):
assert xy.shape == (150, 5)
def test_lazy_loading_metadata(self):
- # Initial Setup
- did_cache_dir = openml.utils._create_cache_directory_for_id(
- openml.datasets.functions.DATASETS_CACHE_DIR_NAME,
- 2,
- )
_compare_dataset = openml.datasets.get_dataset(
2,
download_data=False,
download_features_meta_data=True,
download_qualities=True,
)
- change_time = os.stat(did_cache_dir).st_mtime # noqa: PTH116
+ did_cache_file = self._get_cache_filename(2)
+ change_time = os.stat(did_cache_file).st_mtime # noqa: PTH116
# Test with cache
_dataset = openml.datasets.get_dataset(
@@ -250,15 +248,12 @@ def test_lazy_loading_metadata(self):
download_features_meta_data=False,
download_qualities=False,
)
- assert change_time == os.stat(did_cache_dir).st_mtime # noqa: PTH116
+ assert change_time == os.stat(did_cache_file).st_mtime # noqa: PTH116
assert _dataset.features == _compare_dataset.features
assert _dataset.qualities == _compare_dataset.qualities
# -- Test without cache
- openml.utils._remove_cache_dir_for_id(
- openml.datasets.functions.DATASETS_CACHE_DIR_NAME,
- did_cache_dir,
- )
+ did_cache_file.unlink()
_dataset = openml.datasets.get_dataset(
2,
@@ -266,8 +261,9 @@ def test_lazy_loading_metadata(self):
download_features_meta_data=False,
download_qualities=False,
)
- assert [f.name for f in Path(did_cache_dir).iterdir()] == ["description.xml"]
- assert change_time != os.stat(did_cache_dir).st_mtime # noqa: PTH116
+
+ assert did_cache_file.exists()
+ assert change_time != os.stat(did_cache_file).st_mtime # noqa: PTH116
assert _dataset.features == _compare_dataset.features
assert _dataset.qualities == _compare_dataset.qualities
@@ -412,7 +408,7 @@ def test__read_features(mocker, workdir, static_cache_dir):
This test also does some simple checks to verify that the features are read correctly
"""
filename_mock = mocker.patch("openml.datasets.dataset._get_features_pickle_file")
- pickle_mock = mocker.patch("openml.datasets.dataset.pickle")
+ pickle_mock = mocker.patch("openml._api.resources.dataset.pickle")
filename_mock.return_value = os.path.join(workdir, "features.xml.pkl") # noqa: PTH118
pickle_mock.load.side_effect = FileNotFoundError
@@ -423,9 +419,13 @@ def test__read_features(mocker, workdir, static_cache_dir):
"org",
"openml",
"test",
- "datasets",
+ "api",
+ "v1",
+ "xml",
+ "data",
+ "features",
"2",
- "features.xml",
+ "body.xml",
),
)
assert isinstance(features, dict)
@@ -444,7 +444,7 @@ def test__read_qualities(static_cache_dir, workdir, mocker):
This test also does some minor checks to ensure that the qualities are read correctly.
"""
filename_mock = mocker.patch("openml.datasets.dataset._get_qualities_pickle_file")
- pickle_mock = mocker.patch("openml.datasets.dataset.pickle")
+ pickle_mock = mocker.patch("openml._api.resources.dataset.pickle")
filename_mock.return_value = os.path.join(workdir, "qualities.xml.pkl") # noqa: PTH118
pickle_mock.load.side_effect = FileNotFoundError
@@ -455,26 +455,16 @@ def test__read_qualities(static_cache_dir, workdir, mocker):
"org",
"openml",
"test",
- "datasets",
+ "api",
+ "v1",
+ "xml",
+ "data",
+ "qualities",
"2",
- "qualities.xml",
+ "body.xml",
),
)
assert isinstance(qualities, dict)
assert len(qualities) == 106
assert pickle_mock.load.call_count == 0
assert pickle_mock.dump.call_count == 1
-
-
-def test__check_qualities():
- qualities = [{"oml:name": "a", "oml:value": "0.5"}]
- qualities = openml.datasets.dataset._check_qualities(qualities)
- assert qualities["a"] == 0.5
-
- qualities = [{"oml:name": "a", "oml:value": "null"}]
- qualities = openml.datasets.dataset._check_qualities(qualities)
- assert qualities["a"] != qualities["a"]
-
- qualities = [{"oml:name": "a", "oml:value": None}]
- qualities = openml.datasets.dataset._check_qualities(qualities)
- assert qualities["a"] != qualities["a"]
diff --git a/tests/test_datasets/test_dataset_functions.py b/tests/test_datasets/test_dataset_functions.py
index fae4a1c4ed..49adab20e6 100644
--- a/tests/test_datasets/test_dataset_functions.py
+++ b/tests/test_datasets/test_dataset_functions.py
@@ -4,7 +4,6 @@
import itertools
import os
import random
-import shutil
import time
import uuid
from collections.abc import Iterable
@@ -22,16 +21,14 @@
import openml
from openml import OpenMLDataset
-from openml._api_calls import _download_minio_file
+from openml._api.clients.minio import MinIOClient
from openml.datasets import edit_dataset, fork_dataset
from openml.datasets.functions import (
DATASETS_CACHE_DIR_NAME,
_get_dataset_arff,
- _get_dataset_description,
_get_dataset_features_file,
_get_dataset_parquet,
_get_dataset_qualities_file,
- _get_online_dataset_arff,
_get_online_dataset_format,
_topic_add_dataset,
_topic_delete_dataset,
@@ -50,6 +47,11 @@
from openml.utils import _create_cache_directory_for_id, _tag_entity
+def _get_online_dataset_arff(dataset_id: int) -> str:
+ dataset = openml.datasets.get_dataset(dataset_id, download_data=False)
+ return _get_dataset_arff(dataset).read_text(encoding="utf8")
+
+
class TestOpenMLDataset(TestBase):
_multiprocess_can_split_ = True
@@ -106,6 +108,9 @@ def _check_datasets(self, datasets):
for did in datasets:
self._check_dataset(datasets[did])
+ def _get_cache_filename(self, did):
+ return self.http_client.cache_path_from_url(f"data/{did}")
+
@pytest.mark.test_server
def test_tag_untag_dataset(self):
tag = f"test_tag_{random.randint(1, 1000000)}" # noqa: S311
@@ -182,7 +187,7 @@ def test__name_to_id_with_deactivated(self):
"""Check that an activated dataset is returned if an earlier deactivated one exists."""
self.use_production_server()
# /d/1 was deactivated
- assert openml.datasets.functions._name_to_id("anneal") == 2
+ assert openml.datasets.functions._name_to_id("anneal", version=1) == 2
@pytest.mark.production_server
def test__name_to_id_with_multiple_active(self):
@@ -235,7 +240,7 @@ def test_get_datasets_by_name(self):
dids = ["anneal", "kr-vs-kp"]
datasets = openml.datasets.get_datasets(dids)
assert len(datasets) == 2
- _assert_datasets_retrieved_successfully([1, 2])
+ _assert_datasets_retrieved_successfully(datasets)
@pytest.mark.test_server
def test_get_datasets_by_mixed(self):
@@ -243,21 +248,21 @@ def test_get_datasets_by_mixed(self):
dids = ["anneal", 2]
datasets = openml.datasets.get_datasets(dids)
assert len(datasets) == 2
- _assert_datasets_retrieved_successfully([1, 2])
+ _assert_datasets_retrieved_successfully(datasets)
@pytest.mark.test_server
def test_get_datasets(self):
dids = [1, 2]
datasets = openml.datasets.get_datasets(dids)
assert len(datasets) == 2
- _assert_datasets_retrieved_successfully([1, 2])
+ _assert_datasets_retrieved_successfully(datasets)
@pytest.mark.test_server
def test_get_dataset_by_name(self):
dataset = openml.datasets.get_dataset("anneal")
assert type(dataset) == OpenMLDataset # noqa: E721
assert dataset.dataset_id == 1
- _assert_datasets_retrieved_successfully([1])
+ _assert_datasets_retrieved_successfully([dataset])
assert len(dataset.features) > 1
assert len(dataset.qualities) > 4
@@ -334,87 +339,46 @@ def test_download_rowid(self):
dataset = openml.datasets.get_dataset(did)
assert dataset.row_id_attribute == "Counter"
- @pytest.mark.test_server
- def test__get_dataset_description(self):
- description = _get_dataset_description(self.workdir, 2)
- assert isinstance(description, dict)
- description_xml_path = os.path.join(self.workdir, "description.xml") # noqa: PTH118
- assert os.path.exists(description_xml_path) # noqa: PTH110
-
@pytest.mark.test_server
def test__getarff_path_dataset_arff(self):
openml.config.set_root_cache_directory(self.static_cache_dir)
- description = _get_dataset_description(self.workdir, 2)
- arff_path = _get_dataset_arff(description, cache_directory=self.workdir)
+ dataset = openml.datasets.get_dataset(2)
+ arff_path = _get_dataset_arff(dataset)
assert isinstance(arff_path, Path)
assert arff_path.exists()
def test__download_minio_file_object_does_not_exist(self):
+ minio = MinIOClient()
self.assertRaisesRegex( # noqa: PT027
FileNotFoundError,
r"Object at .* does not exist",
- _download_minio_file,
+ minio.download_minio_file,
source="http://data.openml.org/dataset20/i_do_not_exist.pq",
- destination=self.workdir,
- exists_ok=True,
- )
-
- def test__download_minio_file_to_directory(self):
- _download_minio_file(
- source="http://data.openml.org/dataset20/dataset_20.pq",
- destination=self.workdir,
- exists_ok=True,
- )
- assert os.path.isfile(os.path.join(self.workdir, "dataset_20.pq")), ( # noqa: PTH113, PTH118
- "_download_minio_file can save to a folder by copying the object name"
- )
-
- def test__download_minio_file_to_path(self):
- file_destination = os.path.join(self.workdir, "custom.pq") # noqa: PTH118
- _download_minio_file(
- source="http://data.openml.org/dataset20/dataset_20.pq",
- destination=file_destination,
exists_ok=True,
)
- assert os.path.isfile(file_destination), ( # noqa: PTH113
- "_download_minio_file can save to a folder by copying the object name"
- )
-
- def test__download_minio_file_raises_FileExists_if_destination_in_use(self):
- file_destination = Path(self.workdir, "custom.pq")
- file_destination.touch()
-
- self.assertRaises( # noqa: PT027
- FileExistsError,
- _download_minio_file,
- source="http://data.openml.org/dataset20/dataset_20.pq",
- destination=str(file_destination),
- exists_ok=False,
- )
def test__download_minio_file_works_with_bucket_subdirectory(self):
- file_destination = Path(self.workdir, "custom.pq")
- _download_minio_file(
+ minio = MinIOClient()
+ file_destination = minio.download_minio_file(
source="http://data.openml.org/dataset61/dataset_61.pq",
- destination=file_destination,
exists_ok=True,
)
assert os.path.isfile(file_destination), ( # noqa: PTH113
- "_download_minio_file can download from subdirectories"
+ "download_minio_file can download from subdirectories"
)
- @mock.patch("openml._api_calls._download_minio_file")
+ @mock.patch.object(MinIOClient, "download_minio_file")
@pytest.mark.test_server
def test__get_dataset_parquet_is_cached(self, patch):
openml.config.set_root_cache_directory(self.static_cache_dir)
patch.side_effect = RuntimeError(
- "_download_parquet_url should not be called when loading from cache",
+ "download_parquet_url should not be called when loading from cache",
)
description = {
"oml:parquet_url": "http://data.openml.org/dataset30/dataset_30.pq",
"oml:id": "30",
}
- path = _get_dataset_parquet(description, cache_directory=None)
+ path = _get_dataset_parquet(description)
assert isinstance(path, Path), "_get_dataset_parquet returns a path"
assert path.is_file(), "_get_dataset_parquet returns path to real file"
@@ -423,7 +387,7 @@ def test__get_dataset_parquet_file_does_not_exist(self):
"oml:parquet_url": "http://data.openml.org/dataset20/does_not_exist.pq",
"oml:id": "20",
}
- path = _get_dataset_parquet(description, cache_directory=self.workdir)
+ path = _get_dataset_parquet(description)
assert path is None, "_get_dataset_parquet returns None if no file is found"
def test__getarff_md5_issue(self):
@@ -437,8 +401,8 @@ def test__getarff_md5_issue(self):
self.assertRaisesRegex( # noqa: PT027
OpenMLHashException,
- "Checksum of downloaded file is unequal to the expected checksum abc when downloading "
- "https://www.openml.org/data/download/61. Raised when downloading dataset 5.",
+ "Checksum of downloaded file is unequal to the expected checksum abc "
+ "when downloading https://www.openml.org/data/download/61.",
_get_dataset_arff,
description,
)
@@ -447,62 +411,38 @@ def test__getarff_md5_issue(self):
@pytest.mark.test_server
def test__get_dataset_features(self):
- features_file = _get_dataset_features_file(self.workdir, 2)
+ features_file = _get_dataset_features_file(2)
assert isinstance(features_file, Path)
- features_xml_path = self.workdir / "features.xml"
- assert features_xml_path.exists()
+ assert features_file.exists()
@pytest.mark.test_server
def test__get_dataset_qualities(self):
- qualities = _get_dataset_qualities_file(self.workdir, 2)
+ qualities = _get_dataset_qualities_file(2)
assert isinstance(qualities, Path)
- qualities_xml_path = self.workdir / "qualities.xml"
- assert qualities_xml_path.exists()
+ assert qualities.exists()
@pytest.mark.test_server
def test_get_dataset_force_refresh_cache(self):
- did_cache_dir = _create_cache_directory_for_id(
- DATASETS_CACHE_DIR_NAME,
- 2,
- )
openml.datasets.get_dataset(2)
- change_time = os.stat(did_cache_dir).st_mtime # noqa: PTH116
+ did_cache_file = self._get_cache_filename(2)
+ change_time = os.stat(did_cache_file).st_mtime # noqa: PTH116
# Test default
openml.datasets.get_dataset(2)
- assert change_time == os.stat(did_cache_dir).st_mtime # noqa: PTH116
+ assert change_time == os.stat(did_cache_file).st_mtime # noqa: PTH116
# Test refresh
openml.datasets.get_dataset(2, force_refresh_cache=True)
- assert change_time != os.stat(did_cache_dir).st_mtime # noqa: PTH116
-
- # Final clean up
- openml.utils._remove_cache_dir_for_id(
- DATASETS_CACHE_DIR_NAME,
- did_cache_dir,
- )
+ assert change_time != os.stat(did_cache_file).st_mtime # noqa: PTH116
@pytest.mark.test_server
def test_get_dataset_force_refresh_cache_clean_start(self):
- did_cache_dir = _create_cache_directory_for_id(
- DATASETS_CACHE_DIR_NAME,
- 2,
- )
- # Clean up
- openml.utils._remove_cache_dir_for_id(
- DATASETS_CACHE_DIR_NAME,
- did_cache_dir,
- )
+ with pytest.raises(FileNotFoundError):
+ self._get_cache_filename(2)
- # Test clean start
openml.datasets.get_dataset(2, force_refresh_cache=True)
- assert os.path.exists(did_cache_dir) # noqa: PTH110
- # Final clean up
- openml.utils._remove_cache_dir_for_id(
- DATASETS_CACHE_DIR_NAME,
- did_cache_dir,
- )
+ assert self._get_cache_filename(2).exists()
def test_deletion_of_cache_dir(self):
# Simple removal
@@ -517,19 +457,18 @@ def test_deletion_of_cache_dir(self):
)
assert not os.path.exists(did_cache_dir) # noqa: PTH110
- # get_dataset_description is the only data guaranteed to be downloaded
- @mock.patch("openml.datasets.functions._get_dataset_description")
- @pytest.mark.test_server
- def test_deletion_of_cache_dir_faulty_download(self, patch):
- patch.side_effect = Exception("Boom!")
- self.assertRaisesRegex(Exception, "Boom!", openml.datasets.get_dataset, dataset_id=1) # noqa: PT027
- datasets_cache_dir = os.path.join(openml.config.get_cache_directory(), "datasets") # noqa: PTH118
- assert len(list(Path(datasets_cache_dir).iterdir())) == 0
-
@pytest.mark.test_server
def test_publish_dataset(self):
arff_file_path = (
- self.static_cache_dir / "org" / "openml" / "test" / "datasets" / "2" / "dataset.arff"
+ self.static_cache_dir
+ / "org"
+ / "openml"
+ / "test"
+ / "data"
+ / "download"
+ / "1666876"
+ / "phpFsFYVN"
+ / "body.arff"
)
dataset = OpenMLDataset(
"anneal",
@@ -564,6 +503,7 @@ def test__retrieve_class_labels(self):
labels = custom_ds.retrieve_class_labels(target_name=custom_ds.features[31].name)
assert labels == ["COIL", "SHEET"]
+ @pytest.mark.skip(reason="URL uploading is disabled on the server: OpenML#1333")
@pytest.mark.test_server
def test_upload_dataset_with_url(self):
dataset = OpenMLDataset(
@@ -1392,13 +1332,13 @@ def test_get_dataset_cache_format_feather(self):
dataset.get_data()
# Check if dataset is written to cache directory using feather
- cache_dir = openml.config.get_cache_directory()
- cache_dir_for_id = os.path.join(cache_dir, "datasets", "128") # noqa: PTH118
- feather_file = os.path.join(cache_dir_for_id, "dataset.feather") # noqa: PTH118
- pickle_file = os.path.join(cache_dir_for_id, "dataset.feather.attributes.pkl.py3") # noqa: PTH118
+ assert dataset.data_file is not None
+ arff_file = Path(dataset.data_file)
+ feather_file = arff_file.with_name("body.feather")
+ pickle_file = arff_file.with_name("body.feather.attributes.pkl.py3")
data = pd.read_feather(feather_file)
- assert os.path.isfile(feather_file), "Feather file is missing" # noqa: PTH113
- assert os.path.isfile(pickle_file), "Attributes pickle file is missing" # noqa: PTH113
+ assert feather_file.is_file(), "Feather file is missing"
+ assert pickle_file.is_file(), "Attributes pickle file is missing"
assert data.shape == (150, 5)
# Check if get_data is able to retrieve feather data
@@ -1425,7 +1365,7 @@ def test_data_edit_critical_field(self):
n_tries = 10
# we need to wait for the edit to be reflected on the server
for i in range(n_tries):
- edited_dataset = openml.datasets.get_dataset(did)
+ edited_dataset = openml.datasets.get_dataset(did, force_refresh_cache=True)
try:
assert edited_dataset.default_target_attribute == "shape", edited_dataset
assert edited_dataset.ignore_attribute == ["oil"], edited_dataset
@@ -1436,10 +1376,6 @@ def test_data_edit_critical_field(self):
time.sleep(10)
# Delete the cache dir to get the newer version of the dataset
- shutil.rmtree(
- os.path.join(openml.config.get_cache_directory(), "datasets", str(did)), # noqa: PTH118
- )
-
def test_data_edit_requires_field(self):
# Check server exception when no field to edit is provided
with pytest.raises(
@@ -1696,7 +1632,7 @@ def test_delete_dataset(self):
assert openml.datasets.delete_dataset(_dataset_id)
-@mock.patch.object(requests.Session, "delete")
+@mock.patch.object(requests.Session, "request")
def test_delete_dataset_not_owned(
mock_delete, test_files_directory, test_server_v1, test_apikey_v1
):
@@ -1715,11 +1651,12 @@ def test_delete_dataset_not_owned(
openml.datasets.delete_dataset(40_000)
dataset_url = test_server_v1 + "data/40000"
- assert dataset_url == mock_delete.call_args.args[0]
+ assert dataset_url == mock_delete.call_args.kwargs.get("url")
+ assert mock_delete.call_args.kwargs.get("method") == "DELETE"
assert test_apikey_v1 == mock_delete.call_args.kwargs.get("params", {}).get("api_key")
-@mock.patch.object(requests.Session, "delete")
+@mock.patch.object(requests.Session, "request")
def test_delete_dataset_with_run(mock_delete, test_files_directory, test_server_v1, test_apikey_v1):
content_file = (
test_files_directory / "mock_responses" / "datasets" / "data_delete_has_tasks.xml"
@@ -1736,11 +1673,12 @@ def test_delete_dataset_with_run(mock_delete, test_files_directory, test_server_
openml.datasets.delete_dataset(40_000)
dataset_url = test_server_v1 + "data/40000"
- assert dataset_url == mock_delete.call_args.args[0]
+ assert dataset_url == mock_delete.call_args.kwargs.get("url")
+ assert mock_delete.call_args.kwargs.get("method") == "DELETE"
assert test_apikey_v1 == mock_delete.call_args.kwargs.get("params", {}).get("api_key")
-@mock.patch.object(requests.Session, "delete")
+@mock.patch.object(requests.Session, "request")
def test_delete_dataset_success(mock_delete, test_files_directory, test_server_v1, test_apikey_v1):
content_file = (
test_files_directory / "mock_responses" / "datasets" / "data_delete_successful.xml"
@@ -1754,11 +1692,12 @@ def test_delete_dataset_success(mock_delete, test_files_directory, test_server_v
assert success
dataset_url = test_server_v1 + "data/40000"
- assert dataset_url == mock_delete.call_args.args[0]
+ assert dataset_url == mock_delete.call_args.kwargs.get("url")
+ assert mock_delete.call_args.kwargs.get("method") == "DELETE"
assert test_apikey_v1 == mock_delete.call_args.kwargs.get("params", {}).get("api_key")
-@mock.patch.object(requests.Session, "delete")
+@mock.patch.object(requests.Session, "request")
def test_delete_unknown_dataset(mock_delete, test_files_directory, test_server_v1, test_apikey_v1):
content_file = (
test_files_directory / "mock_responses" / "datasets" / "data_delete_not_exist.xml"
@@ -1775,7 +1714,8 @@ def test_delete_unknown_dataset(mock_delete, test_files_directory, test_server_v
openml.datasets.delete_dataset(9_999_999)
dataset_url = test_server_v1 + "data/9999999"
- assert dataset_url == mock_delete.call_args.args[0]
+ assert dataset_url == mock_delete.call_args.kwargs.get("url")
+ assert mock_delete.call_args.kwargs.get("method") == "DELETE"
assert test_apikey_v1 == mock_delete.call_args.kwargs.get("params", {}).get("api_key")
@@ -1852,34 +1792,44 @@ def test_list_datasets_combined_filters(all_datasets: pd.DataFrame):
def _dataset_file_is_downloaded(did: int, file: str):
- cache_directory = Path(openml.config.get_cache_directory()) / "datasets" / str(did)
+ cache_directory = Path(openml.config.get_cache_directory()) / "api/v1/xml/data" / str(did)
return (cache_directory / file).exists()
def _dataset_description_is_downloaded(did: int):
- return _dataset_file_is_downloaded(did, "description.xml")
+ return _dataset_file_is_downloaded(did, "body.xml")
def _dataset_qualities_is_downloaded(did: int):
- return _dataset_file_is_downloaded(did, "qualities.xml")
+ cache_directory = Path(openml.config.get_cache_directory()) / "api/v1/xml/data/qualities/"
+ return (cache_directory / str(did) / "body.xml").exists()
def _dataset_features_is_downloaded(did: int):
- return _dataset_file_is_downloaded(did, "features.xml")
+ cache_directory = Path(openml.config.get_cache_directory()) / "api/v1/xml/data/features/"
+ return (cache_directory / str(did) / "body.xml").exists()
-def _dataset_data_file_is_downloaded(did: int):
- cache_directory = Path(openml.config.get_cache_directory()) / "datasets" / str(did)
- return any(f.suffix in (".pq", ".arff") for f in cache_directory.iterdir())
+def _dataset_data_file_is_downloaded(dataset: OpenMLDataset):
+ if dataset._parquet_url is not None:
+ pq_directory = Path(openml.config.get_minio_download_path(dataset._parquet_url)).parent
+ if pq_directory.exists():
+ return any(f.suffix == ".pq" for f in pq_directory.iterdir())
+ arff_directory = (
+ Path(openml.config.get_cache_directory()) / "data/v1/download" / str(dataset.id)
+ )
+ if arff_directory.exists():
+ return any(f.suffix == ".arff" for f in arff_directory.iterdir())
+ return False
def _assert_datasets_retrieved_successfully(
- dids: Iterable[int],
+ datasets: Iterable[OpenMLDataset],
with_qualities: bool = False, # noqa: FBT002
with_features: bool = False, # noqa: FBT002
with_data: bool = False, # noqa: FBT002
):
- """Checks that all files for the given dids have been downloaded.
+ """Checks that all files for the given datasets have been downloaded.
This includes:
- description
@@ -1887,16 +1837,16 @@ def _assert_datasets_retrieved_successfully(
- features
- absence of data arff if metadata_only, else it must be present too.
"""
- for did in dids:
- assert _dataset_description_is_downloaded(did)
+ for dataset in datasets:
+ assert _dataset_description_is_downloaded(dataset.id)
- has_qualities = _dataset_qualities_is_downloaded(did)
+ has_qualities = _dataset_qualities_is_downloaded(dataset.id)
assert has_qualities if with_qualities else not has_qualities
- has_features = _dataset_features_is_downloaded(did)
+ has_features = _dataset_features_is_downloaded(dataset.id)
assert has_features if with_features else not has_features
- has_data = _dataset_data_file_is_downloaded(did)
+ has_data = _dataset_data_file_is_downloaded(dataset)
assert has_data if with_data else not has_data
@@ -1924,12 +1874,13 @@ def test_get_dataset_lazy_behavior(
download_data=with_data,
download_qualities=with_qualities,
download_features_meta_data=with_features,
+ force_refresh_cache=True,
)
assert type(dataset) == OpenMLDataset # noqa: E721
assert dataset.name == "anneal"
_assert_datasets_retrieved_successfully(
- [1],
+ [dataset],
with_qualities=with_qualities,
with_features=with_features,
with_data=with_data,
@@ -1938,7 +1889,7 @@ def test_get_dataset_lazy_behavior(
assert dataset.qualities, "Qualities should be downloaded on-demand if not during get_dataset"
assert dataset.get_data(), "Data should be downloaded on-demand if not during get_dataset"
_assert_datasets_retrieved_successfully(
- [1], with_qualities=True, with_features=True, with_data=True
+ [dataset], with_qualities=True, with_features=True, with_data=True
)
@@ -1955,9 +1906,7 @@ def test__get_dataset_parquet_not_cached():
"oml:parquet_url": "http://data.openml.org/dataset20/dataset_20.pq",
"oml:id": "20",
}
- path = _get_dataset_parquet(
- description, cache_directory=Path(openml.config.get_cache_directory())
- )
+ path = _get_dataset_parquet(description)
assert isinstance(path, Path), "_get_dataset_parquet returns a path"
assert path.is_file(), "_get_dataset_parquet returns path to real file"
diff --git a/tests/test_flows/test_flow.py b/tests/test_flows/test_flow.py
index fca2da9be4..8372feab60 100644
--- a/tests/test_flows/test_flow.py
+++ b/tests/test_flows/test_flow.py
@@ -143,6 +143,8 @@ def test_from_xml_to_xml(self):
.strip()
.replace("\n\n", "\n")
.replace(""", '"')
+ # php-rest-api >= v1.3 escapes apostrophes in the returned XML
+ .replace("'", "'")
)
flow_xml = re.sub(r"^$", "", flow_xml)
new_xml = (
diff --git a/tests/test_tasks/test_task_functions.py b/tests/test_tasks/test_task_functions.py
index ee337308fe..639ad58cde 100644
--- a/tests/test_tasks/test_task_functions.py
+++ b/tests/test_tasks/test_task_functions.py
@@ -149,6 +149,7 @@ def test__get_task_live(self):
@pytest.mark.test_server
def test_get_task(self):
task = openml.tasks.get_task(1, download_data=True) # anneal; crossvalidation
+ dataset = openml.datasets.get_dataset(task.dataset_id)
assert isinstance(task, OpenMLTask)
assert os.path.exists( # noqa: PTH110
os.path.join(openml.config.get_cache_directory(), "tasks", "1", "task.xml") # noqa: PTH118
@@ -157,7 +158,7 @@ def test_get_task(self):
os.path.join(openml.config.get_cache_directory(), "tasks", "1", "datasplits.arff") # noqa: PTH118
)
assert os.path.exists( # noqa: PTH110
- os.path.join(openml.config.get_cache_directory(), "datasets", "1", "dataset_1.pq") # noqa: PTH118
+ openml.config.get_minio_download_path(dataset._parquet_url)
)
@pytest.mark.test_server
diff --git a/tests/test_utils/test_utils.py b/tests/test_utils/test_utils.py
index 60d80a82e9..b252e98d22 100644
--- a/tests/test_utils/test_utils.py
+++ b/tests/test_utils/test_utils.py
@@ -4,6 +4,7 @@
import unittest.mock
import pytest
+import requests
import openml
from openml.testing import _check_dataset
@@ -45,11 +46,6 @@ def min_number_evaluations_on_test_server() -> int:
return 8
-def _mocked_perform_api_call(call, request_method): # noqa: ARG001
- url = openml.config.server + call
- return openml._api_calls._download_text_file(url)
-
-
@pytest.mark.test_server
def test_list_all():
openml.utils._list_all(listing_call=openml.tasks.functions._list_tasks)
@@ -117,12 +113,14 @@ def test_list_all_for_evaluations(min_number_evaluations_on_test_server):
assert min_number_evaluations_on_test_server == len(evaluations)
-@unittest.mock.patch("openml._api_calls._perform_api_call", side_effect=_mocked_perform_api_call)
+@unittest.mock.patch.object(
+ requests.Session, "request", autospec=True, wraps=requests.Session.request
+)
@pytest.mark.test_server
-def test_list_all_few_results_available(_perform_api_call): # noqa: PT019
+def test_list_all_few_results_available(mocked_request):
datasets = openml.datasets.list_datasets(size=1000, data_name="iris", data_version=1)
assert len(datasets) == 1, "only one iris dataset version 1 should be present"
- assert _perform_api_call.call_count == 1, "expect just one call to get one dataset"
+ assert mocked_request.call_count == 1, "expect just one call to get one dataset"
@unittest.skipIf(os.name == "nt", "https://github.com/openml/openml-python/issues/1033")