diff --git a/tests/conftest.py b/tests/conftest.py index 0e863fba0..aabe8b245 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -246,6 +246,11 @@ def assert_static_test_cache_correct(root_dir: Path) -> None: assert p.exists(), f"Expected path {p} exists" +@pytest.fixture(scope="session") +def long_version_new(request): + return request.config.getoption("--long") + + @pytest.fixture(scope="class") def long_version(request): request.cls.long_version = request.config.getoption("--long") diff --git a/tests/test_evaluations/test_evaluation_functions.py b/tests/test_evaluations/test_evaluation_functions.py index 8aee7b838..de65ca0b8 100644 --- a/tests/test_evaluations/test_evaluation_functions.py +++ b/tests/test_evaluations/test_evaluation_functions.py @@ -5,267 +5,255 @@ import openml import openml.evaluations -from openml.testing import TestBase - - -@pytest.mark.usefixtures("long_version") -class TestEvaluationFunctions(TestBase): - _multiprocess_can_split_ = True - - def _check_list_evaluation_setups(self, **kwargs): - evals_setups = openml.evaluations.list_evaluations_setups( - "predictive_accuracy", - **kwargs, - sort_order="desc", - ) - evals = openml.evaluations.list_evaluations( - "predictive_accuracy", - **kwargs, - sort_order="desc", - output_format="dataframe", - ) - - # Check if list is non-empty - assert len(evals_setups) > 0 - # Check if length is accurate - assert len(evals_setups) == len(evals) - # Check if output from sort is sorted in the right order - self.assertSequenceEqual( # noqa: PT009 - sorted(evals_setups["value"].tolist(), reverse=True), - evals_setups["value"].tolist(), - ) - - # Check if output and order of list_evaluations is preserved - self.assertSequenceEqual(evals_setups["run_id"].tolist(), evals["run_id"].tolist()) # noqa: PT009 - - if not self.long_version: - evals_setups = evals_setups.head(1) - - # Check if the hyper-parameter column is as accurate and flow_id - for _index, row in evals_setups.iterrows(): - params = openml.runs.get_run(row["run_id"]).parameter_settings - list1 = [param["oml:value"] for param in params] - list2 = list(row["parameters"].values()) - # check if all values are equal - self.assertSequenceEqual(sorted(list1), sorted(list2)) # noqa: PT009 - return evals_setups - - @pytest.mark.production_server - def test_evaluation_list_filter_task(self): - self.use_production_server() - - task_id = 7312 - - evaluations = openml.evaluations.list_evaluations( - "predictive_accuracy", - size=110, - tasks=[task_id], - ) - - assert len(evaluations) > 100 - for run_id in evaluations: - assert evaluations[run_id].task_id == task_id - # default behaviour of this method: return aggregated results (not - # per fold) - assert evaluations[run_id].value is not None - assert evaluations[run_id].values is None - - @pytest.mark.production_server - def test_evaluation_list_filter_uploader_ID_16(self): - self.use_production_server() - - uploader_id = 16 - evaluations = openml.evaluations.list_evaluations( - "predictive_accuracy", - size=60, - uploaders=[uploader_id], - output_format="dataframe", - ) - assert evaluations["uploader"].unique() == [uploader_id] - - assert len(evaluations) > 50 - - @pytest.mark.production_server - def test_evaluation_list_filter_uploader_ID_10(self): - self.use_production_server() - - setup_id = 10 - evaluations = openml.evaluations.list_evaluations( - "predictive_accuracy", - size=60, - setups=[setup_id], - ) - - assert len(evaluations) > 50 - for run_id in evaluations: - assert evaluations[run_id].setup_id == setup_id - # default behaviour of this method: return aggregated results (not - # per fold) - assert evaluations[run_id].value is not None - assert evaluations[run_id].values is None - - @pytest.mark.production_server - def test_evaluation_list_filter_flow(self): - self.use_production_server() - - flow_id = 100 - - evaluations = openml.evaluations.list_evaluations( - "predictive_accuracy", - size=10, - flows=[flow_id], - ) - - assert len(evaluations) > 2 - for run_id in evaluations: - assert evaluations[run_id].flow_id == flow_id - # default behaviour of this method: return aggregated results (not - # per fold) - assert evaluations[run_id].value is not None - assert evaluations[run_id].values is None - - @pytest.mark.production_server - def test_evaluation_list_filter_run(self): - self.use_production_server() - - run_id = 12 - - evaluations = openml.evaluations.list_evaluations( - "predictive_accuracy", - size=2, - runs=[run_id], - ) - - assert len(evaluations) == 1 - for run_id in evaluations: - assert evaluations[run_id].run_id == run_id - # default behaviour of this method: return aggregated results (not - # per fold) - assert evaluations[run_id].value is not None - assert evaluations[run_id].values is None - - @pytest.mark.production_server - def test_evaluation_list_limit(self): - self.use_production_server() - - evaluations = openml.evaluations.list_evaluations( - "predictive_accuracy", - size=100, - offset=100, - ) - assert len(evaluations) == 100 - - @pytest.mark.skip(reason="Affected by lowered result limit #1757") - @pytest.mark.test_server - def test_list_evaluations_empty(self): - evaluations = openml.evaluations.list_evaluations("unexisting_measure") - if len(evaluations) > 0: - raise ValueError("UnitTest Outdated, got somehow results") - - assert isinstance(evaluations, dict) - - @pytest.mark.production_server - def test_evaluation_list_per_fold(self): - self.use_production_server() - size = 1000 - task_ids = [6] - uploader_ids = [1] - flow_ids = [6969] - runs = [6070667, 6070666] - - evaluations = openml.evaluations.list_evaluations( - "predictive_accuracy", - size=size, - offset=0, - tasks=task_ids, - flows=flow_ids, - uploaders=uploader_ids, - runs=runs, - per_fold=True, - ) - - assert len(evaluations) == len(runs) - for run_id in evaluations: - assert evaluations[run_id].value is None - assert evaluations[run_id].values is not None - # potentially we could also test array values, but these might be - # added in the future - - evaluations = openml.evaluations.list_evaluations( - "predictive_accuracy", - size=size, - offset=0, - tasks=task_ids, - flows=flow_ids, - uploaders=uploader_ids, - runs=runs, - per_fold=False, - ) - for run_id in evaluations: - assert evaluations[run_id].value is not None - assert evaluations[run_id].values is None - - @pytest.mark.skip(reason="Affected by lowered result limit #1757") - @pytest.mark.production_server - def test_evaluation_list_sort(self): - self.use_production_server() - size = 10 - task_id = 6 - # Get all evaluations of the task - unsorted_eval = openml.evaluations.list_evaluations( - "predictive_accuracy", - size=None, - offset=0, - tasks=[task_id], - ) - # Get top 10 evaluations of the same task - sorted_eval = openml.evaluations.list_evaluations( - "predictive_accuracy", - size=size, - offset=0, - tasks=[task_id], - sort_order="desc", - ) - assert len(sorted_eval) == size - assert len(unsorted_eval) > 0 - sorted_output = [evaluation.value for evaluation in sorted_eval.values()] - unsorted_output = [evaluation.value for evaluation in unsorted_eval.values()] - - # Check if output from sort is sorted in the right order - assert sorted(sorted_output, reverse=True) == sorted_output - - # Compare manual sorting against sorted output - test_output = sorted(unsorted_output, reverse=True) - assert test_output[:size] == sorted_output - - @pytest.mark.test_server - def test_list_evaluation_measures(self): - measures = openml.evaluations.list_evaluation_measures() - assert isinstance(measures, list) is True - assert all(isinstance(s, str) for s in measures) is True - - @pytest.mark.production_server - def test_list_evaluations_setups_filter_flow(self): - self.use_production_server() - flow_id = [405] - size = 100 - evals = self._check_list_evaluation_setups(flows=flow_id, size=size) - # check if parameters in separate columns works - evals_cols = openml.evaluations.list_evaluations_setups( - "predictive_accuracy", - flows=flow_id, - size=size, - sort_order="desc", - parameters_in_separate_columns=True, - ) - columns = list(evals_cols.columns) - keys = list(evals["parameters"].values[0].keys()) - assert all(elem in columns for elem in keys) - - # 2026-10-20: Removed skip marker, test just works (#1544) - @pytest.mark.production_server - def test_list_evaluations_setups_filter_task(self): - self.use_production_server() - task_id = [6] - size = 121 - self._check_list_evaluation_setups(tasks=task_id, size=size) + + +def _check_list_evaluation_setups(long_version, **kwargs): + evals_setups = openml.evaluations.list_evaluations_setups( + "predictive_accuracy", + **kwargs, + sort_order="desc", + ) + evals = openml.evaluations.list_evaluations( + "predictive_accuracy", + **kwargs, + sort_order="desc", + output_format="dataframe", + ) + + # Check if list is non-empty + assert len(evals_setups) > 0 + # Check if length is accurate + assert len(evals_setups) == len(evals) + # Check if output from sort is sorted in the right order + assert sorted(evals_setups["value"].tolist(), reverse=True) == evals_setups["value"].tolist() + + # Check if output and order of list_evaluations is preserved + assert evals_setups["run_id"].tolist() == evals["run_id"].tolist() + + if not long_version: + evals_setups = evals_setups.head(1) + + # Check if the hyper-parameter column is as accurate and flow_id + for _index, row in evals_setups.iterrows(): + params = openml.runs.get_run(row["run_id"]).parameter_settings + list1 = [param["oml:value"] for param in params] + list2 = list(row["parameters"].values()) + # check if all values are equal + assert sorted(list1) == sorted(list2) + return evals_setups + + +@pytest.mark.production_server +def test_evaluation_list_filter_task(): + task_id = 7312 + + evaluations = openml.evaluations.list_evaluations( + "predictive_accuracy", + size=110, + tasks=[task_id], + ) + + assert len(evaluations) > 100 + for run_id in evaluations: + assert evaluations[run_id].task_id == task_id + # default behaviour of this method: return aggregated results (not + # per fold) + assert evaluations[run_id].value is not None + assert evaluations[run_id].values is None + + +@pytest.mark.production_server +def test_evaluation_list_filter_uploader_ID_16(): + uploader_id = 16 + evaluations = openml.evaluations.list_evaluations( + "predictive_accuracy", + size=60, + uploaders=[uploader_id], + output_format="dataframe", + ) + assert evaluations["uploader"].unique() == [uploader_id] + + assert len(evaluations) > 50 + + +@pytest.mark.production_server +def test_evaluation_list_filter_uploader_ID_10(): + setup_id = 10 + evaluations = openml.evaluations.list_evaluations( + "predictive_accuracy", + size=60, + setups=[setup_id], + ) + + assert len(evaluations) > 50 + for run_id in evaluations: + assert evaluations[run_id].setup_id == setup_id + # default behaviour of this method: return aggregated results (not + # per fold) + assert evaluations[run_id].value is not None + assert evaluations[run_id].values is None + + +@pytest.mark.production_server +def test_evaluation_list_filter_flow(): + flow_id = 100 + + evaluations = openml.evaluations.list_evaluations( + "predictive_accuracy", + size=10, + flows=[flow_id], + ) + + assert len(evaluations) > 2 + for run_id in evaluations: + assert evaluations[run_id].flow_id == flow_id + # default behaviour of this method: return aggregated results (not + # per fold) + assert evaluations[run_id].value is not None + assert evaluations[run_id].values is None + + +@pytest.mark.production_server +def test_evaluation_list_filter_run(): + run_id = 12 + + evaluations = openml.evaluations.list_evaluations( + "predictive_accuracy", + size=2, + runs=[run_id], + ) + + assert len(evaluations) == 1 + for run_id in evaluations: + assert evaluations[run_id].run_id == run_id + # default behaviour of this method: return aggregated results (not + # per fold) + assert evaluations[run_id].value is not None + assert evaluations[run_id].values is None + + +@pytest.mark.production_server +def test_evaluation_list_limit(): + evaluations = openml.evaluations.list_evaluations( + "predictive_accuracy", + size=100, + offset=100, + ) + assert len(evaluations) == 100 + + +@pytest.mark.skip(reason="Affected by lowered result limit #1757") +@pytest.mark.test_server +def test_list_evaluations_empty(): + evaluations = openml.evaluations.list_evaluations("unexisting_measure") + if len(evaluations) > 0: + raise ValueError("UnitTest Outdated, got somehow results") + + assert isinstance(evaluations, dict) + + +@pytest.mark.production_server +def test_evaluation_list_per_fold(): + size = 1000 + task_ids = [6] + uploader_ids = [1] + flow_ids = [6969] + runs = [6070667, 6070666] + + evaluations = openml.evaluations.list_evaluations( + "predictive_accuracy", + size=size, + offset=0, + tasks=task_ids, + flows=flow_ids, + uploaders=uploader_ids, + runs=runs, + per_fold=True, + ) + + assert len(evaluations) == len(runs) + for run_id in evaluations: + assert evaluations[run_id].value is None + assert evaluations[run_id].values is not None + # potentially we could also test array values, but these might be + # added in the future + + evaluations = openml.evaluations.list_evaluations( + "predictive_accuracy", + size=size, + offset=0, + tasks=task_ids, + flows=flow_ids, + uploaders=uploader_ids, + runs=runs, + per_fold=False, + ) + for run_id in evaluations: + assert evaluations[run_id].value is not None + assert evaluations[run_id].values is None + + +@pytest.mark.skip(reason="Affected by lowered result limit #1757") +@pytest.mark.production_server +def test_evaluation_list_sort(): + size = 10 + task_id = 6 + # Get all evaluations of the task + unsorted_eval = openml.evaluations.list_evaluations( + "predictive_accuracy", + size=None, + offset=0, + tasks=[task_id], + ) + # Get top 10 evaluations of the same task + sorted_eval = openml.evaluations.list_evaluations( + "predictive_accuracy", + size=size, + offset=0, + tasks=[task_id], + sort_order="desc", + ) + assert len(sorted_eval) == size + assert len(unsorted_eval) > 0 + sorted_output = [evaluation.value for evaluation in sorted_eval.values()] + unsorted_output = [evaluation.value for evaluation in unsorted_eval.values()] + + # Check if output from sort is sorted in the right order + assert sorted(sorted_output, reverse=True) == sorted_output + + # Compare manual sorting against sorted output + test_output = sorted(unsorted_output, reverse=True) + assert test_output[:size] == sorted_output + + +@pytest.mark.test_server +def test_list_evaluation_measures(): + measures = openml.evaluations.list_evaluation_measures() + assert isinstance(measures, list) is True + assert all(isinstance(s, str) for s in measures) is True + + +@pytest.mark.production_server +def test_list_evaluations_setups_filter_flow(long_version_new): + flow_id = [405] + size = 100 + evals = _check_list_evaluation_setups(long_version_new, flows=flow_id, size=size) + # check if parameters in separate columns works + evals_cols = openml.evaluations.list_evaluations_setups( + "predictive_accuracy", + flows=flow_id, + size=size, + sort_order="desc", + parameters_in_separate_columns=True, + ) + columns = list(evals_cols.columns) + keys = list(evals["parameters"].values[0].keys()) + assert all(elem in columns for elem in keys) + + +# 2026-10-20: Removed skip marker, test just works (#1544) +@pytest.mark.production_server +def test_list_evaluations_setups_filter_task(long_version_new): + task_id = [6] + size = 121 + _check_list_evaluation_setups(long_version_new, tasks=task_id, size=size) diff --git a/tests/test_evaluations/test_evaluations_example.py b/tests/test_evaluations/test_evaluations_example.py index eff976389..93af77114 100644 --- a/tests/test_evaluations/test_evaluations_example.py +++ b/tests/test_evaluations/test_evaluations_example.py @@ -1,51 +1,42 @@ # License: BSD 3-Clause from __future__ import annotations -import unittest - import pytest import openml -class TestEvaluationsExample(unittest.TestCase): - @pytest.mark.skip(reason="Affected by lowered result limit #1757") - def test_example_python_paper(self): - # Example script which will appear in the upcoming OpenML-Python paper - # This test ensures that the example will keep running! - with openml.config.overwrite_config_context( # noqa: F823 - { - "server": "https://www.openml.org/api/v1/xml", - "apikey": None, - } - ): - import matplotlib.pyplot as plt - import numpy as np - - import openml - - df = openml.evaluations.list_evaluations_setups( - "predictive_accuracy", - flows=[8353], - tasks=[6], - parameters_in_separate_columns=True, - ) # Choose an SVM flow, for example 8353, and a task. - - assert len(df) > 0, ( - "No evaluation found for flow 8353 on task 6, could " - "be that this task is not available on the test server." - ) - - hp_names = ["sklearn.svm.classes.SVC(16)_C", "sklearn.svm.classes.SVC(16)_gamma"] - df[hp_names] = df[hp_names].astype(float).apply(np.log) - C, gamma, score = df[hp_names[0]], df[hp_names[1]], df["value"] - - cntr = plt.tricontourf(C, gamma, score, levels=12, cmap="RdBu_r") - plt.colorbar(cntr, label="accuracy") - plt.xlim((min(C), max(C))) - plt.ylim((min(gamma), max(gamma))) - plt.xlabel("C (log10)", size=16) - plt.ylabel("gamma (log10)", size=16) - plt.title("SVM performance landscape", size=20) - - plt.tight_layout() +# Uses lower result count than paper example (see #1757) +@pytest.mark.production_server +def test_example_python_paper(): + # Example script which will appear in the upcoming OpenML-Python paper + # This test ensures that the example will keep running! + import matplotlib.pyplot as plt + import numpy as np + + df = openml.evaluations.list_evaluations_setups( + "predictive_accuracy", + tasks=[6], + flows=[8353], + parameters_in_separate_columns=True, + size=1000, + ) # Choose an SVM flow, for example 8353, and a task. + + assert len(df) > 0, ( + "No evaluation found for flow 8353 on task 6, could " + "be that this task is not available on the test server." + ) + + hp_names = ["sklearn.svm.classes.SVC(16)_C", "sklearn.svm.classes.SVC(16)_gamma"] + df[hp_names] = df[hp_names].astype(float).apply(np.log) + C, gamma, score = df[hp_names[0]], df[hp_names[1]], df["value"] + + cntr = plt.tricontourf(C, gamma, score, levels=12, cmap="RdBu_r") + plt.colorbar(cntr, label="accuracy") + plt.xlim((min(C), max(C))) + plt.ylim((min(gamma), max(gamma))) + plt.xlabel("C (log10)", size=16) + plt.ylabel("gamma (log10)", size=16) + plt.title("SVM performance landscape", size=20) + + plt.tight_layout()