test.yml

---
name: Tests

on:
  workflow_dispatch:

  push:
    branches:
      - main
    tags:
      - "v*.*.*"

  pull_request:
    branches:
      - main

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  test:
    name: (${{ matrix.os }},Py${{ matrix.python-version }},sk${{ matrix.scikit-learn }}${{ matrix.pandas-version != '' && format(',pd:{0}', matrix.pandas-version) || '' }},sk-only:${{ matrix.sklearn-only }})
    runs-on: ${{ matrix.os }}

    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
        scikit-learn: ["1.3.*", "1.4.*", "1.5.*", "1.6.*", "1.7.*"]
        os: [ubuntu-latest]
        sklearn-only: ["true"]

        exclude:
          # (python, sklearn) combinations for which there is no PyPI release
          # scikit-learn 1.3
          - python-version: "3.13"
            scikit-learn: "1.3.*"
          - python-version: "3.14"
            scikit-learn: "1.3.*"
          # scikit-learn 1.4
          - python-version: "3.13"
            scikit-learn: "1.4.*"
          - python-version: "3.14"
            scikit-learn: "1.4.*"
          # scikit-learn 1.5
          - python-version: "3.14"
            scikit-learn: "1.5.*"
          # scikit-learn 1.6
          - python-version: "3.14"
            scikit-learn: "1.6.*"
          # scikit-learn 1.7 is installed with pandas 3
          - python-version: "3.10"
            scikit-learn: "1.7.*"


        include:
          # Full test run on ubuntu, 3.14
          - os: ubuntu-latest
            python-version: "3.14"
            scikit-learn: "1.7.*"
            sklearn-only: "false"

          # Full test run on Windows
          - os: windows-latest
            python-version: "3.12"
            scikit-learn: "1.5.*"
            sklearn-only: "false"

          # Coverage run
          - os: ubuntu-latest
            python-version: "3.12"
            scikit-learn: "1.5.*"
            sklearn-only: "false"
            code-cov: true

    steps:
    - uses: actions/checkout@v7
      with:
        fetch-depth: 2

    - name: Setup Python ${{ matrix.python-version }}
      uses: actions/setup-python@v7
      with:
        python-version: ${{ matrix.python-version }}

    - name: Install test dependencies, scikit-learn, and pandas
      shell: bash
      run: |
        python -m pip install --upgrade pip
        pip install -e .[test] scikit-learn==${{ matrix.scikit-learn }}

        # scikit-learn 1.7+ requires pandas 3.x, earlier versions use pandas 2.x
        version="${{ matrix.scikit-learn }}"
        major=$(echo "$version" | cut -d. -f1)
        minor=$(echo "$version" | cut -d. -f2)

        if [[ "$major" -gt 1 ]] || { [[ "$major" -eq 1 ]] && [[ "$minor" -ge 7 ]]; }; then
          pip install "pandas==3.*"
        else
          pip install "pandas==2.*" "pyarrow<26"
        fi

    - name: Store repository status
      id: status-before
      if: matrix.os != 'windows-latest'
      run: |
        git_status=$(git status --porcelain -b)
        echo "BEFORE=$git_status" >> $GITHUB_ENV
        echo "Repository status before tests: $git_status"

    - name: Clone Services
      if: matrix.os == 'ubuntu-latest'
      id: clone-services
      run: |
        git clone --depth 1 https://github.com/openml/services.git

    - name: Start Docker Services
      id: start-services
      if: matrix.os == 'ubuntu-latest'
      working-directory: ./services
      run: |
        chmod -R a+rw ./data
        chmod -R a+rw ./logs
        docker compose --profile rest-api --profile minio --profile evaluation-engine up -d

        echo "Waiting for PHP API to boot..."
        timeout 60s bash -c 'until [ "$(docker inspect -f {{.State.Health.Status}} openml-php-rest-api)" == "healthy" ]; do sleep 5; done'

        echo "Final Verification: Gateway Connectivity..."
        curl -sSfL http://localhost:8000/api/v1/xml/data/1 | head -n 15

        docker container ls

    - name: Show installed dependencies
      run: python -m pip list

    - name: Run tests on Ubuntu Test
      if: matrix.os == 'ubuntu-latest'
      env:
        OPENML_TEST_SERVER_ADMIN_KEY: ${{ secrets.OPENML_TEST_SERVER_ADMIN_KEY }}
        OPENML_USE_LOCAL_SERVICES: "true"
      run: |
        if [ "${{ matrix.code-cov }}" = "true" ]; then
          codecov="--cov=openml --long --cov-report=xml"
        fi

        if [ "${{ matrix.sklearn-only }}" = "true" ]; then
          marks="sklearn and not production_server"
        else
          marks="not production_server"
        fi

        pytest -n 4 --durations=20 --dist load -sv $codecov -o log_cli=true -m "$marks"

    - name: Run tests on Ubuntu Production
      if: matrix.os == 'ubuntu-latest'
      env:
        OPENML_TEST_SERVER_ADMIN_KEY: ${{ secrets.OPENML_TEST_SERVER_ADMIN_KEY }}
        OPENML_USE_LOCAL_SERVICES: "true"
      run: |
        if [ "${{ matrix.code-cov }}" = "true" ]; then
          codecov="--cov=openml --long --cov-report=xml"
        fi

        if [ "${{ matrix.sklearn-only }}" = "true" ]; then
          marks="sklearn and production_server"
        else
          marks="production_server"
        fi

        pytest -n 4 --durations=20 --dist load -sv $codecov -o log_cli=true -m "$marks"

    - name: Run tests on Windows
      if: matrix.os == 'windows-latest'
      env:
        OPENML_TEST_SERVER_ADMIN_KEY: ${{ secrets.OPENML_TEST_SERVER_ADMIN_KEY }}
      run: |  # we need a separate step because of the bash-specific if-statement in the previous one.
        pytest -n 4 --durations=20 --dist load -sv --reruns 5 --reruns-delay 1 -m "not test_server"

    - name: Upload coverage
      if: matrix.code-cov && always()
      uses: codecov/codecov-action@v7
      with:
        files: coverage.xml
        token: ${{ secrets.CODECOV_TOKEN }}
        cleanup: true
        fail_ci_if_error: true
        verbose: true

    - name: Dump server logs
      if: always() && steps.start-services.outcome == 'success'
      run: |
        docker logs openml-php-rest-api -t

    - name: Cleanup Docker setup
      if: always() && steps.clone-services.outcome == 'success'
      working-directory: ./services
      run: |
        docker compose --profile rest-api --profile minio --profile evaluation-engine down -v
        cd ..
        sudo rm -rf services

    - name: Check for files left behind by test
      if: matrix.os != 'windows-latest' && always()
      run: |
        before="${{ env.BEFORE }}"
        after="$(git status --porcelain -b)"
        if [[ "$before" != "$after" ]]; then
            echo "git status from before: $before"
            echo "git status from after: $after"
            echo "Not all generated files have been deleted!"
            exit 1
        fi