From 7f57378d50c825d4fa1316be0f91ebb03bdae1dc Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 16 Oct 2026 11:44:36 +0200 Subject: [PATCH 01/11] Extract elastic search indexing as a separate service --- config/php/.env | 3 ++- docker-compose.override.yaml | 10 ---------- docker-compose.yaml | 13 +++++++++++++ 3 files changed, 15 insertions(+), 11 deletions(-) delete mode 100644 docker-compose.override.yaml diff --git a/config/php/.env b/config/php/.env index 6b12ef8..1e2c03b 100644 --- a/config/php/.env +++ b/config/php/.env @@ -11,4 +11,5 @@ DB_USER_EXPDB_READ=root DB_PASS_EXPDB_READ=ok ES_URL=elasticsearch:9200 ES_PASSWORD=default -INDEX_ES_DURING_STARTUP=true +INDEX_ES_DURING_STARTUP=false +ES_ENABLED=false diff --git a/docker-compose.override.yaml b/docker-compose.override.yaml deleted file mode 100644 index 28aa7f5..0000000 --- a/docker-compose.override.yaml +++ /dev/null @@ -1,10 +0,0 @@ -services: - php-api: - environment: - ES_ENABLED: true - depends_on: - elasticsearch: - condition: service_healthy - - - diff --git a/docker-compose.yaml b/docker-compose.yaml index 278961f..eef9102 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -48,6 +48,17 @@ services: cpus: '0.2' memory: 250M + elasticsearch-indexer: + profiles: ["all", "rest-api", "frontend", "elasticsearch"] + image: openml/php-rest-api:v1.2.11a + container_name: "openml-elasticsearch-indexer" + env_file: config/php/.env + environment: + INDEX_ES: true + depends_on: + elasticsearch: + condition: service_healthy + nginx: profiles: ["all", "minio", "rest-api", "frontend", "elasticsearch", "evaluation-engine"] build: @@ -149,6 +160,8 @@ services: depends_on: php-api: condition: service_healthy + elasticsearch-indexer: + condition: service_completed_successfully minio: profiles: ["all", "minio", "rest-api", "evaluation-engine"] From 592f6cf561ede8e3e6979562b198c5c2c1358908 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 16 Oct 2026 11:55:01 +0200 Subject: [PATCH 02/11] Update readme for change --- README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index a72ee43..b0369eb 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,11 @@ docker compose down ``` While it is up, the API is available under `http://localhost:8082`, e.g., `http://localhost:8082/datasets/1`. +Similarly, you can start the php-api directly without a profile to start it without elasticsearch: +```bash +docker compose up php-api -d +docker compose down +``` ## Known issues See the Github Issue list for the known issues. @@ -128,10 +133,6 @@ To test if dataset processing works, we can then run: ``` where `DATASET_ID` is replaced by the identifier printed in the previous step. -It is also possible to run the php-api without elasticsearch, this is useful to avoid the considerable overhead of starting the elasticsearch container and creating indices. -First, ensure that `INDEX_ES_DURING_STARTUP=false` in `config/php/.env` otherwise it will try to connect to elasticsearch and fail. -Then explicitly specify the compose file so that the `docker-compose.override.yaml` file will be ignored: `docker compose -f docker-compose.yaml php-api -d`. - ### PHP, Parquet and Croissant converter If you want to do local development on containers that are part of the docker-compose, you want those containers to change based on your code. You should have the relevant code somewhere on your system. You will need to tell the docker-compose where to find it. You can do so by setting environment variables. From caf25f2a85c066dfb0ede6059848ab31f19ab40a Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 16 Oct 2026 11:56:22 +0200 Subject: [PATCH 03/11] use correct variable name --- config/php/.env | 2 +- docker-compose.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/php/.env b/config/php/.env index 1e2c03b..13229b9 100644 --- a/config/php/.env +++ b/config/php/.env @@ -11,5 +11,5 @@ DB_USER_EXPDB_READ=root DB_PASS_EXPDB_READ=ok ES_URL=elasticsearch:9200 ES_PASSWORD=default -INDEX_ES_DURING_STARTUP=false +INDEX_ES=false ES_ENABLED=false diff --git a/docker-compose.yaml b/docker-compose.yaml index eef9102..ca41967 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -73,7 +73,7 @@ services: php-api: profiles: ["all", "minio", "rest-api", "frontend", "evaluation-engine"] - image: openml/php-rest-api:v1.2.10 + image: openml/php-rest-api:v1.2.11a container_name: "openml-php-rest-api" ports: - "8080:80" # also known as /api (nginx) From 1e91fdefdf312f581befd08f9d697e6f48f458b9 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Mon, 19 Oct 2026 15:00:37 +0300 Subject: [PATCH 04/11] make sure db is prepared before indexing --- docker-compose.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yaml b/docker-compose.yaml index ca41967..879ae69 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -58,6 +58,8 @@ services: depends_on: elasticsearch: condition: service_healthy + database-setup: + condition: service_completed_successfully nginx: profiles: ["all", "minio", "rest-api", "frontend", "elasticsearch", "evaluation-engine"] From d7d45d522624d83805de5f29f8c5c00ed1541d5b Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Mon, 19 Oct 2026 15:27:36 +0300 Subject: [PATCH 05/11] Update php version to (to be released) versions --- docker-compose.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 879ae69..178e8e2 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -50,7 +50,7 @@ services: elasticsearch-indexer: profiles: ["all", "rest-api", "frontend", "elasticsearch"] - image: openml/php-rest-api:v1.2.11a + image: openml/php-rest-api:v1.2.11 container_name: "openml-elasticsearch-indexer" env_file: config/php/.env environment: @@ -75,7 +75,7 @@ services: php-api: profiles: ["all", "minio", "rest-api", "frontend", "evaluation-engine"] - image: openml/php-rest-api:v1.2.11a + image: openml/php-rest-api:v1.2.11 container_name: "openml-php-rest-api" ports: - "8080:80" # also known as /api (nginx) From 23cb328be16599a89deb44f23b6518d7fa465c45 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Wed, 21 Oct 2026 11:18:18 +0300 Subject: [PATCH 06/11] Exclude `elasticsearch-startup` from common profiles It doesn't work well with `--wait`. --- README.md | 6 +++--- docker-compose.yaml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index b0369eb..2084e2d 100644 --- a/README.md +++ b/README.md @@ -45,10 +45,10 @@ You can use different profiles: - `[no profile]`: databases - `"elasticsearch"`: databases + nginx + elasticsearch -- `"rest-api"`: databases + nginx + elasticsearch + REST API +- `"rest-api"`: databases + nginx + REST API - `"frontend"`: databases + nginx + elasticsearch + REST API + frontend + email-server -- `"minio"`: databases + nginx + elasticsearch + REST APP + MinIO + parquet and croissant conversion -- `"evaluation-engine"`: databases + nginx + elastichsearc + REST API + MinIO + evaluation engine +- `"minio"`: databases + nginx + REST APP + MinIO + parquet and croissant conversion +- `"evaluation-engine"`: databases + nginx + REST API + MinIO + evaluation engine - `"all"`: everything Usage examples: diff --git a/docker-compose.yaml b/docker-compose.yaml index 178e8e2..17ab8b3 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -49,7 +49,7 @@ services: memory: 250M elasticsearch-indexer: - profiles: ["all", "rest-api", "frontend", "elasticsearch"] + profiles: ["elasticsearch"] image: openml/php-rest-api:v1.2.11 container_name: "openml-elasticsearch-indexer" env_file: config/php/.env From a1fe6b059e545c2cc3d5796f835f8a29bc5f43a6 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Wed, 21 Oct 2026 14:29:43 +0300 Subject: [PATCH 07/11] Use PHP API 1.3.0 and be more prudent with profiles --- docker-compose.yaml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 17ab8b3..b1c360b 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -25,7 +25,7 @@ services: condition: service_healthy elasticsearch: - profiles: ["all", "minio", "rest-api", "frontend", "elasticsearch", "evaluation-engine"] + profiles: ["all", "elasticsearch"] image: docker.elastic.co/elasticsearch/elasticsearch:6.8.23 # This version is not available on arm. Newer version do not play well with the PHP code. See https://github.com/openml/OpenML/tree/feature/elasticsearch8 container_name: "openml-elasticsearch" platform: "linux/amd64" @@ -50,10 +50,11 @@ services: elasticsearch-indexer: profiles: ["elasticsearch"] - image: openml/php-rest-api:v1.2.11 + image: openml/php-rest-api:v1.3.0 container_name: "openml-elasticsearch-indexer" env_file: config/php/.env environment: + ES_ENABLED: true INDEX_ES: true depends_on: elasticsearch: @@ -74,14 +75,12 @@ services: php-api: - profiles: ["all", "minio", "rest-api", "frontend", "evaluation-engine"] - image: openml/php-rest-api:v1.2.11 + profiles: ["all", "rest-api", "minio"] + image: openml/php-rest-api:v1.3.0 container_name: "openml-php-rest-api" ports: - "8080:80" # also known as /api (nginx) env_file: config/php/.env - environment: - ES_ENABLED: false depends_on: database-setup: condition: service_completed_successfully From b2e33ac4c88db5f28f2043f879b708b40794d4b7 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Wed, 21 Oct 2026 14:52:31 +0300 Subject: [PATCH 08/11] Use here-string to avoid pipefail --- scripts/wait_for_run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/wait_for_run.sh b/scripts/wait_for_run.sh index 9cecbd4..b5bc231 100755 --- a/scripts/wait_for_run.sh +++ b/scripts/wait_for_run.sh @@ -11,7 +11,7 @@ ELAPSED=0 while [ "$ELAPSED" -lt "$TIMEOUT" ]; do RESPONSE=$(curl -sS "http://localhost:8000/api/v1/xml/run/${RUN_ID}") - if echo "$RESPONSE" | grep -q "area_under_roc_curve"; then + if grep -q "area_under_roc_curve" <<< "$RESPONSE"; then echo "Run $RUN_ID has evaluations." exit 0 fi From 7ab8d5e1d3ac8faf8247dd2a28b4951a26edda78 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Thu, 22 Oct 2026 08:59:23 +0300 Subject: [PATCH 09/11] Bump to PHP v1.3.1 --- docker-compose.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index b1c360b..7b1255c 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -50,7 +50,7 @@ services: elasticsearch-indexer: profiles: ["elasticsearch"] - image: openml/php-rest-api:v1.3.0 + image: openml/php-rest-api:v1.3.1 container_name: "openml-elasticsearch-indexer" env_file: config/php/.env environment: @@ -76,7 +76,7 @@ services: php-api: profiles: ["all", "rest-api", "minio"] - image: openml/php-rest-api:v1.3.0 + image: openml/php-rest-api:v1.3.1 container_name: "openml-php-rest-api" ports: - "8080:80" # also known as /api (nginx) From 594dacebb062ae0282c914aa4d2a2373a9e788a4 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Thu, 22 Oct 2026 09:56:35 +0300 Subject: [PATCH 10/11] Restore profiles to ES services --- README.md | 6 +++--- docker-compose.yaml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 2084e2d..8060cd9 100644 --- a/README.md +++ b/README.md @@ -45,10 +45,10 @@ You can use different profiles: - `[no profile]`: databases - `"elasticsearch"`: databases + nginx + elasticsearch -- `"rest-api"`: databases + nginx + REST API +- `"rest-api"`: databases + nginx + elasticsearch + REST API - `"frontend"`: databases + nginx + elasticsearch + REST API + frontend + email-server -- `"minio"`: databases + nginx + REST APP + MinIO + parquet and croissant conversion -- `"evaluation-engine"`: databases + nginx + REST API + MinIO + evaluation engine +- `"minio"`: databases + nginx + elasticsearch + REST APP + MinIO + parquet and croissant conversion +- `"evaluation-engine"`: databases + nginx + elasticsearch + REST API + MinIO + evaluation engine - `"all"`: everything Usage examples: diff --git a/docker-compose.yaml b/docker-compose.yaml index 7b1255c..aec7305 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -25,7 +25,7 @@ services: condition: service_healthy elasticsearch: - profiles: ["all", "elasticsearch"] + profiles: ["all", "minio", "rest-api", "frontend", "elasticsearch", "evaluation-engine"] image: docker.elastic.co/elasticsearch/elasticsearch:6.8.23 # This version is not available on arm. Newer version do not play well with the PHP code. See https://github.com/openml/OpenML/tree/feature/elasticsearch8 container_name: "openml-elasticsearch" platform: "linux/amd64" @@ -49,7 +49,7 @@ services: memory: 250M elasticsearch-indexer: - profiles: ["elasticsearch"] + profiles: ["all", "minio", "rest-api", "frontend","elasticsearch", "evaluation-engine"] image: openml/php-rest-api:v1.3.1 container_name: "openml-elasticsearch-indexer" env_file: config/php/.env @@ -75,7 +75,7 @@ services: php-api: - profiles: ["all", "rest-api", "minio"] + profiles: ["all", "minio", "rest-api", "frontend", "evaluation-engine"] image: openml/php-rest-api:v1.3.1 container_name: "openml-php-rest-api" ports: From b796293b27c6065a8df091e350997ec5a3a4c671 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Thu, 22 Oct 2026 12:09:31 +0300 Subject: [PATCH 11/11] Share common PHP API definitions --- docker-compose.yaml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index aec7305..d8fac54 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,4 +1,8 @@ services: + x-php-image: &php-image + image: openml/php-rest-api:v1.3.1 + env_file: config/php/.env + database: image: "openml/test-database:v0.1.20260204" container_name: "openml-test-database" @@ -49,10 +53,9 @@ services: memory: 250M elasticsearch-indexer: - profiles: ["all", "minio", "rest-api", "frontend","elasticsearch", "evaluation-engine"] - image: openml/php-rest-api:v1.3.1 + profiles: ["all", "minio", "rest-api", "frontend", "elasticsearch", "evaluation-engine"] + <<: *php-image container_name: "openml-elasticsearch-indexer" - env_file: config/php/.env environment: ES_ENABLED: true INDEX_ES: true @@ -76,11 +79,10 @@ services: php-api: profiles: ["all", "minio", "rest-api", "frontend", "evaluation-engine"] - image: openml/php-rest-api:v1.3.1 + <<: *php-image container_name: "openml-php-rest-api" ports: - "8080:80" # also known as /api (nginx) - env_file: config/php/.env depends_on: database-setup: condition: service_completed_successfully