From d3210a132ec6b7cef267e644d77b44031ab05b49 Mon Sep 17 00:00:00 2001 From: Ivor Bosloper Date: Fri, 11 Sep 2026 22:00:16 +0200 Subject: [PATCH] BE-WAL: the crops were in the table all along, under their names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 23,216 of the collection's 341,968 fields — 6.79%, over 59 codes — carried no HCAT code: Jachère, Luzerne, Trèfles, Triticale d'hiver and the cereal mixes among them. They were not missing from EuroCrops' table. That table leaves original_code empty in 208 of its 298 rows and identifies the crop by name there, and the converter matched on the code alone. `ec_mapping_name_fallback` matches on the crop name where the code has no row, and only against rows the table itself leaves uncoded, so a code that is deliberately unmapped stays unmapped. It is off by default and on for be_wal, the only table known to be keyed this way — checked against de.csv, de_nrw_2021.csv, be_vlg_2021.csv, si_2021.csv, dk_2019.csv, lv_2021.csv and fr_2018.csv, none of which has an uncoded row. Names are stripped on both sides: three Walloon crops are spelled with a trailing space in the table, which is the difference between 0.06% and 0.000%. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01DVx9uQV2QPM8ecPAY3ZjXG --- CHANGELOG.md | 1 + fiboa_cli/datasets/be_wal.py | 3 +++ fiboa_cli/datasets/commons/hcat.py | 17 +++++++++++++++++ 3 files changed, 21 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a654d7dd..7a38da4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - ES-CL: the ITACyL server is https-only, the 2025 shapefiles sit in province subfolders, and C_REFREC is the identifier - A test refuses a fixture above 5 MB, committed or merely lying in the fixture folder, because a failing convert test downloads the real source there - ES-MD: find RECINTO.shp wherever the archive puts it +- BE-WAL: match the crop by name where EuroCrops' table leaves the code empty, which covered 6.79% of the collection - Update vecorel-cli to v0.2.17: - GeoJSON is read as UTF-8 as the format mandates, instead of the platform locale (cp1252 on Windows mangled umlauts) - GeoJSON files with a byte order mark no longer fail to read diff --git a/fiboa_cli/datasets/be_wal.py b/fiboa_cli/datasets/be_wal.py index 99eb957e..047d8965 100644 --- a/fiboa_cli/datasets/be_wal.py +++ b/fiboa_cli/datasets/be_wal.py @@ -38,6 +38,9 @@ class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter): "determination:datetime": "determination:datetime", } ec_mapping_csv = "be_wal_all_years.csv" + # 208 of that table's 298 rows carry no code, only the crop name, so every + # crop whose coded row is missing went unmapped: 23,216 fields over 59 codes. + ec_mapping_name_fallback = True column_additions = { "determination:datetime": "2022-01-01T00:00:00Z", } diff --git a/fiboa_cli/datasets/commons/hcat.py b/fiboa_cli/datasets/commons/hcat.py index 5a94d219..525997f0 100644 --- a/fiboa_cli/datasets/commons/hcat.py +++ b/fiboa_cli/datasets/commons/hcat.py @@ -18,6 +18,9 @@ class AddHCATMixin: """ ec_mapping_csv: Optional[str] = None # TODO rename to hcat_mapping_csv + # Match on the crop name where the table has no row for the code: + # be_wal_all_years.csv leaves original_code empty in 208 of its 298 rows. + ec_mapping_name_fallback = False mapping_file = None ec_mapping: Optional[list[dict]] = None # TODO rename to hcat_mapping @@ -70,12 +73,26 @@ def add_hcat(self, gdf): def map_to(attribute): return {e[from_code]: e[attribute] or None for e in self.ec_mapping} + name_col = None + if self.ec_mapping_name_fallback and from_code == "original_code": + name_col = self.get_code_column(gdf, "crop:name") + + def map_by_name(attribute): + # Three Walloon crops carry a trailing space in the table. + return { + (e["original_name"] or "").strip(): e[attribute] or None + for e in self.ec_mapping + if not (e["original_code"] or "").strip() + } + col = None for k, v in zip( self.hcat_columns.keys(), ("translated_name", "HCAT3_name", "HCAT3_code") ): if v in self.ec_mapping[0]: col = crop_code_col.map(map_to(v)) + if name_col is not None: + col = col.fillna(name_col.str.strip().map(map_by_name(v))) gdf[k] = col assert np.unique(col[~col.isna()]).size > 1, "No HCAT crops mapped"