diff --git a/CHANGELOG.md b/CHANGELOG.md index 311c9986..507c5c03 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - ES-CL: the ITACyL server is https-only, the 2025 shapefiles sit in province subfolders, and C_REFREC is the identifier - A test refuses a fixture above 5 MB, committed or merely lying in the fixture folder, because a failing convert test downloads the real source there - ES-MD: find RECINTO.shp wherever the archive puts it +- BE-WAL: match the crop by name where EuroCrops' table leaves the code empty, which covered 6.79% of the collection - CZ: find the shapefile in nested archive folders (2026) - CZ: read the 2019-2022 editions (GPZ_DP: renamed crop and area columns, no application date, ENTITA_ID is the land block), recover the crop codes the 2020 edition leaves empty, map the 167 crop codes EuroCrops does not carry, and publish a declaration that straddles two land blocks once - BE-VLG: editions 2018-2026; REF_ID is published as block_id and the row index identifies the field, the QGIS styles table in the 2026 GeoPackage is skipped, and the 2020 archive is read as cp1252 with its CRS declared diff --git a/fiboa_cli/datasets/be_wal.py b/fiboa_cli/datasets/be_wal.py index 99eb957e..047d8965 100644 --- a/fiboa_cli/datasets/be_wal.py +++ b/fiboa_cli/datasets/be_wal.py @@ -38,6 +38,9 @@ class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter): "determination:datetime": "determination:datetime", } ec_mapping_csv = "be_wal_all_years.csv" + # 208 of that table's 298 rows carry no code, only the crop name, so every + # crop whose coded row is missing went unmapped: 23,216 fields over 59 codes. + ec_mapping_name_fallback = True column_additions = { "determination:datetime": "2022-01-01T00:00:00Z", } diff --git a/fiboa_cli/datasets/commons/hcat.py b/fiboa_cli/datasets/commons/hcat.py index 5a94d219..525997f0 100644 --- a/fiboa_cli/datasets/commons/hcat.py +++ b/fiboa_cli/datasets/commons/hcat.py @@ -18,6 +18,9 @@ class AddHCATMixin: """ ec_mapping_csv: Optional[str] = None # TODO rename to hcat_mapping_csv + # Match on the crop name where the table has no row for the code: + # be_wal_all_years.csv leaves original_code empty in 208 of its 298 rows. + ec_mapping_name_fallback = False mapping_file = None ec_mapping: Optional[list[dict]] = None # TODO rename to hcat_mapping @@ -70,12 +73,26 @@ def add_hcat(self, gdf): def map_to(attribute): return {e[from_code]: e[attribute] or None for e in self.ec_mapping} + name_col = None + if self.ec_mapping_name_fallback and from_code == "original_code": + name_col = self.get_code_column(gdf, "crop:name") + + def map_by_name(attribute): + # Three Walloon crops carry a trailing space in the table. + return { + (e["original_name"] or "").strip(): e[attribute] or None + for e in self.ec_mapping + if not (e["original_code"] or "").strip() + } + col = None for k, v in zip( self.hcat_columns.keys(), ("translated_name", "HCAT3_name", "HCAT3_code") ): if v in self.ec_mapping[0]: col = crop_code_col.map(map_to(v)) + if name_col is not None: + col = col.fillna(name_col.str.strip().map(map_by_name(v))) gdf[k] = col assert np.unique(col[~col.isna()]).size > 1, "No HCAT crops mapped"