PT: the 2025 edition, and the right id

CHANGELOG.md

@@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - ES-CL: the ITACyL server is https-only, the 2025 shapefiles sit in province subfolders, and C_REFREC is the identifier
 - A test refuses a fixture above 5 MB, committed or merely lying in the fixture folder, because a failing convert test downloads the real source there
 - ES-MD: find RECINTO.shp wherever the archive puts it
+- PT: the 2025 edition, whose layers, crop column and area units all changed, and CUL_ID as the identifier
 - Update vecorel-cli to v0.2.17:
   - GeoJSON is read as UTF-8 as the format mandates, instead of the platform locale (cp1252 on Windows mangled umlauts)
   - GeoJSON files with a byte order mark no longer fail to read

fiboa_cli/datasets/pt.py

@@ -1,8 +1,16 @@
+import re
+
+import geopandas as gpd
 from vecorel_cli.conversion.admin import AdminConverterMixin
 
 from ..conversion.fiboa_converter import FiboaBaseConverter
 from .commons.hcat import AddHCATMixin
 
+# Up to 2023 the country is split into "Culturas_<district>" layers, from 2025 into
+# "T<NUTS 3 code>" layers. Both files carry other layers too (parcel blocks, land cover,
+# an empty "Culturas" container, a non-spatial "Codes" table) that are not field boundaries.
+DATA_LAYER = re.compile(r"^(Culturas_.+|T[0-9A-Z]{3})$")
+
 
 class PTConverter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter):
     id = "pt"
@@ -12,6 +20,7 @@ class PTConverter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter):
     # see https://www.ifap.pt/isip/ows/
     BASE = "https://www.ifap.pt/isip/ows/resources/"
     variants = {
+        "2025": BASE + "2025/culturas.gpkg",
         "2023": BASE + "2023/Continente.gpkg",
         "2022": BASE + "2022/2022.zip",
         "2021": BASE + "2021/2021.zip",
@@ -25,29 +34,51 @@ class PTConverter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter):
     }
 
     def layer_filter(self, layer, uri):
-        return layer.startswith("Culturas_")
+        return bool(DATA_LAYER.match(layer))
 
     provider = (
         "IPAP - Instituto de Financiamento da Agricultura e Pescas <https://www.ifap.pt/isip/ows/>"
     )
     license = "No conditions apply <https://inspire.ec.europa.eu/metadata-codelist/ConditionsApplyingToAccessAndUse/noConditionsApply>"
     columns = {
         "geometry": "geometry",
-        "OSA_ID": "id",
-        "CUL_ID": "block_id",
+        # CUL_ID identifies the crop parcel and is unique across every layer of
+        # both editions (3,568,852 of 3,568,852 in 2025); OSA_ID is the land
+        # occupation polygon it lies in, which several parcels can share.
+        "CUL_ID": "id",
+        "OSA_ID": "block_id",
         "CUL_CODIGO": "crop:code",
+        # The crop name is only published up to 2023; from 2025 the code is all there is.
         "CT_português": "crop:name",
         "Shape_Area": "metrics:area",
         "Shape_Length": "metrics:perimeter",
     }
     extensions = {"https://fiboa.org/crop-extension/v0.2.0/schema.yaml"}
     ec_mapping_csv = "https://fiboa.org/code/pt/pt.csv"
-    column_additions = {
-        "determination:datetime": "2023-01-01T00:00:00Z",
-    }
+    use_variant_as_determination = True
     area_is_in_ha = False
     missing_schemas = {
         "properties": {
             "block_id": {"type": "int64"},
         }
     }
+
+    def migrate(self, gdf) -> gpd.GeoDataFrame:
+        # 2025 renamed the crop code column and dropped the crop name.
+        if "PUN_CUL_CO" in gdf.columns:
+            gdf = gdf.rename(columns={"PUN_CUL_CO": "CUL_CODIGO"})
+
+        if gdf.crs is not None and gdf.crs.is_geographic:
+            # 2025 is published in WGS 84, with Shape_Area and Shape_Length computed in
+            # degrees. Recompute both in metres; up to 2023 the file is in ETRS89 /
+            # Portugal TM06 and the published values are already metric.
+            metric = gdf.geometry.to_crs("EPSG:6933")
+            gdf["Shape_Area"] = metric.area
+            gdf["Shape_Length"] = metric.length
+
+        # 2025 types the identifiers as floats, which would stringify id as "28398800.0".
+        for column in ("OSA_ID", "CUL_ID"):
+            if column in gdf.columns and gdf[column].dtype.kind == "f":
+                gdf[column] = gdf[column].astype("int64")
+
+        return super().migrate(gdf)

tests/data-files/convert/pt/README.md

@@ -1,10 +1,19 @@
-The gpkg is built up out of multiple layers, only the 'Culturas_'-prefixed layers should be loaded
+The gpkg is built up out of multiple layers, only the field boundary layers should be loaded:
+`Culturas_<district>` up to 2023, `T<NUTS 3 code>` from 2025 on.
 
-The test file is created by taking the first 100 features from these different layers. 
-We now have a representable test-case. Downloaded data is assumed in $DOWNLOADED_SOURCE/Continente.gpkg. Run
+The test files are created by taking the first 100 features from these different layers.
+Downloaded data is assumed in $DOWNLOADED_SOURCE. Run
 
 ```
 ogr2ogr Continente.gpkg $DOWNLOADED_SOURCE/Continente.gpkg Continente -limit 100
 ogr2ogr -update Continente.gpkg $DOWNLOADED_SOURCE/Continente.gpkg Culturas_Aveiro -limit 100
 ogr2ogr -update Continente.gpkg $DOWNLOADED_SOURCE/Continente.gpkg OcupacoesSolo_Aveiro -limit 100
+
+ogr2ogr culturas.gpkg $DOWNLOADED_SOURCE/culturas.gpkg T111 -limit 100
+for layer in T150 Culturas Codes; do
+  ogr2ogr -update culturas.gpkg $DOWNLOADED_SOURCE/culturas.gpkg $layer -limit 100
+done
 ```
+
+`Culturas` (empty) and `Codes` (the NUTS 3 code list) are kept in the 2025 file so the layer
+filter is exercised on the layers it has to skip.