diff --git a/CHANGELOG.md b/CHANGELOG.md index 1003027a..a654d7dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] - Declare the beautifulsoup4 dependency the ES-PV and ES-VC converters import +- ES-CL: the ITACyL server is https-only, the 2025 shapefiles sit in province subfolders, and C_REFREC is the identifier - A test refuses a fixture above 5 MB, committed or merely lying in the fixture folder, because a failing convert test downloads the real source there - ES-MD: find RECINTO.shp wherever the archive puts it - Update vecorel-cli to v0.2.17: diff --git a/fiboa_cli/datasets/es_cl.py b/fiboa_cli/datasets/es_cl.py index 1d2cf4d9..499f7caa 100644 --- a/fiboa_cli/datasets/es_cl.py +++ b/fiboa_cli/datasets/es_cl.py @@ -26,27 +26,34 @@ class ESCLConverter(ESBaseConverter): license = "CC-NC: Free use of the data is permitted, but commercial exploitation is prohibited " columns = { - "DN_OID": "id", + # DN_OID is 0 in every row of every province; C_REFREC is SIGPAC's own + # 23-character recinto reference (province, municipality, agregado, zona, + # polĂ­gono, parcela, recinto), which identifies the field and follows it + # across editions. + "C_REFREC": "id", "geometry": "geometry", - "determination:datetime": "determination:datetime", "USO_SIGPAC": "crop:code", "crop:name": "crop:name", "crop:name_en": "crop:name_en", } use_code_attribute = "USO_SIGPAC" - column_additions = ESBaseConverter.column_additions | { - "determination:datetime": "2024-01-01T00:00:00Z" - } + use_variant_as_determination = True def download_files(self, uris, cache_folder=None): paths = super().download_files(uris, cache_folder) new = [] for path, uri in paths: directory = os.path.dirname(path) - ps = [z for z in os.listdir(directory) if regex.search(z)] + # the 2025 archives nest the shapefiles in a province folder + ps = [ + os.path.join(root, z) + for root, _, files in os.walk(directory) + for z in files + if regex.search(z) + ] assert len(ps), f"Missing matching shapefile in {directory}" for p in ps: - new.append((os.path.join(directory, p), uri)) + new.append((p, uri)) return new def get_urls(self): @@ -55,7 +62,7 @@ def get_urls(self): logger.warning(f"Choosing first year {self.variant}") else: assert 2019 <= int(self.variant) <= 2025, f"Wrong year {self.variant}" - base = f"http://ftp.itacyl.es/cartografia/05_SIGPAC/{self.variant}_ETRS89/Parcelario_SIGPAC_CyL_Provincias/" + base = f"https://ftp.itacyl.es/cartografia/05_SIGPAC/{self.variant}_ETRS89/Parcelario_SIGPAC_CyL_Provincias/" response = requests.get(base) assert response.status_code == 200, f"Error getting urls {response}\n{response.content}" uris = { diff --git a/tests/data-files/convert/es_cl/AVILA.zip b/tests/data-files/convert/es_cl/AVILA.zip new file mode 100644 index 00000000..a1741b33 Binary files /dev/null and b/tests/data-files/convert/es_cl/AVILA.zip differ diff --git a/tests/test_convert.py b/tests/test_convert.py index 3ac24a11..d3ce8c7e 100644 --- a/tests/test_convert.py +++ b/tests/test_convert.py @@ -44,6 +44,7 @@ "lv", "ie", "es_cat", + "es_cl", "es", "nz", "lt", @@ -78,6 +79,10 @@ def _input_files(converter, *names): "de_he": _input_files("de_he", "de_he.json"), "br_ba_lem": _input_files("br_ba_lem", "LEM_dataset.zip"), "ch": _input_files("ch", "lwb_nutzungsflaechen_v2_0_lv95.gpkg"), + "es_cl": { + "variant": "2025", + "input_files": {f"{test_path}/es_cl/AVILA.zip": ["replaceme.zip"]}, + }, "es_cat": _input_files("es_cat", "Cultius_DUN2023_GPKG.zip"), "es": {"input_files": {f"{test_path}/es/1501_ALAVA_cd_2025_20250105.gpkg.zip": ["*.gpkg"]}}, "lv": _input_files("lv", "1_100.xml"),