diff --git a/digital_land/phase/harmonise.py b/digital_land/phase/harmonise.py index 2f963c28..276309b4 100644 --- a/digital_land/phase/harmonise.py +++ b/digital_land/phase/harmonise.py @@ -4,8 +4,10 @@ from digital_land.datatype.point import PointDataType from digital_land.datatype.factory import datatype_factory import shapely.wkt +import re import logging + logger = logging.getLogger(__name__) # Storing mandatory fields in dict for now until added to specification @@ -202,7 +204,11 @@ def process(self, stream): if value: harmonised_values = [] for raw_value in self.category_values(value): - normalised_value = raw_value.replace(" ", "-") + normalised_value = re.sub( + r"(\b\s\-+\s\b)|(\b\s\b)", + "-", + raw_value.replace(r"\s", "-"), + ) matching_value = next( ( v diff --git a/tests/unit/phase/test_harmonise.py b/tests/unit/phase/test_harmonise.py index a1d2a3b7..e9680bf9 100755 --- a/tests/unit/phase/test_harmonise.py +++ b/tests/unit/phase/test_harmonise.py @@ -321,12 +321,16 @@ def test_validate_categorical_field_dataset(): "reference": "7", "document-type": "MAP-TO-VALID-VALUE", }, + { + "reference": "8", + "document-type": "MAP - TO-VALID-VALUE", + }, ], ) output = list(h.process(reader)) - assert len(output) == 7 + assert len(output) == 8 # check the fields are set in the output assert output[0]["row"]["document-type"] == "notice" assert output[1]["row"]["document-type"] == "other" @@ -335,6 +339,7 @@ def test_validate_categorical_field_dataset(): assert output[4]["row"]["document-type"] == "area-appraisal" assert output[5]["row"]["document-type"] == "MAP-TO-VALID-VALUE" assert output[6]["row"]["document-type"] == "MAP-TO-VALID-VALUE" + assert output[7]["row"]["document-type"] == "MAP-TO-VALID-VALUE" assert len(issues.rows) == 1 # but we get an issue generated