From 3b1c98503003437212163f19ee106738c6306da1 Mon Sep 17 00:00:00 2001 From: Kelvin Chilvers-Jones <119540748+kcj-coding@users.noreply.github.com> Date: Thu, 29 Oct 2026 08:37:34 +0000 Subject: [PATCH 01/13] Update harmonise.py changed normalised_value code --- digital_land/phase/harmonise.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/digital_land/phase/harmonise.py b/digital_land/phase/harmonise.py index 2f963c28..6ae77a76 100644 --- a/digital_land/phase/harmonise.py +++ b/digital_land/phase/harmonise.py @@ -5,6 +5,7 @@ from digital_land.datatype.factory import datatype_factory import shapely.wkt import logging +import re logger = logging.getLogger(__name__) @@ -202,7 +203,8 @@ def process(self, stream): if value: harmonised_values = [] for raw_value in self.category_values(value): - normalised_value = raw_value.replace(" ", "-") + #normalised_value = raw_value.replace(" ", "-") + normalised_value = re.sub(r"(\b\s\-\s\b)|(\b\s\b)","-",raw_value.replace(r"\s","-"))#.lower()) matching_value = next( ( v From 1bc8ec1bfdf4f3b4b5db43288a134d7682f389af Mon Sep 17 00:00:00 2001 From: Kelvin Chilvers-Jones <119540748+kcj-coding@users.noreply.github.com> Date: Thu, 29 Oct 2026 08:49:56 +0000 Subject: [PATCH 02/13] Update harmonise.py added plus symbol --- digital_land/phase/harmonise.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/digital_land/phase/harmonise.py b/digital_land/phase/harmonise.py index 6ae77a76..a2130730 100644 --- a/digital_land/phase/harmonise.py +++ b/digital_land/phase/harmonise.py @@ -204,7 +204,7 @@ def process(self, stream): harmonised_values = [] for raw_value in self.category_values(value): #normalised_value = raw_value.replace(" ", "-") - normalised_value = re.sub(r"(\b\s\-\s\b)|(\b\s\b)","-",raw_value.replace(r"\s","-"))#.lower()) + normalised_value = re.sub(r"(\b\s\-+\s\b)|(\b\s\b)","-",raw_value.replace(r"\s","-"))#.lower()) matching_value = next( ( v From cf4ed37146c64ab0c240309d3cb85c872b24b001 Mon Sep 17 00:00:00 2001 From: Kelvin Chilvers-Jones <119540748+kcj-coding@users.noreply.github.com> Date: Thu, 29 Oct 2026 10:54:26 +0000 Subject: [PATCH 03/13] Update harmonise.py removed commented code --- digital_land/phase/harmonise.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/digital_land/phase/harmonise.py b/digital_land/phase/harmonise.py index a2130730..dc502010 100644 --- a/digital_land/phase/harmonise.py +++ b/digital_land/phase/harmonise.py @@ -203,8 +203,7 @@ def process(self, stream): if value: harmonised_values = [] for raw_value in self.category_values(value): - #normalised_value = raw_value.replace(" ", "-") - normalised_value = re.sub(r"(\b\s\-+\s\b)|(\b\s\b)","-",raw_value.replace(r"\s","-"))#.lower()) + normalised_value = re.sub(r"(\b\s\-+\s\b)|(\b\s\b)","-",raw_value.replace(r"\s","-")) matching_value = next( ( v From 02585bb1f55ca17af322cb117529a8221a227b82 Mon Sep 17 00:00:00 2001 From: Kelvin Chilvers-Jones <119540748+kcj-coding@users.noreply.github.com> Date: Thu, 29 Oct 2026 10:58:54 +0000 Subject: [PATCH 04/13] Update test_harmonise.py test cases for normalise_value hypen change --- tests/unit/phase/test_harmonise.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/unit/phase/test_harmonise.py b/tests/unit/phase/test_harmonise.py index a1d2a3b7..cfe146e1 100755 --- a/tests/unit/phase/test_harmonise.py +++ b/tests/unit/phase/test_harmonise.py @@ -255,16 +255,23 @@ def test_validate_categorical_fields(): "description": "Test", "tree-preservation-zone-type": "", }, + { + "reference": "4", + "name": "Test TPO 4", + "description": "Test", + "tree-preservation-zone-type": "abc def", + }, ], ) output = list(h.process(reader)) - assert len(output) == 3 + assert len(output) == 4 # check the fields are set in the output assert output[0]["row"]["tree-preservation-zone-type"] == "area" assert output[1]["row"]["tree-preservation-zone-type"] == "other" assert output[2]["row"]["tree-preservation-zone-type"] == "" + assert output[3]["row"]["tree-preservation-zone-type"] == "abc-def" assert len(issues.rows) == 1 # but we get an issue generated @@ -321,12 +328,16 @@ def test_validate_categorical_field_dataset(): "reference": "7", "document-type": "MAP-TO-VALID-VALUE", }, + { + "reference": "8", + "document-type": "MAP - TO-VALID-VALUE", + }, ], ) output = list(h.process(reader)) - assert len(output) == 7 + assert len(output) == 8 # check the fields are set in the output assert output[0]["row"]["document-type"] == "notice" assert output[1]["row"]["document-type"] == "other" @@ -335,6 +346,7 @@ def test_validate_categorical_field_dataset(): assert output[4]["row"]["document-type"] == "area-appraisal" assert output[5]["row"]["document-type"] == "MAP-TO-VALID-VALUE" assert output[6]["row"]["document-type"] == "MAP-TO-VALID-VALUE" + assert output[6]["row"]["document-type"] == "MAP-TO-VALID-VALUE" assert len(issues.rows) == 1 # but we get an issue generated From 2260c71cad48c425109724510aae63e8328a2679 Mon Sep 17 00:00:00 2001 From: Kelvin Chilvers-Jones <119540748+kcj-coding@users.noreply.github.com> Date: Thu, 29 Oct 2026 11:05:11 +0000 Subject: [PATCH 05/13] Update harmonise.py aesthetic changes --- digital_land/phase/harmonise.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/digital_land/phase/harmonise.py b/digital_land/phase/harmonise.py index dc502010..4e380f02 100644 --- a/digital_land/phase/harmonise.py +++ b/digital_land/phase/harmonise.py @@ -203,7 +203,8 @@ def process(self, stream): if value: harmonised_values = [] for raw_value in self.category_values(value): - normalised_value = re.sub(r"(\b\s\-+\s\b)|(\b\s\b)","-",raw_value.replace(r"\s","-")) + normalised_value = re.sub(r"(\b\s\-+\s\b)|(\b\s\b)", + "-",raw_value.replace(r"\s","-")) matching_value = next( ( v From 052089f188d65aea520d625c8d4ea91bb2c8f3c9 Mon Sep 17 00:00:00 2001 From: Kelvin Chilvers-Jones <119540748+kcj-coding@users.noreply.github.com> Date: Thu, 29 Oct 2026 11:22:38 +0000 Subject: [PATCH 06/13] Update harmonise.py more aesthetic changes --- digital_land/phase/harmonise.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/digital_land/phase/harmonise.py b/digital_land/phase/harmonise.py index 4e380f02..8dca9999 100644 --- a/digital_land/phase/harmonise.py +++ b/digital_land/phase/harmonise.py @@ -203,8 +203,9 @@ def process(self, stream): if value: harmonised_values = [] for raw_value in self.category_values(value): - normalised_value = re.sub(r"(\b\s\-+\s\b)|(\b\s\b)", - "-",raw_value.replace(r"\s","-")) + normalised_value = re.sub( + r"(\b\s\-+\s\b)|(\b\s\b)","-",raw_value.replace(r"\s","-") + ) matching_value = next( ( v From 875d476dae0f67a7d1af68f5eff40c88784eb24e Mon Sep 17 00:00:00 2001 From: Kelvin Chilvers-Jones <119540748+kcj-coding@users.noreply.github.com> Date: Thu, 29 Oct 2026 11:28:49 +0000 Subject: [PATCH 07/13] Update harmonise.py more aesthetic changes --- digital_land/phase/harmonise.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/digital_land/phase/harmonise.py b/digital_land/phase/harmonise.py index 8dca9999..d98af896 100644 --- a/digital_land/phase/harmonise.py +++ b/digital_land/phase/harmonise.py @@ -204,7 +204,9 @@ def process(self, stream): harmonised_values = [] for raw_value in self.category_values(value): normalised_value = re.sub( - r"(\b\s\-+\s\b)|(\b\s\b)","-",raw_value.replace(r"\s","-") + r"(\b\s\-+\s\b)|(\b\s\b)", + "-", + raw_value.replace(r"\s","-"), ) matching_value = next( ( From 66586f6aac99f565699428f8b7811541a518db89 Mon Sep 17 00:00:00 2001 From: Kelvin Chilvers-Jones <119540748+kcj-coding@users.noreply.github.com> Date: Thu, 29 Oct 2026 11:37:39 +0000 Subject: [PATCH 08/13] Update test_harmonise.py fixed user error --- tests/unit/phase/test_harmonise.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/phase/test_harmonise.py b/tests/unit/phase/test_harmonise.py index cfe146e1..4f85849c 100755 --- a/tests/unit/phase/test_harmonise.py +++ b/tests/unit/phase/test_harmonise.py @@ -346,7 +346,7 @@ def test_validate_categorical_field_dataset(): assert output[4]["row"]["document-type"] == "area-appraisal" assert output[5]["row"]["document-type"] == "MAP-TO-VALID-VALUE" assert output[6]["row"]["document-type"] == "MAP-TO-VALID-VALUE" - assert output[6]["row"]["document-type"] == "MAP-TO-VALID-VALUE" + assert output[7]["row"]["document-type"] == "MAP-TO-VALID-VALUE" assert len(issues.rows) == 1 # but we get an issue generated From d6d18759c1726e666379f4bc7608682405f9d105 Mon Sep 17 00:00:00 2001 From: kcj-coding Date: Thu, 29 Oct 2026 13:10:05 +0000 Subject: [PATCH 09/13] tst --- digital_land/phase/harmonise.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/digital_land/phase/harmonise.py b/digital_land/phase/harmonise.py index d98af896..933d78bc 100644 --- a/digital_land/phase/harmonise.py +++ b/digital_land/phase/harmonise.py @@ -4,8 +4,9 @@ from digital_land.datatype.point import PointDataType from digital_land.datatype.factory import datatype_factory import shapely.wkt -import logging import re +import logging + logger = logging.getLogger(__name__) From 7bc850385dbcc55ae304a4170c355a8b90c6ae4a Mon Sep 17 00:00:00 2001 From: kcj-coding Date: Thu, 29 Oct 2026 13:24:23 +0000 Subject: [PATCH 10/13] tst --- digital_land/phase/harmonise.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/digital_land/phase/harmonise.py b/digital_land/phase/harmonise.py index 933d78bc..276309b4 100644 --- a/digital_land/phase/harmonise.py +++ b/digital_land/phase/harmonise.py @@ -207,7 +207,7 @@ def process(self, stream): normalised_value = re.sub( r"(\b\s\-+\s\b)|(\b\s\b)", "-", - raw_value.replace(r"\s","-"), + raw_value.replace(r"\s", "-"), ) matching_value = next( ( From f3614386838899b855ccc4aec7eda965a4f7ae14 Mon Sep 17 00:00:00 2001 From: kcj-coding Date: Thu, 29 Oct 2026 13:39:03 +0000 Subject: [PATCH 11/13] tst --- tests/unit/phase/test_harmonise.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/unit/phase/test_harmonise.py b/tests/unit/phase/test_harmonise.py index 4f85849c..ff8c10ee 100755 --- a/tests/unit/phase/test_harmonise.py +++ b/tests/unit/phase/test_harmonise.py @@ -255,12 +255,6 @@ def test_validate_categorical_fields(): "description": "Test", "tree-preservation-zone-type": "", }, - { - "reference": "4", - "name": "Test TPO 4", - "description": "Test", - "tree-preservation-zone-type": "abc def", - }, ], ) From 930c0468c8388615400188be5f2b70a895cdd51f Mon Sep 17 00:00:00 2001 From: kcj-coding Date: Thu, 29 Oct 2026 13:40:43 +0000 Subject: [PATCH 12/13] tst --- tests/unit/phase/test_harmonise.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit/phase/test_harmonise.py b/tests/unit/phase/test_harmonise.py index ff8c10ee..eb9458f6 100755 --- a/tests/unit/phase/test_harmonise.py +++ b/tests/unit/phase/test_harmonise.py @@ -265,7 +265,6 @@ def test_validate_categorical_fields(): assert output[0]["row"]["tree-preservation-zone-type"] == "area" assert output[1]["row"]["tree-preservation-zone-type"] == "other" assert output[2]["row"]["tree-preservation-zone-type"] == "" - assert output[3]["row"]["tree-preservation-zone-type"] == "abc-def" assert len(issues.rows) == 1 # but we get an issue generated From ad62ea8bffa5e92b6649a6942e3dee257b4d8aa1 Mon Sep 17 00:00:00 2001 From: kcj-coding Date: Thu, 29 Oct 2026 13:43:26 +0000 Subject: [PATCH 13/13] tst --- tests/unit/phase/test_harmonise.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/phase/test_harmonise.py b/tests/unit/phase/test_harmonise.py index eb9458f6..e9680bf9 100755 --- a/tests/unit/phase/test_harmonise.py +++ b/tests/unit/phase/test_harmonise.py @@ -260,7 +260,7 @@ def test_validate_categorical_fields(): output = list(h.process(reader)) - assert len(output) == 4 + assert len(output) == 3 # check the fields are set in the output assert output[0]["row"]["tree-preservation-zone-type"] == "area" assert output[1]["row"]["tree-preservation-zone-type"] == "other"