fixed pipeline for UNDATA

statvar_imports/undata/UNData_places_resolved_csv.csv

@@ -2688,6 +2688,7 @@ Jaleeb Al-Shuykh Kuwait,
 Salmiya Kuwait,
 BISHKEK Kyrgyzstan,wikidataId/Q9361
 Jalal-Abad Kyrgyzstan,wikidataId/Q487689
+Manas (Jalal-Abad) Kyrgyzstan,wikidataId/Q487689
 Osh Kyrgyzstan,wikidataId/Q47282
 VIENTIANE Lao People's Democratic Republic,wikidataId/Q9326
 Daugavpils Latvia,wikidataId/Q80021
@@ -4092,14 +4093,18 @@ Tashauz Turkmenistan,wikidataId/Q487672
 GRAND TURK Turks and Caicos Islands,wikidataId/Q34205
 FUNAFUTI Tuvalu,wikidataId/Q34126
 Gulu Uganda,wikidataId/Q581379
+Gulu City Uganda,wikidataId/Q581379
 Hoima Uganda,wikidataId/Q1624192
+Hoima City Uganda,wikidataId/Q1624192
 KAMPALA Uganda,wikidataId/Q3894
 Kasese Uganda,wikidataId/Q1232016
 Kira Uganda,wikidataId/Q966819
 Lugazi Uganda,wikidataId/Q6699738
 Makindye Ssabagabo Uganda,
 Masaka Uganda,wikidataId/Q1032233
+Masaka City Uganda,wikidataId/Q1032233
 Mbarara Uganda,wikidataId/Q731039
+Mbarara City Uganda,wikidataId/Q731039
 Mukono Uganda,wikidataId/Q179198
 Nansana Uganda,wikidataId/Q965956
 Alchevsk Ukraine,wikidataId/Q161144

statvar_imports/undata/config.ini

@@ -1,2 +1,2 @@
 [DEFAULT]
-UN_ZIP_URL = https://data.un.org/Handlers/DownloadHandler.ashx?DataFilter=tableCode:240&DataMartId=POP&Format=csv&c=2,3,6,8,10,12,14,16,17,18&s=_countryEnglishNameOrderBy:asc,refYear:desc,areaCode:asc
+UN_ZIP_URL = https://data.un.org/legacy/Handlers/DownloadHandler.ashx?DataFilter=tableCode:240&DataMartId=POP&Format=csv&c=2,3,6,8,10,12,14,16,17,18&s=_countryEnglishNameOrderBy:asc,refYear:desc,areaCode:asc

statvar_imports/undata/manifest.json

@@ -21,6 +21,7 @@
                     "cleaned_csv": "output/filter_undata.csv"
                 }
             ],
+            "validation_config_file": "validation_config.json",
             "cron_schedule": "0 05 * * 3"
         }
     ]

statvar_imports/undata/validation_config.json

@@ -0,0 +1,22 @@
+{
+  "schema_version": "1.0",
+  "rules": [
+    {
+      "rule_id": "check_deleted_records_percent",
+      "description": "Checks that the percentage of deleted points is within the threshold.",
+      "validator": "DELETED_RECORDS_PERCENT",
+      "params": {
+        "threshold": 0.1
+      }
+    },
+    {
+      "rule_id": "check_max_date_freshness",
+      "description": "Checks that the latest observation year is at least 2024 for all StatVars.",
+      "validator": "SQL_VALIDATOR",
+      "params": {
+        "query": "SELECT StatVar, MaxDate FROM stats",
+        "condition": "CAST(MaxDate AS INTEGER) >= date_part('year', CURRENT_DATE) - 2"
+      }
+    }
+  ]
+}