From afc4c0c10b7ba6e366e57be53b10b39eb87e6261 Mon Sep 17 00:00:00 2001 From: vishwab1 Date: Thu, 15 Oct 2026 09:38:26 +0530 Subject: [PATCH 1/9] feat(AMM-2491): automatic data sync and sync failure review for StopTB laptops - Upload (van to central) every 30 min and master download every 6 h, using the central connection of the automatic BenID download (no stored password; SyncedBy = operator who connected the laptop). Own thread and lock; skips while a manual master download runs. - GET /dataSyncActivity/autoSyncStatus: last runs, last error, and rows central rejected (Processed='F') per table and SyncFailureReason. - Active only where stoptb.enforce.vanid=true and stoptb.van.id>0; manual Upload / Download Masters unchanged. 6 unit tests. Co-Authored-By: Claude Opus 5.5 --- .../environment/common_example.properties | 6 + .../dataSyncActivity/StartSyncActivity.java | 17 ++ .../SyncUtilityClassRepo.java | 2 + .../dataSyncActivity/AutoSyncService.java | 196 ++++++++++++++++++ .../dataSyncActivity/DataSyncRepository.java | 39 ++++ .../dataSyncActivity/AutoSyncServiceTest.java | 137 ++++++++++++ 6 files changed, 397 insertions(+) create mode 100644 src/main/java/com/iemr/mmu/service/dataSyncActivity/AutoSyncService.java create mode 100644 src/test/java/com/iemr/mmu/service/dataSyncActivity/AutoSyncServiceTest.java diff --git a/src/main/environment/common_example.properties b/src/main/environment/common_example.properties index 63dd2fa8..bd95f22c 100644 --- a/src/main/environment/common_example.properties +++ b/src/main/environment/common_example.properties @@ -135,3 +135,9 @@ benid.pool.bootstrapSize=10000 benid.token.dir=/opt/amrit/data benid.autoTopUp.initialDelayMs=120000 benid.autoTopUp.checkIntervalMs=300000 + +## Automatic Data Sync (StopTB laptops only; needs the BenID central connection above) +sync.autoUpload.enabled=true +sync.autoUpload.intervalMs=1800000 +sync.autoMasterDownload.enabled=true +sync.autoMasterDownload.intervalMs=21600000 diff --git a/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java b/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java index 76a9477b..973fdf41 100644 --- a/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java +++ b/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java @@ -41,6 +41,7 @@ import com.iemr.mmu.service.dataSyncActivity.DownSyncDataFromServerImpl; import com.iemr.mmu.service.dataSyncActivity.DownloadDataFromServerImpl; import com.iemr.mmu.service.dataSyncActivity.DownloadDataFromServerTransactionalImpl; +import com.iemr.mmu.service.dataSyncActivity.AutoSyncService; import com.iemr.mmu.service.dataSyncActivity.UploadDataToServerImpl; import com.iemr.mmu.service.dataSyncLayerCentral.DiagnosticDocumentFetchService; import com.iemr.mmu.service.dataSyncLayerCentral.DiagnosticDocumentPushServiceImpl; @@ -75,6 +76,9 @@ public class StartSyncActivity { @Autowired private BenIdAutoTopUpService benIdAutoTopUpService; + @Autowired + private AutoSyncService autoSyncService; + private static final String GROUP_ID = "groupID"; private static final String PROVIDER_SERVICE_MAP_ID = "providerServiceMapID"; @@ -205,6 +209,19 @@ public String startMasterDownload(@RequestBody String requestOBJ, return response.toString(); } + @Operation(summary = "Automatic data sync status and the rows central rejected on upload") + @GetMapping(value = { "/autoSyncStatus" }) + public String autoSyncStatus() { + OutputResponse response = new OutputResponse(); + try { + response.setResponse(autoSyncService.status().toString()); + } catch (Exception e) { + logger.error("Error while getting automatic sync status : " + e); + response.setError(e); + } + return response.toString(); + } + @Operation(summary = "Data sync master download progress check") @GetMapping(value = { "/checkMastersDownloadProgress" }) public String checkMastersDownloadProgress() { diff --git a/src/main/java/com/iemr/mmu/repo/syncActivity_syncLayer/SyncUtilityClassRepo.java b/src/main/java/com/iemr/mmu/repo/syncActivity_syncLayer/SyncUtilityClassRepo.java index c146a7a9..dd054c41 100644 --- a/src/main/java/com/iemr/mmu/repo/syncActivity_syncLayer/SyncUtilityClassRepo.java +++ b/src/main/java/com/iemr/mmu/repo/syncActivity_syncLayer/SyncUtilityClassRepo.java @@ -31,4 +31,6 @@ @Repository public interface SyncUtilityClassRepo extends CrudRepository { List findBySyncTableGroupIDAndDeletedOrderBySyncTableDetailID(Integer groupID, Boolean deleted); + + List findByDeletedOrderBySyncTableDetailID(Boolean deleted); } diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/AutoSyncService.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/AutoSyncService.java new file mode 100644 index 00000000..8f378d74 --- /dev/null +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/AutoSyncService.java @@ -0,0 +1,196 @@ +/* +* AMRIT – Accessible Medical Records via Integrated Technology +* Integrated EHR (Electronic Health Records) Solution +* +* Copyright (C) "Piramal Swasthya Management and Research Institute" +* +* This file is part of AMRIT. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ +package com.iemr.mmu.service.dataSyncActivity; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.locks.ReentrantLock; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +import com.iemr.mmu.service.benid.BenIdAutoTopUpService; +import com.iemr.mmu.service.benid.CentralAuthService; + +import jakarta.annotation.PreDestroy; + +/** + * StopTB laptops: runs Data Sync by itself, so central stays current and laptops get master changes without + * anyone pressing Upload or Download Masters. + * + * - Upload (van to central) every sync.autoUpload.intervalMs (default 30 min). + * - Master download (central to van) every sync.autoMasterDownload.intervalMs (default 6 h). + * + * Uses the central connection of the automatic BenID download (CentralAuthService): no password is stored, and + * SyncedBy is the operator who connected the laptop. Active only where CentralAuthService is enabled + * (stoptb.enforce.vanid=true and stoptb.van.id>0); central and other deployments are unaffected. The manual + * buttons keep working as before. + */ +@Service +public class AutoSyncService { + + private final Logger logger = LoggerFactory.getLogger(this.getClass().getSimpleName()); + + /** Upload and master download never run at the same time from here. */ + private static final ReentrantLock SYNC_LOCK = new ReentrantLock(); + + @Autowired + private CentralAuthService centralAuthService; + + @Autowired + private UploadDataToServerImpl uploadDataToServerImpl; + + @Autowired + private DownloadDataFromServerImpl downloadDataFromServerImpl; + + @Autowired + private DataSyncRepository dataSyncRepository; + + @Autowired + private BenIdAutoTopUpService benIdAutoTopUpService; + + @Value("${sync.autoUpload.enabled:true}") + private boolean uploadEnabled; + + @Value("${sync.autoMasterDownload.enabled:true}") + private boolean masterDownloadEnabled; + + private volatile long lastUploadAt; + private volatile String lastUploadResult; + private volatile long lastMasterDownloadAt; + private volatile String lastMasterDownloadResult; + private volatile String lastError; + + // own thread: a long upload must not hold the shared scheduler thread + private final ExecutorService background = Executors.newSingleThreadExecutor(r -> { + Thread t = new Thread(r, "auto-data-sync"); + t.setDaemon(true); + return t; + }); + + @PreDestroy + public void shutdown() { + background.shutdownNow(); + } + + @Scheduled(initialDelayString = "${sync.autoUpload.initialDelayMs:600000}", fixedDelayString = "${sync.autoUpload.intervalMs:1800000}") + public void scheduledUpload() { + if (uploadEnabled && centralAuthService.isEnabled()) { + background.submit(this::uploadOnce); + } + } + + @Scheduled(initialDelayString = "${sync.autoMasterDownload.initialDelayMs:900000}", fixedDelayString = "${sync.autoMasterDownload.intervalMs:21600000}") + public void scheduledMasterDownload() { + if (masterDownloadEnabled && centralAuthService.isEnabled()) { + background.submit(this::masterDownloadOnce); + } + } + + void uploadOnce() { + if (!SYNC_LOCK.tryLock()) { + return; + } + try { + String key = centralAuthService.getCentralSessionKey(); + if (key == null) { + lastError = "No central connection - waiting for a user login"; + return; + } + int vanID = centralAuthService.getVanID(); + String user = centralAuthService.getConnectedUser(); + String result = uploadDataToServerImpl.getDataToSyncToServer(vanID, + user != null && !user.isBlank() ? user : "auto-sync", key, null); + lastUploadAt = System.currentTimeMillis(); + lastUploadResult = result; + lastError = null; + logger.info("Automatic upload for vanID {} finished: {}", vanID, result); + // internet is up: top up the BenID pool if it is low (same as the manual upload) + benIdAutoTopUpService.triggerNow(); + } catch (Exception e) { + lastError = "Upload: " + e.getMessage(); + logger.warn("Automatic upload failed: {}", e.getMessage()); + } finally { + SYNC_LOCK.unlock(); + } + } + + void masterDownloadOnce() { + if (!SYNC_LOCK.tryLock()) { + return; + } + try { + String key = centralAuthService.getCentralSessionKey(); + if (key == null) { + lastError = "No central connection - waiting for a user login"; + return; + } + int vanID = centralAuthService.getVanID(); + Integer psm = dataSyncRepository.getVanProviderServiceMapID(vanID); + if (psm == null) { + lastError = "Master download: van " + vanID + " not found in m_van"; + return; + } + String result = downloadDataFromServerImpl.downloadMasterDataFromServer(key, null, vanID, psm); + if ("inProgress".equalsIgnoreCase(result)) { + logger.info("Master download already running (manual) - skipping the automatic one"); + return; + } + lastMasterDownloadAt = System.currentTimeMillis(); + lastMasterDownloadResult = result; + lastError = null; + logger.info("Automatic master download for vanID {} finished: {}", vanID, result); + } catch (Exception e) { + lastError = "Master download: " + e.getMessage(); + logger.warn("Automatic master download failed: {}", e.getMessage()); + } finally { + SYNC_LOCK.unlock(); + } + } + + /** For the Data Sync screen: last runs and the rows central has rejected. */ + public JSONObject status() { + JSONObject s = new JSONObject(); + s.put("enabled", centralAuthService.isEnabled()); + s.put("uploadEnabled", uploadEnabled); + s.put("masterDownloadEnabled", masterDownloadEnabled); + s.put("lastUploadAt", lastUploadAt); + s.put("lastUploadResult", lastUploadResult == null ? JSONObject.NULL : lastUploadResult); + s.put("lastMasterDownloadAt", lastMasterDownloadAt); + s.put("lastMasterDownloadResult", + lastMasterDownloadResult == null ? JSONObject.NULL : lastMasterDownloadResult); + s.put("lastError", lastError == null ? JSONObject.NULL : lastError); + List> failures = dataSyncRepository.getSyncFailures(); + long failedRows = failures.stream().mapToLong(f -> ((Number) f.get("failedRows")).longValue()).sum(); + s.put("failedRows", failedRows); + s.put("failures", new JSONArray(failures)); + return s; + } +} diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java index 75a74842..0d918b82 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java @@ -23,9 +23,11 @@ import java.sql.Timestamp; import java.util.ArrayList; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.sql.PreparedStatement; @@ -40,6 +42,7 @@ import org.springframework.jdbc.support.KeyHolder; import org.springframework.stereotype.Service; +import com.iemr.mmu.data.syncActivity_syncLayer.SyncUtilityClass; import com.iemr.mmu.repo.syncActivity_syncLayer.SyncUtilityClassRepo; import com.iemr.mmu.utils.validator.SqlIdentifierValidator; @@ -127,6 +130,42 @@ public int updateProcessedFlagInVan(String schemaName, String tableName, List psm = getJdbcTemplate().queryForList( + "SELECT ProviderServiceMapID FROM db_iemr.m_van WHERE VanID = ? AND Deleted = false", Integer.class, + vanID); + return psm.isEmpty() ? null : psm.get(0); + } + + /** + * Rows central rejected on upload (Processed = 'F'), per sync table and reason. They are sent again on every + * upload until central accepts them. + */ + public List> getSyncFailures() { + jdbcTemplate = getJdbcTemplate(); + List> result = new ArrayList<>(); + Set seen = new HashSet<>(); + for (SyncUtilityClass table : syncutilityClassRepo.findByDeletedOrderBySyncTableDetailID(false)) { + if (Boolean.TRUE.equals(table.getIsMaster()) || table.getSchemaName() == null + || table.getTableName() == null) + continue; + String qualified = SqlIdentifierValidator.validatedSchemaName(table.getSchemaName()) + "." + + SqlIdentifierValidator.validatedTableName(table.getTableName()); + if (!seen.add(qualified.toLowerCase())) + continue; + try { + List> rows = jdbcTemplate.queryForList("SELECT '" + table.getTableName() + + "' AS tableName, SyncFailureReason AS reason, COUNT(*) AS failedRows, MAX(SyncedDate) AS lastTried " + + "FROM " + qualified + " WHERE Processed = 'F' GROUP BY SyncFailureReason"); + result.addAll(rows); + } catch (Exception e) { + logger.warn("Sync failure check skipped for {}: {}", qualified, e.getMessage()); + } + } + return result; + } + // ---------------------------------- End of Upload repository // ---------------------------------- Download Repository diff --git a/src/test/java/com/iemr/mmu/service/dataSyncActivity/AutoSyncServiceTest.java b/src/test/java/com/iemr/mmu/service/dataSyncActivity/AutoSyncServiceTest.java new file mode 100644 index 00000000..a997e756 --- /dev/null +++ b/src/test/java/com/iemr/mmu/service/dataSyncActivity/AutoSyncServiceTest.java @@ -0,0 +1,137 @@ +/* +* AMRIT – Accessible Medical Records via Integrated Technology +* Integrated EHR (Electronic Health Records) Solution +* +* Copyright (C) "Piramal Swasthya Management and Research Institute" +* +* This file is part of AMRIT. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ +package com.iemr.mmu.service.dataSyncActivity; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.List; +import java.util.Map; + +import org.json.JSONObject; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.test.util.ReflectionTestUtils; + +import com.iemr.mmu.service.benid.BenIdAutoTopUpService; +import com.iemr.mmu.service.benid.CentralAuthService; + +class AutoSyncServiceTest { + + private AutoSyncService service; + private CentralAuthService centralAuth; + private UploadDataToServerImpl upload; + private DownloadDataFromServerImpl download; + private DataSyncRepository repo; + private BenIdAutoTopUpService benId; + + @BeforeEach + void setUp() throws Exception { + centralAuth = mock(CentralAuthService.class); + upload = mock(UploadDataToServerImpl.class); + download = mock(DownloadDataFromServerImpl.class); + repo = mock(DataSyncRepository.class); + benId = mock(BenIdAutoTopUpService.class); + when(centralAuth.isEnabled()).thenReturn(true); + when(centralAuth.getVanID()).thenReturn(403); + when(centralAuth.getConnectedUser()).thenReturn("stoptb1"); + when(centralAuth.getCentralSessionKey()).thenReturn("SESSION-KEY"); + when(repo.getVanProviderServiceMapID(403)).thenReturn(1726); + + service = new AutoSyncService(); + ReflectionTestUtils.setField(service, "centralAuthService", centralAuth); + ReflectionTestUtils.setField(service, "uploadDataToServerImpl", upload); + ReflectionTestUtils.setField(service, "downloadDataFromServerImpl", download); + ReflectionTestUtils.setField(service, "dataSyncRepository", repo); + ReflectionTestUtils.setField(service, "benIdAutoTopUpService", benId); + ReflectionTestUtils.setField(service, "uploadEnabled", true); + ReflectionTestUtils.setField(service, "masterDownloadEnabled", true); + } + + @Test + void uploadUsesCentralKeyAndConnectedOperator() throws Exception { + when(upload.getDataToSyncToServer(403, "stoptb1", "SESSION-KEY", null)).thenReturn("Data successfully synced"); + + service.uploadOnce(); + + verify(upload).getDataToSyncToServer(403, "stoptb1", "SESSION-KEY", null); + verify(benId).triggerNow(); + assertEquals("Data successfully synced", service.status().get("lastUploadResult")); + } + + @Test + void noCentralConnectionSkipsUpload() throws Exception { + when(centralAuth.getCentralSessionKey()).thenReturn(null); + + service.uploadOnce(); + + verify(upload, never()).getDataToSyncToServer(anyInt(), anyString(), anyString(), any()); + assertEquals("No central connection - waiting for a user login", service.status().get("lastError")); + } + + @Test + void masterDownloadUsesVanServiceLine() throws Exception { + when(download.downloadMasterDataFromServer("SESSION-KEY", null, 403, 1726)).thenReturn("success"); + + service.masterDownloadOnce(); + + verify(download).downloadMasterDataFromServer("SESSION-KEY", null, 403, 1726); + assertEquals("success", service.status().get("lastMasterDownloadResult")); + } + + @Test + void manualMasterDownloadRunningIsNotCountedAsARun() throws Exception { + when(download.downloadMasterDataFromServer("SESSION-KEY", null, 403, 1726)).thenReturn("inProgress"); + + service.masterDownloadOnce(); + + assertEquals(0L, service.status().getLong("lastMasterDownloadAt")); + } + + @Test + void statusSumsRejectedRows() throws Exception { + when(repo.getSyncFailures()).thenReturn(List.of( + Map.of("tableName", "t_patientissue", "reason", "Duplicate", "failedRows", 2L), + Map.of("tableName", "t_prescription", "reason", "Timeout", "failedRows", 3L))); + + JSONObject status = service.status(); + + assertEquals(5L, status.getLong("failedRows")); + assertEquals(2, status.getJSONArray("failures").length()); + } + + @Test + void disabledLaptopDoesNothing() throws Exception { + when(centralAuth.isEnabled()).thenReturn(false); + + service.scheduledUpload(); + service.scheduledMasterDownload(); + + verify(centralAuth, never()).getCentralSessionKey(); + } +} From e66151e8bc7e9040eb6d8b03726d8c24cfe8e06d Mon Sep 17 00:00:00 2001 From: vishwab1 Date: Thu, 15 Oct 2026 09:39:19 +0530 Subject: [PATCH 2/9] fix(AMM-2491): send sync failure time as text Co-Authored-By: Claude Opus 5.5 --- .../iemr/mmu/service/dataSyncActivity/DataSyncRepository.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java index 0d918b82..bee4a5ef 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java @@ -156,7 +156,7 @@ public List> getSyncFailures() { continue; try { List> rows = jdbcTemplate.queryForList("SELECT '" + table.getTableName() - + "' AS tableName, SyncFailureReason AS reason, COUNT(*) AS failedRows, MAX(SyncedDate) AS lastTried " + + "' AS tableName, SyncFailureReason AS reason, COUNT(*) AS failedRows, DATE_FORMAT(MAX(SyncedDate), '%d-%m-%Y %H:%i') AS lastTried " + "FROM " + qualified + " WHERE Processed = 'F' GROUP BY SyncFailureReason"); result.addAll(rows); } catch (Exception e) { From abfe8894ef3db5de5c6c00cd1a7bb068794a4c9c Mon Sep 17 00:00:00 2001 From: vishwab1 Date: Fri, 16 Oct 2026 07:35:16 +0530 Subject: [PATCH 3/9] feat(AMM-2491): automatic inventory data download after each upload Refills and stock issued at central for this van's store (indents, indent issues, stock transfers, stock entries) reach the laptop without pressing Inventory Data Download. Runs after each automatic upload; its own failure does not undo the upload; shown in autoSyncStatus. Switch: sync.autoInventoryDownload.enabled (default true). 2 new unit tests. Co-Authored-By: Claude Opus 5.5 --- .../environment/common_example.properties | 1 + .../dataSyncActivity/AutoSyncService.java | 29 +++++++++++++++++++ .../dataSyncActivity/AutoSyncServiceTest.java | 26 +++++++++++++++++ 3 files changed, 56 insertions(+) diff --git a/src/main/environment/common_example.properties b/src/main/environment/common_example.properties index bd95f22c..0cb819fb 100644 --- a/src/main/environment/common_example.properties +++ b/src/main/environment/common_example.properties @@ -141,3 +141,4 @@ sync.autoUpload.enabled=true sync.autoUpload.intervalMs=1800000 sync.autoMasterDownload.enabled=true sync.autoMasterDownload.intervalMs=21600000 +sync.autoInventoryDownload.enabled=true diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/AutoSyncService.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/AutoSyncService.java index 8f378d74..3b584fa4 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/AutoSyncService.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/AutoSyncService.java @@ -70,6 +70,9 @@ public class AutoSyncService { @Autowired private DownloadDataFromServerImpl downloadDataFromServerImpl; + @Autowired + private DownloadDataFromServerTransactionalImpl downloadDataFromServerTransactional; + @Autowired private DataSyncRepository dataSyncRepository; @@ -82,10 +85,16 @@ public class AutoSyncService { @Value("${sync.autoMasterDownload.enabled:true}") private boolean masterDownloadEnabled; + // refills / stock issued at central for this van's store ("Inventory Data Download"), after each upload + @Value("${sync.autoInventoryDownload.enabled:true}") + private boolean inventoryDownloadEnabled; + private volatile long lastUploadAt; private volatile String lastUploadResult; private volatile long lastMasterDownloadAt; private volatile String lastMasterDownloadResult; + private volatile long lastInventoryDownloadAt; + private volatile String lastInventoryDownloadResult; private volatile String lastError; // own thread: a long upload must not hold the shared scheduler thread @@ -134,6 +143,8 @@ void uploadOnce() { logger.info("Automatic upload for vanID {} finished: {}", vanID, result); // internet is up: top up the BenID pool if it is low (same as the manual upload) benIdAutoTopUpService.triggerNow(); + if (inventoryDownloadEnabled) + inventoryDownload(vanID, key); } catch (Exception e) { lastError = "Upload: " + e.getMessage(); logger.warn("Automatic upload failed: {}", e.getMessage()); @@ -142,6 +153,20 @@ void uploadOnce() { } } + // Same as the "Inventory Data Download" button: indents, indent issues, stock transfers and stock entries + // that central has for this van's store. Its own failure does not undo the upload. + private void inventoryDownload(int vanID, String key) { + try { + int done = downloadDataFromServerTransactional.downloadTransactionalData(vanID, key, "datasync"); + lastInventoryDownloadAt = System.currentTimeMillis(); + lastInventoryDownloadResult = done > 0 ? "success" : "nothing downloaded"; + logger.info("Automatic inventory download for vanID {} finished: {}", vanID, lastInventoryDownloadResult); + } catch (Exception e) { + lastError = "Inventory download: " + e.getMessage(); + logger.warn("Automatic inventory download failed: {}", e.getMessage()); + } + } + void masterDownloadOnce() { if (!SYNC_LOCK.tryLock()) { return; @@ -186,6 +211,10 @@ public JSONObject status() { s.put("lastMasterDownloadAt", lastMasterDownloadAt); s.put("lastMasterDownloadResult", lastMasterDownloadResult == null ? JSONObject.NULL : lastMasterDownloadResult); + s.put("inventoryDownloadEnabled", inventoryDownloadEnabled); + s.put("lastInventoryDownloadAt", lastInventoryDownloadAt); + s.put("lastInventoryDownloadResult", + lastInventoryDownloadResult == null ? JSONObject.NULL : lastInventoryDownloadResult); s.put("lastError", lastError == null ? JSONObject.NULL : lastError); List> failures = dataSyncRepository.getSyncFailures(); long failedRows = failures.stream().mapToLong(f -> ((Number) f.get("failedRows")).longValue()).sum(); diff --git a/src/test/java/com/iemr/mmu/service/dataSyncActivity/AutoSyncServiceTest.java b/src/test/java/com/iemr/mmu/service/dataSyncActivity/AutoSyncServiceTest.java index a997e756..75c533eb 100644 --- a/src/test/java/com/iemr/mmu/service/dataSyncActivity/AutoSyncServiceTest.java +++ b/src/test/java/com/iemr/mmu/service/dataSyncActivity/AutoSyncServiceTest.java @@ -49,6 +49,7 @@ class AutoSyncServiceTest { private DownloadDataFromServerImpl download; private DataSyncRepository repo; private BenIdAutoTopUpService benId; + private DownloadDataFromServerTransactionalImpl inventory; @BeforeEach void setUp() throws Exception { @@ -57,6 +58,7 @@ void setUp() throws Exception { download = mock(DownloadDataFromServerImpl.class); repo = mock(DataSyncRepository.class); benId = mock(BenIdAutoTopUpService.class); + inventory = mock(DownloadDataFromServerTransactionalImpl.class); when(centralAuth.isEnabled()).thenReturn(true); when(centralAuth.getVanID()).thenReturn(403); when(centralAuth.getConnectedUser()).thenReturn("stoptb1"); @@ -69,6 +71,8 @@ void setUp() throws Exception { ReflectionTestUtils.setField(service, "downloadDataFromServerImpl", download); ReflectionTestUtils.setField(service, "dataSyncRepository", repo); ReflectionTestUtils.setField(service, "benIdAutoTopUpService", benId); + ReflectionTestUtils.setField(service, "downloadDataFromServerTransactional", inventory); + ReflectionTestUtils.setField(service, "inventoryDownloadEnabled", true); ReflectionTestUtils.setField(service, "uploadEnabled", true); ReflectionTestUtils.setField(service, "masterDownloadEnabled", true); } @@ -84,6 +88,28 @@ void uploadUsesCentralKeyAndConnectedOperator() throws Exception { assertEquals("Data successfully synced", service.status().get("lastUploadResult")); } + @Test + void uploadIsFollowedByInventoryDownload() throws Exception { + when(upload.getDataToSyncToServer(403, "stoptb1", "SESSION-KEY", null)).thenReturn("Data successfully synced"); + when(inventory.downloadTransactionalData(403, "SESSION-KEY", "datasync")).thenReturn(1); + + service.uploadOnce(); + + verify(inventory).downloadTransactionalData(403, "SESSION-KEY", "datasync"); + assertEquals("success", service.status().get("lastInventoryDownloadResult")); + } + + @Test + void inventoryDownloadFailureKeepsUploadResult() throws Exception { + when(upload.getDataToSyncToServer(403, "stoptb1", "SESSION-KEY", null)).thenReturn("Data successfully synced"); + when(inventory.downloadTransactionalData(403, "SESSION-KEY", "datasync")).thenThrow(new RuntimeException("offline")); + + service.uploadOnce(); + + assertEquals("Data successfully synced", service.status().get("lastUploadResult")); + assertEquals("Inventory download: offline", service.status().get("lastError")); + } + @Test void noCentralConnectionSkipsUpload() throws Exception { when(centralAuth.getCentralSessionKey()).thenReturn(null); From 826233e3ff4dda54bfb5d4e9251a8711d9cefc2a Mon Sep 17 00:00:00 2001 From: vishwab1 Date: Fri, 16 Oct 2026 07:35:39 +0530 Subject: [PATCH 4/9] fix(datasync): inventory data download skipped indent orders and stock transfers The loop over the 5 tables also did i++ at the end of each pass, so only t_indent, t_indentissue and t_itemstockentry were downloaded; t_indentorder and t_stocktransfer never reached the van. Co-Authored-By: Claude Opus 5.5 --- .../DownloadDataFromServerTransactionalImpl.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownloadDataFromServerTransactionalImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownloadDataFromServerTransactionalImpl.java index ee3c39a5..e441ea6e 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownloadDataFromServerTransactionalImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownloadDataFromServerTransactionalImpl.java @@ -227,8 +227,6 @@ public int downloadTransactionalData(int vanID, String ServerAuthorization, Stri default: } - - i++; } return 1; From a290f38437c7222eccf6eba4e4510d927d0e4cf6 Mon Sep 17 00:00:00 2001 From: vishwab1 Date: Tue, 20 Oct 2026 10:21:44 +0530 Subject: [PATCH 5/9] fix(datasync): central keeps the van row's VanSerialNo and store on upload m_synctabledetail maps van columns to server columns by position (the van's own ID sits where the server list has VanSerialNo); since the by-name record mapping, VanSerialNo was stored NULL, so a re-sent row was inserted again (duplicates) and joins on VanSerialNo broke. VanSerialNo now takes the van row ID and SyncFacilityID the row's store when not sent by name (dataSync.fillVanKeys, default true). Master download on central logs the row count instead of every row (the 661k-row village table exhausted the heap). Co-Authored-By: Claude Opus 5.5 --- .../DataSyncRepositoryCentralDownload.java | 3 +- .../GetDataFromVanAndSyncToDBImpl.java | 33 ++++++++- .../GetDataFromVanAndSyncToDBVanKeysTest.java | 71 +++++++++++++++++++ 3 files changed, 105 insertions(+), 2 deletions(-) create mode 100644 src/test/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDataFromVanAndSyncToDBVanKeysTest.java diff --git a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java index c76962b4..50c49c0e 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java @@ -205,7 +205,8 @@ else if (masterType.equalsIgnoreCase("P")) { } logger.info("Select query central: " + baseQuery); logger.info("Last Downloaded Date " + lastDownloadDate); - logger.info("Result set Details: " + resultSetList); + // Row count only: logging the rows of a large master (villages, 661k rows) exhausted the heap. + logger.info("Rows returned: " + resultSetList.size()); return resultSetList; } diff --git a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDataFromVanAndSyncToDBImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDataFromVanAndSyncToDBImpl.java index c0e508cd..2b7df046 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDataFromVanAndSyncToDBImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDataFromVanAndSyncToDBImpl.java @@ -53,6 +53,10 @@ public class GetDataFromVanAndSyncToDBImpl implements GetDataFromVanAndSyncToDB @Value("${downSync.markDeliveredOnUpSync:true}") private boolean markDownSyncedOnUpSync; + // fill VanSerialNo / SyncFacilityID from the van row when not sent by name (see fillVanKeys) + @Value("${dataSync.fillVanKeys:true}") + private boolean fillVanKeysEnabled = true; + private static final Map> TABLE_GROUPS = new HashMap<>(); static { TABLE_GROUPS.put(1, @@ -372,7 +376,9 @@ private boolean performGenericTableSync(SyncUploadDataDigester syncUploadDataDig // CRITICAL FIX: Keep null as null, don't convert to string cleanRecord.put(cleanKey.trim(), value); } - + if (fillVanKeysEnabled) + fillVanKeys(cleanRecord, serverColumnsList, vanAutoIncColumnName); + String vanSerialNo = String.valueOf(cleanRecord.get(vanAutoIncColumnName)); // Column may be "vanID" (Stop TB tables) or "VanID" (standard tables) — check both @@ -606,6 +612,31 @@ private boolean performGenericTableSync(SyncUploadDataDigester syncUploadDataDig return overallSuccess; } + /** + * m_synctabledetail maps van columns to server columns by position: the van's own ID (VanAutoIncColumnName, + * e.g. BenVisitID, ItemStockEntryID) sits where the server list has VanSerialNo. Records are filled by name, so + * without this VanSerialNo stays NULL: the row cannot be matched on the next upload (duplicates) and joins on + * VanSerialNo break. Same for SyncFacilityID, which some van lists do not send: it is the row's store. + */ + static void fillVanKeys(Map record, List serverColumns, String vanAutoIncColumnName) { + if (hasColumn(serverColumns, "VanSerialNo") && record.get("VanSerialNo") == null + && vanAutoIncColumnName != null && record.get(vanAutoIncColumnName) != null) { + record.put("VanSerialNo", record.get(vanAutoIncColumnName)); + } + if (hasColumn(serverColumns, "SyncFacilityID") && record.get("SyncFacilityID") == null + && record.get("FacilityID") != null) { + record.put("SyncFacilityID", record.get("FacilityID")); + } + } + + private static boolean hasColumn(List columns, String name) { + for (String column : columns) { + if (column.trim().equals(name)) + return true; + } + return false; + } + private void flagUpSyncedRecordsForDownSync(String schemaName, String tableName, List syncResults, Map vanIDByResultIndex) { if (!markDownSyncedOnUpSync || vanIDByResultIndex.isEmpty()) diff --git a/src/test/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDataFromVanAndSyncToDBVanKeysTest.java b/src/test/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDataFromVanAndSyncToDBVanKeysTest.java new file mode 100644 index 00000000..23a2a513 --- /dev/null +++ b/src/test/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDataFromVanAndSyncToDBVanKeysTest.java @@ -0,0 +1,71 @@ +package com.iemr.mmu.service.dataSyncLayerCentral; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.Test; + +class GetDataFromVanAndSyncToDBVanKeysTest { + + // server list of t_itemstockentry (van list has ItemStockEntryID in VanSerialNo's place) + private static final List STOCK_ENTRY_SERVER = Arrays.asList("FacilityID", "ItemID", "BatchNo", + "QuantityInHand", "VanSerialNo", "VanID", "SyncFacilityID"); + + @Test + void vanSerialNoTakesTheVanRowIdWhenNotSentByName() { + Map record = new HashMap<>(Map.of("ItemStockEntryID", 6, "FacilityID", 428, "VanID", 428)); + + GetDataFromVanAndSyncToDBImpl.fillVanKeys(record, STOCK_ENTRY_SERVER, "ItemStockEntryID"); + + assertEquals(6, record.get("VanSerialNo")); + } + + @Test + void syncFacilityIdTakesTheRowStoreWhenNotSent() { + Map record = new HashMap<>(Map.of("ItemStockExitID", 9, "FacilityID", 428)); + + GetDataFromVanAndSyncToDBImpl.fillVanKeys(record, Arrays.asList("ItemStockEntryID", "FacilityID", + " VanSerialNo", "SyncFacilityID"), "ItemStockExitID"); + + assertEquals(428, record.get("SyncFacilityID")); + assertEquals(9, record.get("VanSerialNo")); + } + + @Test + void valuesSentByTheVanAreKept() { + Map record = new HashMap<>( + Map.of("PatientIssueID", 3, "VanSerialNo", 3, "FacilityID", 428, "SyncFacilityID", 428)); + + GetDataFromVanAndSyncToDBImpl.fillVanKeys(record, Arrays.asList("VanSerialNo", "SyncFacilityID"), + "PatientIssueID"); + + assertEquals(3, record.get("VanSerialNo")); + assertEquals(428, record.get("SyncFacilityID")); + } + + @Test + void columnsNotOnTheServerAreNotAdded() { + Map record = new HashMap<>(Map.of("BenVisitID", 1991, "VanID", 428)); + + GetDataFromVanAndSyncToDBImpl.fillVanKeys(record, Arrays.asList("BeneficiaryRegID", "VanSerialNo", "VanID"), + "BenVisitID"); + + assertEquals(1991, record.get("VanSerialNo")); + assertFalse(record.containsKey("SyncFacilityID")); + } + + @Test + void missingVanIdLeavesVanSerialNoEmpty() { + Map record = new HashMap<>(Map.of("FacilityID", 428)); + + GetDataFromVanAndSyncToDBImpl.fillVanKeys(record, STOCK_ENTRY_SERVER, "ItemStockEntryID"); + + assertNull(record.get("VanSerialNo")); + } +} From 15add9e42589af78210e6574b2d927f775d65240 Mon Sep 17 00:00:00 2001 From: vishwab1 Date: Tue, 20 Oct 2026 10:21:44 +0530 Subject: [PATCH 6/9] feat(AMM-2491): StopTB laptop master download and automatic sync off by default - StopTB laptops (stoptb.enforce.vanid=true): service-line and van masters (hubs, vans, stores, items, item-store mapping, roles) are downloaded in full for the van's line, so a laptop set up from a copied database gets them all; shared masters (villages) stay incremental. Other MMU laptops unchanged. - Automatic upload / master download / inventory download are off by default (manual sync until approved); when on, the automatic master download fetches only the inventory masters (sync.autoMasterDownload.inventoryOnly=true). Co-Authored-By: Claude Opus 5.5 --- .../environment/common_example.properties | 10 ++-- .../dataSyncActivity/AutoSyncService.java | 22 +++++++-- .../DownloadDataFromServerImpl.java | 30 ++++++++++++ .../dataSyncActivity/AutoSyncServiceTest.java | 21 ++++++-- .../DownloadMastersFullForLineTest.java | 49 +++++++++++++++++++ 5 files changed, 120 insertions(+), 12 deletions(-) create mode 100644 src/test/java/com/iemr/mmu/service/dataSyncActivity/DownloadMastersFullForLineTest.java diff --git a/src/main/environment/common_example.properties b/src/main/environment/common_example.properties index 0cb819fb..1009a9d6 100644 --- a/src/main/environment/common_example.properties +++ b/src/main/environment/common_example.properties @@ -136,9 +136,11 @@ benid.token.dir=/opt/amrit/data benid.autoTopUp.initialDelayMs=120000 benid.autoTopUp.checkIntervalMs=300000 -## Automatic Data Sync (StopTB laptops only; needs the BenID central connection above) -sync.autoUpload.enabled=true +## Automatic Data Sync (StopTB laptops only; needs the BenID central connection above). Off by default: manual sync only. +sync.autoUpload.enabled=false sync.autoUpload.intervalMs=1800000 -sync.autoMasterDownload.enabled=true +sync.autoMasterDownload.enabled=false sync.autoMasterDownload.intervalMs=21600000 -sync.autoInventoryDownload.enabled=true +# automatic master download fetches only inventory masters (items, stores, item->store mapping); false = all masters +sync.autoMasterDownload.inventoryOnly=true +sync.autoInventoryDownload.enabled=false diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/AutoSyncService.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/AutoSyncService.java index 3b584fa4..e8971fe7 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/AutoSyncService.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/AutoSyncService.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.locks.ReentrantLock; @@ -52,6 +53,9 @@ * SyncedBy is the operator who connected the laptop. Active only where CentralAuthService is enabled * (stoptb.enforce.vanid=true and stoptb.van.id>0); central and other deployments are unaffected. The manual * buttons keep working as before. + * + * Off by default (manual sync only) until the automatic schedule is approved: set sync.autoUpload.enabled, + * sync.autoMasterDownload.enabled and sync.autoInventoryDownload.enabled to true to switch it on. */ @Service public class AutoSyncService { @@ -79,16 +83,25 @@ public class AutoSyncService { @Autowired private BenIdAutoTopUpService benIdAutoTopUpService; - @Value("${sync.autoUpload.enabled:true}") + @Value("${sync.autoUpload.enabled:false}") private boolean uploadEnabled; - @Value("${sync.autoMasterDownload.enabled:true}") + @Value("${sync.autoMasterDownload.enabled:false}") private boolean masterDownloadEnabled; // refills / stock issued at central for this van's store ("Inventory Data Download"), after each upload - @Value("${sync.autoInventoryDownload.enabled:true}") + @Value("${sync.autoInventoryDownload.enabled:false}") private boolean inventoryDownloadEnabled; + // automatic master download fetches only the inventory masters (items, stores, item->store mapping, Min Stock); + // other masters (users, complaints, villages) come with the manual Download Masters button + @Value("${sync.autoMasterDownload.inventoryOnly:true}") + private boolean masterDownloadInventoryOnly; + + static final Set INVENTORY_MASTERS = Set.of("m_item", "m_itemcategory", "m_itemissueconfig", "m_uom", + "m_manufacturer", "m_pharmacologicalcategory", "m_supplier", "m_facilitytype", "m_facility", + "m_itemfacilitymapping"); + private volatile long lastUploadAt; private volatile String lastUploadResult; private volatile long lastMasterDownloadAt; @@ -183,7 +196,8 @@ void masterDownloadOnce() { lastError = "Master download: van " + vanID + " not found in m_van"; return; } - String result = downloadDataFromServerImpl.downloadMasterDataFromServer(key, null, vanID, psm); + String result = downloadDataFromServerImpl.downloadMasterDataFromServer(key, null, vanID, psm, + masterDownloadInventoryOnly ? INVENTORY_MASTERS : null); if ("inProgress".equalsIgnoreCase(result)) { logger.info("Master download already running (manual) - skipping the automatic one"); return; diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownloadDataFromServerImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownloadDataFromServerImpl.java index 3ad4eaa8..572724be 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownloadDataFromServerImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownloadDataFromServerImpl.java @@ -27,6 +27,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -62,6 +63,9 @@ public class DownloadDataFromServerImpl implements DownloadDataFromServer { // rest URLs from server to download master data for van @Value("${dataSyncDownloadUrl}") private String dataSyncDownloadUrl; + // StopTB camp laptop (laptops are set up from copied databases); other MMU deployments keep standard behaviour + @Value("${stoptb.enforce.vanid:false}") + private boolean stopTBLaptop; @Autowired private SyncDownloadMasterRepo syncDownloadMasterRepo; @Autowired @@ -97,6 +101,14 @@ public class DownloadDataFromServerImpl implements DownloadDataFromServer { */ public String downloadMasterDataFromServer(String ServerAuthorization, String jwtToken, Integer vanID, Integer psmID) throws Exception { + return downloadMasterDataFromServer(ServerAuthorization, jwtToken, vanID, psmID, null); + } + + /** + * @param onlyTables master tables to download (case-insensitive), or null for all of m_downloadmastertable + */ + public String downloadMasterDataFromServer(String ServerAuthorization, String jwtToken, Integer vanID, Integer psmID, + Set onlyTables) throws Exception { if (totalCounter != progressCounter) { return "inProgress"; @@ -105,6 +117,10 @@ public String downloadMasterDataFromServer(String ServerAuthorization, String jw String successFlag = " Master download started "; ArrayList downloadMasterList = syncDownloadMasterRepo.getDownloadTables(); + if (onlyTables != null && downloadMasterList != null) { + downloadMasterList.removeIf(t -> t.getTableName() == null + || !onlyTables.contains(t.getTableName().toLowerCase())); + } totalCounter = downloadMasterList.size(); progressCounter = 0; @@ -180,6 +196,8 @@ public String call() { private int downloadDataFromServer(SyncDownloadMaster syncDownloadMaster,String ServerAuthorization, String jwtToken) throws Exception { int successFlag = 0; + if (stopTBLaptop) + fullDownloadForLineAndVanMasters(syncDownloadMaster); // initializing RestTemplate RestTemplate restTemplate = new RestTemplate(); // Provide the required second argument, e.g., an empty string or appropriate authorization token @@ -212,6 +230,18 @@ private int downloadDataFromServer(SyncDownloadMaster syncDownloadMaster,String return successFlag; } + /** + * StopTB laptops only. Service-line (P) and van (V) masters - hubs, vans, service points, stores, items, item-store mapping, roles - + * are always downloaded in full for this van's line: they are small, and a laptop set up from a copied database + * can carry download dates newer than central rows it never received (missing hub -> its vans fail on the + * foreign key). Rows are inserted/updated by ID, so repeating them is harmless. Masters shared by all lines (A, + * e.g. 661k villages) stay incremental. + */ + static void fullDownloadForLineAndVanMasters(SyncDownloadMaster table) { + if ("P".equalsIgnoreCase(table.getMasterType()) || "V".equalsIgnoreCase(table.getMasterType())) + table.setLastDownloadDate(null); + } + private int updateMastersWithLatestData(MasterDownloadDataDigester masterDownloadDataDigester, SyncDownloadMaster syncDownloadMaster) { int successFlag = 0; diff --git a/src/test/java/com/iemr/mmu/service/dataSyncActivity/AutoSyncServiceTest.java b/src/test/java/com/iemr/mmu/service/dataSyncActivity/AutoSyncServiceTest.java index 75c533eb..07fa285f 100644 --- a/src/test/java/com/iemr/mmu/service/dataSyncActivity/AutoSyncServiceTest.java +++ b/src/test/java/com/iemr/mmu/service/dataSyncActivity/AutoSyncServiceTest.java @@ -75,6 +75,7 @@ void setUp() throws Exception { ReflectionTestUtils.setField(service, "inventoryDownloadEnabled", true); ReflectionTestUtils.setField(service, "uploadEnabled", true); ReflectionTestUtils.setField(service, "masterDownloadEnabled", true); + ReflectionTestUtils.setField(service, "masterDownloadInventoryOnly", true); } @Test @@ -121,18 +122,30 @@ void noCentralConnectionSkipsUpload() throws Exception { } @Test - void masterDownloadUsesVanServiceLine() throws Exception { - when(download.downloadMasterDataFromServer("SESSION-KEY", null, 403, 1726)).thenReturn("success"); + void masterDownloadUsesVanServiceLineAndOnlyInventoryMasters() throws Exception { + when(download.downloadMasterDataFromServer("SESSION-KEY", null, 403, 1726, AutoSyncService.INVENTORY_MASTERS)) + .thenReturn("success"); service.masterDownloadOnce(); - verify(download).downloadMasterDataFromServer("SESSION-KEY", null, 403, 1726); + verify(download).downloadMasterDataFromServer("SESSION-KEY", null, 403, 1726, AutoSyncService.INVENTORY_MASTERS); assertEquals("success", service.status().get("lastMasterDownloadResult")); } + @Test + void masterDownloadCanFetchAllMasters() throws Exception { + ReflectionTestUtils.setField(service, "masterDownloadInventoryOnly", false); + when(download.downloadMasterDataFromServer("SESSION-KEY", null, 403, 1726, null)).thenReturn("success"); + + service.masterDownloadOnce(); + + verify(download).downloadMasterDataFromServer("SESSION-KEY", null, 403, 1726, null); + } + @Test void manualMasterDownloadRunningIsNotCountedAsARun() throws Exception { - when(download.downloadMasterDataFromServer("SESSION-KEY", null, 403, 1726)).thenReturn("inProgress"); + when(download.downloadMasterDataFromServer("SESSION-KEY", null, 403, 1726, AutoSyncService.INVENTORY_MASTERS)) + .thenReturn("inProgress"); service.masterDownloadOnce(); diff --git a/src/test/java/com/iemr/mmu/service/dataSyncActivity/DownloadMastersFullForLineTest.java b/src/test/java/com/iemr/mmu/service/dataSyncActivity/DownloadMastersFullForLineTest.java new file mode 100644 index 00000000..d92182b2 --- /dev/null +++ b/src/test/java/com/iemr/mmu/service/dataSyncActivity/DownloadMastersFullForLineTest.java @@ -0,0 +1,49 @@ +package com.iemr.mmu.service.dataSyncActivity; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import java.sql.Timestamp; + +import org.junit.jupiter.api.Test; + +import com.iemr.mmu.data.syncActivity_syncLayer.SyncDownloadMaster; + +class DownloadMastersFullForLineTest { + + private static final Timestamp COPIED_DATE = Timestamp.valueOf("2026-10-16 13:17:31"); + + private static SyncDownloadMaster table(String masterType) { + SyncDownloadMaster t = new SyncDownloadMaster(); + t.setMasterType(masterType); + t.setLastDownloadDate(COPIED_DATE); + return t; + } + + @Test + void serviceLineMastersAreDownloadedInFull() { + SyncDownloadMaster parkingPlace = table("P"); + + DownloadDataFromServerImpl.fullDownloadForLineAndVanMasters(parkingPlace); + + assertNull(parkingPlace.getLastDownloadDate()); + } + + @Test + void vanMastersAreDownloadedInFull() { + SyncDownloadMaster vanServicePoint = table("v"); + + DownloadDataFromServerImpl.fullDownloadForLineAndVanMasters(vanServicePoint); + + assertNull(vanServicePoint.getLastDownloadDate()); + } + + @Test + void mastersSharedByAllLinesStayIncremental() { + SyncDownloadMaster villages = table("A"); + + DownloadDataFromServerImpl.fullDownloadForLineAndVanMasters(villages); + + assertEquals(COPIED_DATE, villages.getLastDownloadDate()); + } +} From 3c2c86d2dccec463f1ef85a45be326d761fb7318 Mon Sep 17 00:00:00 2001 From: vishwab1 Date: Tue, 20 Oct 2026 10:45:03 +0530 Subject: [PATCH 7/9] fix(datasync): central links uploaded rows to central visit/prescription IDs A van sends its own IDs in BenVisitID / PrescriptionID; on central those numbers belong to other rows, so prescriptions and prescribed drugs were linked to another van's visit/prescription, or rejected by the foreign key when the number did not exist. With dataSync.translateVanLinks=true the link is replaced by the central ID of the row the same van uploaded with that VanSerialNo; if that row is not on central yet the record is not saved and is sent again on the next upload. Off by default (other MMU servers unchanged); to be switched on for the StopTB central. Co-Authored-By: Claude Opus 5.5 --- .../environment/common_example.properties | 5 + .../DataSyncRepositoryCentral.java | 16 +++ .../GetDataFromVanAndSyncToDBImpl.java | 50 +++++++++ ...GetDataFromVanAndSyncToDBVanLinksTest.java | 101 ++++++++++++++++++ 4 files changed, 172 insertions(+) create mode 100644 src/test/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDataFromVanAndSyncToDBVanLinksTest.java diff --git a/src/main/environment/common_example.properties b/src/main/environment/common_example.properties index 1009a9d6..46996a54 100644 --- a/src/main/environment/common_example.properties +++ b/src/main/environment/common_example.properties @@ -144,3 +144,8 @@ sync.autoMasterDownload.intervalMs=21600000 # automatic master download fetches only inventory masters (items, stores, item->store mapping); false = all masters sync.autoMasterDownload.inventoryOnly=true sync.autoInventoryDownload.enabled=false + +## Central upload: replace van IDs in link columns (BenVisitID, PrescriptionID) by central IDs. +## Off by default (other MMU servers unchanged); on for the StopTB central. +## Env alternative: DATASYNC_TRANSLATEVANLINKS=true +dataSync.translateVanLinks=false diff --git a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentral.java b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentral.java index 0da5bf4b..75aa52a4 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentral.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentral.java @@ -173,6 +173,22 @@ public int checkRecordIsAlreadyPresentOrNot(String schemaName, String tableName, } } + /** + * Central ID of a row a van uploaded earlier, found by the van's own ID (VanSerialNo) and the van; null if that + * row is not on central (yet). + */ + public Long findCentralIdOfVanRow(String schemaName, String tableName, String idColumn, Object vanSerialNo, + String vanID) { + if (!isValidSchemaName(schemaName) || !isValidDatabaseIdentifierCharacter(tableName) + || !isValidDatabaseIdentifierCharacter(idColumn)) { + throw new IllegalArgumentException("Invalid identifiers provided."); + } + List ids = getJdbcTemplate().queryForList("SELECT " + idColumn + " FROM " + schemaName + "." + tableName + + " WHERE VanSerialNo = ? AND VanID = ? ORDER BY " + idColumn + " DESC LIMIT 1", Long.class, + String.valueOf(vanSerialNo), vanID); + return ids.isEmpty() ? null : ids.get(0); + } + public int[] syncDataToCentralDB(String schema, String tableName, String serverColumns, String query, List syncDataList) { jdbcTemplate = getJdbcTemplate(); diff --git a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDataFromVanAndSyncToDBImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDataFromVanAndSyncToDBImpl.java index 2b7df046..4db30fff 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDataFromVanAndSyncToDBImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDataFromVanAndSyncToDBImpl.java @@ -25,6 +25,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -57,6 +58,17 @@ public class GetDataFromVanAndSyncToDBImpl implements GetDataFromVanAndSyncToDB @Value("${dataSync.fillVanKeys:true}") private boolean fillVanKeysEnabled = true; + // replace van IDs in link columns by central IDs (see translateVanLinks); off by default, on for StopTB central + @Value("${dataSync.translateVanLinks:false}") + private boolean translateVanLinksEnabled; + + // link column -> {table it points to, that table's ID column} + private static final Map VAN_LINKS = new LinkedHashMap<>(); + static { + VAN_LINKS.put("BenVisitID", new String[] { "t_benvisitdetail", "BenVisitID" }); + VAN_LINKS.put("PrescriptionID", new String[] { "t_prescription", "PrescriptionID" }); + } + private static final Map> TABLE_GROUPS = new HashMap<>(); static { TABLE_GROUPS.put(1, @@ -378,6 +390,18 @@ private boolean performGenericTableSync(SyncUploadDataDigester syncUploadDataDig } if (fillVanKeysEnabled) fillVanKeys(cleanRecord, serverColumnsList, vanAutoIncColumnName); + if (translateVanLinksEnabled) { + Object linkVanID = cleanRecord.get("VanID") != null ? cleanRecord.get("VanID") : cleanRecord.get("vanID"); + String linkError = translateVanLinks(schemaName, syncTableName, cleanRecord, serverColumnsList, + linkVanID == null ? null : String.valueOf(linkVanID)); + if (linkError != null) { + syncResults.add(new SyncResult(schemaName, syncTableName, + String.valueOf(cleanRecord.get(vanAutoIncColumnName)), syncUploadDataDigester.getSyncedBy(), + false, linkError)); + overallSuccess = false; + continue; + } + } String vanSerialNo = String.valueOf(cleanRecord.get(vanAutoIncColumnName)); @@ -629,6 +653,32 @@ static void fillVanKeys(Map record, List serverColumns, } } + /** + * A van sends its own IDs in link columns (BenVisitID, PrescriptionID). On central those numbers belong to other + * rows (other vans, central data): the link pointed to the wrong visit/prescription, or the foreign key rejected + * the row. The link is replaced by the central ID of the row the van uploaded with that VanSerialNo. If that row + * is not on central yet, the record is not saved (returned reason) and is sent again on the next upload. + * + * @return null when done, or the reason the record cannot be saved yet + */ + String translateVanLinks(String schemaName, String tableName, Map record, + List serverColumns, String vanID) { + for (Map.Entry link : VAN_LINKS.entrySet()) { + String column = link.getKey(); + String parentTable = link.getValue()[0]; + Object vanValue = record.get(column); + if (vanValue == null || vanID == null || tableName.equalsIgnoreCase(parentTable) + || !hasColumn(serverColumns, column)) + continue; + Long centralID = dataSyncRepositoryCentral.findCentralIdOfVanRow(schemaName, parentTable, + link.getValue()[1], vanValue, vanID); + if (centralID == null) + return "Linked " + parentTable + " (van ID " + vanValue + ") not on central yet; sent again on next upload"; + record.put(column, centralID); + } + return null; + } + private static boolean hasColumn(List columns, String name) { for (String column : columns) { if (column.trim().equals(name)) diff --git a/src/test/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDataFromVanAndSyncToDBVanLinksTest.java b/src/test/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDataFromVanAndSyncToDBVanLinksTest.java new file mode 100644 index 00000000..0b5e6017 --- /dev/null +++ b/src/test/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDataFromVanAndSyncToDBVanLinksTest.java @@ -0,0 +1,101 @@ +package com.iemr.mmu.service.dataSyncLayerCentral; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.test.util.ReflectionTestUtils; + +class GetDataFromVanAndSyncToDBVanLinksTest { + + private GetDataFromVanAndSyncToDBImpl service; + private DataSyncRepositoryCentral repo; + + @BeforeEach + void setUp() { + service = new GetDataFromVanAndSyncToDBImpl(); + repo = mock(DataSyncRepositoryCentral.class); + ReflectionTestUtils.setField(service, "dataSyncRepositoryCentral", repo); + } + + private static Map row(Object... kv) { + Map m = new HashMap<>(); + for (int i = 0; i < kv.length; i += 2) + m.put((String) kv[i], kv[i + 1]); + return m; + } + + @Test + void prescriptionGetsTheCentralIdOfItsVisit() { + // laptop visit 1261 of van 428 is central visit 5003 (central's own 1261 is another van's visit) + when(repo.findCentralIdOfVanRow("db_iemr", "t_benvisitdetail", "BenVisitID", 1261, "428")).thenReturn(5003L); + Map rx = row("PrescriptionID", 378, "BenVisitID", 1261, "VanID", 428); + + String error = service.translateVanLinks("db_iemr", "t_prescription", rx, + List.of("BenVisitID", "VanSerialNo", "VanID"), "428"); + + assertNull(error); + assertEquals(5003L, rx.get("BenVisitID")); + assertEquals(378, rx.get("PrescriptionID")); // own ID of t_prescription is not a link + } + + @Test + void prescribedDrugGetsCentralVisitAndPrescription() { + when(repo.findCentralIdOfVanRow("db_iemr", "t_benvisitdetail", "BenVisitID", 1261, "428")).thenReturn(5003L); + when(repo.findCentralIdOfVanRow("db_iemr", "t_prescription", "PrescriptionID", 378, "428")).thenReturn(9001L); + Map drug = row("PrescribedDrugID", 5, "BenVisitID", 1261, "PrescriptionID", 378); + + assertNull(service.translateVanLinks("db_iemr", "t_prescribeddrug", drug, + Arrays.asList("BenVisitID", "PrescriptionID", "VanSerialNo"), "428")); + + assertEquals(5003L, drug.get("BenVisitID")); + assertEquals(9001L, drug.get("PrescriptionID")); + } + + @Test + void parentNotOnCentralYetIsNotSaved() { + when(repo.findCentralIdOfVanRow(anyString(), anyString(), anyString(), any(), anyString())).thenReturn(null); + Map rx = row("BenVisitID", 1991); + + String error = service.translateVanLinks("db_iemr", "t_prescription", rx, List.of("BenVisitID"), "428"); + + assertNotNull(error); + assertEquals(1991, rx.get("BenVisitID")); // left as it was, row not saved + } + + @Test + void visitItselfAndRowsWithoutLinksAreUnchanged() { + Map visit = row("BenVisitID", 1261, "VisitCode", 10042800001261L); + assertNull(service.translateVanLinks("db_iemr", "t_benvisitdetail", visit, List.of("BenVisitID"), "428")); + assertEquals(1261, visit.get("BenVisitID")); + + Map batch = row("ItemStockEntryID", 41, "FacilityID", 428); + assertNull(service.translateVanLinks("db_iemr", "t_itemstockentry", batch, List.of("FacilityID"), "428")); + + Map noVan = row("BenVisitID", 1261); + assertNull(service.translateVanLinks("db_iemr", "t_prescription", noVan, List.of("BenVisitID"), null)); + verify(repo, never()).findCentralIdOfVanRow(anyString(), anyString(), anyString(), any(), anyString()); + } + + @Test + void linkColumnNotUploadedIsLeftAlone() { + Map issue = row("PrescriptionID", 378); + + assertNull(service.translateVanLinks("db_iemr", "t_patientissue", issue, List.of("VisitCode"), "428")); + + verify(repo, never()).findCentralIdOfVanRow(anyString(), anyString(), anyString(), any(), anyString()); + } +} From 7e00e5d20bf94418b735d408b52f74acd1fc96a1 Mon Sep 17 00:00:00 2001 From: vishwab1 Date: Tue, 20 Oct 2026 11:11:20 +0530 Subject: [PATCH 8/9] fix(datasync): StopTB laptop uploads stock tables only through the Inventory group Inventory tables are configured in group 5 (old lists: stock-outs without FacilityID/SyncFacilityID, patient issues without SyncFacilityID, batch insert fails) and in group 9 Inventory (complete). Group 5 runs first and marks the rows uploaded, so central stored stock-outs without their camp store and per-store reports missed them. On StopTB laptops (stoptb.enforce.vanid=true) a table that a later group also uploads is skipped. Other MMU laptops unchanged. Co-Authored-By: Claude Opus 5.5 --- .../UploadDataToServerImpl.java | 29 +++++++++++ .../UploadSkipDuplicateGroupTest.java | 51 +++++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 src/test/java/com/iemr/mmu/service/dataSyncActivity/UploadSkipDuplicateGroupTest.java diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/UploadDataToServerImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/UploadDataToServerImpl.java index d9567861..ab74ca16 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/UploadDataToServerImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/UploadDataToServerImpl.java @@ -80,6 +80,11 @@ public class UploadDataToServerImpl implements UploadDataToServer { @Value("${BATCH_SIZE}") private int BATCH_SIZE; + // StopTB camp laptop: a table configured in several sync groups is uploaded only by its last group (see + // skipTablesUploadedByLaterGroup) + @Value("${stoptb.enforce.vanid:false}") + private boolean stopTBLaptop; + @Autowired private DataSyncRepository dataSyncRepository; @Autowired @@ -407,10 +412,34 @@ private String startDataSync(int vanID, String user, String Authorization, Strin private List getVanAndServerColumns(Integer groupID) throws Exception { List syncUtilityClassList = getVanAndServerColumnList(groupID); + if (stopTBLaptop) + syncUtilityClassList = skipTablesUploadedByLaterGroup(syncUtilityClassList, groupID, + syncutilityClassRepo.findByDeletedOrderBySyncTableDetailID(false)); logger.debug("Fetched SyncUtilityClass list for groupID {}: {}", groupID, syncUtilityClassList); return syncUtilityClassList; } + /** + * Inventory tables are configured twice: group 5 (old lists without the store columns of stock-outs and patient + * issues, batch insert fails) and group 9 "Inventory" (complete). Group 5 ran first and marked the rows uploaded, + * so central got stock-outs without their camp store. A table that a later group also uploads is skipped here. + */ + static List skipTablesUploadedByLaterGroup(List groupTables, int groupID, + List allTables) { + Map lastGroup = new HashMap<>(); + for (SyncUtilityClass t : allTables) { + if (t.getTableName() != null && t.getSyncTableGroupID() != null) + lastGroup.merge(t.getTableName().toLowerCase(), t.getSyncTableGroupID(), Math::max); + } + List kept = new ArrayList<>(); + for (SyncUtilityClass t : groupTables) { + Integer last = t.getTableName() == null ? null : lastGroup.get(t.getTableName().toLowerCase()); + if (last == null || last <= groupID) + kept.add(t); + } + return kept; + } + public List getVanAndServerColumnList(Integer groupID) throws Exception { List syncUtilityClassList = syncutilityClassRepo .findBySyncTableGroupIDAndDeletedOrderBySyncTableDetailID(groupID, false); diff --git a/src/test/java/com/iemr/mmu/service/dataSyncActivity/UploadSkipDuplicateGroupTest.java b/src/test/java/com/iemr/mmu/service/dataSyncActivity/UploadSkipDuplicateGroupTest.java new file mode 100644 index 00000000..438922a5 --- /dev/null +++ b/src/test/java/com/iemr/mmu/service/dataSyncActivity/UploadSkipDuplicateGroupTest.java @@ -0,0 +1,51 @@ +package com.iemr.mmu.service.dataSyncActivity; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.List; +import java.util.stream.Collectors; + +import org.junit.jupiter.api.Test; + +import com.iemr.mmu.data.syncActivity_syncLayer.SyncUtilityClass; + +class UploadSkipDuplicateGroupTest { + + private static SyncUtilityClass table(int group, String name) { + SyncUtilityClass t = new SyncUtilityClass(); + t.setSyncTableGroupID(group); + t.setTableName(name); + return t; + } + + private static List names(List tables) { + return tables.stream().map(SyncUtilityClass::getTableName).collect(Collectors.toList()); + } + + // as on the laptops: inventory tables in group 5 (old lists) and group 9 (complete) + private final List all = List.of(table(4, "t_Prescription"), table(5, "t_Lab_TestResult"), + table(5, "t_ItemStockEntry"), table(5, "t_ItemStockExit"), table(5, "t_PatientIssue"), + table(9, "t_itemstockentry"), table(9, "t_itemstockexit"), table(9, "t_patientissue"), + table(9, "t_stockadjustment")); + + @Test + void inventoryTablesAreSkippedInGroup5() { + List group5 = all.stream().filter(t -> t.getSyncTableGroupID() == 5).collect(Collectors.toList()); + + assertEquals(List.of("t_Lab_TestResult"), names(UploadDataToServerImpl.skipTablesUploadedByLaterGroup(group5, 5, all))); + } + + @Test + void group9StillUploadsThem() { + List group9 = all.stream().filter(t -> t.getSyncTableGroupID() == 9).collect(Collectors.toList()); + + assertEquals(names(group9), names(UploadDataToServerImpl.skipTablesUploadedByLaterGroup(group9, 9, all))); + } + + @Test + void tablesInOneGroupAreUnchanged() { + List group4 = List.of(table(4, "t_Prescription")); + + assertEquals(List.of("t_Prescription"), names(UploadDataToServerImpl.skipTablesUploadedByLaterGroup(group4, 4, all))); + } +} From 184a7fdfa744d0d7acfe6b508450f0086f59caed Mon Sep 17 00:00:00 2001 From: vishwab1 Date: Tue, 20 Oct 2026 11:30:22 +0530 Subject: [PATCH 9/9] fix(datasync): StopTB laptop upload uses the renamed reason_to_close column and continues past a failing table The laptop sync list still names tb_diagnostic_order.reason_for_refusal; V125 renamed it to reason_to_close and the whole upload stopped with bad SQL grammar. The column name the table really has is used for the van and server column lists, and a table that cannot be read is recorded as failed while the other tables still upload. Both only on a StopTB laptop (stoptb.enforce.vanid). Co-Authored-By: Claude Opus 5.5 --- .../dataSyncActivity/DataSyncRepository.java | 11 ++++ .../UploadDataToServerImpl.java | 60 +++++++++++++++++-- .../UploadRenamedColumnTest.java | 48 +++++++++++++++ 3 files changed, 114 insertions(+), 5 deletions(-) create mode 100644 src/test/java/com/iemr/mmu/service/dataSyncActivity/UploadRenamedColumnTest.java diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java index bee4a5ef..f4850923 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java @@ -215,6 +215,17 @@ public String resolveLastModColumn(String schema, String table) { return (found == null || found.isEmpty()) ? null : found.get(0); } + public Set getTableColumns(String schema, String table) { + jdbcTemplate = getJdbcTemplate(); + + String query = " SELECT COLUMN_NAME FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? "; + + Set columns = new HashSet<>(); + for (String c : jdbcTemplate.queryForList(query, String.class, schema, table)) + columns.add(c.toLowerCase()); + return columns; + } + public List getDownSyncColumns(String schema, String table) { jdbcTemplate = getJdbcTemplate(); diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/UploadDataToServerImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/UploadDataToServerImpl.java index ab74ca16..b0aa115d 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/UploadDataToServerImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/UploadDataToServerImpl.java @@ -26,6 +26,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import org.json.JSONArray; import org.json.JSONException; @@ -156,9 +157,33 @@ private String startDataSync(int vanID, String user, String Authorization, Strin String tableKey = obj.getSchemaName() + "." + obj.getTableName(); boolean tableHasError = false; - // get data from DB to sync to server - List> syncData = getDataToSync(obj.getSchemaName(), obj.getTableName(), - obj.getVanColumnName()); + String[] columns = { obj.getVanColumnName(), obj.getServerColumnName() }; + List> syncData; + try { + if (stopTBLaptop) + columns = useRenamedColumns(obj.getTableName(), columns[0], columns[1], + dataSyncRepository.getTableColumns(obj.getSchemaName(), obj.getTableName())); + + // get data from DB to sync to server + syncData = getDataToSync(obj.getSchemaName(), obj.getTableName(), columns[0]); + } catch (Exception e) { + if (!stopTBLaptop) + throw e; + // StopTB camp laptop: one table that cannot be read must not stop the upload of the others; + // its rows stay unprocessed and go on the next upload + logger.error("Upload of table {} skipped: {}", tableKey, e.getMessage()); + groupHasFailures = true; + Map tableDetails = new HashMap<>(); + tableDetails.put("tableName", obj.getTableName()); + tableDetails.put("schemaName", obj.getSchemaName()); + tableDetails.put("status", "failed"); + tableDetails.put("totalRecords", 0); + tableDetails.put("successfulRecords", 0); + tableDetails.put("failedRecords", 0); + tableDetails.put("failureReasons", List.of("Table could not be read: " + e.getMessage())); + tableDetailsList.add(tableDetails); + continue; + } logger.info("Fetched syncData for schema {} and table {}: {}", obj.getSchemaName(), obj.getTableName(), objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(syncData)); @@ -184,7 +209,7 @@ private String startDataSync(int vanID, String user, String Authorization, Strin Map syncResult = syncDataToServer(vanID, obj.getSchemaName(), obj.getTableName(), - obj.getVanAutoIncColumnName(), obj.getServerColumnName(), syncDataBatch, user, + obj.getVanAutoIncColumnName(), columns[1], syncDataBatch, user, Authorization, token); if (syncResult == null) { @@ -226,7 +251,7 @@ private String startDataSync(int vanID, String user, String Authorization, Strin Map syncResult = syncDataToServer(vanID, obj.getSchemaName(), obj.getTableName(), - obj.getVanAutoIncColumnName(), obj.getServerColumnName(), syncDataBatch, user, + obj.getVanAutoIncColumnName(), columns[1], syncDataBatch, user, Authorization, token); if (syncResult == null) { @@ -440,6 +465,31 @@ static List skipTablesUploadedByLaterGroup(List> RENAMED_COLUMNS = Map.of("tb_diagnostic_order", + Map.of("reason_for_refusal", "reason_to_close", "reason_to_close", "reason_for_refusal")); + + /** + * The sync table list may still name a column by its old name (laptop m_synctabledetail has reason_for_refusal, + * the table has reason_to_close) and the whole upload stopped with "bad SQL grammar". The name the table really + * has is used in both the van and the server column lists (central has the same DB version). + */ + static String[] useRenamedColumns(String tableName, String vanColumns, String serverColumns, + Set tableColumns) { + Map renamed = tableName == null ? null : RENAMED_COLUMNS.get(tableName.toLowerCase()); + if (renamed == null || tableColumns == null || tableColumns.isEmpty()) + return new String[] { vanColumns, serverColumns }; + for (Map.Entry r : renamed.entrySet()) { + if (!tableColumns.contains(r.getKey()) && tableColumns.contains(r.getValue())) { + String listed = "(?i)\\b" + r.getKey() + "\\b"; + vanColumns = vanColumns == null ? null : vanColumns.replaceAll(listed, r.getValue()); + serverColumns = serverColumns == null ? null : serverColumns.replaceAll(listed, r.getValue()); + } + } + return new String[] { vanColumns, serverColumns }; + } + public List getVanAndServerColumnList(Integer groupID) throws Exception { List syncUtilityClassList = syncutilityClassRepo .findBySyncTableGroupIDAndDeletedOrderBySyncTableDetailID(groupID, false); diff --git a/src/test/java/com/iemr/mmu/service/dataSyncActivity/UploadRenamedColumnTest.java b/src/test/java/com/iemr/mmu/service/dataSyncActivity/UploadRenamedColumnTest.java new file mode 100644 index 00000000..adbc39de --- /dev/null +++ b/src/test/java/com/iemr/mmu/service/dataSyncActivity/UploadRenamedColumnTest.java @@ -0,0 +1,48 @@ +package com.iemr.mmu.service.dataSyncActivity; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; + +import java.util.Set; + +import org.junit.jupiter.api.Test; + +class UploadRenamedColumnTest { + + private static final String VAN = "status,error_message,reason_for_refusal,push_response_json," + + "date_format(last_mod_date,'%Y-%m-%d %H:%i:%s'),vanSerialNo"; + private static final String SERVER = "status,error_message,reason_for_refusal,push_response_json,last_mod_date,vanSerialNo"; + + @Test + void oldNameInSyncListUsesTheTableName() { + Set table = Set.of("status", "error_message", "reason_to_close", "push_response_json", "last_mod_date", + "vanserialno"); + + String[] cols = UploadDataToServerImpl.useRenamedColumns("tb_diagnostic_order", VAN, SERVER, table); + + assertArrayEquals(new String[] { VAN.replace("reason_for_refusal", "reason_to_close"), + SERVER.replace("reason_for_refusal", "reason_to_close") }, cols); + } + + @Test + void tableStillWithOldNameIsUnchanged() { + Set table = Set.of("status", "error_message", "reason_for_refusal", "push_response_json"); + + assertArrayEquals(new String[] { VAN, SERVER }, + UploadDataToServerImpl.useRenamedColumns("tb_diagnostic_order", VAN, SERVER, table)); + } + + @Test + void newNameInSyncListOnOlderTable() { + String van = "status,reason_to_close", server = "status,reason_to_close"; + + assertArrayEquals(new String[] { "status,reason_for_refusal", "status,reason_for_refusal" }, + UploadDataToServerImpl.useRenamedColumns("tb_diagnostic_order", van, server, + Set.of("status", "reason_for_refusal"))); + } + + @Test + void otherTablesAreUnchanged() { + assertArrayEquals(new String[] { VAN, SERVER }, UploadDataToServerImpl.useRenamedColumns("t_tbsuspected", VAN, + SERVER, Set.of("reason_to_close"))); + } +}