StopTB camp laptop data sync – = sync, inventory upload fixes and master download

src/main/environment/common_example.properties

@@ -135,3 +135,17 @@ benid.pool.bootstrapSize=10000
 benid.token.dir=/opt/amrit/data
 benid.autoTopUp.initialDelayMs=120000
 benid.autoTopUp.checkIntervalMs=300000
+
+## Automatic Data Sync (StopTB laptops only; needs the BenID central connection above). Off by default: manual sync only.
+sync.autoUpload.enabled=false
+sync.autoUpload.intervalMs=1800000
+sync.autoMasterDownload.enabled=false
+sync.autoMasterDownload.intervalMs=21600000
+# automatic master download fetches only inventory masters (items, stores, item->store mapping); false = all masters
+sync.autoMasterDownload.inventoryOnly=true
+sync.autoInventoryDownload.enabled=false
+
+## Central upload: replace van IDs in link columns (BenVisitID, PrescriptionID) by central IDs.
+## Off by default (other MMU servers unchanged); on for the StopTB central.
+## Env alternative: DATASYNC_TRANSLATEVANLINKS=true
+dataSync.translateVanLinks=false

src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java

@@ -41,6 +41,7 @@
 import com.iemr.mmu.service.dataSyncActivity.DownSyncDataFromServerImpl;
 import com.iemr.mmu.service.dataSyncActivity.DownloadDataFromServerImpl;
 import com.iemr.mmu.service.dataSyncActivity.DownloadDataFromServerTransactionalImpl;
+import com.iemr.mmu.service.dataSyncActivity.AutoSyncService;
 import com.iemr.mmu.service.dataSyncActivity.UploadDataToServerImpl;
 import com.iemr.mmu.service.dataSyncLayerCentral.DiagnosticDocumentFetchService;
 import com.iemr.mmu.service.dataSyncLayerCentral.DiagnosticDocumentPushServiceImpl;
@@ -75,6 +76,9 @@
 	@Autowired
 	private BenIdAutoTopUpService benIdAutoTopUpService;
 
+	@Autowired
+	private AutoSyncService autoSyncService;
+
 	private static final String GROUP_ID = "groupID";
 	private static final String PROVIDER_SERVICE_MAP_ID = "providerServiceMapID";
 
@@ -205,6 +209,19 @@
 		return response.toString();
 	}
 
+	@Operation(summary = "Automatic data sync status and the rows central rejected on upload")
+	@GetMapping(value = { "/autoSyncStatus" })
+	public String autoSyncStatus() {
+		OutputResponse response = new OutputResponse();
+		try {
+			response.setResponse(autoSyncService.status().toString());
+		} catch (Exception e) {
+			logger.error("Error while getting automatic sync status : " + e);
+			response.setError(e);
+		}
+		return response.toString();
+	}
+
 	@Operation(summary = "Data sync master download progress check")
 	@GetMapping(value = { "/checkMastersDownloadProgress" })
 	public String checkMastersDownloadProgress() {

src/main/java/com/iemr/mmu/repo/syncActivity_syncLayer/SyncUtilityClassRepo.java

@@ -31,4 +31,6 @@
 @Repository
 public interface SyncUtilityClassRepo extends CrudRepository<SyncUtilityClass, Integer> {
 	List<SyncUtilityClass> findBySyncTableGroupIDAndDeletedOrderBySyncTableDetailID(Integer groupID, Boolean deleted);
+
+	List<SyncUtilityClass> findByDeletedOrderBySyncTableDetailID(Boolean deleted);
 }

src/main/java/com/iemr/mmu/service/dataSyncActivity/AutoSyncService.java

@@ -0,0 +1,239 @@
+/*
+* AMRIT – Accessible Medical Records via Integrated Technology
+* Integrated EHR (Electronic Health Records) Solution
+*
+* Copyright (C) "Piramal Swasthya Management and Research Institute"
+*
+* This file is part of AMRIT.
+*
+* This program is free software: you can redistribute it and/or modify
+* it under the terms of the GNU General Public License as published by
+* the Free Software Foundation, either version 3 of the License, or
+* (at your option) any later version.
+*
+* This program is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+* GNU General Public License for more details.
+*
+* You should have received a copy of the GNU General Public License
+* along with this program.  If not, see https://www.gnu.org/licenses/.
+*/
+package com.iemr.mmu.service.dataSyncActivity;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.json.JSONArray;
+import org.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Service;
+
+import com.iemr.mmu.service.benid.BenIdAutoTopUpService;
+import com.iemr.mmu.service.benid.CentralAuthService;
+
+import jakarta.annotation.PreDestroy;
+
+/**
+ * StopTB laptops: runs Data Sync by itself, so central stays current and laptops get master changes without
+ * anyone pressing Upload or Download Masters.
+ *
+ * - Upload (van to central) every sync.autoUpload.intervalMs (default 30 min).
+ * - Master download (central to van) every sync.autoMasterDownload.intervalMs (default 6 h).
+ *
+ * Uses the central connection of the automatic BenID download (CentralAuthService): no password is stored, and
+ * SyncedBy is the operator who connected the laptop. Active only where CentralAuthService is enabled
+ * (stoptb.enforce.vanid=true and stoptb.van.id>0); central and other deployments are unaffected. The manual
+ * buttons keep working as before.
+ *
+ * Off by default (manual sync only) until the automatic schedule is approved: set sync.autoUpload.enabled,
+ * sync.autoMasterDownload.enabled and sync.autoInventoryDownload.enabled to true to switch it on.
+ */
+@Service
+public class AutoSyncService {
+
+	private final Logger logger = LoggerFactory.getLogger(this.getClass().getSimpleName());
+
+	/** Upload and master download never run at the same time from here. */
+	private static final ReentrantLock SYNC_LOCK = new ReentrantLock();
+
+	@Autowired
+	private CentralAuthService centralAuthService;
+
+	@Autowired
+	private UploadDataToServerImpl uploadDataToServerImpl;
+
+	@Autowired
+	private DownloadDataFromServerImpl downloadDataFromServerImpl;
+
+	@Autowired
+	private DownloadDataFromServerTransactionalImpl downloadDataFromServerTransactional;
+
+	@Autowired
+	private DataSyncRepository dataSyncRepository;
+
+	@Autowired
+	private BenIdAutoTopUpService benIdAutoTopUpService;
+
+	@Value("${sync.autoUpload.enabled:false}")
+	private boolean uploadEnabled;
+
+	@Value("${sync.autoMasterDownload.enabled:false}")
+	private boolean masterDownloadEnabled;
+
+	// refills / stock issued at central for this van's store ("Inventory Data Download"), after each upload
+	@Value("${sync.autoInventoryDownload.enabled:false}")
+	private boolean inventoryDownloadEnabled;
+
+	// automatic master download fetches only the inventory masters (items, stores, item->store mapping, Min Stock);
+	// other masters (users, complaints, villages) come with the manual Download Masters button
+	@Value("${sync.autoMasterDownload.inventoryOnly:true}")
+	private boolean masterDownloadInventoryOnly;
+
+	static final Set<String> INVENTORY_MASTERS = Set.of("m_item", "m_itemcategory", "m_itemissueconfig", "m_uom",
+			"m_manufacturer", "m_pharmacologicalcategory", "m_supplier", "m_facilitytype", "m_facility",
+			"m_itemfacilitymapping");
+
+	private volatile long lastUploadAt;
+	private volatile String lastUploadResult;
+	private volatile long lastMasterDownloadAt;
+	private volatile String lastMasterDownloadResult;
+	private volatile long lastInventoryDownloadAt;
+	private volatile String lastInventoryDownloadResult;
+	private volatile String lastError;
+
+	// own thread: a long upload must not hold the shared scheduler thread
+	private final ExecutorService background = Executors.newSingleThreadExecutor(r -> {
+		Thread t = new Thread(r, "auto-data-sync");
+		t.setDaemon(true);
+		return t;
+	});
+
+	@PreDestroy
+	public void shutdown() {
+		background.shutdownNow();
+	}
+
+	@Scheduled(initialDelayString = "${sync.autoUpload.initialDelayMs:600000}", fixedDelayString = "${sync.autoUpload.intervalMs:1800000}")
+	public void scheduledUpload() {
+		if (uploadEnabled && centralAuthService.isEnabled()) {
+			background.submit(this::uploadOnce);
+		}
+	}
+
+	@Scheduled(initialDelayString = "${sync.autoMasterDownload.initialDelayMs:900000}", fixedDelayString = "${sync.autoMasterDownload.intervalMs:21600000}")
+	public void scheduledMasterDownload() {
+		if (masterDownloadEnabled && centralAuthService.isEnabled()) {
+			background.submit(this::masterDownloadOnce);
+		}
+	}
+
+	void uploadOnce() {
+		if (!SYNC_LOCK.tryLock()) {
+			return;
+		}
+		try {
+			String key = centralAuthService.getCentralSessionKey();
+			if (key == null) {
+				lastError = "No central connection - waiting for a user login";
+				return;
+			}
+			int vanID = centralAuthService.getVanID();
+			String user = centralAuthService.getConnectedUser();
+			String result = uploadDataToServerImpl.getDataToSyncToServer(vanID,
+					user != null && !user.isBlank() ? user : "auto-sync", key, null);
+			lastUploadAt = System.currentTimeMillis();
+			lastUploadResult = result;
+			lastError = null;
+			logger.info("Automatic upload for vanID {} finished: {}", vanID, result);
+			// internet is up: top up the BenID pool if it is low (same as the manual upload)
+			benIdAutoTopUpService.triggerNow();
+			if (inventoryDownloadEnabled)
+				inventoryDownload(vanID, key);
+		} catch (Exception e) {
+			lastError = "Upload: " + e.getMessage();
+			logger.warn("Automatic upload failed: {}", e.getMessage());
+		} finally {
+			SYNC_LOCK.unlock();
+		}
+	}
+
+	// Same as the "Inventory Data Download" button: indents, indent issues, stock transfers and stock entries
+	// that central has for this van's store. Its own failure does not undo the upload.
+	private void inventoryDownload(int vanID, String key) {
+		try {
+			int done = downloadDataFromServerTransactional.downloadTransactionalData(vanID, key, "datasync");
+			lastInventoryDownloadAt = System.currentTimeMillis();
+			lastInventoryDownloadResult = done > 0 ? "success" : "nothing downloaded";
+			logger.info("Automatic inventory download for vanID {} finished: {}", vanID, lastInventoryDownloadResult);
+		} catch (Exception e) {
+			lastError = "Inventory download: " + e.getMessage();
+			logger.warn("Automatic inventory download failed: {}", e.getMessage());
+		}
+	}
+
+	void masterDownloadOnce() {
+		if (!SYNC_LOCK.tryLock()) {
+			return;
+		}
+		try {
+			String key = centralAuthService.getCentralSessionKey();
+			if (key == null) {
+				lastError = "No central connection - waiting for a user login";
+				return;
+			}
+			int vanID = centralAuthService.getVanID();
+			Integer psm = dataSyncRepository.getVanProviderServiceMapID(vanID);
+			if (psm == null) {
+				lastError = "Master download: van " + vanID + " not found in m_van";
+				return;
+			}
+			String result = downloadDataFromServerImpl.downloadMasterDataFromServer(key, null, vanID, psm,
+					masterDownloadInventoryOnly ? INVENTORY_MASTERS : null);
+			if ("inProgress".equalsIgnoreCase(result)) {
+				logger.info("Master download already running (manual) - skipping the automatic one");
+				return;
+			}
+			lastMasterDownloadAt = System.currentTimeMillis();
+			lastMasterDownloadResult = result;
+			lastError = null;
+			logger.info("Automatic master download for vanID {} finished: {}", vanID, result);
+		} catch (Exception e) {
+			lastError = "Master download: " + e.getMessage();
+			logger.warn("Automatic master download failed: {}", e.getMessage());
+		} finally {
+			SYNC_LOCK.unlock();
+		}
+	}
+
+	/** For the Data Sync screen: last runs and the rows central has rejected. */
+	public JSONObject status() {
+		JSONObject s = new JSONObject();
+		s.put("enabled", centralAuthService.isEnabled());
+		s.put("uploadEnabled", uploadEnabled);
+		s.put("masterDownloadEnabled", masterDownloadEnabled);
+		s.put("lastUploadAt", lastUploadAt);
+		s.put("lastUploadResult", lastUploadResult == null ? JSONObject.NULL : lastUploadResult);
+		s.put("lastMasterDownloadAt", lastMasterDownloadAt);
+		s.put("lastMasterDownloadResult",
+				lastMasterDownloadResult == null ? JSONObject.NULL : lastMasterDownloadResult);
+		s.put("inventoryDownloadEnabled", inventoryDownloadEnabled);
+		s.put("lastInventoryDownloadAt", lastInventoryDownloadAt);
+		s.put("lastInventoryDownloadResult",
+				lastInventoryDownloadResult == null ? JSONObject.NULL : lastInventoryDownloadResult);
+		s.put("lastError", lastError == null ? JSONObject.NULL : lastError);
+		List<Map<String, Object>> failures = dataSyncRepository.getSyncFailures();
+		long failedRows = failures.stream().mapToLong(f -> ((Number) f.get("failedRows")).longValue()).sum();
+		s.put("failedRows", failedRows);
+		s.put("failures", new JSONArray(failures));
+		return s;
+	}
+}

src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java

@@ -23,9 +23,11 @@
 
 import java.sql.Timestamp;
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import java.sql.PreparedStatement;
 
@@ -40,6 +42,7 @@
 import org.springframework.jdbc.support.KeyHolder;
 import org.springframework.stereotype.Service;
 
+import com.iemr.mmu.data.syncActivity_syncLayer.SyncUtilityClass;
 import com.iemr.mmu.repo.syncActivity_syncLayer.SyncUtilityClassRepo;
 import com.iemr.mmu.utils.validator.SqlIdentifierValidator;
 
@@ -127,6 +130,42 @@
 
 	}
 
+	/** Service line (MMU) of the van, used for an automatic master download. */
+	public Integer getVanProviderServiceMapID(int vanID) {
+		List<Integer> psm = getJdbcTemplate().queryForList(
+				"SELECT ProviderServiceMapID FROM db_iemr.m_van WHERE VanID = ? AND Deleted = false", Integer.class,
+				vanID);
+		return psm.isEmpty() ? null : psm.get(0);
+	}
+
+	/**
+	 * Rows central rejected on upload (Processed = 'F'), per sync table and reason. They are sent again on every
+	 * upload until central accepts them.
+	 */
+	public List<Map<String, Object>> getSyncFailures() {
+		jdbcTemplate = getJdbcTemplate();
+		List<Map<String, Object>> result = new ArrayList<>();
+		Set<String> seen = new HashSet<>();
+		for (SyncUtilityClass table : syncutilityClassRepo.findByDeletedOrderBySyncTableDetailID(false)) {
+			if (Boolean.TRUE.equals(table.getIsMaster()) || table.getSchemaName() == null
+					|| table.getTableName() == null)
+				continue;
+			String qualified = SqlIdentifierValidator.validatedSchemaName(table.getSchemaName()) + "."
+					+ SqlIdentifierValidator.validatedTableName(table.getTableName());
+			if (!seen.add(qualified.toLowerCase()))
+				continue;
+			try {
+				List<Map<String, Object>> rows = jdbcTemplate.queryForList("SELECT '" + table.getTableName()
+						+ "' AS tableName, SyncFailureReason AS reason, COUNT(*) AS failedRows, DATE_FORMAT(MAX(SyncedDate), '%d-%m-%Y %H:%i') AS lastTried "
+						+ "FROM " + qualified + " WHERE Processed = 'F' GROUP BY SyncFailureReason");
+				result.addAll(rows);
+			} catch (Exception e) {
+				logger.warn("Sync failure check skipped for {}: {}", qualified, e.getMessage());
+			}
+		}
+		return result;
+	}
+
 	// ---------------------------------- End of Upload repository
 
 	// ---------------------------------- Download Repository
@@ -176,6 +215,17 @@
 		return (found == null || found.isEmpty()) ? null : found.get(0);
 	}
 
+	public Set<String> getTableColumns(String schema, String table) {
+		jdbcTemplate = getJdbcTemplate();
+
+		String query = " SELECT COLUMN_NAME FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? ";
+
+		Set<String> columns = new HashSet<>();
+		for (String c : jdbcTemplate.queryForList(query, String.class, schema, table))
+			columns.add(c.toLowerCase());
+		return columns;
+	}
+
 	public List<String> getDownSyncColumns(String schema, String table) {
 		jdbcTemplate = getJdbcTemplate();