Add bank freshness datasets (48 queries, three DB layouts)

common_scaffold/prompts/prompt_builder.py

@@ -58,6 +58,7 @@
 - query_db:
 {"tool": "query_db", "args": {"db_name": "some_db_name", "query": "SELECT * FROM some_table LIMIT some_limit;"}} # for SQL databases
 {"tool": "query_db", "args": {"db_name": "some_db_name", "query": "{\"collection\": \"some_collection\", \"filter\": {some_filter}, \"projection\": {some_projection}, \"limit\": some_limit}"}} # for MongoDB databases
+{"tool": "query_db", "args": {"db_name": "some_db_name", "query": "{\"collection\": \"some_collection\", \"pipeline\": [some_aggregation_stages]}"}} # MongoDB aggregation
 
 - list_db:
 {"tool": "list_db", "args": {"db_name": "some_db_name"}}

common_scaffold/tools/QueryDBTool.py

@@ -46,7 +46,7 @@ def get_spec(self):
                         },
                         "query": {
                             "type": "string",
-                            "description": "The query to execute. For SQL databases, provide a SQL query string. For MongoDB, provide a JSON string representing the query, including `collection` field, and optional `filter`, `projection`, and `limit` fields."
+                            "description": "The query to execute. For SQL databases, provide a SQL query string. For MongoDB, provide a JSON string representing the query, including `collection` field, and optional `filter`, `projection`, and `limit` fields; or `collection` plus `pipeline` (a list of aggregation stages) to run an aggregation instead."
                         }
                     },
                     "required": ["db_name", "query"]

common_scaffold/tools/db_utils/mongo_utils.py

@@ -1,4 +1,5 @@
 import json
+import os
 import pandas as pd
 from pymongo import MongoClient
 from common_scaffold.tools.db_utils import db_config
@@ -17,6 +18,8 @@
 # Reference: https://www.mongodb.com/docs/manual/administration/install-community/?linux-distribution=ubuntu&linux-package=default&operating-system=linux&search-linux=with-search-linux#start-mongodb-12
 
 def load_db(dump_folder: str, db_name: str):
+    if os.getenv("DAB_PERSISTENT_DBS") and check_db_exists(db_name):
+        return
     if check_db_exists(db_name) == True:
         logging.getLogger(__name__).warning(f"Database '{db_name}' already exists. Cleaning up before loading dump.")
         clean_up(db_name)
@@ -53,6 +56,8 @@ def check_db_exists(db_name):
             raise FatalError(f"MongoDB existence check error ({type(e).__name__}): {str(e)}")
 
 def clean_up(db_name):
+    if os.getenv("DAB_PERSISTENT_DBS"):
+        return
     logging.getLogger(__name__).debug(f"Cleaning up MongoDB database '{db_name}'")
     with MongoClient(db_config.MONGO_URI) as mongo_client:
         try:
@@ -77,28 +82,35 @@ def check_args(db_client: dict, query: str):
         collection = query_json["collection"]
         filter = query_json.get("filter", {})
         projection = query_json.get("projection", None)
-        limit = query_json.get("limit", 5)
+        limit = query_json.get("limit", None)  # no silent cap: matches SQL, which returns every row
+        pipeline = query_json.get("pipeline", None)
+        if pipeline is not None and not isinstance(pipeline, list):
+            raise ValueError("Invalid Mongo query: `pipeline` must be a list of aggregation stages")
         return {
             "db_name": db_client['db_name'],
             "collection": collection,
             "filter": filter,
             "projection": projection,
-            "limit": limit
+            "limit": limit,
+            "pipeline": pipeline,
         }
 
     @staticmethod
-    def exec(db_name, collection, filter, projection, limit):
+    def exec(db_name, collection, filter, projection, limit, pipeline=None):
         with MongoClient(db_config.MONGO_URI) as mongo_client:
             db = mongo_client[db_name]
 
             if collection not in db.list_collection_names():
                 raise ValueError(f"Collection does not exist: {collection}")
 
             try:
-                cursor = db[collection].find(filter, projection)
-                if limit is not None:
-                    cursor = cursor.limit(limit)
-                result = list(cursor)
+                if pipeline is not None:
+                    result = list(db[collection].aggregate(pipeline))
+                else:
+                    cursor = db[collection].find(filter, projection)
+                    if limit is not None:
+                        cursor = cursor.limit(limit)
+                    result = list(cursor)
             except Exception as e:
                 raise ValueError(f"MongoDB query execution error ({type(e).__name__}): {str(e)}")
 

common_scaffold/tools/db_utils/postgres_utils.py

@@ -22,6 +22,8 @@ def pg_ident(identifier: str) -> str:
 
 
 def load_db(sql_file: str, db_name: str):
+    if os.getenv("DAB_PERSISTENT_DBS") and check_db_exists(db_name):
+        return
     if check_db_exists(db_name):
         logging.getLogger(__name__).warning(f"Database '{db_name}' already exists. Cleaning up before loading dump.")
         clean_up(db_name)
@@ -91,6 +93,8 @@ def check_db_exists(db_name):
 
 
 def clean_up(db_name):
+    if os.getenv("DAB_PERSISTENT_DBS"):
+        return
     logging.getLogger(__name__).debug(f"Cleaning up PostgreSQL database '{db_name}'")
     try:
         conn = psycopg2.connect(

query_bank_fresh_direct/db_config.yaml

@@ -0,0 +1,21 @@
+db_clients:
+  core_banking:
+    db_type: postgres
+    db_name: bank_cbs
+    sql_file: query_dataset/cbs.sql
+  cards:
+    db_type: sqlite
+    db_path: query_dataset/cards.db
+  lending:
+    db_type: duckdb
+    db_path: query_dataset/lending.duckdb
+  payments:
+    db_type: sqlite
+    db_path: query_dataset/payments.db
+  crm:
+    db_type: mongo
+    db_name: bank_crm
+    dump_folder: query_dataset/bank_crm_dump
+  insurance:
+    db_type: duckdb
+    db_path: query_dataset/insurance.duckdb

query_bank_fresh_direct/db_description.txt

@@ -0,0 +1,55 @@
+You are working with six databases to solve this query. They are the operational systems of one retail bank in India. Amounts are in Indian rupees (INR). They are the bank's live systems: every transaction is visible in them as soon as it is posted.
+
+Here are the descriptions of these six databases:
+
+1. core_banking
+   - This database is stored in a PostgreSQL database and is the bank's core banking system: customers, their deposit accounts, account transactions, cheques, and fixed and recurring deposits.
+   - This database consists of six tables:
+     - customers: Bank customers
+       - Fields: customer_id, name, age, gender, senior_citizen, kyc_status, monthly_income, comm_line1, comm_line2, comm_city, comm_state, comm_pincode, perm_line1, perm_line2, perm_city, perm_state, perm_pincode (comm_ = communication address, perm_ = permanent address)
+     - accounts: Savings and current accounts
+       - Fields: account_id, customer_id, account_type, status, ifsc, home_branch, nominee_name, nominee_relationship, open_date, available_balance, hold_amount, currency
+     - account_holders: Holders of each account, in order (holder_seq 1 is the primary holder)
+       - Fields: account_id, holder_seq, holder_name
+     - transactions: Account transactions
+       - Fields: reference_id, account_id, txn_date, description, amount, direction, channel, balance_after
+     - cheques: Cheque leaves issued on accounts
+       - Fields: account_id, cheque_number, status
+     - deposits: Fixed deposits (kind FD) and recurring deposits (kind RD)
+       - Fields: deposit_id, payout_account, kind, principal, rate, tenure_months, start_date, maturity_date, maturity_amount, payout_type, maturity_instruction, status, nominee_name, nominee_relationship, monthly_installment, installments_paid, total_deposited, source_account
+
+2. cards
+   - This database is stored in a SQLite database and is the bank's card management system.
+   - This database consists of two tables:
+     - cards: Debit and credit cards
+       - Fields: card_id, account_no, card_type, network, status, credit_limit, available_limit
+     - card_controls: Per-card usage controls, one row per channel (atm, online, pos) and region (domestic, international)
+       - Fields: card_id, account_no, channel, region, enabled, daily_limit
+
+3. lending
+   - This database is stored in a DuckDB database and is the bank's loan management system.
+   - This database consists of one table:
+     - loans: Home, personal and gold loans
+       - Fields: loan_id, borrower_ref, loan_type, outstanding, rate, emi, remaining_tenure_months, next_due_date, status
+
+4. payments
+   - This database is stored in a SQLite database and is the bank's payments hub.
+   - This database consists of one table:
+     - mandates: Standing payment mandates (UPI autopay, NACH and similar) that debit an account
+       - Fields: mandate_id, debit_account, payee, amount, frequency, next_debit_date, status, mandate_type
+
+5. crm
+   - This database is stored in a MongoDB database and is the bank's customer relationship management system.
+   - This database consists of three collections:
+     - customers: Customer relationship data
+       - Fields: _id, segment, profession, sector, employer, marital_status, email, phone
+     - service_requests: Service requests raised by customers
+       - Fields: _id, request_id, category, description, related_transaction_id, account_id, status, priority, created_at, eta
+     - offers: Product offers shown to customers
+       - Fields: _id, customer, offer_id, type, title, detail, indicative
+
+6. insurance
+   - This database is stored in a DuckDB database and is the feed of insurance policies the bank distributes for its partner insurers.
+   - This database consists of one table:
+     - policies: Insurance policies held by bank customers
+       - Fields: policy_id, policyholder_ref, policy_name, insurer, sum_assured, premium, frequency, status, next_premium_date, nominee_name, nominee_relationship

query_bank_fresh_direct/query1/ground_truth.csv

@@ -0,0 +1 @@
+1113900.00

query_bank_fresh_direct/query1/query.json

@@ -0,0 +1 @@
+"Current time: 2026-07-01 08:13 IST. What is the available balance of account SB4516247616 right now?"

query_bank_fresh_direct/query1/validate.py

@@ -0,0 +1,81 @@
+"""Bank freshness question 1 (control, 2026-07-01 08:13). Generated by make_fresh_datasets.py; do not edit."""
+import re
+
+SPEC = ['money', 1113900.0, 1.0]
+DATETIME = re.compile('\\b\\d{4}-\\d{2}-\\d{2}(?:[ T]\\d{1,2}:\\d{2}(?::\\d{2})?)?\\b|\\b\\d{1,2}:\\d{2}\\b')
+
+import re
+
+NUMBER = re.compile(r"(?<![\w])\d[\d,]*(?:\.\d+)?(?![\w])")   # not inside an id such as CARD60216
+WINDOW = 80
+
+
+def numbers(text):
+    out = []
+    for m in NUMBER.finditer(text):
+        try:
+            out.append(float(m.group().replace(",", "")))
+        except ValueError:
+            pass
+    return out
+
+
+def squash(s):
+    return re.sub(r"\s+", " ", s).strip().lower()
+
+
+def holds(spec, text):
+    kind = spec[0]
+    if kind in ("count", "money"):
+        nums = numbers(text)
+        if kind == "count":
+            return any(v == spec[1] for v in nums)
+        return any(abs(v - spec[1]) <= spec[2] for v in nums)
+    if kind == "text":
+        return squash(spec[1]) in squash(text)
+    if kind == "ids":
+        return all(re.search(rf"(?<![\w]){re.escape(i)}(?![\w])", text, re.I) for i in spec[1])
+    if kind == "labeled":
+        alts = [lb if isinstance(lb, list) else [lb] for lb, _ in spec[1]]
+        every = [x for a in alts for x in a]
+        for labels, (_, sub) in zip(alts, spec[1]):
+            others = [x for x in every if x not in labels]
+            ok = False
+            for lb in labels:
+                for m in re.finditer(re.escape(lb), text, re.I):
+                    window = text[m.end():m.end() + WINDOW].split("\n", 1)[0]
+                    cut = [i for i in (re.search(re.escape(o), window, re.I) for o in others) if i]
+                    if cut:
+                        window = window[:min(i.start() for i in cut)]
+                    if holds(sub, window):
+                        ok = True
+                        break
+                if ok:
+                    break
+            if not ok:
+                return False
+        return True
+    if kind == "all":
+        return all(holds(s, text) for s in spec[1])
+    raise ValueError(f"unknown spec {kind}")
+
+
+def describe(spec):
+    kind = spec[0]
+    if kind == "count":
+        return str(spec[1])
+    if kind == "money":
+        return f"{spec[1]:.2f}"
+    if kind == "text":
+        return spec[1]
+    if kind == "ids":
+        return ", ".join(spec[1])
+    if kind == "labeled":
+        return "; ".join(f"{(l[0] if isinstance(l, list) else l)}: {describe(s)}" for l, s in spec[1])
+    return "; ".join(describe(s) for s in spec[1])
+
+def validate(llm_output: str):
+    # dates and times are removed first: every question states the current time (fresh_check.py)
+    if holds(SPEC, DATETIME.sub(" ", llm_output)):
+        return True, "Expected answer found: " + describe(SPEC)
+    return False, "Expected answer not found: " + describe(SPEC)

query_bank_fresh_direct/query10/ground_truth.csv

@@ -0,0 +1 @@
+3158000.00

query_bank_fresh_direct/query10/query.json

@@ -0,0 +1 @@
+"Current time: 2026-07-03 08:13 IST. What is the total available balance across all of customer CUST_7D3D21E0's accounts right now?"

query_bank_fresh_direct/query10/validate.py

@@ -0,0 +1,81 @@
+"""Bank freshness question 10 (control, 2026-07-03 08:13). Generated by make_fresh_datasets.py; do not edit."""
+import re
+
+SPEC = ['money', 3158000.0, 1.0]
+DATETIME = re.compile('\\b\\d{4}-\\d{2}-\\d{2}(?:[ T]\\d{1,2}:\\d{2}(?::\\d{2})?)?\\b|\\b\\d{1,2}:\\d{2}\\b')
+
+import re
+
+NUMBER = re.compile(r"(?<![\w])\d[\d,]*(?:\.\d+)?(?![\w])")   # not inside an id such as CARD60216
+WINDOW = 80
+
+
+def numbers(text):
+    out = []
+    for m in NUMBER.finditer(text):
+        try:
+            out.append(float(m.group().replace(",", "")))
+        except ValueError:
+            pass
+    return out
+
+
+def squash(s):
+    return re.sub(r"\s+", " ", s).strip().lower()
+
+
+def holds(spec, text):
+    kind = spec[0]
+    if kind in ("count", "money"):
+        nums = numbers(text)
+        if kind == "count":
+            return any(v == spec[1] for v in nums)
+        return any(abs(v - spec[1]) <= spec[2] for v in nums)
+    if kind == "text":
+        return squash(spec[1]) in squash(text)
+    if kind == "ids":
+        return all(re.search(rf"(?<![\w]){re.escape(i)}(?![\w])", text, re.I) for i in spec[1])
+    if kind == "labeled":
+        alts = [lb if isinstance(lb, list) else [lb] for lb, _ in spec[1]]
+        every = [x for a in alts for x in a]
+        for labels, (_, sub) in zip(alts, spec[1]):
+            others = [x for x in every if x not in labels]
+            ok = False
+            for lb in labels:
+                for m in re.finditer(re.escape(lb), text, re.I):
+                    window = text[m.end():m.end() + WINDOW].split("\n", 1)[0]
+                    cut = [i for i in (re.search(re.escape(o), window, re.I) for o in others) if i]
+                    if cut:
+                        window = window[:min(i.start() for i in cut)]
+                    if holds(sub, window):
+                        ok = True
+                        break
+                if ok:
+                    break
+            if not ok:
+                return False
+        return True
+    if kind == "all":
+        return all(holds(s, text) for s in spec[1])
+    raise ValueError(f"unknown spec {kind}")
+
+
+def describe(spec):
+    kind = spec[0]
+    if kind == "count":
+        return str(spec[1])
+    if kind == "money":
+        return f"{spec[1]:.2f}"
+    if kind == "text":
+        return spec[1]
+    if kind == "ids":
+        return ", ".join(spec[1])
+    if kind == "labeled":
+        return "; ".join(f"{(l[0] if isinstance(l, list) else l)}: {describe(s)}" for l, s in spec[1])
+    return "; ".join(describe(s) for s in spec[1])
+
+def validate(llm_output: str):
+    # dates and times are removed first: every question states the current time (fresh_check.py)
+    if holds(SPEC, DATETIME.sub(" ", llm_output)):
+        return True, "Expected answer found: " + describe(SPEC)
+    return False, "Expected answer not found: " + describe(SPEC)

query_bank_fresh_direct/query11/ground_truth.csv

@@ -0,0 +1 @@
+756300.00

query_bank_fresh_direct/query11/query.json

@@ -0,0 +1 @@
+"Current time: 2026-07-03 11:02 IST. What is the available balance of account SB7976773201 right now?"