From 4100c6a2eff89ca8b237159281d780ffe7184b7f Mon Sep 17 00:00:00 2001 From: iamvirul Date: Fri, 25 Sep 2026 08:14:01 +0530 Subject: [PATCH] feat(diff): match tables and hash rows case-insensitively across engines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cross-engine data diffs (e.g. PostgreSQL -> Oracle) silently reported "No data differences found" because identifier case folding differs between engines: PostgreSQL stores unquoted names lower-case ("customers") while Oracle stores them upper-case ("CUSTOMERS"). Table matching and the row hash keyed on the raw identifier, so no tables matched, zero rows were compared, and real missing/changed rows went undetected — a dangerous false negative for a migration-verification tool. Fix the data-diff path to compare on a canonical (case-folded) identifier while preserving the real identifier for SQL generation, since quoted identifiers are case-sensitive in most engines (notably Oracle): - schema.CanonicalIdent: case/whitespace-normalized identifier for matching - content.orderedColumns: order columns by canonical name so both engines produce a comparable column sequence - content.hashRow: emit canonical column names so identical row data hashes equal regardless of identifier case - content.BuildDataDiff: match tables by canonical name, look up hashes by each side's real name, label output with the prod-side name - runDiff: build the shared-table set by canonical name so both sides hash Schema drift (legitimate cross-engine type differences such as integer vs number, varchar vs varchar2) is still reported honestly and unchanged. gen-pack/apply SQL generation is unaffected and remains a follow-up for cross-engine migration packs. Verified end-to-end against live PostgreSQL and Oracle XE containers: the diff now reports rows missing in Oracle (removed_keys) and value drift (updated_keys) correctly. Tests: - schema.TestCanonicalIdent (+ cross-engine match invariant) - content.TestHashTable_CrossEngineCaseInsensitive - content.TestBuildDataDiff_CaseInsensitiveTableMatch --- cmd/deepdiffdb/main.go | 14 ++++++-- internal/content/diff.go | 35 +++++++++++++++---- internal/content/hash.go | 13 ++++++-- internal/schema/identifiers.go | 19 +++++++++++ internal/schema/identifiers_test.go | 33 ++++++++++++++++++ tests/content/diff_test.go | 47 ++++++++++++++++++++++++++ tests/content/hash_test.go | 52 +++++++++++++++++++++++++++++ 7 files changed, 202 insertions(+), 11 deletions(-) create mode 100644 internal/schema/identifiers.go create mode 100644 internal/schema/identifiers_test.go diff --git a/cmd/deepdiffdb/main.go b/cmd/deepdiffdb/main.go index 3767dbe..a23b864 100644 --- a/cmd/deepdiffdb/main.go +++ b/cmd/deepdiffdb/main.go @@ -423,13 +423,21 @@ func runFullDiff(args []string) error { maxParallel := resolveParallel(*parallelFlag, cfg.Performance.MaxParallelTables) log.Debug("hashing parameters", "batch_size", batchSize, "max_parallel", maxParallel) - // Build the set of shared tables (exist in both prod and dev). + // Build the set of shared tables (exist in both prod and dev). Matching is + // case-insensitive on the identifier so a table survives a cross-engine + // migration that folds names differently (PostgreSQL "customers" vs Oracle + // "CUSTOMERS"). Each side keeps its real name so table hashing generates + // correctly-cased SQL. + devByCanonical := make(map[string]schema.Table, len(devSchema.Tables)) + for name, devTable := range devSchema.Tables { + devByCanonical[schema.CanonicalIdent(name)] = devTable + } sharedProdTables := make(map[string]schema.Table) sharedDevTables := make(map[string]schema.Table) for name, prodTable := range prodSchema.Tables { - if devTable, ok := devSchema.Tables[name]; ok { + if devTable, ok := devByCanonical[schema.CanonicalIdent(name)]; ok { sharedProdTables[name] = prodTable - sharedDevTables[name] = devTable + sharedDevTables[devTable.Name] = devTable } } diff --git a/internal/content/diff.go b/internal/content/diff.go index 5bace68..c278661 100644 --- a/internal/content/diff.go +++ b/internal/content/diff.go @@ -1,6 +1,10 @@ package content -import "github.com/iamvirul/deepdiff-db/internal/schema" +import ( + "sort" + + "github.com/iamvirul/deepdiff-db/internal/schema" +) // TableDataDiff captures row-level differences for one table. type TableDataDiff struct { @@ -72,22 +76,41 @@ func DiffTableHashes(table string, prod, dev map[string]string) TableDataDiff { func BuildDataDiff(prodSchema, devSchema *schema.Schema, prodHashes, devHashes map[string]map[string]string) (DataDiff, Conflicts) { diff := DataDiff{} conflicts := Conflicts{} + + // Index dev tables by canonical name so a table is matched to its prod + // counterpart even when the engines fold identifier case differently + // (PostgreSQL "customers" vs Oracle "CUSTOMERS"). Hashes are still looked + // up by each side's real table name. + devByCanonical := make(map[string]string, len(devSchema.Tables)) + for devName := range devSchema.Tables { + devByCanonical[schema.CanonicalIdent(devName)] = devName + } + + // Iterate prod tables in deterministic order for stable report output. + prodNames := make([]string, 0, len(prodSchema.Tables)) for name := range prodSchema.Tables { - if _, ok := devSchema.Tables[name]; !ok { + prodNames = append(prodNames, name) + } + sort.Strings(prodNames) + + for _, prodName := range prodNames { + devName, ok := devByCanonical[schema.CanonicalIdent(prodName)] + if !ok { continue } - pHashes := prodHashes[name] - dHashes := devHashes[name] + pHashes := prodHashes[prodName] + dHashes := devHashes[devName] - td := DiffTableHashes(name, pHashes, dHashes) + // Label the table with the prod-side name (the source of truth). + td := DiffTableHashes(prodName, pHashes, dHashes) diff.Tables = append(diff.Tables, td) // Detect conflicts (rows that exist in both but differ) for k, prodHash := range pHashes { if devHash, ok := dHashes[k]; ok && devHash != prodHash { conflicts.Conflicts = append(conflicts.Conflicts, Conflict{ - Table: name, + Table: prodName, Key: k, ProdHash: prodHash, DevHash: devHash, diff --git a/internal/content/hash.go b/internal/content/hash.go index ca0edbf..50bf1c3 100644 --- a/internal/content/hash.go +++ b/internal/content/hash.go @@ -339,7 +339,12 @@ func orderedColumns(tbl schema.Table, ignoreFn func(table, column string) bool) } nonPK = append(nonPK, name) } - sort.Strings(nonPK) + // Sort by canonical (case-folded) name so two engines that differ only in + // identifier case (e.g. PostgreSQL "email" vs Oracle "EMAIL") produce the + // same column order, and therefore comparable row hashes. + sort.Slice(nonPK, func(i, j int) bool { + return schema.CanonicalIdent(nonPK[i]) < schema.CanonicalIdent(nonPK[j]) + }) cols = append(cols, nonPK...) return cols } @@ -412,7 +417,11 @@ func buildKey(cols []string, values []any, pk []string) (string, error) { func hashRow(cols []string, values []any) string { var b strings.Builder for i, col := range cols { - fmt.Fprintf(&b, "%s=%v\n", col, values[i]) + // Use the canonical (case-folded) column name so a row with identical + // values hashes the same across engines that differ only in identifier + // case (PostgreSQL "email" vs Oracle "EMAIL"). Column order is already + // canonicalized by orderedColumns. + fmt.Fprintf(&b, "%s=%v\n", schema.CanonicalIdent(col), values[i]) } sum := sha256.Sum256([]byte(b.String())) return hex.EncodeToString(sum[:]) diff --git a/internal/schema/identifiers.go b/internal/schema/identifiers.go new file mode 100644 index 0000000..4599d49 --- /dev/null +++ b/internal/schema/identifiers.go @@ -0,0 +1,19 @@ +package schema + +import "strings" + +// CanonicalIdent returns a case- and whitespace-normalized form of a SQL +// identifier, used to match the same logical object across database engines. +// +// Engines disagree on how unquoted identifiers are folded: PostgreSQL lower-cases +// them, while Oracle and DB2 upper-case them. The same logical table is therefore +// stored as "customers" in PostgreSQL but "CUSTOMERS" in Oracle. Canonicalizing to +// lower case lets the diff engine treat them as the same object. +// +// This form is used ONLY for matching objects and for building the engine-agnostic +// row hash. The original identifier is always preserved on the Table/Column structs +// for SQL generation, because quoted identifiers are case-sensitive in most engines +// (notably Oracle): querying "customers" would not find the real table CUSTOMERS. +func CanonicalIdent(ident string) string { + return strings.ToLower(strings.TrimSpace(ident)) +} diff --git a/internal/schema/identifiers_test.go b/internal/schema/identifiers_test.go new file mode 100644 index 0000000..43c82bd --- /dev/null +++ b/internal/schema/identifiers_test.go @@ -0,0 +1,33 @@ +package schema + +import "testing" + +func TestCanonicalIdent(t *testing.T) { + cases := []struct { + name string + in string + want string + }{ + {"postgres lower", "customers", "customers"}, + {"oracle upper", "CUSTOMERS", "customers"}, + {"mixed case", "Customers", "customers"}, + {"surrounding whitespace", " customers ", "customers"}, + {"empty", "", ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := CanonicalIdent(tc.in); got != tc.want { + t.Errorf("CanonicalIdent(%q) = %q, want %q", tc.in, got, tc.want) + } + }) + } +} + +// TestCanonicalIdent_CrossEngineMatch documents the core invariant the diff +// engine relies on: identifiers that a migration folds to a different case +// (PostgreSQL lower vs Oracle upper) canonicalize to the same key. +func TestCanonicalIdent_CrossEngineMatch(t *testing.T) { + if CanonicalIdent("customers") != CanonicalIdent("CUSTOMERS") { + t.Error("expected PostgreSQL and Oracle folded names to canonicalize equal") + } +} diff --git a/tests/content/diff_test.go b/tests/content/diff_test.go index 06b8d25..947342d 100644 --- a/tests/content/diff_test.go +++ b/tests/content/diff_test.go @@ -250,6 +250,53 @@ func TestBuildDataDiff(t *testing.T) { } } +// TestBuildDataDiff_CaseInsensitiveTableMatch verifies that a table which a +// cross-engine migration folds to a different case (PostgreSQL "customers" -> +// Oracle "CUSTOMERS") is still matched, so missing and changed rows are +// detected instead of the table being silently skipped. +func TestBuildDataDiff_CaseInsensitiveTableMatch(t *testing.T) { + prodSchema := &schema.Schema{ + Tables: map[string]schema.Table{ + "customers": {Name: "customers"}, + }, + } + devSchema := &schema.Schema{ + Tables: map[string]schema.Table{ + "CUSTOMERS": {Name: "CUSTOMERS"}, + }, + } + + // Prod (source of truth) has ids 1,2,5; dev (Oracle) has 1,2 with 2 changed. + prodHashes := map[string]map[string]string{ + "customers": {"1": "h1", "2": "h2_old", "5": "h5"}, + } + devHashes := map[string]map[string]string{ + "CUSTOMERS": {"1": "h1", "2": "h2_new"}, + } + + diff, conflicts := content.BuildDataDiff(prodSchema, devSchema, prodHashes, devHashes) + + if len(diff.Tables) != 1 { + t.Fatalf("expected 1 matched table, got %d", len(diff.Tables)) + } + td := diff.Tables[0] + if td.Table != "customers" { + t.Errorf("expected prod-side label 'customers', got %q", td.Table) + } + if len(td.Removed) != 1 || td.Removed[0] != "5" { + t.Errorf("expected removed (missing in dev) [5], got %v", td.Removed) + } + if len(td.Updated) != 1 || td.Updated[0] != "2" { + t.Errorf("expected updated [2], got %v", td.Updated) + } + if len(td.Added) != 0 { + t.Errorf("expected no added rows, got %v", td.Added) + } + if len(conflicts.Conflicts) != 1 || conflicts.Conflicts[0].Table != "customers" || conflicts.Conflicts[0].Key != "2" { + t.Errorf("expected 1 conflict on customers.2, got %v", conflicts.Conflicts) + } +} + func TestHasChanges(t *testing.T) { tests := []struct { name string diff --git a/tests/content/hash_test.go b/tests/content/hash_test.go index f0fe27a..599f6b6 100644 --- a/tests/content/hash_test.go +++ b/tests/content/hash_test.go @@ -77,6 +77,58 @@ func TestHashTable(t *testing.T) { } } +// TestHashTable_CrossEngineCaseInsensitive verifies that identical row data +// hashes the same regardless of identifier case. A cross-engine migration +// (PostgreSQL lower-case -> Oracle upper-case) must not make matching rows look +// different purely because column/table names are folded differently. +func TestHashTable_CrossEngineCaseInsensitive(t *testing.T) { + ctx := context.Background() + db := openMemDB(t) + + if _, err := db.ExecContext(ctx, `CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT NOT NULL, email TEXT)`); err != nil { + t.Fatalf("create table: %v", err) + } + if _, err := db.ExecContext(ctx, `INSERT INTO users (id, name, email) VALUES (1, 'Alice', 'alice@example.com'), (2, 'Bob', 'bob@example.com')`); err != nil { + t.Fatalf("insert: %v", err) + } + + // Same underlying table, described with lower-case (PostgreSQL-style) and + // upper-case (Oracle-style) identifiers. SQLite resolves both case-insensitively. + lower := usersTable() + upper := schema.Table{ + Name: "USERS", + Columns: map[string]schema.Column{ + "ID": {Name: "ID"}, + "NAME": {Name: "NAME"}, + "EMAIL": {Name: "EMAIL"}, + }, + PrimaryKey: []string{"ID"}, + } + + lowerHashes, err := content.HashTable(ctx, db, "sqlite", lower, nil, 0) + if err != nil { + t.Fatalf("hash lower: %v", err) + } + upperHashes, err := content.HashTable(ctx, db, "sqlite", upper, nil, 0) + if err != nil { + t.Fatalf("hash upper: %v", err) + } + + if len(lowerHashes) != len(upperHashes) { + t.Fatalf("key count mismatch: lower=%d upper=%d", len(lowerHashes), len(upperHashes)) + } + for key, lh := range lowerHashes { + uh, ok := upperHashes[key] + if !ok { + t.Errorf("key %q missing from upper-case hashing", key) + continue + } + if lh != uh { + t.Errorf("row hash for key %q differs across identifier case: %s vs %s", key, lh, uh) + } + } +} + func TestHashTable_WithIgnore(t *testing.T) { ctx := context.Background() db := openMemDB(t)