diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9c64e41..ad05127 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go-version: ['1.24', '1.25.10'] + go-version: ['1.24', '1.25.13'] steps: - name: Checkout code @@ -44,7 +44,7 @@ jobs: continue-on-error: false - name: Merge coverage files - if: matrix.go-version == '1.25.10' + if: matrix.go-version == '1.25.13' run: | # Merge coverage files using go tool cover echo "mode: atomic" > coverage.txt @@ -52,7 +52,7 @@ jobs: continue-on-error: false - name: Verify coverage file exists - if: matrix.go-version == '1.25.10' + if: matrix.go-version == '1.25.13' run: | if [ ! -f coverage.txt ]; then echo "Error: coverage.txt not found" @@ -62,7 +62,7 @@ jobs: head -5 coverage.txt - name: Upload coverage artifact for merging - if: matrix.go-version == '1.25.10' + if: matrix.go-version == '1.25.13' uses: actions/upload-artifact@v4 with: name: coverage-unit-${{ github.run_id }} @@ -81,7 +81,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: '1.25.10' + go-version: '1.25.13' cache-dependency-path: go.sum - name: Verify dependencies @@ -133,7 +133,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: '1.25.10' + go-version: '1.25.13' cache-dependency-path: go.sum - name: Download unit test coverage artifact @@ -176,7 +176,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: '1.25.10' + go-version: '1.25.13' cache-dependency-path: go.sum - name: Run linter @@ -198,7 +198,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: '1.25.10' + go-version: '1.25.13' cache-dependency-path: go.sum - name: Download dependencies diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index 69d4a98..d860717 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -21,7 +21,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: '1.25.9' + go-version-file: go.mod cache-dependency-path: go.sum - name: Run gosec diff --git a/CHANGELOG.md b/CHANGELOG.md index bccc9a4..1d89d8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,47 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- **Cross-engine case-insensitive table discovery & row hashing**: + - Tables are discovered and paired across database engines that enforce different default identifier cases (e.g., PostgreSQL lowercase `"customers"` $\leftrightarrow$ Oracle UPPERCASE `"CUSTOMERS"`). + - Normalizes identifiers via canonical name folding (`schema.CanonicalIdent`) while generating dialect-specific SQL quoting for each engine. +- **Column-level row inspection (`row-diff` command and `--row-diff` flag)**: + - New CLI command `deepdiffdb row-diff` and `--row-diff` flag for `deepdiffdb diff`. + - Side-by-side terminal comparison table showing column values between production and development with differing columns flagged with `*`. + - Supports `--table`, `--key`, `--status` (`updated`, `added`, `removed`, `all`), `--limit`, and `--format` (`table` or `json`). + - Writes machine-readable `row_diff.json` and formatted text `row_diff.txt` to the output directory. +- **HTML report viewer enhancements**: + - **Data Changes Tab**: Opens automatically when data drift exists; expanding any table displays the complete column comparison table for every modified, added, or removed row. + - **Dedicated Row Data Tab**: Card view of every differing row across all tables with real-time table filtering and diff counters. + - **Conflicts Tab**: Displays conflicting keys with source and target SHA-256 hashes alongside side-by-side column difference tables. +- **Documentation & Sample Project**: + - New feature guide: `website/docs/features/cross-engine-diff.md` explaining cross-engine identifier folding and pipeline verification. + - New command guide: `website/docs/commands/row-diff.md`. + - Live interactive sample HTML report updated at `/samples/report.html`. + - New **Sample 18** (`samples/18-cross-engine-row-diff/`): PostgreSQL 16 $\leftrightarrow$ Oracle XE 21c migration verification with seeded `um_user` and `um_user_attribute` drift. + +### Fixed + +- **False negative "No differences found" on cross-engine migrations**: + - Fixed an issue where `deepdiffdb diff` reported zero differences between PostgreSQL and Oracle because exact string matching failed on uppercase vs. lowercase table names (e.g. `um_user_attribute` vs. `UM_USER_ATTRIBUTE`). + +### Security + +- **Bumped Go to 1.25.13**: Resolves Go standard library vulnerabilities reported by `govulncheck`: + - **GO-2026-6218**: Quadratic complexity in `net/url` + - **GO-2026-6091**: Context tracking in `html/template` + - **GO-2026-6090** & **GO-2026-5856**: Handshake and Encrypted Client Hello handling in `crypto/tls` + - **GO-2026-6088**: Recursion depth guard in `encoding/xml` + - **GO-2026-5972**: Maximum recursion depth in `encoding/asn1` + - **GO-2026-5039**: Input formatting in `net/textproto` + - **GO-2026-5037**: Inefficient hostname parsing in `crypto/x509` + - **GO-2026-5026**: Punycode label validation in `net/http` +- **Bumped `golang.org/x/text` to v0.39.0**: + - **GO-2026-5970**: Infinite loop on invalid input in `golang.org/x/text` +- **Updated `security.yml` workflow**: + - Switched from hardcoded older Go version (`1.25.9`) to `go-version-file: go.mod` to ensure CI security scans always run with the active patched toolchain. + ## [1.4.5] - 2026-05-19 ### Security diff --git a/cmd/deepdiffdb/main.go b/cmd/deepdiffdb/main.go index a23b864..f243dfa 100644 --- a/cmd/deepdiffdb/main.go +++ b/cmd/deepdiffdb/main.go @@ -201,6 +201,8 @@ func run(args []string) error { return runSchemaMigrate(args[1:]) case "diff": return runFullDiff(args[1:]) + case "row-diff": + return runRowDiff(args[1:]) case "gen-pack": return runGenPack(args[1:]) case "apply": @@ -325,6 +327,7 @@ func runFullDiff(args []string) error { fs := flag.NewFlagSet("diff", flag.ContinueOnError) configPath := fs.String("config", "deepdiffdb.config.yaml", "Path to configuration file") generateHTML := fs.Bool("html", false, "Generate interactive HTML report") + showRowDiff := fs.Bool("row-diff", false, "Display and generate column-level diffs for changed rows") batchSizeFlag := fs.Int("batch-size", 0, "Rows per keyset-paginated query (0 = use config default)") parallelFlag := fs.Int("parallel", 0, "Max tables hashed concurrently (0 = use config default)") verbose := fs.Bool("verbose", false, "Enable verbose logging") @@ -463,11 +466,39 @@ func runFullDiff(args []string) error { log.Info("full diff complete", "tables_scanned", tablesScanned, "has_changes", dataDiff.HasChanges()) fmt.Println("Schema OK. Data diff complete.") + var rowReport *resolve.RowDiffReport if dataDiff.HasChanges() { fmt.Printf("Changes detected. See %s, %s, and %s\n", filepath.Join(cfg.Output.Dir, "content_diff.json"), filepath.Join(cfg.Output.Dir, "conflicts.json"), filepath.Join(cfg.Output.Dir, "summary.txt")) if conflicts.HasConflicts() { fmt.Printf("Warning: %d conflicts detected. Review %s\n", len(conflicts.Conflicts), filepath.Join(cfg.Output.Dir, "conflicts.json")) } + + if *showRowDiff || *generateHTML { + log.Info("fetching column-level row differences") + fetchedReport, err := resolve.FetchRowDiffReport( + ctx, prodDB, devDB, cfg.Prod.Driver, cfg.Dev.Driver, + prodSchema, devSchema, dataDiff, conflicts, resolve.RowDiffOptions{ + StatusFilter: "all", + }, + ) + if err != nil { + log.Warn("failed to fetch row diffs", "error", err) + } else if fetchedReport.HasChanges() { + rowReport = &fetchedReport + if *showRowDiff { + if err := resolve.WriteRowDiffReport(*rowReport, cfg.Output.Dir); err != nil { + log.Warn("failed to write row diff report", "error", err) + } + display := cli.NewDisplay() + for _, trd := range rowReport.Tables { + for _, rd := range trd.Rows { + display.PrintRowDiff(rd.Table, rd.Key, string(rd.Status), rd.Columns) + } + } + fmt.Printf("Row-level column diffs written to %s and %s\n", filepath.Join(cfg.Output.Dir, "row_diff.json"), filepath.Join(cfg.Output.Dir, "row_diff.txt")) + } + } + } } else { fmt.Println("No data differences found.") } @@ -475,6 +506,9 @@ func runFullDiff(args []string) error { // Generate HTML report if requested if *generateHTML { htmlPath := filepath.Join(cfg.Output.Dir, "report.html") + reportOpts := htmlreport.DefaultReportOptions() + reportOpts.RowDiffReport = rowReport + reportData := htmlreport.BuildReportData( fmt.Sprintf("%s:%d/%s", cfg.Prod.Host, cfg.Prod.Port, cfg.Prod.Database), fmt.Sprintf("%s:%d/%s", cfg.Dev.Host, cfg.Dev.Port, cfg.Dev.Database), @@ -486,11 +520,11 @@ func runFullDiff(args []string) error { "", // No migration SQL for diff command "", // No migration pack for diff command tablesScanned, - nil, + reportOpts, ) reportData.Version = version - generator := htmlreport.NewGenerator(nil) + generator := htmlreport.NewGenerator(reportOpts) if err := generator.GenerateReport(reportData, htmlPath); err != nil { return fmt.Errorf("generate HTML report: %w", err) } @@ -502,6 +536,158 @@ func runFullDiff(args []string) error { return schemaDriftErr } +// runRowDiff executes the "row-diff" command to inspect column-level differences for changed rows. +func runRowDiff(args []string) error { + fs := flag.NewFlagSet("row-diff", flag.ContinueOnError) + configPath := fs.String("config", "deepdiffdb.config.yaml", "Path to configuration file") + tableFlag := fs.String("table", "", "Filter by specific table name") + keyFlag := fs.String("key", "", "Filter by primary key value") + statusFlag := fs.String("status", "updated", "Row status to inspect: updated, added, removed, or all") + limitFlag := fs.Int("limit", 0, "Max rows to inspect per table (0 = unlimited)") + formatFlag := fs.String("format", "table", "Output format: table or json") + outputFlag := fs.String("output", "", "Output directory to save reports (default: config output dir)") + batchSizeFlag := fs.Int("batch-size", 0, "Rows per keyset-paginated query (0 = use config default)") + parallelFlag := fs.Int("parallel", 0, "Max tables hashed concurrently (0 = use config default)") + verbose := fs.Bool("verbose", false, "Enable verbose logging") + logFile := fs.String("log-file", "", "Write logs to file") + logLevel := fs.String("log-level", "info", "Log level: debug, info, warn, error") + logFormat := fs.String("log-format", "text", "Log format: text or json") + if err := fs.Parse(args); err != nil { + return err + } + + log, logCloser, err := initializeLogger(*verbose, *logFile, *logLevel, *logFormat) + if err != nil { + return err + } + if logCloser != nil { + defer logCloser.Close() + } + + log.Info("starting row-level data diff") + + cfg, err := config.Load(*configPath) + if err != nil { + return fmt.Errorf("failed to load config: %w", err) + } + + outDir := cfg.Output.Dir + if *outputFlag != "" { + outDir = *outputFlag + } + + ctx := logger.ToContext(context.Background(), log) + + prodDB, err := openDatabaseWithSpinner(ctx, cfg.Prod, "production") + if err != nil { + return fmt.Errorf("prod connection failed: %w", err) + } + defer prodDB.Close() + + devDB, err := openDatabaseWithSpinner(ctx, cfg.Dev, "development") + if err != nil { + return fmt.Errorf("dev connection failed: %w", err) + } + defer devDB.Close() + + log.Info("loading database schemas") + prodSchema, err := schema.LoadSchema(ctx, prodDB, cfg.Prod.Driver, cfg.Prod.Database, cfg.Ignore.Tables, schema.LoadSchemaOptions{IgnoreViews: cfg.Ignore.Views, IgnoreRoutines: cfg.Ignore.Routines, IgnoreTriggers: cfg.Ignore.Triggers, IgnoreSequences: cfg.Ignore.Sequences}) + if err != nil { + return fmt.Errorf("load prod schema: %w", err) + } + devSchema, err := schema.LoadSchema(ctx, devDB, cfg.Dev.Driver, cfg.Dev.Database, cfg.Ignore.Tables, schema.LoadSchemaOptions{IgnoreViews: cfg.Ignore.Views, IgnoreRoutines: cfg.Ignore.Routines, IgnoreTriggers: cfg.Ignore.Triggers, IgnoreSequences: cfg.Ignore.Sequences}) + if err != nil { + return fmt.Errorf("load dev schema: %w", err) + } + + var dataDiff content.DataDiff + var conflicts content.Conflicts + + // If table and key are specified, we can query the row directly without full table hashing + if *tableFlag != "" && *keyFlag != "" { + log.Info("direct row inspection for key", "table", *tableFlag, "key", *keyFlag) + } else { + ignoreColumn := content.IgnoreMatcher(cfg.Ignore.Columns) + batchSize := resolveBatchSize(*batchSizeFlag, cfg.Performance.HashBatchSize) + maxParallel := resolveParallel(*parallelFlag, cfg.Performance.MaxParallelTables) + + devByCanonical := make(map[string]schema.Table, len(devSchema.Tables)) + for name, devTable := range devSchema.Tables { + devByCanonical[schema.CanonicalIdent(name)] = devTable + } + sharedProdTables := make(map[string]schema.Table) + sharedDevTables := make(map[string]schema.Table) + for name, prodTable := range prodSchema.Tables { + if *tableFlag != "" && schema.CanonicalIdent(name) != schema.CanonicalIdent(*tableFlag) { + continue + } + if devTable, ok := devByCanonical[schema.CanonicalIdent(name)]; ok { + sharedProdTables[name] = prodTable + sharedDevTables[devTable.Name] = devTable + } + } + + if len(sharedProdTables) == 0 { + if *tableFlag != "" { + return fmt.Errorf("table %q not found in both databases", *tableFlag) + } + fmt.Println("No matching tables found to compare.") + return nil + } + + log.Info("hashing tables to detect row differences", "table_count", len(sharedProdTables)) + prodHashes, err := hashTablesParallel(ctx, sharedProdTables, prodDB, cfg.Prod.Driver, ignoreColumn, batchSize, maxParallel) + if err != nil { + return fmt.Errorf("hash prod tables: %w", err) + } + devHashes, err := hashTablesParallel(ctx, sharedDevTables, devDB, cfg.Dev.Driver, ignoreColumn, batchSize, maxParallel) + if err != nil { + return fmt.Errorf("hash dev tables: %w", err) + } + dataDiff, conflicts = content.BuildDataDiff(prodSchema, devSchema, prodHashes, devHashes) + } + + opts := resolve.RowDiffOptions{ + TableFilter: *tableFlag, + KeyFilter: *keyFlag, + StatusFilter: *statusFlag, + Limit: *limitFlag, + } + + report, err := resolve.FetchRowDiffReport(ctx, prodDB, devDB, cfg.Prod.Driver, cfg.Dev.Driver, prodSchema, devSchema, dataDiff, conflicts, opts) + if err != nil { + return fmt.Errorf("fetch row diff report: %w", err) + } + + if err := resolve.WriteRowDiffReport(report, outDir); err != nil { + log.Warn("failed to write row diff report", "error", err) + } + + if *formatFlag == "json" { + data, err := json.MarshalIndent(report, "", " ") + if err != nil { + return fmt.Errorf("marshal json: %w", err) + } + fmt.Println(string(data)) + return nil + } + + if !report.HasChanges() { + fmt.Println("No row-level differences found matching criteria.") + return nil + } + + display := cli.NewDisplay() + for _, trd := range report.Tables { + for _, rd := range trd.Rows { + display.PrintRowDiff(rd.Table, rd.Key, string(rd.Status), rd.Columns) + } + } + fmt.Printf("Total differing rows inspected: %d\n", report.TotalRows()) + fmt.Printf("Row diff report written to %s and %s\n", filepath.Join(outDir, "row_diff.json"), filepath.Join(outDir, "row_diff.txt")) + return nil +} + // runGenPack performs the "gen-pack" command: it loads configuration, compares prod and dev schemas, and generates a migration pack for any detected data differences while writing schema and data reports to the configured output directory. // It opens prod and dev database connections, ensures the output directory exists, fails if schema drift is detected (after writing schema reports), computes per-table content hashes with configured ignore rules, builds the data diff and any conflicts, generates a migration pack when changes exist, writes data reports including the pack path and conflicts, and prints a brief status summary. // It returns a non-nil error for failures during configuration loading, database connections, schema loading or reporting, hashing, pack generation, or report writing; it also returns an error when schema drift is detected so the pack is not produced. @@ -1090,8 +1276,8 @@ func runResolveConflicts(args []string) error { } // Fetch row data for comparison - prod, dev, err := resolve.FetchConflictRows( - ctx, prodDB, devDB, cfg.Prod.Driver, + prod, dev, err := resolve.FetchConflictRowsCrossEngine( + ctx, prodDB, devDB, cfg.Prod.Driver, cfg.Dev.Driver, prodSchema, devSchema, res.Conflict, ) if err != nil { @@ -1484,7 +1670,8 @@ Commands: check Validate configuration and show quick summary schema-diff Detect schema drift schema-migrate Generate schema migration script - diff Full diff: schema + data (supports --html for interactive report) + diff Full diff: schema + data (supports --html, --row-diff) + row-diff Inspect changed row data in columns (supports --table, --key, --status, --format) gen-pack Generate SQL migration pack (supports --html, --resume) apply Apply migration pack (supports --resume) resolve-conflicts Interactively resolve pending conflicts (supports --resume) diff --git a/go.mod b/go.mod index 664c904..ec9913c 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/iamvirul/deepdiff-db -go 1.25.10 +go 1.25.13 require ( github.com/go-sql-driver/mysql v1.9.3 @@ -12,7 +12,7 @@ require ( github.com/testcontainers/testcontainers-go/modules/mssql v0.41.0 github.com/testcontainers/testcontainers-go/modules/mysql v0.40.0 github.com/testcontainers/testcontainers-go/modules/postgres v0.40.0 - golang.org/x/sync v0.20.0 + golang.org/x/sync v0.21.0 gopkg.in/yaml.v3 v3.0.1 modernc.org/sqlite v1.40.1 ) @@ -85,7 +85,7 @@ require ( golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b // indirect golang.org/x/sys v0.45.0 // indirect golang.org/x/term v0.43.0 // indirect - golang.org/x/text v0.37.0 // indirect + golang.org/x/text v0.39.0 // indirect google.golang.org/grpc v1.82.1 // indirect google.golang.org/protobuf v1.36.11 // indirect modernc.org/libc v1.66.10 // indirect diff --git a/go.sum b/go.sum index 1c4b2af..c748cf8 100644 --- a/go.sum +++ b/go.sum @@ -204,12 +204,12 @@ golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b h1:M2rDM6z3Fhozi9O7NWsxAkg/yqS/lQJ6PmkyIV3YP+o= golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8= -golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= -golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w= golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -219,12 +219,12 @@ golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= -golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= -golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus= +golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM= golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= -golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= +golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= +golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 h1:yQugLulqltosq0B/f8l4w9VryjV+N/5gcW0jQ3N8Qec= google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478/go.mod h1:C6ADNqOxbgdUUeRTU+LCHDPB9ttAMCTff6auwCVa4uc= diff --git a/internal/cli/display.go b/internal/cli/display.go index 4978649..6c179c3 100644 --- a/internal/cli/display.go +++ b/internal/cli/display.go @@ -107,6 +107,51 @@ func (d *Display) PrintConflictComparison(prod, dev *resolve.RowData, diffs []re fmt.Fprintln(d.writer) } +// PrintRowDiff displays a side-by-side comparison of a row's column differences. +func (d *Display) PrintRowDiff(table, key, status string, diffs []resolve.ColumnDiff) { + d.PrintDoubleLine() + fmt.Fprintf(d.writer, "Table: %s | Key: %s (%s)\n", table, key, status) + d.PrintDoubleLine() + + // Calculate column widths + colWidth := 20 + valWidth := 24 + + // Print header + fmt.Fprintf(d.writer, " %-*s | %-*s | %-*s\n", + colWidth, "Column", + valWidth, "Production", + valWidth, "Development") + fmt.Fprintf(d.writer, " %s+%s+%s\n", + strings.Repeat("-", colWidth+1), + strings.Repeat("-", valWidth+2), + strings.Repeat("-", valWidth+2)) + + // Print each column + for _, diff := range diffs { + prodStr := formatValueForDisplay(diff.ProdVal, valWidth) + devStr := formatValueForDisplay(diff.DevVal, valWidth) + + marker := "" + if diff.Differs { + marker = " *" + } + + fmt.Fprintf(d.writer, " %-*s | %-*s | %-*s%s\n", + colWidth, truncate(diff.Column, colWidth), + valWidth, prodStr, + valWidth, devStr, + marker) + } + + fmt.Fprintln(d.writer) + if hasDifferences(diffs) { + fmt.Fprintln(d.writer, " * indicates columns that differ") + } + d.PrintDoubleLine() + fmt.Fprintln(d.writer) +} + // PrintSummary displays the resolution summary. func (d *Display) PrintSummary(summary resolve.ResolutionSummary, outputPath string) { d.PrintDoubleLine() diff --git a/internal/content/resolve/fetch.go b/internal/content/resolve/fetch.go index 3885609..403638e 100644 --- a/internal/content/resolve/fetch.go +++ b/internal/content/resolve/fetch.go @@ -29,8 +29,26 @@ type ColumnDiff struct { Differs bool } -// FetchConflictRows fetches both production and development row data for a conflict. -// Returns nil for a row if it doesn't exist in that database. +// FindTableByCanonicalName searches a schema for a table matching tableName, +// trying exact match first, then canonical (case-folded) match. +func FindTableByCanonicalName(sch *schema.Schema, tableName string) (schema.Table, bool) { + if sch == nil || len(sch.Tables) == 0 { + return schema.Table{}, false + } + if tbl, ok := sch.Tables[tableName]; ok { + return tbl, true + } + canon := schema.CanonicalIdent(tableName) + for name, tbl := range sch.Tables { + if schema.CanonicalIdent(name) == canon { + return tbl, true + } + } + return schema.Table{}, false +} + +// FetchConflictRows fetches both production and development row data for a conflict +// using a single driver. Delegates to FetchConflictRowsCrossEngine. func FetchConflictRows( ctx context.Context, prodDB, devDB *sql.DB, @@ -38,36 +56,40 @@ func FetchConflictRows( prodSchema, devSchema *schema.Schema, conflict content.Conflict, ) (prod, dev *RowData, err error) { - // Get table schema from both databases - prodTable, prodExists := prodSchema.Tables[conflict.Table] - devTable, devExists := devSchema.Tables[conflict.Table] + return FetchConflictRowsCrossEngine(ctx, prodDB, devDB, driver, driver, prodSchema, devSchema, conflict) +} + +// FetchConflictRowsCrossEngine fetches both production and development row data for a conflict, +// supporting independent drivers (e.g. postgres vs oracle) and case-insensitive table matching. +// Returns nil for a row if it doesn't exist in that database. +func FetchConflictRowsCrossEngine( + ctx context.Context, + prodDB, devDB *sql.DB, + prodDriver, devDriver string, + prodSchema, devSchema *schema.Schema, + conflict content.Conflict, +) (prod, dev *RowData, err error) { + // Match table schema from both databases case-insensitively + prodTable, prodExists := FindTableByCanonicalName(prodSchema, conflict.Table) + devTable, devExists := FindTableByCanonicalName(devSchema, conflict.Table) if !prodExists && !devExists { return nil, nil, fmt.Errorf("table %s not found in either database", conflict.Table) } - // Use dev table schema as reference (it should have all columns) - var refTable schema.Table - if devExists { - refTable = devTable - } else { - refTable = prodTable - } - - // Get ordered column list - columns := getOrderedColumns(refTable) - - // Fetch from production + // Fetch from production using prod-specific table and columns if prodExists { - prod, err = fetchRowData(ctx, prodDB, driver, prodTable, columns, conflict.Key) + columns := getOrderedColumns(prodTable) + prod, err = FetchRowData(ctx, prodDB, prodDriver, prodTable, columns, conflict.Key) if err != nil && !isNoRowsError(err) { return nil, nil, fmt.Errorf("fetch prod row: %w", err) } } - // Fetch from development + // Fetch from development using dev-specific table and columns if devExists { - dev, err = fetchRowData(ctx, devDB, driver, devTable, columns, conflict.Key) + columns := getOrderedColumns(devTable) + dev, err = FetchRowData(ctx, devDB, devDriver, devTable, columns, conflict.Key) if err != nil && !isNoRowsError(err) { return nil, nil, fmt.Errorf("fetch dev row: %w", err) } @@ -76,8 +98,8 @@ func FetchConflictRows( return prod, dev, nil } -// fetchRowData fetches a single row from the database. -func fetchRowData( +// FetchRowData fetches a single row from the database using its primary key. +func FetchRowData( ctx context.Context, db *sql.DB, driver string, @@ -139,52 +161,65 @@ func fetchRowData( } // CompareRows compares two rows and returns the differences. +// It matches columns case-insensitively across engines (e.g. PostgreSQL "email" +// matches Oracle "EMAIL") while labeling with the prod-side column name. // Returns a list of column differences, with Differs=true for columns that don't match. func CompareRows(prod, dev *RowData) []ColumnDiff { if prod == nil && dev == nil { return nil } - // Collect all columns from both rows - colSet := make(map[string]bool) - if prod != nil { - for _, col := range prod.Columns { - colSet[col] = true - } - } + var diffs []ColumnDiff + matchedDev := make(map[string]bool) + + // Build dev canonical lookup map + devByCanonical := make(map[string]string) if dev != nil { for _, col := range dev.Columns { - colSet[col] = true + devByCanonical[schema.CanonicalIdent(col)] = col } } - // Sort columns for consistent output - columns := make([]string, 0, len(colSet)) - for col := range colSet { - columns = append(columns, col) - } - sort.Strings(columns) - - // Compare each column - diffs := make([]ColumnDiff, 0, len(columns)) - for _, col := range columns { - var prodVal, devVal any - if prod != nil { - prodVal = prod.Values[col] - } - if dev != nil { - devVal = dev.Values[col] + if prod != nil { + for _, pCol := range prod.Columns { + canon := schema.CanonicalIdent(pCol) + var devVal any + differs := true + if dCol, ok := devByCanonical[canon]; ok { + matchedDev[dCol] = true + if dev != nil { + devVal = dev.Values[dCol] + differs = !valuesEqual(prod.Values[pCol], devVal) + } + } + diffs = append(diffs, ColumnDiff{ + Column: pCol, + ProdVal: prod.Values[pCol], + DevVal: devVal, + Differs: differs, + }) } + } - differs := !valuesEqual(prodVal, devVal) - diffs = append(diffs, ColumnDiff{ - Column: col, - ProdVal: prodVal, - DevVal: devVal, - Differs: differs, - }) + // Dev columns that do not exist in prod + if dev != nil { + for _, dCol := range dev.Columns { + if !matchedDev[dCol] { + diffs = append(diffs, ColumnDiff{ + Column: dCol, + ProdVal: nil, + DevVal: dev.Values[dCol], + Differs: true, + }) + } + } } + // Sort columns alphabetically for deterministic output + sort.Slice(diffs, func(i, j int) bool { + return strings.ToLower(diffs[i].Column) < strings.ToLower(diffs[j].Column) + }) + return diffs } @@ -226,9 +261,11 @@ func splitKey(key string, expected int) ([]string, error) { func quoteIdent(driver, ident string) string { switch driver { case "mysql": - return "`" + ident + "`" - case "postgres", "postgresql": - return `"` + ident + `"` + return "`" + strings.ReplaceAll(ident, "`", "``") + "`" + case "postgres", "postgresql", "sqlite", "oracle": + return "\"" + strings.ReplaceAll(ident, "\"", "\"\"") + "\"" + case "mssql": + return "[" + strings.ReplaceAll(ident, "]", "]]") + "]" default: return ident } diff --git a/internal/content/resolve/rowdiff.go b/internal/content/resolve/rowdiff.go new file mode 100644 index 0000000..14b0699 --- /dev/null +++ b/internal/content/resolve/rowdiff.go @@ -0,0 +1,289 @@ +package resolve + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/iamvirul/deepdiff-db/internal/content" + "github.com/iamvirul/deepdiff-db/internal/schema" +) + +// RowDiffStatus represents the state of a row difference. +type RowDiffStatus string + +const ( + // StatusUpdated indicates the row exists in both prod and dev with differing values. + StatusUpdated RowDiffStatus = "updated" + // StatusAdded indicates the row exists only in development. + StatusAdded RowDiffStatus = "added" + // StatusRemoved indicates the row exists only in production. + StatusRemoved RowDiffStatus = "removed" +) + +// RowDiff represents the column-level differences for a single row. +type RowDiff struct { + Table string `json:"table"` + Key string `json:"key"` + Status RowDiffStatus `json:"status"` + Columns []ColumnDiff `json:"columns"` + DiffColumns []string `json:"differing_columns,omitempty"` +} + +// TableRowDiff represents all row differences for a specific table. +type TableRowDiff struct { + Table string `json:"table"` + Rows []RowDiff `json:"rows"` +} + +// RowDiffReport contains all row differences grouped by table. +type RowDiffReport struct { + Tables []TableRowDiff `json:"tables"` +} + +// HasChanges reports whether any row differences exist in the report. +func (r RowDiffReport) HasChanges() bool { + for _, t := range r.Tables { + if len(t.Rows) > 0 { + return true + } + } + return false +} + +// TotalRows returns the total count of differing rows in the report. +func (r RowDiffReport) TotalRows() int { + count := 0 + for _, t := range r.Tables { + count += len(t.Rows) + } + return count +} + +// RowDiffOptions controls which row diffs are fetched. +type RowDiffOptions struct { + TableFilter string // optional filter for a specific table + KeyFilter string // optional filter for a specific primary key + StatusFilter string // "updated", "added", "removed", or "all" (default: "updated") + Limit int // max rows to inspect per table (0 = unlimited) +} + +// FetchRowDiffReport fetches column-level row data and differences for conflicting / changed rows. +// It supports cross-engine setups where table or column casing differs between prod and dev. +func FetchRowDiffReport( + ctx context.Context, + prodDB, devDB *sql.DB, + prodDriver, devDriver string, + prodSchema, devSchema *schema.Schema, + dataDiff content.DataDiff, + conflicts content.Conflicts, + opts RowDiffOptions, +) (RowDiffReport, error) { + report := RowDiffReport{} + statusFilter := strings.ToLower(strings.TrimSpace(opts.StatusFilter)) + if statusFilter == "" { + statusFilter = string(StatusUpdated) + } + + includeUpdated := statusFilter == "all" || statusFilter == string(StatusUpdated) + includeAdded := statusFilter == "all" || statusFilter == string(StatusAdded) + includeRemoved := statusFilter == "all" || statusFilter == string(StatusRemoved) + + // Direct single-key lookup optimization when table and key are specified + if opts.TableFilter != "" && opts.KeyFilter != "" && len(dataDiff.Tables) == 0 { + conflict := content.Conflict{Table: opts.TableFilter, Key: opts.KeyFilter} + prod, dev, err := FetchConflictRowsCrossEngine( + ctx, prodDB, devDB, prodDriver, devDriver, prodSchema, devSchema, conflict, + ) + if err != nil { + return report, fmt.Errorf("fetch row data for %s key %s: %w", opts.TableFilter, opts.KeyFilter, err) + } + if prod != nil || dev != nil { + diffs := CompareRows(prod, dev) + var diffCols []string + for _, cd := range diffs { + if cd.Differs { + diffCols = append(diffCols, cd.Column) + } + } + st := StatusUpdated + if prod == nil { + st = StatusAdded + } else if dev == nil { + st = StatusRemoved + } + report.Tables = append(report.Tables, TableRowDiff{ + Table: opts.TableFilter, + Rows: []RowDiff{ + { + Table: opts.TableFilter, + Key: opts.KeyFilter, + Status: st, + Columns: diffs, + DiffColumns: diffCols, + }, + }, + }) + } + return report, nil + } + + for _, td := range dataDiff.Tables { + if opts.TableFilter != "" && schema.CanonicalIdent(td.Table) != schema.CanonicalIdent(opts.TableFilter) { + continue + } + + tableRows := TableRowDiff{Table: td.Table} + count := 0 + + // Helper to fetch and append a row diff + fetchRow := func(key string, status RowDiffStatus) error { + if opts.KeyFilter != "" && key != opts.KeyFilter { + return nil + } + if opts.Limit > 0 && count >= opts.Limit { + return nil + } + + conflict := content.Conflict{Table: td.Table, Key: key} + prod, dev, err := FetchConflictRowsCrossEngine( + ctx, prodDB, devDB, prodDriver, devDriver, prodSchema, devSchema, conflict, + ) + if err != nil { + return fmt.Errorf("fetch row %s.%s: %w", td.Table, key, err) + } + + diffs := CompareRows(prod, dev) + var diffCols []string + for _, cd := range diffs { + if cd.Differs { + diffCols = append(diffCols, cd.Column) + } + } + + tableRows.Rows = append(tableRows.Rows, RowDiff{ + Table: td.Table, + Key: key, + Status: status, + Columns: diffs, + DiffColumns: diffCols, + }) + count++ + return nil + } + + if includeUpdated { + for _, k := range td.Updated { + if err := fetchRow(k, StatusUpdated); err != nil { + return report, err + } + if opts.Limit > 0 && count >= opts.Limit { + break + } + } + } + + if includeRemoved { + for _, k := range td.Removed { + if err := fetchRow(k, StatusRemoved); err != nil { + return report, err + } + if opts.Limit > 0 && count >= opts.Limit { + break + } + } + } + + if includeAdded { + for _, k := range td.Added { + if err := fetchRow(k, StatusAdded); err != nil { + return report, err + } + if opts.Limit > 0 && count >= opts.Limit { + break + } + } + } + + if len(tableRows.Rows) > 0 { + report.Tables = append(report.Tables, tableRows) + } + } + + return report, nil +} + +// WriteRowDiffReport writes the row diff report to row_diff.json and row_diff.txt in outDir. +func WriteRowDiffReport(report RowDiffReport, outDir string) error { + if err := os.MkdirAll(outDir, 0o750); err != nil { + return fmt.Errorf("ensure output dir: %w", err) + } + + // 1. JSON + data, err := json.MarshalIndent(report, "", " ") + if err != nil { + return fmt.Errorf("marshal row diff json: %w", err) + } + if err := os.WriteFile(filepath.Join(outDir, "row_diff.json"), data, 0o600); err != nil { + return fmt.Errorf("write row_diff.json: %w", err) + } + + // 2. Text + txt := FormatRowDiffText(report) + if err := os.WriteFile(filepath.Join(outDir, "row_diff.txt"), []byte(txt), 0o600); err != nil { + return fmt.Errorf("write row_diff.txt: %w", err) + } + + return nil +} + +// FormatRowDiffText returns a human-readable text representation of row differences. +func FormatRowDiffText(report RowDiffReport) string { + var b strings.Builder + if !report.HasChanges() { + b.WriteString("No row-level differences found.\n") + return b.String() + } + + colWidth := 20 + valWidth := 24 + + for _, trd := range report.Tables { + for _, rd := range trd.Rows { + b.WriteString(strings.Repeat("=", 75) + "\n") + fmt.Fprintf(&b, "Table: %s | Key: %s (%s)\n", rd.Table, rd.Key, rd.Status) + b.WriteString(strings.Repeat("=", 75) + "\n") + fmt.Fprintf(&b, " %-*s | %-*s | %-*s\n", + colWidth, "Column", + valWidth, "Production", + valWidth, "Development") + fmt.Fprintf(&b, " %s+%s+%s\n", + strings.Repeat("-", colWidth+1), + strings.Repeat("-", valWidth+2), + strings.Repeat("-", valWidth+2)) + + for _, cd := range rd.Columns { + prodStr := FormatValue(cd.ProdVal) + devStr := FormatValue(cd.DevVal) + marker := "" + if cd.Differs { + marker = " *" + } + fmt.Fprintf(&b, " %-*s | %-*s | %-*s%s\n", + colWidth, cd.Column, + valWidth, prodStr, + valWidth, devStr, + marker) + } + + b.WriteString("\n * indicates columns that differ\n") + b.WriteString(strings.Repeat("=", 75) + "\n\n") + } + } + + return b.String() +} diff --git a/internal/report/html/generator.go b/internal/report/html/generator.go index fe9852a..f648247 100644 --- a/internal/report/html/generator.go +++ b/internal/report/html/generator.go @@ -63,6 +63,9 @@ func templateFuncs() template.FuncMap { "join": func(items []string, sep string) string { return strings.Join(items, sep) }, + "formatVal": func(v any) string { + return resolve.FormatValue(v) + }, "formatTime": func(t time.Time) string { return t.Format("2006-01-02 15:04:05 MST") }, @@ -175,11 +178,17 @@ func BuildReportData( data.TableDiffs = buildTableDiffs(dataDiff, opts) } + // Process row diff report + if opts.RowDiffReport != nil && opts.RowDiffReport.HasChanges() { + data.RowDiffReport = opts.RowDiffReport + data.HasRowDiffs = true + } + // Process conflicts with resolution details if conflicts != nil && conflicts.HasConflicts() { data.Conflicts = conflicts data.HasConflicts = true - data.ConflictItems = buildConflictItemsWithResolutions(conflicts, resolutions) + data.ConflictItems = buildConflictItemsWithResolutionsAndRowDiffs(conflicts, resolutions, opts.RowDiffReport) } // Process resolutions and build breakdown @@ -438,6 +447,16 @@ func buildTableDiffs(dataDiff *content.DataDiff, opts *ReportOptions) []TableDif td.UpdatedKeys = limitKeys(t.Updated, maxKeys) } + if opts.RowDiffReport != nil { + for _, trd := range opts.RowDiffReport.Tables { + if schema.CanonicalIdent(trd.Table) == schema.CanonicalIdent(t.Table) { + td.RowDiffs = trd.Rows + td.HasRowDiffs = len(trd.Rows) > 0 + break + } + } + } + diffs = append(diffs, td) } @@ -449,8 +468,9 @@ func buildTableDiffs(dataDiff *content.DataDiff, opts *ReportOptions) []TableDif return diffs } -// buildConflictItemsWithResolutions converts conflicts to display format with resolution details. -func buildConflictItemsWithResolutions(conflicts *content.Conflicts, resolutions []resolve.Resolution) []ConflictDisplay { + +// buildConflictItemsWithResolutionsAndRowDiffs converts conflicts to display format with resolution and row diff details. +func buildConflictItemsWithResolutionsAndRowDiffs(conflicts *content.Conflicts, resolutions []resolve.Resolution, rowReport *resolve.RowDiffReport) []ConflictDisplay { var items []ConflictDisplay // Build a map of resolutions by table+key for quick lookup @@ -460,6 +480,17 @@ func buildConflictItemsWithResolutions(conflicts *content.Conflicts, resolutions resolutionMap[key] = r } + // Build a map of row diffs by table+key for quick lookup + rowDiffMap := make(map[string]resolve.RowDiff) + if rowReport != nil { + for _, t := range rowReport.Tables { + for _, rd := range t.Rows { + rowDiffMap[rd.Table+":"+rd.Key] = rd + rowDiffMap[schema.CanonicalIdent(rd.Table)+":"+rd.Key] = rd + } + } + } + for _, c := range conflicts.Conflicts { item := ConflictDisplay{ Table: c.Table, @@ -477,6 +508,16 @@ func buildConflictItemsWithResolutions(conflicts *content.Conflicts, resolutions item.IsResolved = res.Resolved } + if rd, ok := rowDiffMap[key]; ok { + item.ColumnDiffs = rd.Columns + item.DiffColumns = rd.DiffColumns + item.HasColumnDiffs = len(rd.Columns) > 0 + } else if rd, ok := rowDiffMap[schema.CanonicalIdent(c.Table)+":"+c.Key]; ok { + item.ColumnDiffs = rd.Columns + item.DiffColumns = rd.DiffColumns + item.HasColumnDiffs = len(rd.Columns) > 0 + } + items = append(items, item) } diff --git a/internal/report/html/template.go b/internal/report/html/template.go index f3e5d9b..1650735 100644 --- a/internal/report/html/template.go +++ b/internal/report/html/template.go @@ -592,6 +592,15 @@ const reportTemplate = ` overflow: hidden; } + .conflict-item-card { + border-bottom: 1px solid var(--border); + background: var(--bg); + } + + .conflict-item-card:last-child { + border-bottom: none; + } + .conflict-item { display: grid; grid-template-columns: 140px 1fr 200px 100px; @@ -600,12 +609,78 @@ const reportTemplate = ` padding: 12px 16px; background: var(--bg); font-size: 13px; + cursor: pointer; } .conflict-item:hover { background: var(--bg-secondary); } + .conflict-item-body { + padding: 0 16px 14px 16px; + background: var(--bg-secondary); + border-top: 1px dashed var(--border); + } + + .row-data-table { + width: 100%; + border-collapse: collapse; + font-size: 12px; + margin-top: 10px; + background: var(--bg); + border: 1px solid var(--border); + border-radius: 4px; + overflow: hidden; + } + + .row-data-table th, .row-data-table td { + padding: 7px 12px; + text-align: left; + border-bottom: 1px solid var(--border); + } + + .row-data-table th { + background: var(--bg-tertiary); + font-weight: 600; + color: var(--text-secondary); + } + + .row-data-table tr:last-child td { + border-bottom: none; + } + + .diff-row-changed { + background: rgba(234, 88, 12, 0.08); + } + + .diff-star { + color: #ea580c; + font-weight: bold; + } + + .rowdiff-card { + background: var(--bg); + border: 1px solid var(--border); + border-radius: 6px; + margin-bottom: 16px; + overflow: hidden; + } + + .rowdiff-header { + display: flex; + align-items: center; + gap: 16px; + padding: 10px 16px; + background: var(--bg-secondary); + border-bottom: 1px solid var(--border); + font-size: 13px; + font-weight: 500; + } + + .rowdiff-body { + padding: 12px 16px; + } + .conflict-table { font-family: var(--font-mono); font-weight: 500; @@ -873,21 +948,26 @@ const reportTemplate = `
-
+
{{if .HasSchemaDiff}} {{range .SchemaChanges}}
@@ -1035,7 +1115,7 @@ const reportTemplate = ` {{end}}
-
+
{{if .HasDataDiff}}
+ + +
+
+ {{range .RowDiffReport.Tables}} + {{range .Rows}} +
+
+ {{.Table}} + Key: {{.Key}} + {{.Status}} +
+
+ + + + + + + + + + {{range .Columns}} + + + + + + {{end}} + +
ColumnProductionDevelopment
{{.Column}}{{if .Differs}} *{{end}}{{formatVal .ProdVal}}{{formatVal .DevVal}}
+
+
+ {{end}} + {{end}} +
+
+ {{end}} + {{if .HasMigration}}
@@ -1185,8 +1366,17 @@ const reportTemplate = ` function filterConflicts() { const v = document.getElementById('conflict-filter').value; - document.querySelectorAll('#conflict-list .conflict-item').forEach(r => { - r.style.display = !v || r.dataset.table === v ? '' : 'none'; + document.querySelectorAll('#conflict-list .conflict-item-card').forEach(r => { + const show = !v || r.dataset.table === v; + r.style.display = show ? '' : 'none'; + }); + } + + function filterRowDiffs() { + const v = document.getElementById('rowdiff-filter').value; + document.querySelectorAll('#rowdiff-list .rowdiff-card').forEach(r => { + const show = !v || r.dataset.table === v; + r.style.display = show ? '' : 'none'; }); } @@ -1216,9 +1406,21 @@ const reportTemplate = ` const cf = document.getElementById('conflict-filter'); if (cf) { const ts = new Set(); - document.querySelectorAll('#conflict-list .conflict-item').forEach(i => ts.add(i.dataset.table)); + document.querySelectorAll('#conflict-list .conflict-item-card').forEach(i => ts.add(i.dataset.table)); ts.forEach(t => { const o = document.createElement('option'); o.value = t; o.textContent = t; cf.appendChild(o); }); } + + const rf = document.getElementById('rowdiff-filter'); + if (rf) { + const rts = new Set(); + document.querySelectorAll('#rowdiff-list .rowdiff-card').forEach(i => rts.add(i.dataset.table)); + rts.forEach(t => { const o = document.createElement('option'); o.value = t; o.textContent = t; rf.appendChild(o); }); + } + + const hash = window.location.hash.replace('#', ''); + if (hash && document.getElementById('tab-' + hash)) { + switchTab(hash); + } }); diff --git a/internal/report/html/types.go b/internal/report/html/types.go index 01e3730..9d874a4 100644 --- a/internal/report/html/types.go +++ b/internal/report/html/types.go @@ -4,6 +4,7 @@ import ( "time" "github.com/iamvirul/deepdiff-db/internal/content" + "github.com/iamvirul/deepdiff-db/internal/content/resolve" "github.com/iamvirul/deepdiff-db/internal/schema" ) @@ -91,6 +92,10 @@ type ReportData struct { // SQL Migration MigrationSQL string `json:"migration_sql,omitempty"` HasMigration bool `json:"has_migration"` + + // Row data differences + RowDiffReport *resolve.RowDiffReport `json:"row_diff_report,omitempty"` + HasRowDiffs bool `json:"has_row_diffs"` } // ReportSummary contains aggregate statistics. @@ -160,14 +165,16 @@ type ForeignKeyChangeDisplay struct { // TableDiffDisplay represents a table's data differences for display. type TableDiffDisplay struct { - Table string `json:"table"` - AddedCount int `json:"added_count"` - RemovedCount int `json:"removed_count"` - UpdatedCount int `json:"updated_count"` - AddedKeys []string `json:"added_keys,omitempty"` - RemovedKeys []string `json:"removed_keys,omitempty"` - UpdatedKeys []string `json:"updated_keys,omitempty"` - HasChanges bool `json:"has_changes"` + Table string `json:"table"` + AddedCount int `json:"added_count"` + RemovedCount int `json:"removed_count"` + UpdatedCount int `json:"updated_count"` + AddedKeys []string `json:"added_keys,omitempty"` + RemovedKeys []string `json:"removed_keys,omitempty"` + UpdatedKeys []string `json:"updated_keys,omitempty"` + HasChanges bool `json:"has_changes"` + RowDiffs []resolve.RowDiff `json:"row_diffs,omitempty"` + HasRowDiffs bool `json:"has_row_diffs"` } // ConflictDisplay represents a conflict for display. @@ -179,7 +186,10 @@ type ConflictDisplay struct { Resolution string `json:"resolution,omitempty"` // "keep_prod", "use_dev", "pending" Strategy string `json:"strategy,omitempty"` // "ours", "theirs", "manual" Decision string `json:"decision,omitempty"` // "keep_prod", "use_dev", "pending" - IsResolved bool `json:"is_resolved"` + IsResolved bool `json:"is_resolved"` + ColumnDiffs []resolve.ColumnDiff `json:"column_diffs,omitempty"` + DiffColumns []string `json:"differing_columns,omitempty"` + HasColumnDiffs bool `json:"has_column_diffs"` } // TableStrategyDisplay shows the resolution strategy for a table. @@ -213,6 +223,9 @@ type ReportOptions struct { // Enable PDF export button (requires browser print) EnablePDFExport bool + + // RowDiffReport contains column-level row differences for display + RowDiffReport *resolve.RowDiffReport } // DefaultReportOptions returns sensible defaults for report generation. diff --git a/samples/18-cross-engine-row-diff/Makefile b/samples/18-cross-engine-row-diff/Makefile new file mode 100644 index 0000000..e7806b9 --- /dev/null +++ b/samples/18-cross-engine-row-diff/Makefile @@ -0,0 +1,51 @@ +BIN = ../../deepdiffdb + +.PHONY: help up wait-healthy seed diff row-diff html down clean + +help: + @echo "Sample 18: Cross-Engine Row Diff & Migration Verification" + @echo "" + @echo "Targets:" + @echo " make up - Start PostgreSQL 16 and Oracle XE 21c containers" + @echo " make wait-healthy - Wait for both containers to report healthy" + @echo " make seed - Initialize schemas and seed drifted data" + @echo " make diff - Run full cross-engine diff (schema + data)" + @echo " make row-diff - Inspect column-level differences in terminal" + @echo " make html - Generate and open interactive HTML report" + @echo " make down - Stop and remove Docker containers" + @echo " make clean - Remove diff-output artifacts" + +up: + docker compose up -d + +wait-healthy: + @echo "Waiting for PostgreSQL to be healthy..." + @until [ "$$(docker inspect --format='{{.State.Health.Status}}' sample18-postgres 2>/dev/null)" = "healthy" ]; do sleep 2; done + @echo "PostgreSQL is ready." + @echo "Waiting for Oracle to be healthy (takes ~45-90s on initial start)..." + @until [ "$$(docker inspect --format='{{.State.Health.Status}}' sample18-oracle 2>/dev/null)" = "healthy" ]; do sleep 5; done + @echo "Oracle XE is ready." + +seed: + @echo "Seeding PostgreSQL..." + docker exec -i sample18-postgres psql -U postgres -d appdb < init-scripts/01-postgres-init.sql + @echo "Seeding Oracle XE..." + docker exec -i sample18-oracle sqlplus -S appuser/secret@XEPDB1 < init-scripts/02-oracle-init.sql + @echo "Seeding complete." + +diff: + $(BIN) diff --config deepdiffdb.config.yaml + +row-diff: + $(BIN) row-diff --config deepdiffdb.config.yaml --status all + +html: + $(BIN) diff --config deepdiffdb.config.yaml --html --row-diff + @echo "Opening diff-output/report.html..." + @open diff-output/report.html 2>/dev/null || xdg-open diff-output/report.html 2>/dev/null || echo "Report available at diff-output/report.html" + +down: + docker compose down -v + +clean: + rm -rf diff-output diff --git a/samples/18-cross-engine-row-diff/README.md b/samples/18-cross-engine-row-diff/README.md new file mode 100644 index 0000000..e6144f6 --- /dev/null +++ b/samples/18-cross-engine-row-diff/README.md @@ -0,0 +1,138 @@ +# Sample 18: Cross-Engine Row Diff & Migration Verification + +This sample demonstrates **cross-engine migration verification** between **PostgreSQL 16** (Source / Production) and **Oracle XE 21c** (Target / Development). + +It replicates common real-world migration discrepancies where: +1. **Identifier case differs**: PostgreSQL stores unquoted identifiers in lowercase (`um_user`, `um_user_attribute`), whereas Oracle stores them in UPPERCASE (`UM_USER`, `UM_USER_ATTRIBUTE`). +2. **Data drift exists**: Certain rows have modified columns, missing records, or newly inserted rows. +3. **Column-level inspection is needed**: You need to see exactly *which* column drifted (e.g. `email` or `attr_value`) rather than just knowing that a row checksum failed. + +--- + +## Architecture & Scenario + +```text +┌─────────────────────────────────┐ ┌─────────────────────────────────┐ +│ PostgreSQL 16 (Prod) │ │ Oracle XE 21c (Dev) │ +│ Port: 55432 │ │ Port: 51521 │ +├─────────────────────────────────┤ ├─────────────────────────────────┤ +│ um_user (6 rows, lowercase) │ ◄───────► │ UM_USER (5 rows, UPPERCASE) │ +│ um_user_attribute (7 rows) │ │ UM_USER_ATTRIBUTE (6 rows) │ +└─────────────────────────────────┘ └─────────────────────────────────┘ + │ + ▼ + ┌───────────────────────────┐ + │ DeepDiff DB │ + │ Canonical Case Folding │ + │ Column-Level Row Diffing │ + └───────────────────────────┘ +``` + +--- + +## Quick Start + +### 1. Start Containers + +```bash +make up +make wait-healthy +``` + +*Note: Oracle XE 21c takes approximately 45–90 seconds to initialize on the first launch.* + +### 2. Seed Data + +```bash +make seed +``` + +This script seeds: +- **`um_user` / `UM_USER`**: + - Key `1`, `2`, `3`: Identical in both databases. + - Key `4` (`dan`): Email in Postgres is `dan@corp.internal`, but in Oracle is `dan+updated@corp.internal`. + - Keys `5`, `6` (`eve`, `frank`): Present in Postgres, missing in Oracle. + - Key `7` (`grace`): Present in Oracle, missing in Postgres. +- **`um_user_attribute` / `UM_USER_ATTRIBUTE`**: + - Key `104`: Attribute `department` is `'Finance'` in Postgres, but `'Accounting'` in Oracle. + - Keys `106`, `107`: Missing in Oracle. + - Key `108`: New in Oracle. + +### 3. Run Cross-Engine Diff + +```bash +make diff +``` + +DeepDiff DB canonically pairs `um_user` with `UM_USER` and `um_user_attribute` with `UM_USER_ATTRIBUTE`: + +```text +Changes detected. See diff-output/content_diff.json, diff-output/conflicts.json, and diff-output/summary.txt +Warning: 2 conflicts detected. Review diff-output/conflicts.json +``` + +### 4. Inspect Column-Level Row Differences (`row-diff`) + +Run `make row-diff` to view the exact column values side-by-side: + +```bash +make row-diff +``` + +Output: + +```text +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +Table: um_user | Key: 4 (updated) +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + Column | Production (PostgreSQL) | Development (Oracle) + ---------------------+--------------------------+-------------------------- + email | dan@corp.internal | dan+updated@corp.internal * + id | 4 | 4 + status | ACTIVE | ACTIVE + username | dan | dan + + * indicates columns that differ +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +Table: um_user_attribute | Key: 104 (updated) +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + Column | Production (PostgreSQL) | Development (Oracle) + ---------------------+--------------------------+-------------------------- + attr_name | department | department + attr_value | Finance | Accounting * + id | 104 | 104 + user_id | 3 | 3 + + * indicates columns that differ +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +``` + +### 5. Targeted Key Inspection + +To inspect a single record directly without scanning the whole table: + +```bash +../../deepdiffdb row-diff --table um_user --key 4 +``` + +### 6. Interactive HTML Report + +Generate the HTML viewer with column diffing embedded: + +```bash +make html +``` + +The report opens in your browser at `diff-output/report.html`, featuring: +- **Data Changes Tab**: Click `um_user` or `um_user_attribute` to see the column diff tables right inside the expanded row. +- **Row Data Tab**: Browse all changed records with real-time table filtering. +- **Conflicts Tab**: Inspect conflicting hashes with side-by-side values. + +### 7. Teardown + +```bash +make down +make clean +``` diff --git a/samples/18-cross-engine-row-diff/deepdiffdb.config.yaml b/samples/18-cross-engine-row-diff/deepdiffdb.config.yaml new file mode 100644 index 0000000..580a152 --- /dev/null +++ b/samples/18-cross-engine-row-diff/deepdiffdb.config.yaml @@ -0,0 +1,32 @@ +# DeepDiff DB Configuration: PostgreSQL (Prod) <-> Oracle (Dev) +# Demonstrates cross-engine case-insensitive table discovery and row diffing. + +prod: + driver: postgres + host: localhost + port: 55432 + user: postgres + password: secret + database: appdb + ssl_mode: disable + +dev: + driver: oracle + host: localhost + port: 51521 + user: appuser + password: secret + database: XEPDB1 + +ignore: + tables: [] + columns: + - created_at + - CREATED_AT + +performance: + hash_batch_size: 1000 + max_parallel_tables: 4 + +output: + dir: ./diff-output diff --git a/samples/18-cross-engine-row-diff/docker-compose.yml b/samples/18-cross-engine-row-diff/docker-compose.yml new file mode 100644 index 0000000..7bb7aaa --- /dev/null +++ b/samples/18-cross-engine-row-diff/docker-compose.yml @@ -0,0 +1,31 @@ +services: + postgres: + image: postgres:16-alpine + container_name: sample18-postgres + environment: + POSTGRES_DB: appdb + POSTGRES_USER: postgres + POSTGRES_PASSWORD: secret + ports: + - "55432:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d appdb"] + interval: 5s + timeout: 5s + retries: 5 + + oracle: + image: gvenzl/oracle-xe:21-slim-faststart + container_name: sample18-oracle + environment: + ORACLE_PASSWORD: secret + APP_USER: appuser + APP_USER_PASSWORD: secret + ports: + - "51521:1521" + healthcheck: + test: ["CMD-SHELL", "/opt/oracle/healthcheck.sh"] + interval: 10s + timeout: 10s + retries: 20 + start_period: 45s diff --git a/samples/18-cross-engine-row-diff/init-scripts/01-postgres-init.sql b/samples/18-cross-engine-row-diff/init-scripts/01-postgres-init.sql new file mode 100644 index 0000000..5594884 --- /dev/null +++ b/samples/18-cross-engine-row-diff/init-scripts/01-postgres-init.sql @@ -0,0 +1,39 @@ +-- PostgreSQL initialization (Source / Production) +-- Default PostgreSQL table and column identifiers are folded to lowercase. + +DROP TABLE IF EXISTS um_user_attribute; +DROP TABLE IF EXISTS um_user; + +CREATE TABLE um_user ( + id SERIAL PRIMARY KEY, + username VARCHAR(100) NOT NULL, + email VARCHAR(255) NOT NULL, + status VARCHAR(20) NOT NULL DEFAULT 'ACTIVE', + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE um_user_attribute ( + id SERIAL PRIMARY KEY, + user_id INT NOT NULL REFERENCES um_user(id) ON DELETE CASCADE, + attr_name VARCHAR(100) NOT NULL, + attr_value VARCHAR(255) NOT NULL +); + +-- Seed production users +INSERT INTO um_user (id, username, email, status) VALUES +(1, 'alice', 'alice@corp.internal', 'ACTIVE'), +(2, 'bob', 'bob@corp.internal', 'ACTIVE'), +(3, 'carol', 'carol@corp.internal', 'INACTIVE'), +(4, 'dan', 'dan@corp.internal', 'ACTIVE'), +(5, 'eve', 'eve@corp.internal', 'ACTIVE'), +(6, 'frank', 'frank@corp.internal', 'ACTIVE'); + +-- Seed user attributes +INSERT INTO um_user_attribute (id, user_id, attr_name, attr_value) VALUES +(101, 1, 'department', 'Engineering'), +(102, 1, 'role', 'Lead Developer'), +(103, 2, 'department', 'DevOps'), +(104, 3, 'department', 'Finance'), +(105, 4, 'department', 'Security'), +(106, 5, 'department', 'HR'), +(107, 6, 'department', 'Sales'); diff --git a/samples/18-cross-engine-row-diff/init-scripts/02-oracle-init.sql b/samples/18-cross-engine-row-diff/init-scripts/02-oracle-init.sql new file mode 100644 index 0000000..02cf8b5 --- /dev/null +++ b/samples/18-cross-engine-row-diff/init-scripts/02-oracle-init.sql @@ -0,0 +1,55 @@ +-- Oracle initialization (Target / Development) +-- Default Oracle table and column identifiers are folded to UPPERCASE. + +BEGIN + EXECUTE IMMEDIATE 'DROP TABLE UM_USER_ATTRIBUTE CASCADE CONSTRAINTS'; +EXCEPTION + WHEN OTHERS THEN IF SQLCODE != -942 THEN RAISE; END IF; +END; +/ + +BEGIN + EXECUTE IMMEDIATE 'DROP TABLE UM_USER CASCADE CONSTRAINTS'; +EXCEPTION + WHEN OTHERS THEN IF SQLCODE != -942 THEN RAISE; END IF; +END; +/ + +CREATE TABLE UM_USER ( + ID NUMBER(10) PRIMARY KEY, + USERNAME VARCHAR2(100) NOT NULL, + EMAIL VARCHAR2(255) NOT NULL, + STATUS VARCHAR2(20) DEFAULT 'ACTIVE' NOT NULL, + CREATED_AT TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL +); + +CREATE TABLE UM_USER_ATTRIBUTE ( + ID NUMBER(10) PRIMARY KEY, + USER_ID NUMBER(10) NOT NULL REFERENCES UM_USER(ID), + ATTR_NAME VARCHAR2(100) NOT NULL, + ATTR_VALUE VARCHAR2(255) NOT NULL +); + +-- Seed development users with intentional drift: +-- Row 1, 2, 3: identical +-- Row 4: updated email ('dan+updated@corp.internal' vs 'dan@corp.internal') +-- Row 5, 6: missing (deleted or omitted during partial sync) +-- Row 7: new user added in target +INSERT INTO UM_USER (ID, USERNAME, EMAIL, STATUS) VALUES (1, 'alice', 'alice@corp.internal', 'ACTIVE'); +INSERT INTO UM_USER (ID, USERNAME, EMAIL, STATUS) VALUES (2, 'bob', 'bob@corp.internal', 'ACTIVE'); +INSERT INTO UM_USER (ID, USERNAME, EMAIL, STATUS) VALUES (3, 'carol', 'carol@corp.internal', 'INACTIVE'); +INSERT INTO UM_USER (ID, USERNAME, EMAIL, STATUS) VALUES (4, 'dan', 'dan+updated@corp.internal', 'ACTIVE'); +INSERT INTO UM_USER (ID, USERNAME, EMAIL, STATUS) VALUES (7, 'grace', 'grace@corp.internal', 'ACTIVE'); + +-- Seed user attributes with drift: +-- Attr 104: updated attr_value ('Accounting' vs 'Finance') +-- Attr 106, 107: missing +-- Attr 108: new attribute for grace +INSERT INTO UM_USER_ATTRIBUTE (ID, USER_ID, ATTR_NAME, ATTR_VALUE) VALUES (101, 1, 'department', 'Engineering'); +INSERT INTO UM_USER_ATTRIBUTE (ID, USER_ID, ATTR_NAME, ATTR_VALUE) VALUES (102, 1, 'role', 'Lead Developer'); +INSERT INTO UM_USER_ATTRIBUTE (ID, USER_ID, ATTR_NAME, ATTR_VALUE) VALUES (103, 2, 'department', 'DevOps'); +INSERT INTO UM_USER_ATTRIBUTE (ID, USER_ID, ATTR_NAME, ATTR_VALUE) VALUES (104, 3, 'department', 'Accounting'); +INSERT INTO UM_USER_ATTRIBUTE (ID, USER_ID, ATTR_NAME, ATTR_VALUE) VALUES (105, 4, 'department', 'Security'); +INSERT INTO UM_USER_ATTRIBUTE (ID, USER_ID, ATTR_NAME, ATTR_VALUE) VALUES (108, 7, 'department', 'Legal'); + +COMMIT; diff --git a/tests/resolve/fetch_test.go b/tests/resolve/fetch_test.go index f56450d..7d6de6a 100644 --- a/tests/resolve/fetch_test.go +++ b/tests/resolve/fetch_test.go @@ -564,3 +564,114 @@ func TestFetchConflictRows_NoPrimaryKey(t *testing.T) { t.Error("expected error for table with no primary key") } } + +// ============================================================================ +// Cross-engine case-insensitive table and column matching tests +// ============================================================================ + +func TestFetchConflictRowsCrossEngine_CaseInsensitiveTable(t *testing.T) { + ctx := context.Background() + prodDB := openFetchMemDB(t) + devDB := openFetchMemDB(t) + + mustExec(t, prodDB, `CREATE TABLE customers (id INTEGER PRIMARY KEY, name TEXT, email TEXT)`) + mustExec(t, prodDB, `INSERT INTO customers VALUES (1, 'Alice', 'alice@prod.com')`) + + mustExec(t, devDB, `CREATE TABLE CUSTOMERS (ID INTEGER PRIMARY KEY, NAME TEXT, EMAIL TEXT)`) + mustExec(t, devDB, `INSERT INTO CUSTOMERS VALUES (1, 'Alice', 'alice@dev.com')`) + + prodSch := &schema.Schema{ + Tables: map[string]schema.Table{ + "customers": { + Name: "customers", + Columns: map[string]schema.Column{ + "id": {Name: "id"}, + "name": {Name: "name"}, + "email": {Name: "email"}, + }, + PrimaryKey: []string{"id"}, + }, + }, + } + devSch := &schema.Schema{ + Tables: map[string]schema.Table{ + "CUSTOMERS": { + Name: "CUSTOMERS", + Columns: map[string]schema.Column{ + "ID": {Name: "ID"}, + "NAME": {Name: "NAME"}, + "EMAIL": {Name: "EMAIL"}, + }, + PrimaryKey: []string{"ID"}, + }, + }, + } + + conflict := content.Conflict{Table: "customers", Key: "1"} + prod, dev, err := resolve.FetchConflictRowsCrossEngine(ctx, prodDB, devDB, "sqlite", "sqlite", prodSch, devSch, conflict) + if err != nil { + t.Fatalf("FetchConflictRowsCrossEngine: %v", err) + } + if prod == nil || dev == nil { + t.Fatal("expected both prod and dev rows across case-differing table names") + } + + diffs := resolve.CompareRows(prod, dev) + if len(diffs) != 3 { + t.Fatalf("expected 3 column diffs, got %d", len(diffs)) + } + + for _, d := range diffs { + if d.Column == "email" { + if !d.Differs { + t.Errorf("expected email column to differ: prod=%v, dev=%v", d.ProdVal, d.DevVal) + } + } else { + if d.Differs { + t.Errorf("expected column %q to match across case-differing names, but marked differing", d.Column) + } + } + } +} + +func TestCompareRows_CrossEngineColumnCasing(t *testing.T) { + prod := &resolve.RowData{ + Columns: []string{"id", "name", "email", "status"}, + Values: map[string]any{ + "id": int64(42), + "name": "Dan", + "email": "dan@example.com", + "status": "ACTIVE", + }, + } + dev := &resolve.RowData{ + Columns: []string{"ID", "NAME", "EMAIL", "STATUS"}, + Values: map[string]any{ + "ID": int64(42), + "NAME": "Dan", + "EMAIL": "dan+new@example.com", + "STATUS": "ACTIVE", + }, + } + + diffs := resolve.CompareRows(prod, dev) + if len(diffs) != 4 { + t.Fatalf("expected 4 matched columns, got %d", len(diffs)) + } + + differingCount := 0 + var diffCol string + for _, d := range diffs { + if d.Differs { + differingCount++ + diffCol = d.Column + } + } + + if differingCount != 1 { + t.Fatalf("expected exactly 1 differing column, got %d", differingCount) + } + if diffCol != "email" { + t.Errorf("expected differing column to be 'email', got %q", diffCol) + } +} diff --git a/tests/resolve/rowdiff_test.go b/tests/resolve/rowdiff_test.go new file mode 100644 index 0000000..76aa14a --- /dev/null +++ b/tests/resolve/rowdiff_test.go @@ -0,0 +1,174 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/iamvirul/deepdiff-db/internal/content" + "github.com/iamvirul/deepdiff-db/internal/content/resolve" + "github.com/iamvirul/deepdiff-db/internal/schema" +) + +func TestFetchRowDiffReport_UpdatedAndRemovedAndAdded(t *testing.T) { + ctx := context.Background() + prodDB := openFetchMemDB(t) + devDB := openFetchMemDB(t) + + mustExec(t, prodDB, `CREATE TABLE items (id INTEGER PRIMARY KEY, title TEXT, price REAL)`) + mustExec(t, prodDB, `INSERT INTO items VALUES (1, 'Book', 10.5), (2, 'Pen', 1.5), (3, 'Notebook', 4.0)`) + + mustExec(t, devDB, `CREATE TABLE items (id INTEGER PRIMARY KEY, title TEXT, price REAL)`) + mustExec(t, devDB, `INSERT INTO items VALUES (1, 'Book', 12.0), (2, 'Pen', 1.5), (4, 'Eraser', 0.5)`) + + prodSch := &schema.Schema{ + Tables: map[string]schema.Table{ + "items": { + Name: "items", + Columns: map[string]schema.Column{ + "id": {Name: "id"}, + "title": {Name: "title"}, + "price": {Name: "price"}, + }, + PrimaryKey: []string{"id"}, + }, + }, + } + devSch := &schema.Schema{ + Tables: map[string]schema.Table{ + "items": { + Name: "items", + Columns: map[string]schema.Column{ + "id": {Name: "id"}, + "title": {Name: "title"}, + "price": {Name: "price"}, + }, + PrimaryKey: []string{"id"}, + }, + }, + } + + dataDiff := content.DataDiff{ + Tables: []content.TableDataDiff{ + { + Table: "items", + Updated: []string{"1"}, + Removed: []string{"3"}, + Added: []string{"4"}, + }, + }, + } + conflicts := content.Conflicts{ + Conflicts: []content.Conflict{ + {Table: "items", Key: "1"}, + }, + } + + // 1. Test status "all" + opts := resolve.RowDiffOptions{StatusFilter: "all"} + report, err := resolve.FetchRowDiffReport(ctx, prodDB, devDB, "sqlite", "sqlite", prodSch, devSch, dataDiff, conflicts, opts) + if err != nil { + t.Fatalf("FetchRowDiffReport: %v", err) + } + + if len(report.Tables) != 1 { + t.Fatalf("expected 1 table, got %d", len(report.Tables)) + } + if len(report.Tables[0].Rows) != 3 { + t.Fatalf("expected 3 row diffs, got %d", len(report.Tables[0].Rows)) + } + + // Check updated row (key 1) + row1 := report.Tables[0].Rows[0] + if row1.Key != "1" || row1.Status != resolve.StatusUpdated { + t.Errorf("expected row 1 to be updated, got %+v", row1) + } + if len(row1.DiffColumns) != 1 || row1.DiffColumns[0] != "price" { + t.Errorf("expected price to be differing column for row 1, got %v", row1.DiffColumns) + } + + // Check removed row (key 3) + row3 := report.Tables[0].Rows[1] + if row3.Key != "3" || row3.Status != resolve.StatusRemoved { + t.Errorf("expected row 3 to be removed, got %+v", row3) + } + + // Check added row (key 4) + row4 := report.Tables[0].Rows[2] + if row4.Key != "4" || row4.Status != resolve.StatusAdded { + t.Errorf("expected row 4 to be added, got %+v", row4) + } + + // 2. Test status "updated" only + optsUpdated := resolve.RowDiffOptions{StatusFilter: "updated"} + reportUpdated, err := resolve.FetchRowDiffReport(ctx, prodDB, devDB, "sqlite", "sqlite", prodSch, devSch, dataDiff, conflicts, optsUpdated) + if err != nil { + t.Fatalf("FetchRowDiffReport (updated): %v", err) + } + if len(reportUpdated.Tables[0].Rows) != 1 || reportUpdated.Tables[0].Rows[0].Key != "1" { + t.Errorf("expected only key 1 for updated filter, got %d rows", len(reportUpdated.Tables[0].Rows)) + } + + // 3. Test key filter + optsKey := resolve.RowDiffOptions{StatusFilter: "all", KeyFilter: "3"} + reportKey, err := resolve.FetchRowDiffReport(ctx, prodDB, devDB, "sqlite", "sqlite", prodSch, devSch, dataDiff, conflicts, optsKey) + if err != nil { + t.Fatalf("FetchRowDiffReport (key filter): %v", err) + } + if len(reportKey.Tables[0].Rows) != 1 || reportKey.Tables[0].Rows[0].Key != "3" { + t.Errorf("expected only key 3 for key filter, got %d rows", len(reportKey.Tables[0].Rows)) + } +} + +func TestWriteRowDiffReport_AndFormat(t *testing.T) { + report := resolve.RowDiffReport{ + Tables: []resolve.TableRowDiff{ + { + Table: "customers", + Rows: []resolve.RowDiff{ + { + Table: "customers", + Key: "4", + Status: resolve.StatusUpdated, + Columns: []resolve.ColumnDiff{ + {Column: "id", ProdVal: 4, DevVal: 4, Differs: false}, + {Column: "email", ProdVal: "dan@example.com", DevVal: "dan+new@example.com", Differs: true}, + }, + DiffColumns: []string{"email"}, + }, + }, + }, + }, + } + + tmpDir, err := os.MkdirTemp("", "rowdiff_test_*") + if err != nil { + t.Fatalf("MkdirTemp: %v", err) + } + defer os.RemoveAll(tmpDir) + + if err := resolve.WriteRowDiffReport(report, tmpDir); err != nil { + t.Fatalf("WriteRowDiffReport: %v", err) + } + + jsonPath := filepath.Join(tmpDir, "row_diff.json") + if _, err := os.Stat(jsonPath); err != nil { + t.Errorf("row_diff.json was not created: %v", err) + } + + txtPath := filepath.Join(tmpDir, "row_diff.txt") + txtData, err := os.ReadFile(txtPath) + if err != nil { + t.Fatalf("read row_diff.txt: %v", err) + } + + txt := string(txtData) + if !strings.Contains(txt, "Table: customers | Key: 4 (updated)") { + t.Errorf("row_diff.txt missing table/key header: %s", txt) + } + if !strings.Contains(txt, "dan+new@example.com") { + t.Errorf("row_diff.txt missing updated value: %s", txt) + } +} diff --git a/website/docs/commands/diff.md b/website/docs/commands/diff.md index 79aa382..8745e8f 100644 --- a/website/docs/commands/diff.md +++ b/website/docs/commands/diff.md @@ -28,6 +28,7 @@ Tables with schema drift are skipped for data diffing and flagged in the report. |---|---| | `--config` | Path to the configuration file (default: `deepdiffdb.config.yaml`) | | `--html` | Generate an interactive HTML report (`report.html`) in addition to JSON/text outputs | +| `--row-diff` | Inspect column-level differences for changed rows directly in terminal and write `row_diff.json` and `row_diff.txt` | | `--batch-size N` | Rows per keyset-paginated query (overrides `performance.hash_batch_size`) | | `--parallel N` | Max tables hashed concurrently (overrides `performance.max_parallel_tables`) | | `--verbose` | Enable debug-level logging | @@ -45,6 +46,8 @@ All files are written to `output.dir` (default: `./diff-output`). | `schema_diff.txt` | Human-readable schema diff | | `content_diff.json` | Row-level differences: added, removed, updated rows per table | | `conflicts.json` | Rows that exist in both databases with conflicting values | +| `row_diff.json` | Column-level differences for each changed row (with `--row-diff` or `--html`) | +| `row_diff.txt` | Human-readable side-by-side column diff report (with `--row-diff`) | | `summary.txt` | High-level statistics: tables scanned, rows added/updated/removed, conflict count | | `report.html` | Interactive HTML report (only when `--html` is used) | @@ -54,7 +57,10 @@ All files are written to `output.dir` (default: `./diff-output`). # Basic diff deepdiffdb diff --config deepdiffdb.config.yaml -# With HTML report and streaming flags for large tables +# Full diff with column-level row inspection and interactive HTML report +deepdiffdb diff --config deepdiffdb.config.yaml --row-diff --html + +# With streaming flags for large tables deepdiffdb diff --config deepdiffdb.config.yaml --html --batch-size 5000 --parallel 4 ``` diff --git a/website/docs/commands/index.md b/website/docs/commands/index.md index 58c5930..798e682 100644 --- a/website/docs/commands/index.md +++ b/website/docs/commands/index.md @@ -14,6 +14,7 @@ DeepDiff DB exposes a set of focused commands. Each command does one thing; chai | `schema-diff` | Detect schema drift between production and development | | `schema-migrate` | Generate a standalone schema migration SQL script | | `diff` | Full diff: schema introspection + row-level data comparison | +| `row-diff` | Inspect column-level differences for modified/added/removed rows | | `gen-pack` | Generate a complete SQL migration pack (schema + data) | | `apply` | Apply a migration pack to the production database | | `resolve-conflicts` | Interactively (or automatically) resolve data conflicts | diff --git a/website/docs/commands/row-diff.md b/website/docs/commands/row-diff.md new file mode 100644 index 0000000..f61a919 --- /dev/null +++ b/website/docs/commands/row-diff.md @@ -0,0 +1,116 @@ +--- +sidebar_position: 6 +--- + +# row-diff + +Inspects column-level differences for modified, added, or removed rows between production and development databases. Displays a side-by-side terminal comparison with differing columns highlighted, and writes machine-readable reports. + +![row-diff Terminal Inspection](/img/row-diff-terminal.svg) + +## Usage + +```bash +# Inspect all updated rows across all tables +deepdiffdb row-diff --config deepdiffdb.config.yaml + +# Inspect a specific table and primary key directly +deepdiffdb row-diff --table customers --key 4 + +# Inspect removed rows in JSON format +deepdiffdb row-diff --status removed --format json + +# Run as part of a full diff +deepdiffdb diff --config deepdiffdb.config.yaml --row-diff +``` + +## What It Does + +1. **Hashes & Discovers Differences**: Hashes tables across databases using keyset pagination and identifies modified, removed, or added primary keys. +2. **Cross-Engine Identifier Resolution**: Pairs tables and columns case-insensitively across different database engines (for example, PostgreSQL `customers.email` $\leftrightarrow$ Oracle `CUSTOMERS.EMAIL`). +3. **Selective Row Fetching**: Queries the actual raw column values directly using parameterized queries and dialect-specific identifier quoting. When `--table` and `--key` are supplied, it executes a targeted single-row lookup for instant results. +4. **Side-by-Side Column Comparison**: Compares column values across both engines, flags differences with `*`, and formats NULL and type variations cleanly. +5. **Generates Reports**: Outputs results to stdout and saves `row_diff.json` and `row_diff.txt` in the configured output directory. + +## Flags + +| Flag | Description | Default | +|---|---|---| +| `--config` | Path to configuration file | `deepdiffdb.config.yaml` | +| `--table` | Filter inspection to a specific table name | `""` (all tables) | +| `--key` | Filter inspection to a specific primary key value | `""` (all keys) | +| `--status` | Row status to inspect: `updated`, `added`, `removed`, or `all` | `updated` | +| `--limit` | Maximum differing rows to inspect per table (`0` = unlimited) | `0` | +| `--format` | Output format: `table` (terminal ASCII) or `json` | `table` | +| `--output` | Output directory to save reports | Config output directory | +| `--batch-size N` | Rows per keyset-paginated query | Config default | +| `--parallel N` | Max tables hashed concurrently | Config default | +| `--verbose` | Enable debug logging | `false` | + +## Terminal Output Example + +```text +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +Table: customers | Key: 4 (updated) +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + Column | Production | Development + ---------------------+--------------------------+-------------------------- + country | FR | FR + email | dan@example.com | dan+new@example.com * + id | 4 | 4 + name | Dan | Dan + + * indicates columns that differ +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Total differing rows inspected: 1 +Row diff report written to diff-output/row_diff.json and diff-output/row_diff.txt +``` + +## JSON Output Example + +Passing `--format json` outputs structured data ideal for automated CI/CD assertion steps: + +```json +{ + "total_rows": 1, + "tables": [ + { + "table": "customers", + "rows": [ + { + "table": "customers", + "key": "4", + "status": "updated", + "columns": [ + { + "column": "country", + "prod_val": "FR", + "dev_val": "FR", + "differs": false + }, + { + "column": "email", + "prod_val": "dan@example.com", + "dev_val": "dan+new@example.com", + "differs": true + } + ], + "differing_columns": ["email"] + } + ] + } + ] +} +``` + +## Cross-Engine Migrations (PostgreSQL ↔ Oracle / MySQL) + +Different database engines enforce different default identifier casing: +- **PostgreSQL**: folds unquoted identifiers to **lowercase** (`um_user_attribute`) +- **Oracle**: folds unquoted identifiers to **UPPERCASE** (`UM_USER_ATTRIBUTE`) + +`deepdiffdb row-diff` seamlessly bridges this gap: +- Tables are discovered using canonical name folding. +- Each database query preserves the correct engine-specific quoting (`"customers"` in PostgreSQL, `"CUSTOMERS"` in Oracle, `` `customers` `` in MySQL, `[customers]` in MSSQL). +- Column values are paired by canonical name, allowing you to instantly identify which exact columns drifted during an ETL or replication pipeline. diff --git a/website/docs/features/cross-engine-diff.md b/website/docs/features/cross-engine-diff.md new file mode 100644 index 0000000..e86192d --- /dev/null +++ b/website/docs/features/cross-engine-diff.md @@ -0,0 +1,132 @@ +--- +sidebar_position: 1 +--- + +# Cross-Engine Data Diff & Verification + +DeepDiff DB supports **cross-engine data diffing and migration verification**, allowing you to compare schemas and row data between completely different database engines (such as **PostgreSQL $\leftrightarrow$ Oracle**, **MySQL $\leftrightarrow$ PostgreSQL**, or **SQLite $\leftrightarrow$ MSSQL**). + +![Live Migration Pipeline Mismatch](/img/cross-engine-pipeline-mismatch.jpg) + +--- + +## The Cross-Engine Challenge: Identifier Case Folding + +When migrating data between database engines (for example, in an enterprise ETL pipeline migrating from PostgreSQL to Oracle), automated row-count and checksum verification frequently fails or reports false positives: + +```text +TOTAL RECORD COUNTS (entire database) +Source (PG) : 12,923 +Oracle : 19,132 +Delta (Oracle - Source): +6,209 + +MISMATCHED TABLES: +TABLE PG ORACLE DELTA +------------------------------------------------ +UM_USER_ATTRIBUTE 12281 18403 +6122 +UM_USER 640 727 +87 +``` + +### Why Naive Comparisons Fail + +1. **Identifier Folding Discrepancies**: + - **PostgreSQL** folds unquoted identifiers to **lowercase** (`um_user_attribute`, `um_user`). + - **Oracle** folds unquoted identifiers to **UPPERCASE** (`UM_USER_ATTRIBUTE`, `UM_USER`). + - **MySQL** on Linux defaults to case-sensitive table names, while on Windows/macOS it is case-insensitive. + - Traditional tools comparing by exact table name (`"um_user" == "UM_USER"`) find 0 matching tables and report `"No differences found"`, completely missing thousands of drifted rows! + +2. **Column Quoting and Dialect Differences**: + - Querying a PostgreSQL table requires double quotes for mixed-case or lowercase identifiers: `SELECT "id", "email" FROM "customers"`. + - Querying Oracle requires uppercase or quoted identifiers: `SELECT "ID", "EMAIL" FROM "CUSTOMERS"`. + - SQLite and MySQL use backticks or brackets. + +--- + +## How DeepDiff DB Solves This + +DeepDiff DB introduces **engine-aware canonical identifier folding** and **dialect-specific query drivers**: + +```text +┌─────────────────────────┐ ┌─────────────────────────┐ +│ PostgreSQL (Source) │ │ Oracle (Target) │ +│ "um_user" (lowercase) │ │ "UM_USER" (UPPERCASE) │ +└────────────┬────────────┘ └────────────┬────────────┘ + │ │ + ▼ ▼ + Canonical: "um_user" ◄──────────────────► Canonical: "um_user" + (Matched & Paired) + │ + ▼ + ┌─────────────────────────────────────────┐ + │ Keyset-Paginated Row Hashing │ + │ PostgreSQL driver: SELECT "id"... │ + │ Oracle driver: SELECT "ID"... │ + └─────────────────────┬───────────────────┘ + │ + ▼ + ┌─────────────────────────────────────────┐ + │ Side-by-Side Column Diffing │ + │ email: dan@example.com (*) │ + │ email: dan+new@example.com │ + └─────────────────────────────────────────┘ +``` + +1. **Case-Insensitive Table Discovery**: DeepDiff matches tables across engines using case-folded canonical names (`schema.CanonicalIdent`). +2. **Preserved Dialect Execution**: Each database connection uses its own driver and dialect rules, ensuring queries against PostgreSQL use correct lowercase identifiers while queries against Oracle use uppercase identifiers. +3. **Column-Level Value Alignment**: Columns are aligned canonically, so `email` in PostgreSQL is compared directly with `EMAIL` in Oracle regardless of catalog casing. + +--- + +## Inspecting Changed Rows (`row-diff`) + +Once table differences or conflicts are detected, use the `row-diff` command (or pass `--row-diff` to `diff`) to inspect exactly what changed at the column level: + +![row-diff Terminal Output](/img/row-diff-terminal.svg) + +```bash +# Compare PostgreSQL against Oracle with column-level row inspection +deepdiffdb diff --config deepdiffdb.config.yaml --row-diff + +# Or inspect a specific table and primary key directly +deepdiffdb row-diff --table customers --key 4 +``` + +Terminal output highlights differing columns with an asterisk (`*`): + +```text +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +Table: customers | Key: 4 (updated) +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + Column | Production (PostgreSQL) | Development (Oracle) + ---------------------+--------------------------+-------------------------- + country | FR | FR + email | dan@example.com | dan+new@example.com * + id | 4 | 4 + name | Dan | Dan + + * indicates columns that differ +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +``` + +--- + +## Interactive HTML Report + +**[→ View Live Interactive Report](pathname:///samples/report.html)** + +Generating an HTML report (`deepdiffdb diff --html`) provides an interactive side-by-side inspection view: + +- **Data Changes Tab**: Opens automatically when data drift exists. Clicking any table displays both key summaries and the complete column comparison table for every modified row. +- **Row Data Tab**: Provides a dedicated card view of every changed row across all tables with real-time table filtering. +- **Conflicts Tab**: Displays conflicting keys with source and target hash values, resolution strategy, and side-by-side column values. + +```bash +deepdiffdb diff --config deepdiffdb.config.yaml --html +open diff-output/report.html +``` + +--- + +## Ready-to-Run Sample + +Check out **[Sample 18: Cross-Engine Row Diff & Verification](https://github.com/iamvirul/deepdiff-db/tree/main/samples/18-cross-engine-row-diff)** in the repository, which sets up PostgreSQL and Oracle containers with pre-seeded data mismatches to test cross-engine verification. diff --git a/website/docs/features/html-reports.md b/website/docs/features/html-reports.md index de6ea8d..270a1ed 100644 --- a/website/docs/features/html-reports.md +++ b/website/docs/features/html-reports.md @@ -10,7 +10,11 @@ DeepDiff DB can generate a self-contained, interactive HTML report alongside the **[→ View Sample Report](pathname:///samples/report.html)** -The sample report is generated from two real MySQL databases with intentional schema drift: added/removed/modified views, routines, and triggers, plus table-level column and index changes with data conflicts. +The sample report is generated directly from a cross-engine migration diff (**PostgreSQL $\leftrightarrow$ Oracle XE**). It demonstrates: +- The **Data Changes** tab with automatic expansion and embedded column-level comparisons (`*` highlighting differing fields like `email`) +- The dedicated **Row Data** tab with per-table filtering across all differing rows +- The **Conflicts** tab with SHA-256 hash comparison and column difference tables +- Full print and PDF export styling ## Generating a Report @@ -56,18 +60,25 @@ Lists every schema object that changed between prod and dev. - Added, removed, and modified sequences - For modified sequences: shows which numeric properties differ (increment, min/max value, cache, cycle) -### Data Diff Tab +### Data Changes Tab - Summary counts per table: rows added, rows removed, rows updated -- Expandable per-table section showing the primary key of each changed row -- Filtering controls to show only added / removed / updated rows +- Click any table to expand and view both primary key listings and full **side-by-side column-level row differences** for every modified, added, or removed row +- Table filter dropdown for quick navigation + +### Row Data Tab + +- Dedicated card view for every differing row across the entire database +- Displays side-by-side columns: **Column name**, **Production value**, and **Development value** +- Asterisks (`*`) and highlighted background colors immediately draw attention to columns that differ +- Filter by table name using the real-time filter dropdown ### Conflicts Tab -- Lists all conflicts grouped by table -- Each conflict shows the full row comparison with changed columns highlighted -- Resolution status badge (auto-resolved, pending, resolved) when `resolutions.json` is present -- Per-table resolution strategy shown when configured +- Lists all conflicts grouped by table with SHA-256 hash comparison (`prod_hash → dev_hash`) +- Embedded column comparison table showing exactly which fields differ +- Resolution status badge (`Keep Source`, `Use Target`, or `Pending`) when resolutions are applied +- Per-table resolution strategy breakdown table ### SQL Migration Preview Tab diff --git a/website/docs/samples/index.md b/website/docs/samples/index.md index b7590db..6a336c8 100644 --- a/website/docs/samples/index.md +++ b/website/docs/samples/index.md @@ -27,6 +27,7 @@ The `samples/` directory in the repository contains 17 self-contained example pr | 15 | [mssql-support](https://github.com/iamvirul/deepdiff-db/tree/main/samples/15-mssql-support) | Full MSSQL workflow: schema diff, data diff, gen-pack, apply | MSSQL | Yes | | 16 | [oracle-support](https://github.com/iamvirul/deepdiff-db/tree/main/samples/16-oracle-support) | Full Oracle workflow: schema drift, data diff, gen-pack, apply | Oracle | Yes | | 17 | [git-like-versioning](https://github.com/iamvirul/deepdiff-db/tree/main/samples/17-git-like-versioning) | Git-like versioning: commit snapshots, branches, checkout, ASCII tree graph, compare versions, generate rollback SQL | MySQL | Yes | +| 18 | [cross-engine-row-diff](https://github.com/iamvirul/deepdiff-db/tree/main/samples/18-cross-engine-row-diff) | Cross-engine migration verification (PostgreSQL ↔ Oracle), case-folding resolution, and column-level `row-diff` | PostgreSQL / Oracle | Yes | ## Running a SQLite Sample @@ -98,3 +99,18 @@ deepdiffdb version diff deepdiffdb version rollback --out diff-output/rollback_v3.sql docker-compose down -v ``` + +## Sample 18: Cross-Engine Row Diff & Migration Verification + +Sample 18 demonstrates real-world cross-engine migration verification between **PostgreSQL 16** (source) and **Oracle XE 21c** (target). It seeds tables with differing default identifier cases (`um_user` in PG vs `UM_USER` in Oracle, and `um_user_attribute` in PG vs `UM_USER_ATTRIBUTE` in Oracle) with intentional column drift: + +```bash +cd samples/18-cross-engine-row-diff +make up # start PostgreSQL and Oracle containers +make wait-healthy # wait for both containers to be ready +make seed # populate schemas and test datasets +make diff # run cross-engine diff +make row-diff # inspect column-level row differences in terminal +make html # generate and open interactive HTML report +make down # stop containers +``` diff --git a/website/sidebars.js b/website/sidebars.js index 9e4c0f2..199929c 100644 --- a/website/sidebars.js +++ b/website/sidebars.js @@ -8,7 +8,7 @@ const sidebars = { }, { type: 'category', label: 'Commands', - items: ['commands/index', 'commands/check', 'commands/schema-diff', 'commands/schema-migrate', 'commands/diff', 'commands/gen-pack', 'commands/apply', 'commands/resolve-conflicts', 'commands/version'], + items: ['commands/index', 'commands/check', 'commands/schema-diff', 'commands/schema-migrate', 'commands/diff', 'commands/row-diff', 'commands/gen-pack', 'commands/apply', 'commands/resolve-conflicts', 'commands/version'], }, { type: 'category', label: 'Database Support', @@ -16,7 +16,7 @@ const sidebars = { }, { type: 'category', label: 'Features', - items: ['features/git-versioning', 'features/streaming', 'features/html-reports', 'features/conflict-resolution', 'features/checkpoint-resume', 'features/logging'], + items: ['features/cross-engine-diff', 'features/git-versioning', 'features/streaming', 'features/html-reports', 'features/conflict-resolution', 'features/checkpoint-resume', 'features/logging'], }, { type: 'category', label: 'Deployment', diff --git a/website/static/img/cross-engine-pipeline-mismatch.jpg b/website/static/img/cross-engine-pipeline-mismatch.jpg new file mode 100644 index 0000000..6847139 Binary files /dev/null and b/website/static/img/cross-engine-pipeline-mismatch.jpg differ diff --git a/website/static/img/row-diff-terminal.svg b/website/static/img/row-diff-terminal.svg new file mode 100644 index 0000000..134879a --- /dev/null +++ b/website/static/img/row-diff-terminal.svg @@ -0,0 +1,97 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + deepdiffdb row-diff — Cross-Engine Inspection + + + + + + + $ + deepdiffdb row-diff --table customers --key 4 + + + ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + Table: + customers + | + Key: + 4 + + updated + ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + + + Column + | + Production (PostgreSQL) + | + Development (Oracle) + + + --------------------+---------------------------+--------------------------- + + + country + | + FR + | + FR + + + + email + | + dan@example.com + | + dan+new@example.com + * + + + id + | + 4 + | + 4 + + + name + | + Dan + | + Dan + + + * + indicates columns that differ across engines + ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + + + ✓ + 1 row difference inspected (PostgreSQL "customers" ↔ Oracle "CUSTOMERS") + + diff --git a/website/static/samples/report.html b/website/static/samples/report.html index f56d9d3..b7e2a9b 100644 --- a/website/static/samples/report.html +++ b/website/static/samples/report.html @@ -589,6 +589,15 @@ overflow: hidden; } + .conflict-item-card { + border-bottom: 1px solid var(--border); + background: var(--bg); + } + + .conflict-item-card:last-child { + border-bottom: none; + } + .conflict-item { display: grid; grid-template-columns: 140px 1fr 200px 100px; @@ -597,12 +606,78 @@ padding: 12px 16px; background: var(--bg); font-size: 13px; + cursor: pointer; } .conflict-item:hover { background: var(--bg-secondary); } + .conflict-item-body { + padding: 0 16px 14px 16px; + background: var(--bg-secondary); + border-top: 1px dashed var(--border); + } + + .row-data-table { + width: 100%; + border-collapse: collapse; + font-size: 12px; + margin-top: 10px; + background: var(--bg); + border: 1px solid var(--border); + border-radius: 4px; + overflow: hidden; + } + + .row-data-table th, .row-data-table td { + padding: 7px 12px; + text-align: left; + border-bottom: 1px solid var(--border); + } + + .row-data-table th { + background: var(--bg-tertiary); + font-weight: 600; + color: var(--text-secondary); + } + + .row-data-table tr:last-child td { + border-bottom: none; + } + + .diff-row-changed { + background: rgba(234, 88, 12, 0.08); + } + + .diff-star { + color: #ea580c; + font-weight: bold; + } + + .rowdiff-card { + background: var(--bg); + border: 1px solid var(--border); + border-radius: 6px; + margin-bottom: 16px; + overflow: hidden; + } + + .rowdiff-header { + display: flex; + align-items: center; + gap: 16px; + padding: 10px 16px; + background: var(--bg-secondary); + border-bottom: 1px solid var(--border); + font-size: 13px; + font-weight: 500; + } + + .rowdiff-body { + padding: 12px 16px; + } + .conflict-table { font-family: var(--font-mono); font-weight: 500; @@ -774,19 +849,19 @@

Database Diff Report

Generated - 2026-05-03 13:38:34 +0530 + 2026-09-25 08:54:47 +0530
Source - localhost:3316/prod_ecommerce + 127.0.0.1:55432/appdb
Target - localhost:3317/dev_ecommerce + 127.0.0.1:51521/XEPDB1
Version - v1.4.0-dev.20260503T080705Z.59ac4a3 + dev
@@ -797,23 +872,23 @@

Database Diff Report

Schema
-
7
+
1
Tables
-
+20
+
0
Added
-
-7
+
-2
Removed
-
18
+
1
Modified
-
18
+
1
Conflicts
@@ -822,19 +897,24 @@

Database Diff Report

-
+
@@ -842,7 +922,7 @@

Database Diff Report

- feature_flags + CUSTOMERS New Table @@ -855,7 +935,7 @@

Database Diff Report

i
-
Table 'feature_flags' exists in dev but not in prod
+
Table 'CUSTOMERS' exists in dev but not in prod
@@ -868,201 +948,36 @@

Database Diff Report

customers - Modified - -
-
- -
-
~
-
- email - Type changed: varchar(100) -> varchar(255) -
-
- - - - -
-
- -
-
- - - - orders - - Modified - -
-
- - -
-
+
-
- INDEX idx_customer_id - (customer_id) -
-
- -
-
−
-
- INDEX customer_id - (customer_id) -
-
- - - -
-
- -
-
- - - - products - - Modified + Dropped +
-
-
+
-
- discount_percent - Column added: discount_percent decimal(5,2) -
-
- -
-
- - -
-
- - - - Views - 3 change(s) -
-
- -
-
+
-
- v_customer_stats - View 'v_customer_stats' exists in dev but not in prod -
-
- -
-
−
-
- v_customer_summary - View 'v_customer_summary' exists in prod but not in dev -
-
-
-
~
-
- v_active_orders - View definition changed -
+
i
+
Table 'customers' exists in prod but not in dev
-
-
- - - - Routines - 3 change(s) -
-
- -
-
+
-
- FUNCTION: fn_format_price - FUNCTION 'fn_format_price' exists in dev but not in prod -
-
- -
-
~
-
- fn_calculate_total - definition changed; parameters changed -
-
- -
-
~
-
- fn_get_customer_tier - definition changed -
-
- -
-
- -
-
- - - - Triggers - 2 change(s) -
-
- -
-
+
-
- trg_products_updated_at (table: products) - Trigger 'trg_products_updated_at' on table 'products' exists in dev but not in prod -
-
- -
-
~
-
- trg_orders_audit - definition changed -
-
- -
-
-
+
@@ -1077,170 +992,174 @@

Database Diff Report

- - - audit_log - (click to expand) - - — - −7 - — - - - - -
- - -
- Removed Keys: -
89710346
-
- - -
- - - - customers - (click to expand) + (click to view row data) - +1 — - ~5 + −2 + ~1 - +
-
- Added Keys: -
6
-
- - - -
- Modified Keys: -
34512
-
- -
- - - - - - - inventory_log - (click to expand) - - +5 - — - — - - - - -
- -
- Added Keys: -
789106
-
- - - -
- - - - - - - order_items - (click to expand) - - +6 - — - — - - - - -
- -
- Added Keys: -
91113141012
-
- - - -
- - - - - - - orders - (click to expand) - - +5 - — - ~3 - - - - -
- Added Keys: -
106789
+ Removed Keys: +
56
-
Modified Keys: -
453
+
4
-
- - - - - - - products - (click to expand) - - +3 - — - ~10 - - - - -
-
- Added Keys: -
131112
-
- - - -
- Modified Keys: -
13578246910
+
+ Row Data Differences (3 rows): + +
+
+ Key: 4 + updated + Changed: email +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ColumnProductionDevelopment
countryFRFR
email *dan@example.comdan+new@example.com
id44
nameDanDan
+
+
+ +
+
+ Key: 5 + removed + Changed: country, email, id, name +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ColumnProductionDevelopment
country *INNULL
email *eve@example.comNULL
id *5NULL
name *EveNULL
+
+
+ +
+
+ Key: 6 + removed + Changed: country, email, id, name +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ColumnProductionDevelopment
country *JPNULL
email *frank@example.comNULL
id *6NULL
name *FrankNULL
+
+
+
@@ -1263,261 +1182,218 @@

Database Diff Report

-
- orders - 3 -
- 50e98d4f0d2f... - → - 3bc0bb9da318... -
-
- - Pending -
-
- -
- orders - 4 -
- 91c74b904980... - → - 932e3c9169d2... -
-
- - Pending -
-
- -
- orders - 5 -
- 579ea721981c... - → - f71fa3411956... -
-
- - Pending -
-
- -
- products - 10 -
- 056fec0d5c80... - → - 3ebc00ec928f... -
-
- - Pending -
-
- -
- products - 1 -
- 91290ea738a1... - → - 6a0483d92962... -
-
- - Pending -
-
- -
- products - 3 -
- 8d56e3071d26... - → - f0a691c2fd9a... -
-
- - Pending -
-
- -
- products - 5 -
- 809e6e458343... - → - d9e14527579f... -
-
- - Pending -
-
- -
- products - 7 -
- 2c83bd23bb23... - → - 0fceddb13a5d... -
-
- - Pending -
-
- -
- products - 8 -
- 602de4ce149c... - → - 74347a9a8653... -
-
- - Pending -
-
- -
- products - 2 -
- bf9d32823bd2... - → - 0c78ab070b93... -
-
- - Pending -
-
- -
- products - 4 -
- 0a4bb61bbee7... - → - d86101d3905a... -
-
- - Pending -
-
- -
- products - 6 -
- 131705dda71e... - → - fd58eb3590d5... -
-
- - Pending -
-
- -
- products - 9 -
- b58fe1a5cd09... - → - 8fc3ad499b63... -
-
- - Pending -
-
- -
- customers - 2 -
- 99cac2a7c695... - → - 0a949f3c2ddf... +
+
+ customers + 4 +
+ f29abbc96823... + → + e25da4226966... +
+
+ + Pending +
-
- - Pending + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ColumnProductionDevelopment
countryFRFR
email *dan@example.comdan+new@example.com
id44
nameDanDan
+
-
- customers - 3 -
- 8f56cdab82c9... - → - 2a3cf69f64dc... -
-
- - Pending -
-
+
+ +
+ + +
+
+ +
+
+ -
- customers - 4 -
- b4257f97cc9f... - → - a16b7ce17528... +
+
+ customers + Key: 4 + updated
-
- - Pending +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ColumnProductionDevelopment
countryFRFR
email *dan@example.comdan+new@example.com
id44
nameDanDan
-
- customers - 5 -
- 2014b0374b60... - → - 7f8723118572... +
+
+ customers + Key: 5 + removed
-
- - Pending +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ColumnProductionDevelopment
country *INNULL
email *eve@example.comNULL
id *5NULL
name *EveNULL
-
- customers - 1 -
- f30811df6706... - → - dcb1c5d6e44a... +
+
+ customers + Key: 6 + removed
-
- - Pending +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ColumnProductionDevelopment
country *JPNULL
email *frank@example.comNULL
id *6NULL
name *FrankNULL
-
- + +
+
@@ -1552,8 +1428,17 @@

Database Diff Report

function filterConflicts() { const v = document.getElementById('conflict-filter').value; - document.querySelectorAll('#conflict-list .conflict-item').forEach(r => { - r.style.display = !v || r.dataset.table === v ? '' : 'none'; + document.querySelectorAll('#conflict-list .conflict-item-card').forEach(r => { + const show = !v || r.dataset.table === v; + r.style.display = show ? '' : 'none'; + }); + } + + function filterRowDiffs() { + const v = document.getElementById('rowdiff-filter').value; + document.querySelectorAll('#rowdiff-list .rowdiff-card').forEach(r => { + const show = !v || r.dataset.table === v; + r.style.display = show ? '' : 'none'; }); } @@ -1583,9 +1468,21 @@

Database Diff Report

const cf = document.getElementById('conflict-filter'); if (cf) { const ts = new Set(); - document.querySelectorAll('#conflict-list .conflict-item').forEach(i => ts.add(i.dataset.table)); + document.querySelectorAll('#conflict-list .conflict-item-card').forEach(i => ts.add(i.dataset.table)); ts.forEach(t => { const o = document.createElement('option'); o.value = t; o.textContent = t; cf.appendChild(o); }); } + + const rf = document.getElementById('rowdiff-filter'); + if (rf) { + const rts = new Set(); + document.querySelectorAll('#rowdiff-list .rowdiff-card').forEach(i => rts.add(i.dataset.table)); + rts.forEach(t => { const o = document.createElement('option'); o.value = t; o.textContent = t; rf.appendChild(o); }); + } + + const hash = window.location.hash.replace('#', ''); + if (hash && document.getElementById('tab-' + hash)) { + switchTab(hash); + } });