From ddc4f16bb8f892213401007193745595c7a9799f Mon Sep 17 00:00:00 2001 From: Elie Gambache Date: Wed, 28 Oct 2026 22:37:56 +0200 Subject: [PATCH] fix(generator): keep Otzaria links whose target Sefaria does not provide MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Otzaria links were skipped for every book imported from Sefaria (#28), to drop the targum links that drifted a verse on differing headers. That also dropped links Sefaria has no equivalent for, like the 17,980 משנה ברורה → שער הציון links Otzaria aligned on the Sefaria text. Skip only links between two Sefaria books. This brings back ~23,900 links (שער הציון, תרגום ותפסיר רס"ג, ביאור הגר"א on the Zohar), checked against the current DB. Remove the unused processLinkFile duplicate. Signed-off-by: Elie Gambache --- .../seforimlibrary/otzariasqlite/Generator.kt | 138 +----------------- 1 file changed, 8 insertions(+), 130 deletions(-) diff --git a/generator/otzariasqlite/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/otzariasqlite/Generator.kt b/generator/otzariasqlite/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/otzariasqlite/Generator.kt index 5bbf35b6..22645408 100644 --- a/generator/otzariasqlite/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/otzariasqlite/Generator.kt +++ b/generator/otzariasqlite/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/otzariasqlite/Generator.kt @@ -485,7 +485,7 @@ class DatabaseGenerator( booksByTitle = filtered.associateBy { it.title } booksById = filtered.associateBy { it.id } - // Build set of Sefaria book IDs to skip Otzaria links for these books + // Build set of Sefaria book IDs to skip Otzaria links between two of them sefariaBookIds.clear() for (book in filtered) { val src = runCatching { repository.getSourceById(book.sourceId) }.getOrNull() @@ -501,7 +501,7 @@ class DatabaseGenerator( } else { logger.i { "Preloaded ${filtered.size} books into memory for fast link processing" } } - logger.i { "Identified ${sefariaBookIds.size} Sefaria books (Otzaria links will be skipped for these)" } + logger.i { "Identified ${sefariaBookIds.size} Sefaria books (Otzaria links between two of them will be skipped)" } } } @@ -1395,129 +1395,6 @@ class DatabaseGenerator( updateBookHasLinksTable() } - /** - * Processes a single link file, creating links between books. - * - * @param linkFile The path to the link file - * @return The number of links successfully processed - */ - private suspend fun processLinkFile(linkFile: Path): Int { - val bookTitle = linkFile.nameWithoutExtension.removeSuffix("_links") - logger.d { "Processing link file for book: $bookTitle" } - - // Find the source book (use RAM cache) - val sourceBook = booksByTitle[bookTitle] - - if (sourceBook == null) { - logger.w { "Source book not found for links: $bookTitle" } - return 0 - } - - // Skip Otzaria links for books that come from Sefaria (Sefaria links are more accurate) - if (sourceBook.id in sefariaBookIds) { - return 0 - } - - logger.d { "Found source book with ID: ${sourceBook.id}" } - - try { - val content = linkFile.readText() - logger.d { "Link file content length: ${content.length}" } - val links = parseLinksFromJson(content, bookTitle) - logger.d { "Decoded ${links.size} links from file" } - var processed = 0 - - for ((index, linkData) in links.withIndex()) { - try { - // Find the target book - // Handle paths with backslashes - val path = linkData.path_2 - val targetTitle = if (path.contains('\\')) { - // Extract the last component of a backslash-separated path - val lastComponent = path.split('\\').last() - // Remove file extension if present - lastComponent.substringBeforeLast('.', lastComponent) - } else { - // Use the standard path handling for forward slash paths - val targetPath = Paths.get(path) - targetPath.fileName.toString().substringBeforeLast('.') - } - logger.d { "Link ${index + 1}/${links.size} - Target book title: $targetTitle" } - - // Try to find the target book (use RAM cache) - val targetBook = booksByTitle[targetTitle] - if (targetBook == null) { - // Enhanced logging for debugging - logger.i { "Link ${index + 1}/${links.size} - Target book not found: $targetTitle" } - logger.i { "Original path: ${linkData.path_2}" } - continue - } - logger.d { "Using target book with ID: ${targetBook.id}" } - - // Find the lines - // Adjust indices from 1-based to 0-based - val sourceLineIndex = (linkData.line_index_1.toInt() - 1).coerceAtLeast(0) - val targetLineIndex = (linkData.line_index_2.toInt() - 1).coerceAtLeast(0) - - logger.d { "Looking for source line at index: $sourceLineIndex (original: ${linkData.line_index_1}) in book ${sourceBook.id}" } - - // Try to find the source line (use RAM cache) - val sourceLineId = getLineIdCached(sourceBook.id, sourceLineIndex) - if (sourceLineId == null) { - logger.d { "Source line not found at index: $sourceLineIndex, skipping this link but continuing with others" } - continue - } - logger.d { "Using source line with ID: $sourceLineId" } - - logger.d { "Looking for target line at index: $targetLineIndex (original: ${linkData.line_index_2}) in book ${targetBook.id}" } - - // Try to find the target line (use RAM cache) - val targetLineId = getLineIdCached(targetBook.id, targetLineIndex) - if (targetLineId == null) { - logger.d { "Target line not found at index: $targetLineIndex, skipping this link but continuing with others" } - continue - } - logger.d { "Using target line with ID: $targetLineId" } - - // Skip links where source or target is a heading line - if (sourceLineId in headingLineIds || targetLineId in headingLineIds) { - logger.d { "Skipping link to heading line" } - continue - } - - val connectionType = ConnectionType.fromString(linkData.connectionType) - val link = Link( - id = allocator.linkId(sourceLineId, targetLineId, bindings.upsertConnectionType(connectionType.name)), - sourceBookId = sourceBook.id, - targetBookId = targetBook.id, - sourceLineId = sourceLineId, - targetLineId = targetLineId, - targetLineIndex = targetLineIndex, - connectionType = connectionType - ) - - logger.d { "Inserting link from book ${sourceBook.id} to book ${targetBook.id}" } - val linkId = repository.insertLink(link) - logger.d { "Link inserted with ID: $linkId" } - processed++ - } catch (e: IllegalStateException) { - throw e // link id collision: data would be lost, fail the build - } catch (e: Exception) { - // Changed from error to debug level to reduce unnecessary error logs - logger.d(e) { "Error processing link: ${linkData.heRef_2}" } - logger.d { "Error processing link: ${e.message}" } - } - } - logger.d { "Processed $processed links out of ${links.size}" } - return processed - } catch (e: Exception) { - // Changed from error to warning level to reduce unnecessary error logs - logger.w(e) { "Error processing link file: ${linkFile.fileName}" } - logger.d { "Error processing link file: ${e.message}" } - return 0 - } - } - /** * Processes links that were preloaded in memory for a given source book. */ @@ -1528,11 +1405,6 @@ class DatabaseGenerator( return 0 } - // Skip Otzaria links for books that come from Sefaria (Sefaria links are more accurate) - if (sourceBook.id in sefariaBookIds) { - return 0 - } - var processed = 0 for ((index, linkData) in links.withIndex()) { try { @@ -1552,6 +1424,12 @@ class DatabaseGenerator( continue } + // Between two Sefaria books, Sefaria's own links are exact while Otzaria's + // assume matching line indices and drift on differing headers (verse N → + // targum verse N-1). Otzaria stays the only source for any other pair + // (e.g. משנה ברורה → שער הציון), so those links are kept. + if (sourceBook.id in sefariaBookIds && targetBook.id in sefariaBookIds) continue + val sourceLineIndex = (linkData.line_index_1.toInt() - 1).coerceAtLeast(0) val targetLineIndex = (linkData.line_index_2.toInt() - 1).coerceAtLeast(0)