Skip to content

fix(generator): keep Otzaria links whose target Sefaria does not provide #140

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kdroidFilter merged 1 commit into from
Oct 28, 2026
Merged

fix(generator): keep Otzaria links whose target Sefaria does not provide #140

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
138 changes: 8 additions & 130 deletions ...te/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/otzariasqlite/Generator.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -485,7 +485,7 @@ class DatabaseGenerator(
booksByTitle = filtered.associateBy { it.title }
booksById = filtered.associateBy { it.id }

// Build set of Sefaria book IDs to skip Otzaria links for these books
// Build set of Sefaria book IDs to skip Otzaria links between two of them
sefariaBookIds.clear()
for (book in filtered) {
val src = runCatching { repository.getSourceById(book.sourceId) }.getOrNull()
Expand All @@ -501,7 +501,7 @@ class DatabaseGenerator(
} else {
logger.i { "Preloaded ${filtered.size} books into memory for fast link processing" }
}
logger.i { "Identified ${sefariaBookIds.size} Sefaria books (Otzaria links will be skipped for these)" }
logger.i { "Identified ${sefariaBookIds.size} Sefaria books (Otzaria links between two of them will be skipped)" }
}
}

Expand Down Expand Up @@ -1395,129 +1395,6 @@ class DatabaseGenerator(
updateBookHasLinksTable()
}

/**
* Processes a single link file, creating links between books.
*
* @param linkFile The path to the link file
* @return The number of links successfully processed
*/
private suspend fun processLinkFile(linkFile: Path): Int {
val bookTitle = linkFile.nameWithoutExtension.removeSuffix("_links")
logger.d { "Processing link file for book: $bookTitle" }

// Find the source book (use RAM cache)
val sourceBook = booksByTitle[bookTitle]

if (sourceBook == null) {
logger.w { "Source book not found for links: $bookTitle" }
return 0
}

// Skip Otzaria links for books that come from Sefaria (Sefaria links are more accurate)
if (sourceBook.id in sefariaBookIds) {
return 0
}

logger.d { "Found source book with ID: ${sourceBook.id}" }

try {
val content = linkFile.readText()
logger.d { "Link file content length: ${content.length}" }
val links = parseLinksFromJson(content, bookTitle)
logger.d { "Decoded ${links.size} links from file" }
var processed = 0

for ((index, linkData) in links.withIndex()) {
try {
// Find the target book
// Handle paths with backslashes
val path = linkData.path_2
val targetTitle = if (path.contains('\\')) {
// Extract the last component of a backslash-separated path
val lastComponent = path.split('\\').last()
// Remove file extension if present
lastComponent.substringBeforeLast('.', lastComponent)
} else {
// Use the standard path handling for forward slash paths
val targetPath = Paths.get(path)
targetPath.fileName.toString().substringBeforeLast('.')
}
logger.d { "Link ${index + 1}/${links.size} - Target book title: $targetTitle" }

// Try to find the target book (use RAM cache)
val targetBook = booksByTitle[targetTitle]
if (targetBook == null) {
// Enhanced logging for debugging
logger.i { "Link ${index + 1}/${links.size} - Target book not found: $targetTitle" }
logger.i { "Original path: ${linkData.path_2}" }
continue
}
logger.d { "Using target book with ID: ${targetBook.id}" }

// Find the lines
// Adjust indices from 1-based to 0-based
val sourceLineIndex = (linkData.line_index_1.toInt() - 1).coerceAtLeast(0)
val targetLineIndex = (linkData.line_index_2.toInt() - 1).coerceAtLeast(0)

logger.d { "Looking for source line at index: $sourceLineIndex (original: ${linkData.line_index_1}) in book ${sourceBook.id}" }

// Try to find the source line (use RAM cache)
val sourceLineId = getLineIdCached(sourceBook.id, sourceLineIndex)
if (sourceLineId == null) {
logger.d { "Source line not found at index: $sourceLineIndex, skipping this link but continuing with others" }
continue
}
logger.d { "Using source line with ID: $sourceLineId" }

logger.d { "Looking for target line at index: $targetLineIndex (original: ${linkData.line_index_2}) in book ${targetBook.id}" }

// Try to find the target line (use RAM cache)
val targetLineId = getLineIdCached(targetBook.id, targetLineIndex)
if (targetLineId == null) {
logger.d { "Target line not found at index: $targetLineIndex, skipping this link but continuing with others" }
continue
}
logger.d { "Using target line with ID: $targetLineId" }

// Skip links where source or target is a heading line
if (sourceLineId in headingLineIds || targetLineId in headingLineIds) {
logger.d { "Skipping link to heading line" }
continue
}

val connectionType = ConnectionType.fromString(linkData.connectionType)
val link = Link(
id = allocator.linkId(sourceLineId, targetLineId, bindings.upsertConnectionType(connectionType.name)),
sourceBookId = sourceBook.id,
targetBookId = targetBook.id,
sourceLineId = sourceLineId,
targetLineId = targetLineId,
targetLineIndex = targetLineIndex,
connectionType = connectionType
)

logger.d { "Inserting link from book ${sourceBook.id} to book ${targetBook.id}" }
val linkId = repository.insertLink(link)
logger.d { "Link inserted with ID: $linkId" }
processed++
} catch (e: IllegalStateException) {
throw e // link id collision: data would be lost, fail the build
} catch (e: Exception) {
// Changed from error to debug level to reduce unnecessary error logs
logger.d(e) { "Error processing link: ${linkData.heRef_2}" }
logger.d { "Error processing link: ${e.message}" }
}
}
logger.d { "Processed $processed links out of ${links.size}" }
return processed
} catch (e: Exception) {
// Changed from error to warning level to reduce unnecessary error logs
logger.w(e) { "Error processing link file: ${linkFile.fileName}" }
logger.d { "Error processing link file: ${e.message}" }
return 0
}
}

/**
* Processes links that were preloaded in memory for a given source book.
*/
Expand All @@ -1528,11 +1405,6 @@ class DatabaseGenerator(
return 0
}

// Skip Otzaria links for books that come from Sefaria (Sefaria links are more accurate)
if (sourceBook.id in sefariaBookIds) {
return 0
}

var processed = 0
for ((index, linkData) in links.withIndex()) {
try {
Expand All @@ -1552,6 +1424,12 @@ class DatabaseGenerator(
continue
}

// Between two Sefaria books, Sefaria's own links are exact while Otzaria's
// assume matching line indices and drift on differing headers (verse N →
// targum verse N-1). Otzaria stays the only source for any other pair
// (e.g. משנה ברורה → שער הציון), so those links are kept.
if (sourceBook.id in sefariaBookIds && targetBook.id in sefariaBookIds) continue

val sourceLineIndex = (linkData.line_index_1.toInt() - 1).coerceAtLeast(0)
val targetLineIndex = (linkData.line_index_2.toInt() - 1).coerceAtLeast(0)

Expand Down
Loading