Fix Niadd range downloads in DownloadWorker

patches/futon-niadd-range-download.patch

@@ -167,6 +167,171 @@ index ae55a0f..9f4338b 100644
  	class Factory @Inject constructor(
  		@MangaHttpClient private val okHttpClient: OkHttpClient,
  		@PageCache private val pagesCache: LocalStorageCache,
+diff --git a/app/src/main/kotlin/io/github/landwarderer/futon/download/ui/worker/DownloadWorker.kt b/app/src/main/kotlin/io/github/landwarderer/futon/download/ui/worker/DownloadWorker.kt
+index 8227b2a..742341e 100644
+--- a/app/src/main/kotlin/io/github/landwarderer/futon/download/ui/worker/DownloadWorker.kt
++++ b/app/src/main/kotlin/io/github/landwarderer/futon/download/ui/worker/DownloadWorker.kt
+@@ -91,6 +91,7 @@ import okio.use
+ import org.koitharu.kotatsu.parsers.exception.TooManyRequestExceptions
+ import org.koitharu.kotatsu.parsers.model.Manga
+ import org.koitharu.kotatsu.parsers.model.MangaChapter
++import org.koitharu.kotatsu.parsers.model.MangaParserSource
+ import org.koitharu.kotatsu.parsers.model.MangaSource
+ import org.koitharu.kotatsu.parsers.util.ifNullOrEmpty
+ import org.koitharu.kotatsu.parsers.util.mapToSet
+@@ -125,6 +126,10 @@ class DownloadWorker @AssistedInject constructor(
+ 	private val notificationFactory = notificationFactoryFactory.create(uuid = params.id, isSilent = task.isSilent)
+ 	private val notificationManager = appContext.getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager
+ 	private val bandwidthLimiter = BandwidthLimiter { settings.downloadBandwidthLimit * 1024 }
++	private val niaddOkHttp = okHttp.newBuilder()
++		.followRedirects(false)
++		.followSslRedirects(false)
++		.build()
+ 
+ 	@Volatile
+ 	private var lastPublishedState: DownloadState? = null
+@@ -427,6 +432,13 @@ class DownloadWorker @AssistedInject constructor(
+ 			}
+ 			return file
+ 		}
++		if (isNiaddRangeDownload(url, source)) {
++			slowdownDispatcher.delay(source)
++			return NIADD_DOWNLOAD_SEMAPHORE.withPermit {
++				downloadNiaddFileByRanges(url, destination, source)
++			}
++		}
++
+ 		val request = PageLoader.createPageRequest(url, source)
+ 		slowdownDispatcher.delay(source)
+ 		return imageProxyInterceptor.interceptPageRequest(request, okHttp)
+@@ -450,6 +462,117 @@ class DownloadWorker @AssistedInject constructor(
+ 			}
+ 	}
+ 
++	private fun isNiaddRangeDownload(url: String, source: MangaSource): Boolean {
++		return source == MangaParserSource.NINEMANGA_RU &&
++			url.substringBefore('?').endsWith(".webp", ignoreCase = true) &&
++			url.contains("acc=", ignoreCase = true) &&
++			url.contains("exp=", ignoreCase = true)
++	}
++
++	private suspend fun downloadNiaddFileByRanges(
++		url: String,
++		destination: File,
++		source: MangaSource,
++	): File {
++		Log.d("DownloadWorker", "Using Niadd range download for $url")
++		var file: File? = null
++		var offset = 0L
++		var totalBytes = -1L
++
++		try {
++			while (totalBytes < 0L || offset < totalBytes) {
++				val requestedEnd = if (totalBytes > 0L) {
++					minOf(offset + NIADD_RANGE_CHUNK_SIZE - 1L, totalBytes - 1L)
++				} else {
++					offset + NIADD_RANGE_CHUNK_SIZE - 1L
++				}
++				val request = PageLoader.createPageRequest(url, source)
++					.newBuilder()
++					.header("Range", "bytes=$offset-$requestedEnd")
++					.header("Connection", "close")
++					.build()
++
++				niaddOkHttp.newCall(request).execute().use { response ->
++					check(response.code == 206) {
++						val location = response.header("Location")
++						"Niadd CDN range request returned HTTP ${response.code}" +
++							if (location.isNullOrBlank()) {
++								" for $url"
++							} else {
++								", Location=$location for $url"
++							}
++					}
++
++					val contentType = response.header("Content-Type")
++						?.substringBefore(';')
++						?.trim()
++						.orEmpty()
++					check(contentType.equals("image/webp", ignoreCase = true)) {
++						"Niadd CDN returned '$contentType' instead of image/webp for $url"
++					}
++
++					val contentRange = response.header("Content-Range")
++						?: error("Niadd CDN response has no Content-Range for $url")
++					val rangeMatch = NIADD_CONTENT_RANGE_REGEX.matchEntire(contentRange)
++						?: error("Invalid Niadd Content-Range: $contentRange")
++					val responseStart = rangeMatch.groupValues[1].toLong()
++					val responseEnd = rangeMatch.groupValues[2].toLong()
++					val responseTotal = rangeMatch.groupValues[3].toLong()
++					check(responseTotal > 0L && responseStart == offset) {
++						"Unexpected Niadd Content-Range '$contentRange' for offset $offset ($url)"
++					}
++
++					if (totalBytes < 0L) {
++						totalBytes = responseTotal
++					} else {
++						check(totalBytes == responseTotal) {
++							"Niadd CDN changed content size from $totalBytes to $responseTotal for $url"
++						}
++					}
++
++					val expectedEnd = minOf(requestedEnd, totalBytes - 1L)
++					check(responseEnd == expectedEnd) {
++						"Unexpected Niadd range end: expected $expectedEnd, got $responseEnd " +
++							"($contentRange, $url)"
++					}
++
++					val body = response.requireBody()
++					if (file == null) {
++						file = destination.createTempFile(
++							ext = MimeTypes.getExtension(body.contentType()?.toMimeType()),
++						)
++					}
++					val outputFile = checkNotNull(file)
++					check(outputFile.length() == offset) {
++						"Niadd temporary file offset mismatch: expected $offset, got ${outputFile.length()} for $url"
++					}
++
++					val expectedBytes = expectedEnd - offset + 1L
++					outputFile.sink(append = offset > 0L).buffer().use { sink ->
++						sink.writeAllCancellable(body.source(), bandwidthLimiter)
++					}
++					val newLength = outputFile.length()
++					val receivedBytes = newLength - offset
++					check(receivedBytes == expectedBytes) {
++						"Short Niadd range: expected $expectedBytes bytes, got $receivedBytes " +
++							"for bytes=$offset-$expectedEnd ($url)"
++					}
++
++					offset = newLength
++				}
++			}
++
++			val result = checkNotNull(file)
++			check(totalBytes > 0L && result.length() == totalBytes) {
++				"Incomplete Niadd image: expected $totalBytes bytes, got ${result.length()} for $url"
++			}
++			return result
++		} catch (e: Exception) {
++			file?.delete()
++			throw e
++		}
++	}
++
+ 	private fun File.createTempFile(ext: String?) = File(
+ 		this,
+ 		buildString {
+@@ -633,6 +756,9 @@ class DownloadWorker @AssistedInject constructor(
+ 
+ 		const val MAX_FAILSAFE_ATTEMPTS = 2
+ 		const val MAX_PAGES_PARALLELISM = 4
++		private const val NIADD_RANGE_CHUNK_SIZE = 16L * 1024L
++		private val NIADD_CONTENT_RANGE_REGEX = Regex("""bytes (\d+)-(\d+)/(\d+)""")
++		private val NIADD_DOWNLOAD_SEMAPHORE = Semaphore(1)
+ 		const val DOWNLOAD_ERROR_DELAY = 2_000L
+ 		const val MAX_RETRY_DELAY = 7_200_000L // 2 hours
+ 		const val TAG = "download"
 diff --git a/app/src/main/kotlin/io/github/landwarderer/futon/reader/domain/PageLoader.kt b/app/src/main/kotlin/io/github/landwarderer/futon/reader/domain/PageLoader.kt
 index 18d203d..02b1653 100644
 --- a/app/src/main/kotlin/io/github/landwarderer/futon/reader/domain/PageLoader.kt

scripts/patch_ninemanga_niadd.py

@@ -425,38 +425,6 @@ replace_exact(
 
                 val dateText = anchorDate ?: rowDate
 
-                val metadataText = when {
-                    anchorDate != null -> anchorText
-                    rowDate != null -> rowText
-                    else -> ""
-                }
-
-                val textBeforeDate = if (dateText != null) {
-                    metadataText
-                        .substringBefore(dateText)
-                        .trim()
-                } else {
-                    metadataText
-                }
-
-                /*
-                 * Niadd exposes the page count immediately before
-                 * the chapter date, for example:
-                 *
-                 *   Падение русала 1 - 42 41 Aug 02, 2024
-                 *                           ^^
-                 *
-                 * Store that number in a URL fragment. Fragments
-                 * are local metadata and are not sent to the server.
-                 */
-                val pageCount = Regex(
-                    """\\s+(\\d+)\\s*$"""
-                )
-                    .find(textBeforeDate)
-                    ?.groupValues
-                    ?.getOrNull(1)
-                    ?.toIntOrNull()
-
                 val chapterTitle = if (
                     dateText != null &&
                     rawTitle.contains(dateText)
@@ -472,15 +440,6 @@ replace_exact(
                     rawTitle
                 }
 
-                val chapterUrl = if (
-                    pageCount != null &&
-                    pageCount > 0
-                ) {
-                    "$href#niadd-pages=$pageCount"
-                } else {
-                    href
-                }
-
                 MangaChapter(
                     id = generateUid(href),
                     title = chapterTitle.takeIf {
@@ -488,7 +447,7 @@ replace_exact(
                     },
                     number = i + 1f,
                     volume = 0,
-                    url = chapterUrl,
+                    url = href,
                     uploadDate = parseChapterDateByLang(
                         dateText.orEmpty()
                     ),
@@ -641,42 +600,78 @@ replace_exact(
     '''    override suspend fun getPages(
         chapter: MangaChapter,
     ): List<MangaPage> {
+        val chapterUrl = chapter.url
+            .substringBefore("#")
+
+        val doc = webClient
+            .httpGet(
+                chapterUrl.toAbsoluteUrl(domain)
+            )
+            .parseHtml()
+
         if (domain.endsWith("niadd.com")) {
-            val originalUrl = chapter.url
+            val pageLabelRegex = Regex(
-                .substringBefore("#")
+                """^\\s*(\\d+)\\s*/\\s*(\\d+)\\s*$"""
-                .substringBefore("?")
+            )
 
-            val totalPages = chapter.url
+            val pageOptions = doc
-                .substringAfter(
+                .select("option[value]")
-                    "#niadd-pages=",
+                .mapNotNull { option ->
-                    "",
+                    val match = pageLabelRegex.matchEntire(
-                )
+                        option.text().trim()
-                .substringBefore("&")
+                    ) ?: return@mapNotNull null
-                .toIntOrNull()
-                ?.takeIf { it > 0 }
-                ?: error(
-                    "Niadd page count is missing for ${chapter.url}. " +
-                        "Refresh manga details and try again."
-                )
 
-            val normalizedUrl = originalUrl
+                    val pageNumber = match
-                .removeSuffix("/")
+                        .groupValues
+                        .getOrNull(1)
+                        ?.toIntOrNull()
+                        ?: return@mapNotNull null
 
-            val baseUrl = if (
+                    val totalPages = match
-                normalizedUrl.endsWith(".html")
+                        .groupValues
-            ) {
+                        .getOrNull(2)
-                normalizedUrl.removeSuffix(".html")
+                        ?.toIntOrNull()
-            } else {
+                        ?: return@mapNotNull null
-                normalizedUrl
-            }
 
-            return (1..totalPages).map { pageNumber ->
+                    val url = option.attr("value").trim()
-                val url = if (pageNumber == 1) {
+                    if (url.isEmpty()) {
-                    originalUrl
+                        return@mapNotNull null
-                } else {
+                    }
-                    "$baseUrl-$pageNumber.html"
+
+                    Triple(pageNumber, totalPages, url)
+                }
+                .distinctBy { (pageNumber, _, _) ->
+                    pageNumber
+                }
+                .sortedBy { (pageNumber, _, _) ->
+                    pageNumber
                 }
 
+            val totalPages = pageOptions
+                .firstOrNull()
+                ?.second
+                ?: error(
+                    "Cannot find Niadd page selector for $chapterUrl"
+                )
+
+            check(
+                pageOptions.all { (_, total, _) ->
+                    total == totalPages
+                }
+            ) {
+                "Inconsistent Niadd page count for $chapterUrl"
+            }
+
+            check(
+                pageOptions.size == totalPages &&
+                    pageOptions.map { it.first } ==
+                    (1..totalPages).toList()
+            ) {
+                "Incomplete Niadd page selector for $chapterUrl: " +
+                    "found ${pageOptions.size} of $totalPages pages"
+            }
+
+            return pageOptions.map { (_, _, url) ->
                 MangaPage(
                     id = generateUid(url),
                     url = url,
@@ -686,12 +681,6 @@ replace_exact(
             }
         }
 
-        val doc = webClient
-            .httpGet(
-                chapter.url.toAbsoluteUrl(domain)
-            )
-            .parseHtml()
-
         return doc
             .body()
             .requireElementById("page")