v5.1

patches/futon-niadd-range-download.patch

@@ -1,6 +1,177 @@
+diff --git a/app/src/main/kotlin/io/github/landwarderer/futon/details/ui/pager/pages/MangaPageFetcher.kt b/app/src/main/kotlin/io/github/landwarderer/futon/details/ui/pager/pages/MangaPageFetcher.kt
+index ae55a0f..9f4338b 100644
+--- a/app/src/main/kotlin/io/github/landwarderer/futon/details/ui/pager/pages/MangaPageFetcher.kt
++++ b/app/src/main/kotlin/io/github/landwarderer/futon/details/ui/pager/pages/MangaPageFetcher.kt
+@@ -15,21 +15,26 @@ import coil3.request.Options
+ import okhttp3.Headers
+ import okhttp3.OkHttpClient
+ import okhttp3.Response
++import okio.Buffer
+ import okio.FileSystem
+ import okio.Path.Companion.toOkioPath
+ import io.github.landwarderer.futon.core.network.MangaHttpClient
+ import io.github.landwarderer.futon.core.network.imageproxy.ImageProxyInterceptor
+ import io.github.landwarderer.futon.core.parser.MangaRepository
+ import io.github.landwarderer.futon.core.util.MimeTypes
++import io.github.landwarderer.futon.core.util.ext.MimeType
+ import io.github.landwarderer.futon.core.util.ext.fetch
+ import io.github.landwarderer.futon.core.util.ext.isNetworkUri
+ import io.github.landwarderer.futon.core.util.ext.toMimeTypeOrNull
+ import io.github.landwarderer.futon.local.data.LocalStorageCache
+ import io.github.landwarderer.futon.local.data.PageCache
+ import org.koitharu.kotatsu.parsers.model.MangaPage
++import org.koitharu.kotatsu.parsers.model.MangaParserSource
+ import org.koitharu.kotatsu.parsers.util.mimeType
+ import org.koitharu.kotatsu.parsers.util.requireBody
+ import org.koitharu.kotatsu.parsers.util.runCatchingCancellable
++import kotlinx.coroutines.sync.Semaphore
++import kotlinx.coroutines.sync.withPermit
+ import io.github.landwarderer.futon.reader.domain.PageLoader
+ import javax.inject.Inject
+ 
+@@ -43,6 +48,11 @@ class MangaPageFetcher(
+ 	private val imageLoader: ImageLoader,
+ ) : Fetcher {
+ 
++	private val niaddOkHttpClient = okHttpClient.newBuilder()
++		.followRedirects(false)
++		.followSslRedirects(false)
++		.build()
++
+ 	override suspend fun fetch(): FetchResult? {
+ 		if (!page.preview.isNullOrEmpty()) {
+ 			runCatchingCancellable {
+@@ -72,6 +82,12 @@ class MangaPageFetcher(
+ 	}
+ 
+ 	private suspend fun fetchPage(pageUrl: String): FetchResult {
++		if (page.source == MangaParserSource.NINEMANGA_RU) {
++			return niaddSemaphore.withPermit {
++				fetchNiaddPageByRanges(pageUrl)
++			}
++		}
++
+ 		val request = PageLoader.createPageRequest(pageUrl, page.source)
+ 		return imageProxyInterceptor.interceptPageRequest(request, okHttpClient).use { response ->
+ 			if (!response.isSuccessful) {
+@@ -89,6 +105,99 @@ class MangaPageFetcher(
+ 		}
+ 	}
+ 
++	private suspend fun fetchNiaddPageByRanges(pageUrl: String): FetchResult {
++		val buffer = Buffer()
++		var mimeType: MimeType? = null
++		var offset = 0L
++		var totalBytes = -1L
++
++		while (totalBytes < 0L || offset < totalBytes) {
++			val requestedEnd = if (totalBytes > 0L) {
++				minOf(offset + NIADD_RANGE_CHUNK_SIZE - 1L, totalBytes - 1L)
++			} else {
++				offset + NIADD_RANGE_CHUNK_SIZE - 1L
++			}
++			val request = PageLoader.createPageRequest(pageUrl, page.source)
++				.newBuilder()
++				.header("Range", "bytes=$offset-$requestedEnd")
++				.header("Connection", "close")
++				.build()
++
++			niaddOkHttpClient.newCall(request).execute().use { response ->
++				check(response.code == 206) {
++					val location = response.header("Location")
++					"Niadd CDN range request returned HTTP ${response.code}" +
++						if (location.isNullOrBlank()) {
++							" for $pageUrl"
++						} else {
++							", Location=$location for $pageUrl"
++						}
++				}
++
++				val contentType = response.header("Content-Type")
++					?.substringBefore(';')
++					?.trim()
++					.orEmpty()
++				check(contentType.startsWith("image/", ignoreCase = true)) {
++					"Niadd CDN returned non-image Content-Type '$contentType' for $pageUrl"
++				}
++				if (pageUrl.substringBefore('?').endsWith(".webp", ignoreCase = true)) {
++					check(contentType.equals("image/webp", ignoreCase = true)) {
++						"Niadd CDN replaced WebP with '$contentType' for $pageUrl"
++					}
++				}
++
++				val contentRange = response.header("Content-Range")
++					?: error("Niadd CDN response has no Content-Range for $pageUrl")
++				val rangeMatch = NIADD_CONTENT_RANGE_REGEX.matchEntire(contentRange)
++					?: error("Invalid Niadd Content-Range: $contentRange")
++				val responseStart = rangeMatch.groupValues[1].toLong()
++				val responseEnd = rangeMatch.groupValues[2].toLong()
++				val responseTotal = rangeMatch.groupValues[3].toLong()
++				check(responseTotal > 0L && responseStart == offset) {
++					"Unexpected Niadd Content-Range '$contentRange' for offset $offset ($pageUrl)"
++				}
++
++				if (totalBytes < 0L) {
++					totalBytes = responseTotal
++				} else {
++					check(totalBytes == responseTotal) {
++						"Niadd CDN changed content size from $totalBytes to $responseTotal for $pageUrl"
++					}
++				}
++
++				val expectedEnd = minOf(requestedEnd, totalBytes - 1L)
++				check(responseEnd == expectedEnd) {
++					"Unexpected Niadd range end: expected $expectedEnd, got $responseEnd " +
++						"($contentRange, $pageUrl)"
++				}
++				if (mimeType == null) {
++					mimeType = response.mimeType?.toMimeTypeOrNull()
++				}
++
++				val expectedBytes = expectedEnd - offset + 1L
++				val receivedBytes = buffer.writeAll(response.requireBody().source())
++				check(receivedBytes == expectedBytes) {
++					"Short Niadd range: expected $expectedBytes bytes, got $receivedBytes " +
++						"for bytes=$offset-$expectedEnd ($pageUrl)"
++				}
++
++				offset += receivedBytes
++			}
++		}
++
++		check(totalBytes > 0L && buffer.size == totalBytes) {
++			"Incomplete Niadd image: expected $totalBytes bytes, got ${buffer.size} for $pageUrl"
++		}
++
++		val file = pagesCache.set(pageUrl, buffer, mimeType)
++		return SourceFetchResult(
++			source = ImageSource(file.toOkioPath(), FileSystem.SYSTEM),
++			mimeType = mimeType?.toString(),
++			dataSource = DataSource.NETWORK,
++		)
++	}
++
+ 	private fun Response.toNetworkResponse() = NetworkResponse(
+ 		code = code,
+ 		requestMillis = sentRequestAtMillis,
+@@ -106,6 +215,12 @@ class MangaPageFetcher(
+ 		return headers.build()
+ 	}
+ 
++	companion object {
++		private const val NIADD_RANGE_CHUNK_SIZE = 16L * 1024L
++		private val NIADD_CONTENT_RANGE_REGEX = Regex("""bytes (\d+)-(\d+)/(\d+)""")
++		private val niaddSemaphore = Semaphore(1)
++	}
++
+ 	class Factory @Inject constructor(
+ 		@MangaHttpClient private val okHttpClient: OkHttpClient,
+ 		@PageCache private val pagesCache: LocalStorageCache,
+diff --git a/app/src/main/kotlin/io/github/landwarderer/futon/reader/domain/PageLoader.kt b/app/src/main/kotlin/io/github/landwarderer/futon/reader/domain/PageLoader.kt
+index 18d203d..02b1653 100644
 --- a/app/src/main/kotlin/io/github/landwarderer/futon/reader/domain/PageLoader.kt
 +++ b/app/src/main/kotlin/io/github/landwarderer/futon/reader/domain/PageLoader.kt
-@@ -30,6 +30,7 @@
+@@ -30,6 +30,7 @@ import io.github.landwarderer.futon.core.prefs.AppSettings
  import io.github.landwarderer.futon.core.ui.image.TrimTransformation
  import io.github.landwarderer.futon.core.util.FileSize
  import io.github.landwarderer.futon.core.util.MimeTypes
@@ -8,7 +179,7 @@
  import io.github.landwarderer.futon.core.util.ext.URI_SCHEME_ZIP
  import io.github.landwarderer.futon.core.util.ext.cancelChildrenAndJoin
  import io.github.landwarderer.futon.core.util.ext.compressToPNG
-@@ -52,6 +53,7 @@
+@@ -52,6 +53,7 @@ import io.github.landwarderer.futon.download.ui.worker.DownloadSlowdownDispatche
  import io.github.landwarderer.futon.local.data.LocalStorageCache
  import io.github.landwarderer.futon.local.data.PageCache
  import org.koitharu.kotatsu.parsers.model.MangaPage
@@ -16,7 +187,7 @@
  import org.koitharu.kotatsu.parsers.model.MangaSource
  import org.koitharu.kotatsu.parsers.util.requireBody
  import org.koitharu.kotatsu.parsers.util.runCatchingCancellable
-@@ -70,6 +72,9 @@
+@@ -70,6 +72,9 @@ import kotlinx.coroutines.sync.withLock
  import kotlinx.coroutines.sync.withPermit
  import okhttp3.OkHttpClient
  import okhttp3.Request
@@ -26,15 +197,19 @@
  import okio.use
  import org.jetbrains.annotations.Blocking
  import java.io.File
-@@ -97,6 +102,7 @@
+@@ -97,6 +102,11 @@ class PageLoader @Inject constructor(
  
  	private val tasks = LongSparseArray<ProgressDeferred<Uri, Float>>()
  	private val semaphore = Semaphore(3)
 +	private val niaddSemaphore = Semaphore(1)
++	private val niaddOkHttp = okHttp.newBuilder()
++		.followRedirects(false)
++		.followSslRedirects(false)
++		.build()
  	private val convertLock = Mutex()
  	private val prefetchLock = Mutex()
  
-@@ -303,13 +309,100 @@
+@@ -303,13 +313,126 @@ class PageLoader @Inject constructor(
  				if (isPrefetch) {
  					downloadSlowdownDispatcher.delay(page.source)
  				}
@@ -65,9 +240,9 @@
 +		progress: MutableStateFlow<Float>,
 +	): Uri {
 +		val tempFile = File.createTempFile(
-+			prefix = "niadd-page-",
-+			suffix = ".part",
-+			directory = context.cacheDir,
++			"niadd-page-",
++			".part",
++			context.cacheDir,
 +		)
 +		var mimeType: MimeType? = null
 +		try {
@@ -87,18 +262,40 @@
 +						.header("Connection", "close")
 +						.build()
 +
-+					okHttp.newCall(request).execute().use { response ->
++					niaddOkHttp.newCall(request).execute().use { response ->
 +						check(response.code == 206) {
-+							"Niadd CDN ignored range request: HTTP ${response.code} for $pageUrl"
++							val location = response.header("Location")
++							"Niadd CDN range request returned HTTP ${response.code}" +
++								if (location.isNullOrBlank()) {
++									" for $pageUrl"
++								} else {
++									", Location=$location for $pageUrl"
++								}
++						}
++
++						val contentType = response.header("Content-Type")
++							?.substringBefore(';')
++							?.trim()
++							.orEmpty()
++						check(contentType.startsWith("image/", ignoreCase = true)) {
++							"Niadd CDN returned non-image Content-Type '$contentType' for $pageUrl"
++						}
++						if (pageUrl.substringBefore('?').endsWith(".webp", ignoreCase = true)) {
++							check(contentType.equals("image/webp", ignoreCase = true)) {
++								"Niadd CDN replaced WebP with '$contentType' for $pageUrl"
++							}
 +						}
 +
 +						val contentRange = response.header("Content-Range")
 +							?: error("Niadd CDN response has no Content-Range for $pageUrl")
-+						val responseTotal = contentRange
-+							.substringAfterLast('/', "")
-+							.toLongOrNull()
-+							?.takeIf { it > 0L }
++						val rangeMatch = NIADD_CONTENT_RANGE_REGEX.matchEntire(contentRange)
 +							?: error("Invalid Niadd Content-Range: $contentRange")
++						val responseStart = rangeMatch.groupValues[1].toLong()
++						val responseEnd = rangeMatch.groupValues[2].toLong()
++						val responseTotal = rangeMatch.groupValues[3].toLong()
++						check(responseTotal > 0L && responseStart == offset) {
++							"Unexpected Niadd Content-Range '$contentRange' for offset $offset ($pageUrl)"
++						}
 +
 +						if (totalBytes < 0L) {
 +							totalBytes = responseTotal
@@ -108,12 +305,16 @@
 +							}
 +						}
 +
++						val expectedEnd = minOf(requestedEnd, totalBytes - 1L)
++						check(responseEnd == expectedEnd) {
++							"Unexpected Niadd range end: expected $expectedEnd, got $responseEnd " +
++								"($contentRange, $pageUrl)"
++						}
 +						val body = response.requireBody()
 +						if (mimeType == null) {
 +							mimeType = body.contentType()?.toMimeType()
 +						}
 +
-+						val expectedEnd = minOf(requestedEnd, totalBytes - 1L)
 +						val expectedBytes = expectedEnd - offset + 1L
 +						val receivedBytes = sink.writeAll(body.source())
 +						check(receivedBytes == expectedBytes) {
@@ -140,11 +341,12 @@
  		}
  	}
  
-@@ -340,6 +433,7 @@
+@@ -340,6 +463,8 @@ class PageLoader @Inject constructor(
  	companion object {
  
  		private const val PROGRESS_UNDEFINED = -1f
 +		private const val NIADD_RANGE_CHUNK_SIZE = 16L * 1024L
++		private val NIADD_CONTENT_RANGE_REGEX = Regex("""bytes (\d+)-(\d+)/(\d+)""")
  		private const val PREFETCH_LIMIT_DEFAULT = 6
  		private const val PREFETCH_MIN_RAM_MB = 80L
  

scripts/patch_ninemanga_niadd.py

@@ -425,38 +425,6 @@ replace_exact(
 
                 val dateText = anchorDate ?: rowDate
 
-                val metadataText = when {
-                    anchorDate != null -> anchorText
-                    rowDate != null -> rowText
-                    else -> ""
-                }
-
-                val textBeforeDate = if (dateText != null) {
-                    metadataText
-                        .substringBefore(dateText)
-                        .trim()
-                } else {
-                    metadataText
-                }
-
-                /*
-                 * Niadd exposes the page count immediately before
-                 * the chapter date, for example:
-                 *
-                 *   Падение русала 1 - 42 41 Aug 02, 2024
-                 *                           ^^
-                 *
-                 * Store that number in a URL fragment. Fragments
-                 * are local metadata and are not sent to the server.
-                 */
-                val pageCount = Regex(
-                    """\\s+(\\d+)\\s*$"""
-                )
-                    .find(textBeforeDate)
-                    ?.groupValues
-                    ?.getOrNull(1)
-                    ?.toIntOrNull()
-
                 val chapterTitle = if (
                     dateText != null &&
                     rawTitle.contains(dateText)
@@ -472,15 +440,6 @@ replace_exact(
                     rawTitle
                 }
 
-                val chapterUrl = if (
-                    pageCount != null &&
-                    pageCount > 0
-                ) {
-                    "$href#niadd-pages=$pageCount"
-                } else {
-                    href
-                }
-
                 MangaChapter(
                     id = generateUid(href),
                     title = chapterTitle.takeIf {
@@ -488,7 +447,7 @@ replace_exact(
                     },
                     number = i + 1f,
                     volume = 0,
-                    url = chapterUrl,
+                    url = href,
                     uploadDate = parseChapterDateByLang(
                         dateText.orEmpty()
                     ),
@@ -641,42 +600,78 @@ replace_exact(
     '''    override suspend fun getPages(
         chapter: MangaChapter,
     ): List<MangaPage> {
+        val chapterUrl = chapter.url
+            .substringBefore("#")
+
+        val doc = webClient
+            .httpGet(
+                chapterUrl.toAbsoluteUrl(domain)
+            )
+            .parseHtml()
+
         if (domain.endsWith("niadd.com")) {
-            val originalUrl = chapter.url
-                .substringBefore("#")
-                .substringBefore("?")
+            val pageLabelRegex = Regex(
+                """^\\s*(\\d+)\\s*/\\s*(\\d+)\\s*$"""
+            )
 
-            val totalPages = chapter.url
-                .substringAfter(
-                    "#niadd-pages=",
-                    "",
-                )
-                .substringBefore("&")
-                .toIntOrNull()
-                ?.takeIf { it > 0 }
-                ?: error(
-                    "Niadd page count is missing for ${chapter.url}. " +
-                        "Refresh manga details and try again."
-                )
+            val pageOptions = doc
+                .select("option[value]")
+                .mapNotNull { option ->
+                    val match = pageLabelRegex.matchEntire(
+                        option.text().trim()
+                    ) ?: return@mapNotNull null
 
-            val normalizedUrl = originalUrl
-                .removeSuffix("/")
+                    val pageNumber = match
+                        .groupValues
+                        .getOrNull(1)
+                        ?.toIntOrNull()
+                        ?: return@mapNotNull null
 
-            val baseUrl = if (
-                normalizedUrl.endsWith(".html")
-            ) {
-                normalizedUrl.removeSuffix(".html")
-            } else {
-                normalizedUrl
-            }
+                    val totalPages = match
+                        .groupValues
+                        .getOrNull(2)
+                        ?.toIntOrNull()
+                        ?: return@mapNotNull null
 
-            return (1..totalPages).map { pageNumber ->
-                val url = if (pageNumber == 1) {
-                    originalUrl
-                } else {
-                    "$baseUrl-$pageNumber.html"
+                    val url = option.attr("value").trim()
+                    if (url.isEmpty()) {
+                        return@mapNotNull null
+                    }
+
+                    Triple(pageNumber, totalPages, url)
+                }
+                .distinctBy { (pageNumber, _, _) ->
+                    pageNumber
+                }
+                .sortedBy { (pageNumber, _, _) ->
+                    pageNumber
                 }
 
+            val totalPages = pageOptions
+                .firstOrNull()
+                ?.second
+                ?: error(
+                    "Cannot find Niadd page selector for $chapterUrl"
+                )
+
+            check(
+                pageOptions.all { (_, total, _) ->
+                    total == totalPages
+                }
+            ) {
+                "Inconsistent Niadd page count for $chapterUrl"
+            }
+
+            check(
+                pageOptions.size == totalPages &&
+                    pageOptions.map { it.first } ==
+                    (1..totalPages).toList()
+            ) {
+                "Incomplete Niadd page selector for $chapterUrl: " +
+                    "found ${pageOptions.size} of $totalPages pages"
+            }
+
+            return pageOptions.map { (_, _, url) ->
                 MangaPage(
                     id = generateUid(url),
                     url = url,
@@ -686,12 +681,6 @@ replace_exact(
             }
         }
 
-        val doc = webClient
-            .httpGet(
-                chapter.url.toAbsoluteUrl(domain)
-            )
-            .parseHtml()
-
         return doc
             .body()
             .requireElementById("page")
@@ -767,27 +756,9 @@ replace_exact(
                 )
             }
 
-            /*
-             * Fallback in case Niadd removes the "N of M"
-             * image link but leaves the actual image element.
-             */
-            val imageUrlRegex = Regex(
-                """(?i)\\.(?:jpe?g|png|gif|webp|avif)(?:\\?|$)"""
-            )
-
-            val image = root
-                .select("img[src]")
-                .firstOrNull { element ->
-                    val src = element.attr("src")
-
-                    imageUrlRegex.containsMatchIn(src) &&
-                        !src.contains("logo", ignoreCase = true) &&
-                        !src.contains("avatar", ignoreCase = true)
-                }
-
-            if (image != null) {
-                return image.attrAsAbsoluteUrl("src")
-            }
+            // Do not fall back to an arbitrary <img>. Niadd may return a
+            // service/notice page with a perfectly valid static image, which
+            // would otherwise be cached as every manga page.
 
             error(
                 "Cannot find Niadd reader image for ${page.url}"