v5.1

patches/futon-niadd-range-download.patch

@@ -1,5 +1,5 @@
 diff --git a/app/src/main/kotlin/io/github/landwarderer/futon/details/ui/pager/pages/MangaPageFetcher.kt b/app/src/main/kotlin/io/github/landwarderer/futon/details/ui/pager/pages/MangaPageFetcher.kt
-index ae55a0f..ddadcc1 100644
+index ae55a0f..9f4338b 100644
 --- a/app/src/main/kotlin/io/github/landwarderer/futon/details/ui/pager/pages/MangaPageFetcher.kt
 +++ b/app/src/main/kotlin/io/github/landwarderer/futon/details/ui/pager/pages/MangaPageFetcher.kt
 @@ -15,21 +15,26 @@ import coil3.request.Options
@@ -29,7 +29,19 @@ index ae55a0f..ddadcc1 100644
  import io.github.landwarderer.futon.reader.domain.PageLoader
  import javax.inject.Inject
  
-@@ -72,6 +77,12 @@ class MangaPageFetcher(
+@@ -43,6 +48,11 @@ class MangaPageFetcher(
+ 	private val imageLoader: ImageLoader,
+ ) : Fetcher {
+ 
++	private val niaddOkHttpClient = okHttpClient.newBuilder()
++		.followRedirects(false)
++		.followSslRedirects(false)
++		.build()
++
+ 	override suspend fun fetch(): FetchResult? {
+ 		if (!page.preview.isNullOrEmpty()) {
+ 			runCatchingCancellable {
+@@ -72,6 +82,12 @@ class MangaPageFetcher(
  	}
  
  	private suspend fun fetchPage(pageUrl: String): FetchResult {
@@ -42,7 +54,7 @@ index ae55a0f..ddadcc1 100644
  		val request = PageLoader.createPageRequest(pageUrl, page.source)
  		return imageProxyInterceptor.interceptPageRequest(request, okHttpClient).use { response ->
  			if (!response.isSuccessful) {
-@@ -89,6 +100,73 @@ class MangaPageFetcher(
+@@ -89,6 +105,99 @@ class MangaPageFetcher(
  		}
  	}
  
@@ -64,18 +76,40 @@ index ae55a0f..ddadcc1 100644
 +				.header("Connection", "close")
 +				.build()
 +
-+			okHttpClient.newCall(request).execute().use { response ->
++			niaddOkHttpClient.newCall(request).execute().use { response ->
 +				check(response.code == 206) {
-+					"Niadd CDN ignored range request: HTTP ${response.code} for $pageUrl"
++					val location = response.header("Location")
++					"Niadd CDN range request returned HTTP ${response.code}" +
++						if (location.isNullOrBlank()) {
++							" for $pageUrl"
++						} else {
++							", Location=$location for $pageUrl"
++						}
++				}
++
++				val contentType = response.header("Content-Type")
++					?.substringBefore(';')
++					?.trim()
++					.orEmpty()
++				check(contentType.startsWith("image/", ignoreCase = true)) {
++					"Niadd CDN returned non-image Content-Type '$contentType' for $pageUrl"
++				}
++				if (pageUrl.substringBefore('?').endsWith(".webp", ignoreCase = true)) {
++					check(contentType.equals("image/webp", ignoreCase = true)) {
++						"Niadd CDN replaced WebP with '$contentType' for $pageUrl"
++					}
 +				}
 +
 +				val contentRange = response.header("Content-Range")
 +					?: error("Niadd CDN response has no Content-Range for $pageUrl")
-+				val responseTotal = contentRange
++				val rangeMatch = NIADD_CONTENT_RANGE_REGEX.matchEntire(contentRange)
-+					.substringAfterLast('/', "")
-+					.toLongOrNull()
-+					?.takeIf { it > 0L }
 +					?: error("Invalid Niadd Content-Range: $contentRange")
++				val responseStart = rangeMatch.groupValues[1].toLong()
++				val responseEnd = rangeMatch.groupValues[2].toLong()
++				val responseTotal = rangeMatch.groupValues[3].toLong()
++				check(responseTotal > 0L && responseStart == offset) {
++					"Unexpected Niadd Content-Range '$contentRange' for offset $offset ($pageUrl)"
++				}
 +
 +				if (totalBytes < 0L) {
 +					totalBytes = responseTotal
@@ -85,11 +119,15 @@ index ae55a0f..ddadcc1 100644
 +					}
 +				}
 +
++				val expectedEnd = minOf(requestedEnd, totalBytes - 1L)
++				check(responseEnd == expectedEnd) {
++					"Unexpected Niadd range end: expected $expectedEnd, got $responseEnd " +
++						"($contentRange, $pageUrl)"
++				}
 +				if (mimeType == null) {
 +					mimeType = response.mimeType?.toMimeTypeOrNull()
 +				}
 +
-+				val expectedEnd = minOf(requestedEnd, totalBytes - 1L)
 +				val expectedBytes = expectedEnd - offset + 1L
 +				val receivedBytes = buffer.writeAll(response.requireBody().source())
 +				check(receivedBytes == expectedBytes) {
@@ -116,12 +154,13 @@ index ae55a0f..ddadcc1 100644
  	private fun Response.toNetworkResponse() = NetworkResponse(
  		code = code,
  		requestMillis = sentRequestAtMillis,
-@@ -106,6 +184,11 @@ class MangaPageFetcher(
+@@ -106,6 +215,12 @@ class MangaPageFetcher(
  		return headers.build()
  	}
  
 +	companion object {
 +		private const val NIADD_RANGE_CHUNK_SIZE = 16L * 1024L
++		private val NIADD_CONTENT_RANGE_REGEX = Regex("""bytes (\d+)-(\d+)/(\d+)""")
 +		private val niaddSemaphore = Semaphore(1)
 +	}
 +
@@ -129,7 +168,7 @@ index ae55a0f..ddadcc1 100644
  		@MangaHttpClient private val okHttpClient: OkHttpClient,
  		@PageCache private val pagesCache: LocalStorageCache,
 diff --git a/app/src/main/kotlin/io/github/landwarderer/futon/reader/domain/PageLoader.kt b/app/src/main/kotlin/io/github/landwarderer/futon/reader/domain/PageLoader.kt
-index 18d203d..6109efa 100644
+index 18d203d..02b1653 100644
 --- a/app/src/main/kotlin/io/github/landwarderer/futon/reader/domain/PageLoader.kt
 +++ b/app/src/main/kotlin/io/github/landwarderer/futon/reader/domain/PageLoader.kt
 @@ -30,6 +30,7 @@ import io.github.landwarderer.futon.core.prefs.AppSettings
@@ -158,15 +197,19 @@ index 18d203d..6109efa 100644
  import okio.use
  import org.jetbrains.annotations.Blocking
  import java.io.File
-@@ -97,6 +102,7 @@ class PageLoader @Inject constructor(
+@@ -97,6 +102,11 @@ class PageLoader @Inject constructor(
  
  	private val tasks = LongSparseArray<ProgressDeferred<Uri, Float>>()
  	private val semaphore = Semaphore(3)
 +	private val niaddSemaphore = Semaphore(1)
++	private val niaddOkHttp = okHttp.newBuilder()
++		.followRedirects(false)
++		.followSslRedirects(false)
++		.build()
  	private val convertLock = Mutex()
  	private val prefetchLock = Mutex()
  
-@@ -303,13 +309,100 @@ class PageLoader @Inject constructor(
+@@ -303,13 +313,126 @@ class PageLoader @Inject constructor(
  				if (isPrefetch) {
  					downloadSlowdownDispatcher.delay(page.source)
  				}
@@ -219,18 +262,40 @@ index 18d203d..6109efa 100644
 +						.header("Connection", "close")
 +						.build()
 +
-+					okHttp.newCall(request).execute().use { response ->
++					niaddOkHttp.newCall(request).execute().use { response ->
 +						check(response.code == 206) {
-+							"Niadd CDN ignored range request: HTTP ${response.code} for $pageUrl"
++							val location = response.header("Location")
++							"Niadd CDN range request returned HTTP ${response.code}" +
++								if (location.isNullOrBlank()) {
++									" for $pageUrl"
++								} else {
++									", Location=$location for $pageUrl"
++								}
++						}
++
++						val contentType = response.header("Content-Type")
++							?.substringBefore(';')
++							?.trim()
++							.orEmpty()
++						check(contentType.startsWith("image/", ignoreCase = true)) {
++							"Niadd CDN returned non-image Content-Type '$contentType' for $pageUrl"
++						}
++						if (pageUrl.substringBefore('?').endsWith(".webp", ignoreCase = true)) {
++							check(contentType.equals("image/webp", ignoreCase = true)) {
++								"Niadd CDN replaced WebP with '$contentType' for $pageUrl"
++							}
 +						}
 +
 +						val contentRange = response.header("Content-Range")
 +							?: error("Niadd CDN response has no Content-Range for $pageUrl")
-+						val responseTotal = contentRange
++						val rangeMatch = NIADD_CONTENT_RANGE_REGEX.matchEntire(contentRange)
-+							.substringAfterLast('/', "")
-+							.toLongOrNull()
-+							?.takeIf { it > 0L }
 +							?: error("Invalid Niadd Content-Range: $contentRange")
++						val responseStart = rangeMatch.groupValues[1].toLong()
++						val responseEnd = rangeMatch.groupValues[2].toLong()
++						val responseTotal = rangeMatch.groupValues[3].toLong()
++						check(responseTotal > 0L && responseStart == offset) {
++							"Unexpected Niadd Content-Range '$contentRange' for offset $offset ($pageUrl)"
++						}
 +
 +						if (totalBytes < 0L) {
 +							totalBytes = responseTotal
@@ -240,12 +305,16 @@ index 18d203d..6109efa 100644
 +							}
 +						}
 +
++						val expectedEnd = minOf(requestedEnd, totalBytes - 1L)
++						check(responseEnd == expectedEnd) {
++							"Unexpected Niadd range end: expected $expectedEnd, got $responseEnd " +
++								"($contentRange, $pageUrl)"
++						}
 +						val body = response.requireBody()
 +						if (mimeType == null) {
 +							mimeType = body.contentType()?.toMimeType()
 +						}
 +
-+						val expectedEnd = minOf(requestedEnd, totalBytes - 1L)
 +						val expectedBytes = expectedEnd - offset + 1L
 +						val receivedBytes = sink.writeAll(body.source())
 +						check(receivedBytes == expectedBytes) {
@@ -272,11 +341,12 @@ index 18d203d..6109efa 100644
  		}
  	}
  
-@@ -340,6 +433,7 @@ class PageLoader @Inject constructor(
+@@ -340,6 +463,8 @@ class PageLoader @Inject constructor(
  	companion object {
  
  		private const val PROGRESS_UNDEFINED = -1f
 +		private const val NIADD_RANGE_CHUNK_SIZE = 16L * 1024L
++		private val NIADD_CONTENT_RANGE_REGEX = Regex("""bytes (\d+)-(\d+)/(\d+)""")
  		private const val PREFETCH_LIMIT_DEFAULT = 6
  		private const val PREFETCH_MIN_RAM_MB = 80L
  

scripts/patch_ninemanga_niadd.py

@@ -425,38 +425,6 @@ replace_exact(
 
                 val dateText = anchorDate ?: rowDate
 
-                val metadataText = when {
-                    anchorDate != null -> anchorText
-                    rowDate != null -> rowText
-                    else -> ""
-                }
-
-                val textBeforeDate = if (dateText != null) {
-                    metadataText
-                        .substringBefore(dateText)
-                        .trim()
-                } else {
-                    metadataText
-                }
-
-                /*
-                 * Niadd exposes the page count immediately before
-                 * the chapter date, for example:
-                 *
-                 *   Падение русала 1 - 42 41 Aug 02, 2024
-                 *                           ^^
-                 *
-                 * Store that number in a URL fragment. Fragments
-                 * are local metadata and are not sent to the server.
-                 */
-                val pageCount = Regex(
-                    """\\s+(\\d+)\\s*$"""
-                )
-                    .find(textBeforeDate)
-                    ?.groupValues
-                    ?.getOrNull(1)
-                    ?.toIntOrNull()
-
                 val chapterTitle = if (
                     dateText != null &&
                     rawTitle.contains(dateText)
@@ -472,15 +440,6 @@ replace_exact(
                     rawTitle
                 }
 
-                val chapterUrl = if (
-                    pageCount != null &&
-                    pageCount > 0
-                ) {
-                    "$href#niadd-pages=$pageCount"
-                } else {
-                    href
-                }
-
                 MangaChapter(
                     id = generateUid(href),
                     title = chapterTitle.takeIf {
@@ -488,7 +447,7 @@ replace_exact(
                     },
                     number = i + 1f,
                     volume = 0,
-                    url = chapterUrl,
+                    url = href,
                     uploadDate = parseChapterDateByLang(
                         dateText.orEmpty()
                     ),
@@ -641,42 +600,78 @@ replace_exact(
     '''    override suspend fun getPages(
         chapter: MangaChapter,
     ): List<MangaPage> {
+        val chapterUrl = chapter.url
+            .substringBefore("#")
+
+        val doc = webClient
+            .httpGet(
+                chapterUrl.toAbsoluteUrl(domain)
+            )
+            .parseHtml()
+
         if (domain.endsWith("niadd.com")) {
-            val originalUrl = chapter.url
+            val pageLabelRegex = Regex(
-                .substringBefore("#")
+                """^\\s*(\\d+)\\s*/\\s*(\\d+)\\s*$"""
-                .substringBefore("?")
+            )
 
-            val totalPages = chapter.url
+            val pageOptions = doc
-                .substringAfter(
+                .select("option[value]")
-                    "#niadd-pages=",
+                .mapNotNull { option ->
-                    "",
+                    val match = pageLabelRegex.matchEntire(
-                )
+                        option.text().trim()
-                .substringBefore("&")
+                    ) ?: return@mapNotNull null
-                .toIntOrNull()
-                ?.takeIf { it > 0 }
-                ?: error(
-                    "Niadd page count is missing for ${chapter.url}. " +
-                        "Refresh manga details and try again."
-                )
 
-            val normalizedUrl = originalUrl
+                    val pageNumber = match
-                .removeSuffix("/")
+                        .groupValues
+                        .getOrNull(1)
+                        ?.toIntOrNull()
+                        ?: return@mapNotNull null
 
-            val baseUrl = if (
+                    val totalPages = match
-                normalizedUrl.endsWith(".html")
+                        .groupValues
-            ) {
+                        .getOrNull(2)
-                normalizedUrl.removeSuffix(".html")
+                        ?.toIntOrNull()
-            } else {
+                        ?: return@mapNotNull null
-                normalizedUrl
-            }
 
-            return (1..totalPages).map { pageNumber ->
+                    val url = option.attr("value").trim()
-                val url = if (pageNumber == 1) {
+                    if (url.isEmpty()) {
-                    originalUrl
+                        return@mapNotNull null
-                } else {
+                    }
-                    "$baseUrl-$pageNumber.html"
+
+                    Triple(pageNumber, totalPages, url)
+                }
+                .distinctBy { (pageNumber, _, _) ->
+                    pageNumber
+                }
+                .sortedBy { (pageNumber, _, _) ->
+                    pageNumber
                 }
 
+            val totalPages = pageOptions
+                .firstOrNull()
+                ?.second
+                ?: error(
+                    "Cannot find Niadd page selector for $chapterUrl"
+                )
+
+            check(
+                pageOptions.all { (_, total, _) ->
+                    total == totalPages
+                }
+            ) {
+                "Inconsistent Niadd page count for $chapterUrl"
+            }
+
+            check(
+                pageOptions.size == totalPages &&
+                    pageOptions.map { it.first } ==
+                    (1..totalPages).toList()
+            ) {
+                "Incomplete Niadd page selector for $chapterUrl: " +
+                    "found ${pageOptions.size} of $totalPages pages"
+            }
+
+            return pageOptions.map { (_, _, url) ->
                 MangaPage(
                     id = generateUid(url),
                     url = url,
@@ -686,12 +681,6 @@ replace_exact(
             }
         }
 
-        val doc = webClient
-            .httpGet(
-                chapter.url.toAbsoluteUrl(domain)
-            )
-            .parseHtml()
-
         return doc
             .body()
             .requireElementById("page")
@@ -767,27 +756,9 @@ replace_exact(
                 )
             }
 
-            /*
+            // Do not fall back to an arbitrary <img>. Niadd may return a
-             * Fallback in case Niadd removes the "N of M"
+            // service/notice page with a perfectly valid static image, which
-             * image link but leaves the actual image element.
+            // would otherwise be cached as every manga page.
-             */
-            val imageUrlRegex = Regex(
-                """(?i)\\.(?:jpe?g|png|gif|webp|avif)(?:\\?|$)"""
-            )
-
-            val image = root
-                .select("img[src]")
-                .firstOrNull { element ->
-                    val src = element.attr("src")
-
-                    imageUrlRegex.containsMatchIn(src) &&
-                        !src.contains("logo", ignoreCase = true) &&
-                        !src.contains("avatar", ignoreCase = true)
-                }
-
-            if (image != null) {
-                return image.attrAsAbsoluteUrl("src")
-            }
 
             error(
                 "Cannot find Niadd reader image for ${page.url}"