diff --git a/app/src/main/kotlin/io/github/landwarderer/futon/details/ui/pager/pages/MangaPageFetcher.kt b/app/src/main/kotlin/io/github/landwarderer/futon/details/ui/pager/pages/MangaPageFetcher.kt index ae55a0f..9f4338b 100644 --- a/app/src/main/kotlin/io/github/landwarderer/futon/details/ui/pager/pages/MangaPageFetcher.kt +++ b/app/src/main/kotlin/io/github/landwarderer/futon/details/ui/pager/pages/MangaPageFetcher.kt @@ -15,21 +15,26 @@ import coil3.request.Options import okhttp3.Headers import okhttp3.OkHttpClient import okhttp3.Response +import okio.Buffer import okio.FileSystem import okio.Path.Companion.toOkioPath import io.github.landwarderer.futon.core.network.MangaHttpClient import io.github.landwarderer.futon.core.network.imageproxy.ImageProxyInterceptor import io.github.landwarderer.futon.core.parser.MangaRepository import io.github.landwarderer.futon.core.util.MimeTypes +import io.github.landwarderer.futon.core.util.ext.MimeType import io.github.landwarderer.futon.core.util.ext.fetch import io.github.landwarderer.futon.core.util.ext.isNetworkUri import io.github.landwarderer.futon.core.util.ext.toMimeTypeOrNull import io.github.landwarderer.futon.local.data.LocalStorageCache import io.github.landwarderer.futon.local.data.PageCache import org.koitharu.kotatsu.parsers.model.MangaPage +import org.koitharu.kotatsu.parsers.model.MangaParserSource import org.koitharu.kotatsu.parsers.util.mimeType import org.koitharu.kotatsu.parsers.util.requireBody import org.koitharu.kotatsu.parsers.util.runCatchingCancellable +import kotlinx.coroutines.sync.Semaphore +import kotlinx.coroutines.sync.withPermit import io.github.landwarderer.futon.reader.domain.PageLoader import javax.inject.Inject @@ -43,6 +48,11 @@ class MangaPageFetcher( private val imageLoader: ImageLoader, ) : Fetcher { + private val niaddOkHttpClient = okHttpClient.newBuilder() + .followRedirects(false) + .followSslRedirects(false) + .build() + override suspend fun fetch(): FetchResult? { if (!page.preview.isNullOrEmpty()) { runCatchingCancellable { @@ -72,6 +82,12 @@ class MangaPageFetcher( } private suspend fun fetchPage(pageUrl: String): FetchResult { + if (page.source == MangaParserSource.NINEMANGA_RU) { + return niaddSemaphore.withPermit { + fetchNiaddPageByRanges(pageUrl) + } + } + val request = PageLoader.createPageRequest(pageUrl, page.source) return imageProxyInterceptor.interceptPageRequest(request, okHttpClient).use { response -> if (!response.isSuccessful) { @@ -89,6 +105,99 @@ class MangaPageFetcher( } } + private suspend fun fetchNiaddPageByRanges(pageUrl: String): FetchResult { + val buffer = Buffer() + var mimeType: MimeType? = null + var offset = 0L + var totalBytes = -1L + + while (totalBytes < 0L || offset < totalBytes) { + val requestedEnd = if (totalBytes > 0L) { + minOf(offset + NIADD_RANGE_CHUNK_SIZE - 1L, totalBytes - 1L) + } else { + offset + NIADD_RANGE_CHUNK_SIZE - 1L + } + val request = PageLoader.createPageRequest(pageUrl, page.source) + .newBuilder() + .header("Range", "bytes=$offset-$requestedEnd") + .header("Connection", "close") + .build() + + niaddOkHttpClient.newCall(request).execute().use { response -> + check(response.code == 206) { + val location = response.header("Location") + "Niadd CDN range request returned HTTP ${response.code}" + + if (location.isNullOrBlank()) { + " for $pageUrl" + } else { + ", Location=$location for $pageUrl" + } + } + + val contentType = response.header("Content-Type") + ?.substringBefore(';') + ?.trim() + .orEmpty() + check(contentType.startsWith("image/", ignoreCase = true)) { + "Niadd CDN returned non-image Content-Type '$contentType' for $pageUrl" + } + if (pageUrl.substringBefore('?').endsWith(".webp", ignoreCase = true)) { + check(contentType.equals("image/webp", ignoreCase = true)) { + "Niadd CDN replaced WebP with '$contentType' for $pageUrl" + } + } + + val contentRange = response.header("Content-Range") + ?: error("Niadd CDN response has no Content-Range for $pageUrl") + val rangeMatch = NIADD_CONTENT_RANGE_REGEX.matchEntire(contentRange) + ?: error("Invalid Niadd Content-Range: $contentRange") + val responseStart = rangeMatch.groupValues[1].toLong() + val responseEnd = rangeMatch.groupValues[2].toLong() + val responseTotal = rangeMatch.groupValues[3].toLong() + check(responseTotal > 0L && responseStart == offset) { + "Unexpected Niadd Content-Range '$contentRange' for offset $offset ($pageUrl)" + } + + if (totalBytes < 0L) { + totalBytes = responseTotal + } else { + check(totalBytes == responseTotal) { + "Niadd CDN changed content size from $totalBytes to $responseTotal for $pageUrl" + } + } + + val expectedEnd = minOf(requestedEnd, totalBytes - 1L) + check(responseEnd == expectedEnd) { + "Unexpected Niadd range end: expected $expectedEnd, got $responseEnd " + + "($contentRange, $pageUrl)" + } + if (mimeType == null) { + mimeType = response.mimeType?.toMimeTypeOrNull() + } + + val expectedBytes = expectedEnd - offset + 1L + val receivedBytes = buffer.writeAll(response.requireBody().source()) + check(receivedBytes == expectedBytes) { + "Short Niadd range: expected $expectedBytes bytes, got $receivedBytes " + + "for bytes=$offset-$expectedEnd ($pageUrl)" + } + + offset += receivedBytes + } + } + + check(totalBytes > 0L && buffer.size == totalBytes) { + "Incomplete Niadd image: expected $totalBytes bytes, got ${buffer.size} for $pageUrl" + } + + val file = pagesCache.set(pageUrl, buffer, mimeType) + return SourceFetchResult( + source = ImageSource(file.toOkioPath(), FileSystem.SYSTEM), + mimeType = mimeType?.toString(), + dataSource = DataSource.NETWORK, + ) + } + private fun Response.toNetworkResponse() = NetworkResponse( code = code, requestMillis = sentRequestAtMillis, @@ -106,6 +215,12 @@ class MangaPageFetcher( return headers.build() } + companion object { + private const val NIADD_RANGE_CHUNK_SIZE = 16L * 1024L + private val NIADD_CONTENT_RANGE_REGEX = Regex("""bytes (\d+)-(\d+)/(\d+)""") + private val niaddSemaphore = Semaphore(1) + } + class Factory @Inject constructor( @MangaHttpClient private val okHttpClient: OkHttpClient, @PageCache private val pagesCache: LocalStorageCache, diff --git a/app/src/main/kotlin/io/github/landwarderer/futon/reader/domain/PageLoader.kt b/app/src/main/kotlin/io/github/landwarderer/futon/reader/domain/PageLoader.kt index 18d203d..02b1653 100644 --- a/app/src/main/kotlin/io/github/landwarderer/futon/reader/domain/PageLoader.kt +++ b/app/src/main/kotlin/io/github/landwarderer/futon/reader/domain/PageLoader.kt @@ -30,6 +30,7 @@ import io.github.landwarderer.futon.core.prefs.AppSettings import io.github.landwarderer.futon.core.ui.image.TrimTransformation import io.github.landwarderer.futon.core.util.FileSize import io.github.landwarderer.futon.core.util.MimeTypes +import io.github.landwarderer.futon.core.util.ext.MimeType import io.github.landwarderer.futon.core.util.ext.URI_SCHEME_ZIP import io.github.landwarderer.futon.core.util.ext.cancelChildrenAndJoin import io.github.landwarderer.futon.core.util.ext.compressToPNG @@ -52,6 +53,7 @@ import io.github.landwarderer.futon.download.ui.worker.DownloadSlowdownDispatche import io.github.landwarderer.futon.local.data.LocalStorageCache import io.github.landwarderer.futon.local.data.PageCache import org.koitharu.kotatsu.parsers.model.MangaPage +import org.koitharu.kotatsu.parsers.model.MangaParserSource import org.koitharu.kotatsu.parsers.model.MangaSource import org.koitharu.kotatsu.parsers.util.requireBody import org.koitharu.kotatsu.parsers.util.runCatchingCancellable @@ -70,6 +72,9 @@ import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.sync.withPermit import okhttp3.OkHttpClient import okhttp3.Request +import okio.buffer +import okio.sink +import okio.source import okio.use import org.jetbrains.annotations.Blocking import java.io.File @@ -97,6 +102,11 @@ class PageLoader @Inject constructor( private val tasks = LongSparseArray>() private val semaphore = Semaphore(3) + private val niaddSemaphore = Semaphore(1) + private val niaddOkHttp = okHttp.newBuilder() + .followRedirects(false) + .followSslRedirects(false) + .build() private val convertLock = Mutex() private val prefetchLock = Mutex() @@ -303,13 +313,126 @@ class PageLoader @Inject constructor( if (isPrefetch) { downloadSlowdownDispatcher.delay(page.source) } - val request = createPageRequest(pageUrl, page.source) - imageProxyInterceptor.interceptPageRequest(request, okHttp).ensureSuccess().use { response -> - response.requireBody().withProgress(progress).use { - cache.set(pageUrl, it.source(), it.contentType()?.toMimeType()) + if (page.source == MangaParserSource.NINEMANGA_RU) { + niaddSemaphore.withPermit { + downloadNiaddPageByRanges(pageUrl, page.source, progress) } - }.toUri() + } else { + val request = createPageRequest(pageUrl, page.source) + imageProxyInterceptor.interceptPageRequest(request, okHttp).ensureSuccess().use { response -> + response.requireBody().withProgress(progress).use { + cache.set(pageUrl, it.source(), it.contentType()?.toMimeType()) + } + }.toUri() + } + } + } + } + + private suspend fun downloadNiaddPageByRanges( + pageUrl: String, + source: MangaSource, + progress: MutableStateFlow, + ): Uri { + val tempFile = File.createTempFile( + "niadd-page-", + ".part", + context.cacheDir, + ) + var mimeType: MimeType? = null + try { + var offset = 0L + var totalBytes = -1L + + tempFile.sink(append = false).buffer().use { sink -> + while (totalBytes < 0L || offset < totalBytes) { + val requestedEnd = if (totalBytes > 0L) { + minOf(offset + NIADD_RANGE_CHUNK_SIZE - 1L, totalBytes - 1L) + } else { + offset + NIADD_RANGE_CHUNK_SIZE - 1L + } + val request = createPageRequest(pageUrl, source) + .newBuilder() + .header("Range", "bytes=$offset-$requestedEnd") + .header("Connection", "close") + .build() + + niaddOkHttp.newCall(request).execute().use { response -> + check(response.code == 206) { + val location = response.header("Location") + "Niadd CDN range request returned HTTP ${response.code}" + + if (location.isNullOrBlank()) { + " for $pageUrl" + } else { + ", Location=$location for $pageUrl" + } + } + + val contentType = response.header("Content-Type") + ?.substringBefore(';') + ?.trim() + .orEmpty() + check(contentType.startsWith("image/", ignoreCase = true)) { + "Niadd CDN returned non-image Content-Type '$contentType' for $pageUrl" + } + if (pageUrl.substringBefore('?').endsWith(".webp", ignoreCase = true)) { + check(contentType.equals("image/webp", ignoreCase = true)) { + "Niadd CDN replaced WebP with '$contentType' for $pageUrl" + } + } + + val contentRange = response.header("Content-Range") + ?: error("Niadd CDN response has no Content-Range for $pageUrl") + val rangeMatch = NIADD_CONTENT_RANGE_REGEX.matchEntire(contentRange) + ?: error("Invalid Niadd Content-Range: $contentRange") + val responseStart = rangeMatch.groupValues[1].toLong() + val responseEnd = rangeMatch.groupValues[2].toLong() + val responseTotal = rangeMatch.groupValues[3].toLong() + check(responseTotal > 0L && responseStart == offset) { + "Unexpected Niadd Content-Range '$contentRange' for offset $offset ($pageUrl)" + } + + if (totalBytes < 0L) { + totalBytes = responseTotal + } else { + check(totalBytes == responseTotal) { + "Niadd CDN changed content size from $totalBytes to $responseTotal for $pageUrl" + } + } + + val expectedEnd = minOf(requestedEnd, totalBytes - 1L) + check(responseEnd == expectedEnd) { + "Unexpected Niadd range end: expected $expectedEnd, got $responseEnd " + + "($contentRange, $pageUrl)" + } + val body = response.requireBody() + if (mimeType == null) { + mimeType = body.contentType()?.toMimeType() + } + + val expectedBytes = expectedEnd - offset + 1L + val receivedBytes = sink.writeAll(body.source()) + check(receivedBytes == expectedBytes) { + "Short Niadd range: expected $expectedBytes bytes, got $receivedBytes " + + "for bytes=$offset-$expectedEnd ($pageUrl)" + } + + offset += receivedBytes + progress.value = (offset.toFloat() / totalBytes.toFloat()) + .coerceIn(0f, 1f) + } + } + } + + check(totalBytes > 0L && tempFile.length() == totalBytes) { + "Incomplete Niadd image: expected $totalBytes bytes, got ${tempFile.length()} for $pageUrl" + } + + return tempFile.source().use { sourceStream -> + cache.set(pageUrl, sourceStream, mimeType).toUri() } + } finally { + tempFile.delete() } } @@ -340,6 +463,8 @@ class PageLoader @Inject constructor( companion object { private const val PROGRESS_UNDEFINED = -1f + private const val NIADD_RANGE_CHUNK_SIZE = 16L * 1024L + private val NIADD_CONTENT_RANGE_REGEX = Regex("""bytes (\d+)-(\d+)/(\d+)""") private const val PREFETCH_LIMIT_DEFAULT = 6 private const val PREFETCH_MIN_RAM_MB = 80L