13 Commits
Author SHA1 Message Date
gandc 15c05f6bcd Fix Niadd range downloads in DownloadWorker
Build Futon with Niadd patch / Build Futon Niadd debug APK (push) Successful in 3m49s
2026-08-11 23:00:35 +03:00
gandc 9ba7eed81a v5.1
Build Futon with Niadd patch / Build Futon Niadd debug APK (push) Successful in 5m24s
2026-08-11 22:38:20 +03:00
gandc bbccd48bd6 v5
Build Futon with Niadd patch / Build Futon Niadd debug APK (push) Successful in 5m40s
2026-08-11 22:15:12 +03:00
gandc 7c63edc664 fix patch
Build Futon with Niadd patch / Build Futon Niadd debug APK (push) Successful in 3m54s
2026-08-10 01:51:37 +03:00
gandc 83e7c698e1 fix SocketTimeoutException
Build Futon with Niadd patch / Build Futon Niadd debug APK (push) Failing after 26s
2026-08-10 01:48:15 +03:00
gandc d5f6d20e1f fix
Build Futon with Niadd patch / Build Futon Niadd debug APK (push) Successful in 4m3s
2026-08-10 01:35:30 +03:00
gandc 008379a09a fix
Build Futon with Niadd patch / Build Futon Niadd debug APK (push) Failing after 10m25s
2026-08-10 01:22:32 +03:00
gandc 897874169d fix 2026-08-10 00:39:10 +03:00
gandc ffeaa412e8 fix pagecount
Build Futon with Niadd patch / Build Futon Niadd debug APK (push) Successful in 13m49s
2026-08-10 00:18:47 +03:00
gandc 24d71d9407 fix manga loading
Build Futon with Niadd patch / Build Futon Niadd debug APK (push) Successful in 14m25s
2026-08-09 23:52:18 +03:00
gandc 5f6c8cf69e Return type mismatch
Build Futon with Niadd patch / Build Futon Niadd debug APK (push) Successful in 13m27s
2026-08-09 23:20:09 +03:00
gandc 9e6d724865 fix ninemanga page structure parse
Build Futon with Niadd patch / Build Futon Niadd debug APK (push) Failing after 10m18s
2026-08-09 23:07:20 +03:00
gandc bdb8b68c3e fix ul.direlist
Build Futon with Niadd patch / Build Futon Niadd debug APK (push) Successful in 12m49s
2026-08-09 22:45:36 +03:00
3 changed files with 1306 additions and 29 deletions

No files matched your search

+34 -22
View File
@@ -7,6 +7,7 @@ on:
paths: paths:
- ".gitea/workflows/build.yml" - ".gitea/workflows/build.yml"
- "scripts/**" - "scripts/**"
- "patches/**"
workflow_dispatch: workflow_dispatch:
env: env:
@@ -22,39 +23,24 @@ env:
jobs: jobs:
build: build:
name: Build Futon Niadd debug APK name: Build Futon Niadd debug APK
runs-on: ubuntu-latest runs-on: android-futon
steps: steps:
- name: Checkout CI repository - name: Checkout CI repository
uses: https://github.com/actions/checkout@v4 uses: https://github.com/actions/checkout@v4
- name: Set up JDK 17 - name: Verify Android build environment
uses: https://github.com/actions/setup-java@v4
with:
distribution: temurin
java-version: "17"
- name: Set up Android SDK
uses: https://github.com/android-actions/setup-android@v3
- name: Install Android SDK 36
shell: bash shell: bash
run: | run: |
set -euxo pipefail set -euxo pipefail
yes | sdkmanager --licenses >/dev/null 2>&1 || true
sdkmanager \
"platform-tools" \
"platforms;android-${ANDROID_PLATFORM}" \
"build-tools;${ANDROID_BUILD_TOOLS}"
echo "=== Android SDK ===" echo "=== Android SDK ==="
echo "ANDROID_HOME=${ANDROID_HOME}" echo "ANDROID_HOME=${ANDROID_HOME}"
echo "ANDROID_SDK_ROOT=${ANDROID_SDK_ROOT:-}" echo "ANDROID_SDK_ROOT=${ANDROID_SDK_ROOT:-}"
echo "=== Installed build tools ===" echo "=== Installed build tools ==="
ls -la "${ANDROID_HOME}/build-tools/${ANDROID_BUILD_TOOLS}" test -x "${ANDROID_HOME}/build-tools/${ANDROID_BUILD_TOOLS}/aapt"
test -x "${ANDROID_HOME}/build-tools/${ANDROID_BUILD_TOOLS}/aapt2"
echo "=== Java ===" echo "=== Java ==="
java -version java -version
@@ -72,6 +58,7 @@ jobs:
test -f scripts/patch_futon_niadd.py test -f scripts/patch_futon_niadd.py
test -f scripts/patch_ninemanga_niadd.py test -f scripts/patch_ninemanga_niadd.py
test -f patches/futon-niadd-range-download.patch
python3 -m py_compile \ python3 -m py_compile \
scripts/patch_futon_niadd.py \ scripts/patch_futon_niadd.py \
@@ -163,6 +150,32 @@ jobs:
git -C work/Futon diff --check git -C work/Futon diff --check
- name: Apply Niadd range-download workaround
shell: bash
run: |
set -euxo pipefail
patch --dry-run -p1 \
-d work/Futon \
< patches/futon-niadd-range-download.patch
patch -p1 \
-d work/Futon \
< patches/futon-niadd-range-download.patch
PAGE_LOADER="work/Futon/app/src/main/kotlin/io/github/landwarderer/futon/reader/domain/PageLoader.kt"
grep -F 'NIADD_RANGE_CHUNK_SIZE = 16L * 1024L' \
"${PAGE_LOADER}"
grep -F 'MangaParserSource.NINEMANGA_RU' \
"${PAGE_LOADER}"
grep -F '.header("Range", "bytes=$offset-$requestedEnd")' \
"${PAGE_LOADER}"
git -C work/Futon diff --check
- name: Apply Niadd parser modifications - name: Apply Niadd parser modifications
shell: bash shell: bash
run: | run: |
@@ -246,6 +259,7 @@ jobs:
shell: bash shell: bash
working-directory: work/Futon working-directory: work/Futon
env: env:
GRADLE_USER_HOME: /root/.gradle
GRADLE_OPTS: >- GRADLE_OPTS: >-
-Dorg.gradle.daemon=false -Dorg.gradle.daemon=false
-Dorg.gradle.jvmargs=-Xmx4g -Dorg.gradle.jvmargs=-Xmx4g
@@ -254,14 +268,12 @@ jobs:
chmod +x gradlew chmod +x gradlew
echo "=== Gradle version ==="
./gradlew --version
echo "=== Build ===" echo "=== Build ==="
./gradlew \ ./gradlew \
--no-daemon \ --no-daemon \
--stacktrace \ --stacktrace \
--max-workers=6 \
:app:assembleDebug :app:assembleDebug
- name: Verify APK identity - name: Verify APK identity
+517
View File
@@ -0,0 +1,517 @@
diff --git a/app/src/main/kotlin/io/github/landwarderer/futon/details/ui/pager/pages/MangaPageFetcher.kt b/app/src/main/kotlin/io/github/landwarderer/futon/details/ui/pager/pages/MangaPageFetcher.kt
index ae55a0f..9f4338b 100644
--- a/app/src/main/kotlin/io/github/landwarderer/futon/details/ui/pager/pages/MangaPageFetcher.kt
+++ b/app/src/main/kotlin/io/github/landwarderer/futon/details/ui/pager/pages/MangaPageFetcher.kt
@@ -15,21 +15,26 @@ import coil3.request.Options
import okhttp3.Headers
import okhttp3.OkHttpClient
import okhttp3.Response
+import okio.Buffer
import okio.FileSystem
import okio.Path.Companion.toOkioPath
import io.github.landwarderer.futon.core.network.MangaHttpClient
import io.github.landwarderer.futon.core.network.imageproxy.ImageProxyInterceptor
import io.github.landwarderer.futon.core.parser.MangaRepository
import io.github.landwarderer.futon.core.util.MimeTypes
+import io.github.landwarderer.futon.core.util.ext.MimeType
import io.github.landwarderer.futon.core.util.ext.fetch
import io.github.landwarderer.futon.core.util.ext.isNetworkUri
import io.github.landwarderer.futon.core.util.ext.toMimeTypeOrNull
import io.github.landwarderer.futon.local.data.LocalStorageCache
import io.github.landwarderer.futon.local.data.PageCache
import org.koitharu.kotatsu.parsers.model.MangaPage
+import org.koitharu.kotatsu.parsers.model.MangaParserSource
import org.koitharu.kotatsu.parsers.util.mimeType
import org.koitharu.kotatsu.parsers.util.requireBody
import org.koitharu.kotatsu.parsers.util.runCatchingCancellable
+import kotlinx.coroutines.sync.Semaphore
+import kotlinx.coroutines.sync.withPermit
import io.github.landwarderer.futon.reader.domain.PageLoader
import javax.inject.Inject
@@ -43,6 +48,11 @@ class MangaPageFetcher(
private val imageLoader: ImageLoader,
) : Fetcher {
+ private val niaddOkHttpClient = okHttpClient.newBuilder()
+ .followRedirects(false)
+ .followSslRedirects(false)
+ .build()
+
override suspend fun fetch(): FetchResult? {
if (!page.preview.isNullOrEmpty()) {
runCatchingCancellable {
@@ -72,6 +82,12 @@ class MangaPageFetcher(
}
private suspend fun fetchPage(pageUrl: String): FetchResult {
+ if (page.source == MangaParserSource.NINEMANGA_RU) {
+ return niaddSemaphore.withPermit {
+ fetchNiaddPageByRanges(pageUrl)
+ }
+ }
+
val request = PageLoader.createPageRequest(pageUrl, page.source)
return imageProxyInterceptor.interceptPageRequest(request, okHttpClient).use { response ->
if (!response.isSuccessful) {
@@ -89,6 +105,99 @@ class MangaPageFetcher(
}
}
+ private suspend fun fetchNiaddPageByRanges(pageUrl: String): FetchResult {
+ val buffer = Buffer()
+ var mimeType: MimeType? = null
+ var offset = 0L
+ var totalBytes = -1L
+
+ while (totalBytes < 0L || offset < totalBytes) {
+ val requestedEnd = if (totalBytes > 0L) {
+ minOf(offset + NIADD_RANGE_CHUNK_SIZE - 1L, totalBytes - 1L)
+ } else {
+ offset + NIADD_RANGE_CHUNK_SIZE - 1L
+ }
+ val request = PageLoader.createPageRequest(pageUrl, page.source)
+ .newBuilder()
+ .header("Range", "bytes=$offset-$requestedEnd")
+ .header("Connection", "close")
+ .build()
+
+ niaddOkHttpClient.newCall(request).execute().use { response ->
+ check(response.code == 206) {
+ val location = response.header("Location")
+ "Niadd CDN range request returned HTTP ${response.code}" +
+ if (location.isNullOrBlank()) {
+ " for $pageUrl"
+ } else {
+ ", Location=$location for $pageUrl"
+ }
+ }
+
+ val contentType = response.header("Content-Type")
+ ?.substringBefore(';')
+ ?.trim()
+ .orEmpty()
+ check(contentType.startsWith("image/", ignoreCase = true)) {
+ "Niadd CDN returned non-image Content-Type '$contentType' for $pageUrl"
+ }
+ if (pageUrl.substringBefore('?').endsWith(".webp", ignoreCase = true)) {
+ check(contentType.equals("image/webp", ignoreCase = true)) {
+ "Niadd CDN replaced WebP with '$contentType' for $pageUrl"
+ }
+ }
+
+ val contentRange = response.header("Content-Range")
+ ?: error("Niadd CDN response has no Content-Range for $pageUrl")
+ val rangeMatch = NIADD_CONTENT_RANGE_REGEX.matchEntire(contentRange)
+ ?: error("Invalid Niadd Content-Range: $contentRange")
+ val responseStart = rangeMatch.groupValues[1].toLong()
+ val responseEnd = rangeMatch.groupValues[2].toLong()
+ val responseTotal = rangeMatch.groupValues[3].toLong()
+ check(responseTotal > 0L && responseStart == offset) {
+ "Unexpected Niadd Content-Range '$contentRange' for offset $offset ($pageUrl)"
+ }
+
+ if (totalBytes < 0L) {
+ totalBytes = responseTotal
+ } else {
+ check(totalBytes == responseTotal) {
+ "Niadd CDN changed content size from $totalBytes to $responseTotal for $pageUrl"
+ }
+ }
+
+ val expectedEnd = minOf(requestedEnd, totalBytes - 1L)
+ check(responseEnd == expectedEnd) {
+ "Unexpected Niadd range end: expected $expectedEnd, got $responseEnd " +
+ "($contentRange, $pageUrl)"
+ }
+ if (mimeType == null) {
+ mimeType = response.mimeType?.toMimeTypeOrNull()
+ }
+
+ val expectedBytes = expectedEnd - offset + 1L
+ val receivedBytes = buffer.writeAll(response.requireBody().source())
+ check(receivedBytes == expectedBytes) {
+ "Short Niadd range: expected $expectedBytes bytes, got $receivedBytes " +
+ "for bytes=$offset-$expectedEnd ($pageUrl)"
+ }
+
+ offset += receivedBytes
+ }
+ }
+
+ check(totalBytes > 0L && buffer.size == totalBytes) {
+ "Incomplete Niadd image: expected $totalBytes bytes, got ${buffer.size} for $pageUrl"
+ }
+
+ val file = pagesCache.set(pageUrl, buffer, mimeType)
+ return SourceFetchResult(
+ source = ImageSource(file.toOkioPath(), FileSystem.SYSTEM),
+ mimeType = mimeType?.toString(),
+ dataSource = DataSource.NETWORK,
+ )
+ }
+
private fun Response.toNetworkResponse() = NetworkResponse(
code = code,
requestMillis = sentRequestAtMillis,
@@ -106,6 +215,12 @@ class MangaPageFetcher(
return headers.build()
}
+ companion object {
+ private const val NIADD_RANGE_CHUNK_SIZE = 16L * 1024L
+ private val NIADD_CONTENT_RANGE_REGEX = Regex("""bytes (\d+)-(\d+)/(\d+)""")
+ private val niaddSemaphore = Semaphore(1)
+ }
+
class Factory @Inject constructor(
@MangaHttpClient private val okHttpClient: OkHttpClient,
@PageCache private val pagesCache: LocalStorageCache,
diff --git a/app/src/main/kotlin/io/github/landwarderer/futon/download/ui/worker/DownloadWorker.kt b/app/src/main/kotlin/io/github/landwarderer/futon/download/ui/worker/DownloadWorker.kt
index 8227b2a..742341e 100644
--- a/app/src/main/kotlin/io/github/landwarderer/futon/download/ui/worker/DownloadWorker.kt
+++ b/app/src/main/kotlin/io/github/landwarderer/futon/download/ui/worker/DownloadWorker.kt
@@ -91,6 +91,7 @@ import okio.use
import org.koitharu.kotatsu.parsers.exception.TooManyRequestExceptions
import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.model.MangaChapter
+import org.koitharu.kotatsu.parsers.model.MangaParserSource
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.util.ifNullOrEmpty
import org.koitharu.kotatsu.parsers.util.mapToSet
@@ -125,6 +126,10 @@ class DownloadWorker @AssistedInject constructor(
private val notificationFactory = notificationFactoryFactory.create(uuid = params.id, isSilent = task.isSilent)
private val notificationManager = appContext.getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager
private val bandwidthLimiter = BandwidthLimiter { settings.downloadBandwidthLimit * 1024 }
+ private val niaddOkHttp = okHttp.newBuilder()
+ .followRedirects(false)
+ .followSslRedirects(false)
+ .build()
@Volatile
private var lastPublishedState: DownloadState? = null
@@ -427,6 +432,13 @@ class DownloadWorker @AssistedInject constructor(
}
return file
}
+ if (isNiaddRangeDownload(url, source)) {
+ slowdownDispatcher.delay(source)
+ return NIADD_DOWNLOAD_SEMAPHORE.withPermit {
+ downloadNiaddFileByRanges(url, destination, source)
+ }
+ }
+
val request = PageLoader.createPageRequest(url, source)
slowdownDispatcher.delay(source)
return imageProxyInterceptor.interceptPageRequest(request, okHttp)
@@ -450,6 +462,117 @@ class DownloadWorker @AssistedInject constructor(
}
}
+ private fun isNiaddRangeDownload(url: String, source: MangaSource): Boolean {
+ return source == MangaParserSource.NINEMANGA_RU &&
+ url.substringBefore('?').endsWith(".webp", ignoreCase = true) &&
+ url.contains("acc=", ignoreCase = true) &&
+ url.contains("exp=", ignoreCase = true)
+ }
+
+ private suspend fun downloadNiaddFileByRanges(
+ url: String,
+ destination: File,
+ source: MangaSource,
+ ): File {
+ Log.d("DownloadWorker", "Using Niadd range download for $url")
+ var file: File? = null
+ var offset = 0L
+ var totalBytes = -1L
+
+ try {
+ while (totalBytes < 0L || offset < totalBytes) {
+ val requestedEnd = if (totalBytes > 0L) {
+ minOf(offset + NIADD_RANGE_CHUNK_SIZE - 1L, totalBytes - 1L)
+ } else {
+ offset + NIADD_RANGE_CHUNK_SIZE - 1L
+ }
+ val request = PageLoader.createPageRequest(url, source)
+ .newBuilder()
+ .header("Range", "bytes=$offset-$requestedEnd")
+ .header("Connection", "close")
+ .build()
+
+ niaddOkHttp.newCall(request).execute().use { response ->
+ check(response.code == 206) {
+ val location = response.header("Location")
+ "Niadd CDN range request returned HTTP ${response.code}" +
+ if (location.isNullOrBlank()) {
+ " for $url"
+ } else {
+ ", Location=$location for $url"
+ }
+ }
+
+ val contentType = response.header("Content-Type")
+ ?.substringBefore(';')
+ ?.trim()
+ .orEmpty()
+ check(contentType.equals("image/webp", ignoreCase = true)) {
+ "Niadd CDN returned '$contentType' instead of image/webp for $url"
+ }
+
+ val contentRange = response.header("Content-Range")
+ ?: error("Niadd CDN response has no Content-Range for $url")
+ val rangeMatch = NIADD_CONTENT_RANGE_REGEX.matchEntire(contentRange)
+ ?: error("Invalid Niadd Content-Range: $contentRange")
+ val responseStart = rangeMatch.groupValues[1].toLong()
+ val responseEnd = rangeMatch.groupValues[2].toLong()
+ val responseTotal = rangeMatch.groupValues[3].toLong()
+ check(responseTotal > 0L && responseStart == offset) {
+ "Unexpected Niadd Content-Range '$contentRange' for offset $offset ($url)"
+ }
+
+ if (totalBytes < 0L) {
+ totalBytes = responseTotal
+ } else {
+ check(totalBytes == responseTotal) {
+ "Niadd CDN changed content size from $totalBytes to $responseTotal for $url"
+ }
+ }
+
+ val expectedEnd = minOf(requestedEnd, totalBytes - 1L)
+ check(responseEnd == expectedEnd) {
+ "Unexpected Niadd range end: expected $expectedEnd, got $responseEnd " +
+ "($contentRange, $url)"
+ }
+
+ val body = response.requireBody()
+ if (file == null) {
+ file = destination.createTempFile(
+ ext = MimeTypes.getExtension(body.contentType()?.toMimeType()),
+ )
+ }
+ val outputFile = checkNotNull(file)
+ check(outputFile.length() == offset) {
+ "Niadd temporary file offset mismatch: expected $offset, got ${outputFile.length()} for $url"
+ }
+
+ val expectedBytes = expectedEnd - offset + 1L
+ outputFile.sink(append = offset > 0L).buffer().use { sink ->
+ sink.writeAllCancellable(body.source(), bandwidthLimiter)
+ }
+ val newLength = outputFile.length()
+ val receivedBytes = newLength - offset
+ check(receivedBytes == expectedBytes) {
+ "Short Niadd range: expected $expectedBytes bytes, got $receivedBytes " +
+ "for bytes=$offset-$expectedEnd ($url)"
+ }
+
+ offset = newLength
+ }
+ }
+
+ val result = checkNotNull(file)
+ check(totalBytes > 0L && result.length() == totalBytes) {
+ "Incomplete Niadd image: expected $totalBytes bytes, got ${result.length()} for $url"
+ }
+ return result
+ } catch (e: Exception) {
+ file?.delete()
+ throw e
+ }
+ }
+
private fun File.createTempFile(ext: String?) = File(
this,
buildString {
@@ -633,6 +756,9 @@ class DownloadWorker @AssistedInject constructor(
const val MAX_FAILSAFE_ATTEMPTS = 2
const val MAX_PAGES_PARALLELISM = 4
+ private const val NIADD_RANGE_CHUNK_SIZE = 16L * 1024L
+ private val NIADD_CONTENT_RANGE_REGEX = Regex("""bytes (\d+)-(\d+)/(\d+)""")
+ private val NIADD_DOWNLOAD_SEMAPHORE = Semaphore(1)
const val DOWNLOAD_ERROR_DELAY = 2_000L
const val MAX_RETRY_DELAY = 7_200_000L // 2 hours
const val TAG = "download"
diff --git a/app/src/main/kotlin/io/github/landwarderer/futon/reader/domain/PageLoader.kt b/app/src/main/kotlin/io/github/landwarderer/futon/reader/domain/PageLoader.kt
index 18d203d..02b1653 100644
--- a/app/src/main/kotlin/io/github/landwarderer/futon/reader/domain/PageLoader.kt
+++ b/app/src/main/kotlin/io/github/landwarderer/futon/reader/domain/PageLoader.kt
@@ -30,6 +30,7 @@ import io.github.landwarderer.futon.core.prefs.AppSettings
import io.github.landwarderer.futon.core.ui.image.TrimTransformation
import io.github.landwarderer.futon.core.util.FileSize
import io.github.landwarderer.futon.core.util.MimeTypes
+import io.github.landwarderer.futon.core.util.ext.MimeType
import io.github.landwarderer.futon.core.util.ext.URI_SCHEME_ZIP
import io.github.landwarderer.futon.core.util.ext.cancelChildrenAndJoin
import io.github.landwarderer.futon.core.util.ext.compressToPNG
@@ -52,6 +53,7 @@ import io.github.landwarderer.futon.download.ui.worker.DownloadSlowdownDispatche
import io.github.landwarderer.futon.local.data.LocalStorageCache
import io.github.landwarderer.futon.local.data.PageCache
import org.koitharu.kotatsu.parsers.model.MangaPage
+import org.koitharu.kotatsu.parsers.model.MangaParserSource
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.util.requireBody
import org.koitharu.kotatsu.parsers.util.runCatchingCancellable
@@ -70,6 +72,9 @@ import kotlinx.coroutines.sync.withLock
import kotlinx.coroutines.sync.withPermit
import okhttp3.OkHttpClient
import okhttp3.Request
+import okio.buffer
+import okio.sink
+import okio.source
import okio.use
import org.jetbrains.annotations.Blocking
import java.io.File
@@ -97,6 +102,11 @@ class PageLoader @Inject constructor(
private val tasks = LongSparseArray<ProgressDeferred<Uri, Float>>()
private val semaphore = Semaphore(3)
+ private val niaddSemaphore = Semaphore(1)
+ private val niaddOkHttp = okHttp.newBuilder()
+ .followRedirects(false)
+ .followSslRedirects(false)
+ .build()
private val convertLock = Mutex()
private val prefetchLock = Mutex()
@@ -303,13 +313,126 @@ class PageLoader @Inject constructor(
if (isPrefetch) {
downloadSlowdownDispatcher.delay(page.source)
}
- val request = createPageRequest(pageUrl, page.source)
- imageProxyInterceptor.interceptPageRequest(request, okHttp).ensureSuccess().use { response ->
- response.requireBody().withProgress(progress).use {
- cache.set(pageUrl, it.source(), it.contentType()?.toMimeType())
+ if (page.source == MangaParserSource.NINEMANGA_RU) {
+ niaddSemaphore.withPermit {
+ downloadNiaddPageByRanges(pageUrl, page.source, progress)
}
- }.toUri()
+ } else {
+ val request = createPageRequest(pageUrl, page.source)
+ imageProxyInterceptor.interceptPageRequest(request, okHttp).ensureSuccess().use { response ->
+ response.requireBody().withProgress(progress).use {
+ cache.set(pageUrl, it.source(), it.contentType()?.toMimeType())
+ }
+ }.toUri()
+ }
+ }
+ }
+ }
+
+ private suspend fun downloadNiaddPageByRanges(
+ pageUrl: String,
+ source: MangaSource,
+ progress: MutableStateFlow<Float>,
+ ): Uri {
+ val tempFile = File.createTempFile(
+ "niadd-page-",
+ ".part",
+ context.cacheDir,
+ )
+ var mimeType: MimeType? = null
+ try {
+ var offset = 0L
+ var totalBytes = -1L
+
+ tempFile.sink(append = false).buffer().use { sink ->
+ while (totalBytes < 0L || offset < totalBytes) {
+ val requestedEnd = if (totalBytes > 0L) {
+ minOf(offset + NIADD_RANGE_CHUNK_SIZE - 1L, totalBytes - 1L)
+ } else {
+ offset + NIADD_RANGE_CHUNK_SIZE - 1L
+ }
+ val request = createPageRequest(pageUrl, source)
+ .newBuilder()
+ .header("Range", "bytes=$offset-$requestedEnd")
+ .header("Connection", "close")
+ .build()
+
+ niaddOkHttp.newCall(request).execute().use { response ->
+ check(response.code == 206) {
+ val location = response.header("Location")
+ "Niadd CDN range request returned HTTP ${response.code}" +
+ if (location.isNullOrBlank()) {
+ " for $pageUrl"
+ } else {
+ ", Location=$location for $pageUrl"
+ }
+ }
+
+ val contentType = response.header("Content-Type")
+ ?.substringBefore(';')
+ ?.trim()
+ .orEmpty()
+ check(contentType.startsWith("image/", ignoreCase = true)) {
+ "Niadd CDN returned non-image Content-Type '$contentType' for $pageUrl"
+ }
+ if (pageUrl.substringBefore('?').endsWith(".webp", ignoreCase = true)) {
+ check(contentType.equals("image/webp", ignoreCase = true)) {
+ "Niadd CDN replaced WebP with '$contentType' for $pageUrl"
+ }
+ }
+
+ val contentRange = response.header("Content-Range")
+ ?: error("Niadd CDN response has no Content-Range for $pageUrl")
+ val rangeMatch = NIADD_CONTENT_RANGE_REGEX.matchEntire(contentRange)
+ ?: error("Invalid Niadd Content-Range: $contentRange")
+ val responseStart = rangeMatch.groupValues[1].toLong()
+ val responseEnd = rangeMatch.groupValues[2].toLong()
+ val responseTotal = rangeMatch.groupValues[3].toLong()
+ check(responseTotal > 0L && responseStart == offset) {
+ "Unexpected Niadd Content-Range '$contentRange' for offset $offset ($pageUrl)"
+ }
+
+ if (totalBytes < 0L) {
+ totalBytes = responseTotal
+ } else {
+ check(totalBytes == responseTotal) {
+ "Niadd CDN changed content size from $totalBytes to $responseTotal for $pageUrl"
+ }
+ }
+
+ val expectedEnd = minOf(requestedEnd, totalBytes - 1L)
+ check(responseEnd == expectedEnd) {
+ "Unexpected Niadd range end: expected $expectedEnd, got $responseEnd " +
+ "($contentRange, $pageUrl)"
+ }
+ val body = response.requireBody()
+ if (mimeType == null) {
+ mimeType = body.contentType()?.toMimeType()
+ }
+
+ val expectedBytes = expectedEnd - offset + 1L
+ val receivedBytes = sink.writeAll(body.source())
+ check(receivedBytes == expectedBytes) {
+ "Short Niadd range: expected $expectedBytes bytes, got $receivedBytes " +
+ "for bytes=$offset-$expectedEnd ($pageUrl)"
+ }
+
+ offset += receivedBytes
+ progress.value = (offset.toFloat() / totalBytes.toFloat())
+ .coerceIn(0f, 1f)
+ }
+ }
+ }
+
+ check(totalBytes > 0L && tempFile.length() == totalBytes) {
+ "Incomplete Niadd image: expected $totalBytes bytes, got ${tempFile.length()} for $pageUrl"
+ }
+
+ return tempFile.source().use { sourceStream ->
+ cache.set(pageUrl, sourceStream, mimeType).toUri()
}
+ } finally {
+ tempFile.delete()
}
}
@@ -340,6 +463,8 @@ class PageLoader @Inject constructor(
companion object {
private const val PROGRESS_UNDEFINED = -1f
+ private const val NIADD_RANGE_CHUNK_SIZE = 16L * 1024L
+ private val NIADD_CONTENT_RANGE_REGEX = Regex("""bytes (\d+)-(\d+)/(\d+)""")
private const val PREFETCH_LIMIT_DEFAULT = 6
private const val PREFETCH_MIN_RAM_MB = 80L
+755 -7
View File
@@ -5,7 +5,10 @@ import sys
if len(sys.argv) != 2: if len(sys.argv) != 2:
print(f"Usage: {sys.argv[0]} <kotatsu-parsers-redo-dir>", file=sys.stderr) print(
f"Usage: {sys.argv[0]} <kotatsu-parsers-redo-dir>",
file=sys.stderr,
)
sys.exit(2) sys.exit(2)
@@ -17,7 +20,10 @@ parser_file = (
) )
if not parser_file.is_file(): if not parser_file.is_file():
print(f"ERROR: parser file not found: {parser_file}", file=sys.stderr) print(
f"ERROR: parser file not found: {parser_file}",
file=sys.stderr,
)
sys.exit(1) sys.exit(1)
@@ -31,7 +37,8 @@ def replace_exact(name: str, old: str, new: str) -> None:
if count != 1: if count != 1:
print( print(
f"ERROR: patch '{name}' expected exactly 1 match, found {count}", f"ERROR: patch '{name}' expected exactly 1 match, "
f"found {count}",
file=sys.stderr, file=sys.stderr,
) )
print( print(
@@ -41,9 +48,26 @@ def replace_exact(name: str, old: str, new: str) -> None:
sys.exit(1) sys.exit(1)
text = text.replace(old, new, 1) text = text.replace(old, new, 1)
print(f"[OK] {name}") print(f"[OK] {name}")
#
# ---------------------------------------------------------------------------
# 1. CATEGORY PAGINATION
# ---------------------------------------------------------------------------
#
# Old NineManga:
#
# /category/index_1
# /category/index_2
#
# Current Niadd:
#
# /category/
# /category/index_2.html
# /category/index_3.html
#
replace_exact( replace_exact(
"Niadd category pagination", "Niadd category pagination",
''' } else { ''' } else {
@@ -67,6 +91,698 @@ replace_exact(
) )
#
# ---------------------------------------------------------------------------
# 2. MANGA LIST
# ---------------------------------------------------------------------------
#
# Old NineManga expects:
#
# ul.direlist
# li
# dd
# a.bookname
#
# Niadd no longer has ul.direlist.
#
# Manga detail links are still stable:
#
# /manga/<name>.html
#
# We therefore identify entries by their canonical /manga/ URL instead of
# presentation CSS classes.
#
replace_exact(
"Niadd manga list parser",
''' val doc = webClient.httpGet(url).parseHtml()
val root = doc.body().selectFirstOrThrow("ul.direlist")
val baseHost = root.baseUri().toHttpUrl().host
return root.select("li").map { node ->
val href = node.selectFirstOrThrow("a").attrAsAbsoluteUrl("href")
val relUrl = href.toRelativeUrl(baseHost)
val dd = node.selectFirst("dd")
Manga(
id = generateUid(relUrl),
url = relUrl,
publicUrl = href,
title = dd?.selectFirst("a.bookname")?.text()?.toCamelCase().orEmpty(),
altTitles = emptySet(),
coverUrl = node.selectFirst("img")?.src(),
rating = RATING_UNKNOWN,
authors = emptySet(),
contentRating = null,
tags = emptySet(),
state = null,
source = source,
description = dd?.selectFirst("p")?.html(),
)
}
''',
''' val doc = webClient.httpGet(url).parseHtml()
if (domain.endsWith("niadd.com")) {
val mangaLinks = doc.select("a[href]").filter { link ->
val href = link.attr("href")
href.startsWith("/manga/") ||
href.startsWith("https://${domain}/manga/")
}
val linksByUrl = mangaLinks.groupBy { link ->
link.attrAsAbsoluteUrl("href")
}
return linksByUrl.mapNotNull { (href, links) ->
val relUrl = href.toRelativeUrl(domain)
val title = links.asSequence()
.mapNotNull { link ->
link.selectFirst(
"h1, h2, h3, h4, h5, h6, strong, b"
)?.textOrNull()
}
.map { it.trim() }
.firstOrNull { it.isNotEmpty() }
?: links.asSequence()
.map { it.ownText().trim() }
.firstOrNull { it.isNotEmpty() }
?: links.asSequence()
.mapNotNull { link ->
link.selectFirst("img[alt]")
?.attr("alt")
?.trim()
?.takeIf { it.isNotEmpty() }
}
.firstOrNull()
?: return@mapNotNull null
val coverUrl = links.asSequence()
.mapNotNull { link ->
link.selectFirst("img")?.src()
?: link.parent()
?.selectFirst("img")
?.src()
?: link.parent()
?.parent()
?.selectFirst("img")
?.src()
}
.firstOrNull()
val description = links.asSequence()
.mapNotNull { link ->
link.select("p")
.mapNotNull { it.textOrNull() }
.maxByOrNull { it.length }
}
.firstOrNull()
?.takeIf { it.isNotBlank() }
Manga(
id = generateUid(relUrl),
url = relUrl,
publicUrl = href,
title = title.toCamelCase(),
altTitles = emptySet(),
coverUrl = coverUrl,
rating = RATING_UNKNOWN,
authors = emptySet(),
contentRating = null,
tags = emptySet(),
state = null,
source = source,
description = description,
)
}
}
val root = doc.body().selectFirstOrThrow("ul.direlist")
val baseHost = root.baseUri().toHttpUrl().host
return root.select("li").map { node ->
val href = node.selectFirstOrThrow("a").attrAsAbsoluteUrl("href")
val relUrl = href.toRelativeUrl(baseHost)
val dd = node.selectFirst("dd")
Manga(
id = generateUid(relUrl),
url = relUrl,
publicUrl = href,
title = dd?.selectFirst("a.bookname")
?.text()
?.toCamelCase()
.orEmpty(),
altTitles = emptySet(),
coverUrl = node.selectFirst("img")?.src(),
rating = RATING_UNKNOWN,
authors = emptySet(),
contentRating = null,
tags = emptySet(),
state = null,
source = source,
description = dd?.selectFirst("p")?.html(),
)
}
''',
)
#
# ---------------------------------------------------------------------------
# 3. DETAILS + CHAPTERS
# ---------------------------------------------------------------------------
#
# Old parser depends on:
#
# div.manga
# div.bookintro
# div.chapterbox
# ul.sub_vol_ul
# a.chapter_list_a
#
# Niadd no longer has that hierarchy.
#
# Current Niadd however has stable semantic URLs:
#
# Manga:
# /manga/Foo.html
#
# Full chapter list:
# /manga/Foo/chapters.html
#
# Chapter:
# /chapter/3_81/5259607/
# /chapter/2_80_2/5205144.html
#
# So for Niadd we parse by semantic links, not CSS classes.
#
replace_exact(
"Niadd details and chapters parser",
''' override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(
manga.url.toAbsoluteUrl(domain) + "?waring=1",
).parseHtml()
val root = doc.body().selectFirstOrThrow("div.manga")
val infoRoot = root.selectFirstOrThrow("div.bookintro")
val tagMap = getOrCreateTagMap()
val selectTag = infoRoot.getElementsByAttributeValue("itemprop", "genre").first()?.select("a")
val tags = selectTag?.mapNotNullToSet { tagMap[it.text()] }
val author = infoRoot.getElementsByAttributeValue("itemprop", "author").first()?.textOrNull()
return manga.copy(
title = root.selectFirst("h1[itemprop=name]")?.textOrNull()?.removeSuffix("Manga")?.trimEnd()
?: manga.title,
tags = tags.orEmpty(),
authors = setOfNotNull(author),
state = parseStatus(infoRoot.select("li a.red").text()),
description = infoRoot.getElementsByAttributeValue("itemprop", "description").first()?.html()
?.substringAfter("</b>"),
chapters = root.selectFirst("div.chapterbox")?.select("ul.sub_vol_ul > li")
?.mapChapters(reversed = true) { i, li ->
val a = li.selectFirstOrThrow("a.chapter_list_a")
val href = a.attrAsRelativeUrl("href").replace("%20", " ")
MangaChapter(
id = generateUid(href),
title = a.textOrNull(),
number = i + 1f,
volume = 0,
url = href,
uploadDate = parseChapterDateByLang(li.selectFirst("span")?.text().orEmpty()),
source = source,
scanlator = null,
branch = null,
)
},
)
}
''',
''' override suspend fun getDetails(manga: Manga): Manga {
if (domain.endsWith("niadd.com")) {
val mangaPath = manga.url
.substringBefore("?")
.removeSuffix("/")
val chaptersPath = if (mangaPath.endsWith(".html")) {
mangaPath.removeSuffix(".html") + "/chapters.html"
} else {
"$mangaPath/chapters.html"
}
val chaptersDoc = webClient.httpGet(
chaptersPath.toAbsoluteUrl(domain) + "?waring=1",
).parseHtml()
/*
* Current Niadd chapters page already contains
* manga title, status, authors, genres and language,
* so avoid a second HTTP request to the details page.
*/
val detailsDoc = chaptersDoc
val heading = detailsDoc.selectFirst("h1")
?.textOrNull()
?.trim()
.orEmpty()
val title = heading
.replace(
Regex(
"""\\s*\\((?:постоянный|завершенный)\\)\\s*$""",
RegexOption.IGNORE_CASE,
),
"",
)
.trim()
.ifEmpty { manga.title }
val tagMap = getOrCreateTagMap()
val tags = detailsDoc
.select("a[href^=/category/]")
.mapNotNullToSet { element ->
val tagTitle = element.text()
.trim()
.removePrefix(",")
.trim()
tagMap[tagTitle]
}
val authors = detailsDoc
.select("a[href*=author=]")
.mapNotNullToSet { element ->
element.textOrNull()
?.trim()
?.takeIf { it.isNotEmpty() }
}
val pageDescription = detailsDoc
.selectFirst("meta[name=description]")
?.attr("content")
?.trim()
?.takeIf { it.isNotEmpty() }
val chapterDateRegex = Regex(
"""\\b[A-Z][a-z]{2}\\s+\\d{1,2},\\s+\\d{4}\\b"""
)
val chapterLinks = chaptersDoc
.select("a[href]")
.filter { element ->
val href = element.attr("href")
(
href.startsWith("/chapter/") ||
href.startsWith(
"https://${domain}/chapter/"
)
) &&
!href.contains("/manga_for_adults/")
}
.distinctBy { element ->
element.attrAsAbsoluteUrl("href")
}
.reversed()
val chapters = chapterLinks.mapIndexed { i, a ->
val href = a.attrAsRelativeUrl("href")
.replace("%20", " ")
val anchorText = a.text().trim()
val rowText = a.parent()?.text().orEmpty().trim()
val rawTitle = a.ownText()
.trim()
.ifEmpty {
anchorText
}
val anchorDate = chapterDateRegex
.find(anchorText)
?.value
val rowDate = chapterDateRegex
.find(rowText)
?.value
val dateText = anchorDate ?: rowDate
val chapterTitle = if (
dateText != null &&
rawTitle.contains(dateText)
) {
rawTitle
.substringBefore(dateText)
.replace(
Regex("""\\s+\\d+\\s*$"""),
"",
)
.trim()
} else {
rawTitle
}
MangaChapter(
id = generateUid(href),
title = chapterTitle.takeIf {
it.isNotEmpty()
},
number = i + 1f,
volume = 0,
url = href,
uploadDate = parseChapterDateByLang(
dateText.orEmpty()
),
source = source,
scanlator = null,
branch = null,
)
}
return manga.copy(
title = title,
tags = tags,
authors = authors,
state = parseStatus(
detailsDoc.body().text()
),
description = manga.description
?: pageDescription,
chapters = chapters,
)
}
val doc = webClient.httpGet(
manga.url.toAbsoluteUrl(domain) + "?waring=1",
).parseHtml()
val root = doc.body().selectFirstOrThrow("div.manga")
val infoRoot = root.selectFirstOrThrow("div.bookintro")
val tagMap = getOrCreateTagMap()
val selectTag = infoRoot
.getElementsByAttributeValue(
"itemprop",
"genre",
)
.first()
?.select("a")
val tags = selectTag
?.mapNotNullToSet {
tagMap[it.text()]
}
val author = infoRoot
.getElementsByAttributeValue(
"itemprop",
"author",
)
.first()
?.textOrNull()
return manga.copy(
title = root
.selectFirst("h1[itemprop=name]")
?.textOrNull()
?.removeSuffix("Manga")
?.trimEnd()
?: manga.title,
tags = tags.orEmpty(),
authors = setOfNotNull(author),
state = parseStatus(
infoRoot.select("li a.red").text()
),
description = infoRoot
.getElementsByAttributeValue(
"itemprop",
"description",
)
.first()
?.html()
?.substringAfter("</b>"),
chapters = root
.selectFirst("div.chapterbox")
?.select("ul.sub_vol_ul > li")
?.mapChapters(
reversed = true
) { i, li ->
val a = li.selectFirstOrThrow(
"a.chapter_list_a"
)
val href = a
.attrAsRelativeUrl("href")
.replace("%20", " ")
MangaChapter(
id = generateUid(href),
title = a.textOrNull(),
number = i + 1f,
volume = 0,
url = href,
uploadDate = parseChapterDateByLang(
li.selectFirst("span")
?.text()
.orEmpty()
),
source = source,
scanlator = null,
branch = null,
)
},
)
}
''',
)
#
# ---------------------------------------------------------------------------
# 4. CHAPTER PAGE LIST
# ---------------------------------------------------------------------------
#
# Old NineManga:
#
# #page option
#
# Current Niadd has several <select>'s on the reader:
# chapter selector
# batch loading
# image scale
# page selector
#
# Instead of relying on id="page", select only options whose text is:
#
# 1/61
# 2/61
# ...
#
replace_exact(
"Niadd page list parser",
''' override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml()
return doc.body().requireElementById("page").select("option").map { option ->
val url = option.attr("value")
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = source,
)
}
}
''',
''' override suspend fun getPages(
chapter: MangaChapter,
): List<MangaPage> {
val chapterUrl = chapter.url
.substringBefore("#")
val doc = webClient
.httpGet(
chapterUrl.toAbsoluteUrl(domain)
)
.parseHtml()
if (domain.endsWith("niadd.com")) {
val pageLabelRegex = Regex(
"""^\\s*(\\d+)\\s*/\\s*(\\d+)\\s*$"""
)
val pageOptions = doc
.select("option[value]")
.mapNotNull { option ->
val match = pageLabelRegex.matchEntire(
option.text().trim()
) ?: return@mapNotNull null
val pageNumber = match
.groupValues
.getOrNull(1)
?.toIntOrNull()
?: return@mapNotNull null
val totalPages = match
.groupValues
.getOrNull(2)
?.toIntOrNull()
?: return@mapNotNull null
val url = option.attr("value").trim()
if (url.isEmpty()) {
return@mapNotNull null
}
Triple(pageNumber, totalPages, url)
}
.distinctBy { (pageNumber, _, _) ->
pageNumber
}
.sortedBy { (pageNumber, _, _) ->
pageNumber
}
val totalPages = pageOptions
.firstOrNull()
?.second
?: error(
"Cannot find Niadd page selector for $chapterUrl"
)
check(
pageOptions.all { (_, total, _) ->
total == totalPages
}
) {
"Inconsistent Niadd page count for $chapterUrl"
}
check(
pageOptions.size == totalPages &&
pageOptions.map { it.first } ==
(1..totalPages).toList()
) {
"Incomplete Niadd page selector for $chapterUrl: " +
"found ${pageOptions.size} of $totalPages pages"
}
return pageOptions.map { (_, _, url) ->
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = source,
)
}
}
return doc
.body()
.requireElementById("page")
.select("option")
.map { option ->
val url = option.attr("value")
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = source,
)
}
}
''',
)
#
# ---------------------------------------------------------------------------
# 5. IMAGE URL
# ---------------------------------------------------------------------------
#
# Old NineManga:
#
# a.pic_download
#
# Current Niadd serves page images from an external image host.
#
# Example:
#
# https://ruwebp.movietop.cc/comics/.../image.webp?acc=...&exp=...
#
# So find an image-looking URL instead of relying on CSS class.
#
replace_exact(
"Niadd image URL parser",
''' override suspend fun getPageUrl(page: MangaPage): String {
val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml()
val root = doc.body()
return root.selectFirstOrThrow("a.pic_download").attrAsAbsoluteUrl("href")
}
''',
''' override suspend fun getPageUrl(
page: MangaPage,
): String {
val doc = webClient
.httpGet(
page.url.toAbsoluteUrl(domain)
)
.parseHtml()
val root = doc.body()
if (domain.endsWith("niadd.com")) {
val readerImageLinkRegex = Regex(
"""^\\s*\\d+\\s+of\\s+\\d+\\s*$""",
RegexOption.IGNORE_CASE,
)
val imageAnchor = root
.select("a[href]")
.firstOrNull { element ->
readerImageLinkRegex.matches(
element.text()
)
}
if (imageAnchor != null) {
return imageAnchor.attrAsAbsoluteUrl(
"href"
)
}
// Do not fall back to an arbitrary <img>. Niadd may return a
// service/notice page with a perfectly valid static image, which
// would otherwise be cached as every manga page.
error(
"Cannot find Niadd reader image for ${page.url}"
)
}
return root
.selectFirstOrThrow("a.pic_download")
.attrAsAbsoluteUrl("href")
}
''',
)
#
# ---------------------------------------------------------------------------
# 6. RUSSIAN STATUS
# ---------------------------------------------------------------------------
#
# Niadd currently uses e.g.:
#
# постоянный
# Завершенный
#
replace_exact( replace_exact(
"Russian status parsing", "Russian status parsing",
''' //ru ''' //ru
@@ -74,12 +790,28 @@ replace_exact(
status.contains("завершенный") -> MangaState.FINISHED status.contains("завершенный") -> MangaState.FINISHED
''', ''',
''' //ru ''' //ru
status.contains("постоянный", ignoreCase = true) -> MangaState.ONGOING status.contains(
status.contains("завершенный", ignoreCase = true) -> MangaState.FINISHED "постоянный",
ignoreCase = true,
) -> MangaState.ONGOING
status.contains(
"завершенный",
ignoreCase = true,
) -> MangaState.FINISHED
''', ''',
) )
#
# ---------------------------------------------------------------------------
# 7. ENABLE RUSSIAN NIADD
# ---------------------------------------------------------------------------
#
# Keep NINEMANGA_RU unchanged intentionally.
#
# That preserves the existing source ID used by Futon database entries.
#
replace_exact( replace_exact(
"NineManga RU to Niadd RU", "NineManga RU to Niadd RU",
''' @Broken ''' @Broken
@@ -91,7 +823,9 @@ replace_exact(
) )
''', ''',
''' @MangaSourceParser("NINEMANGA_RU", "Niadd Русский", "ru") ''' @MangaSourceParser("NINEMANGA_RU", "Niadd Русский", "ru")
class Russian(context: MangaLoaderContext) : NineMangaParser( class Russian(
context: MangaLoaderContext,
) : NineMangaParser(
context, context,
MangaParserSource.NINEMANGA_RU, MangaParserSource.NINEMANGA_RU,
"ru.niadd.com", "ru.niadd.com",
@@ -100,7 +834,21 @@ replace_exact(
) )
parser_file.write_text(text, encoding="utf-8") parser_file.write_text(
text,
encoding="utf-8",
)
print() print()
print(f"[OK] Patched: {parser_file}") print(f"[OK] Patched: {parser_file}")
print()
print("[OK] Niadd domain")
print("[OK] Niadd category pagination")
print("[OK] Niadd manga list")
print("[OK] Niadd details")
print("[OK] Niadd chapter list")
print("[OK] Niadd reader page list")
print("[OK] Niadd image URL")
print("[OK] Russian statuses")
print("[OK] Russian source enabled")