fix pagecount

scripts/patch_ninemanga_niadd.py

@@ -5,7 +5,10 @@ import sys
 
 
 if len(sys.argv) != 2:
-    print(f"Usage: {sys.argv[0]} <kotatsu-parsers-redo-dir>", file=sys.stderr)
+    print(
+        f"Usage: {sys.argv[0]} <kotatsu-parsers-redo-dir>",
+        file=sys.stderr,
+    )
     sys.exit(2)
 
 
@@ -17,7 +20,10 @@ parser_file = (
 )
 
 if not parser_file.is_file():
-    print(f"ERROR: parser file not found: {parser_file}", file=sys.stderr)
+    print(
+        f"ERROR: parser file not found: {parser_file}",
+        file=sys.stderr,
+    )
     sys.exit(1)
 
 
@@ -31,7 +37,8 @@ def replace_exact(name: str, old: str, new: str) -> None:
 
     if count != 1:
         print(
-            f"ERROR: patch '{name}' expected exactly 1 match, found {count}",
+            f"ERROR: patch '{name}' expected exactly 1 match, "
+            f"found {count}",
             file=sys.stderr,
         )
         print(
@@ -41,9 +48,26 @@ def replace_exact(name: str, old: str, new: str) -> None:
         sys.exit(1)
 
     text = text.replace(old, new, 1)
+
     print(f"[OK] {name}")
 
 
+#
+# ---------------------------------------------------------------------------
+# 1. CATEGORY PAGINATION
+# ---------------------------------------------------------------------------
+#
+# Old NineManga:
+#
+#   /category/index_1
+#   /category/index_2
+#
+# Current Niadd:
+#
+#   /category/
+#   /category/index_2.html
+#   /category/index_3.html
+#
 replace_exact(
     "Niadd category pagination",
     '''            } else {
@@ -67,6 +91,727 @@ replace_exact(
 )
 
 
+#
+# ---------------------------------------------------------------------------
+# 2. MANGA LIST
+# ---------------------------------------------------------------------------
+#
+# Old NineManga expects:
+#
+#   ul.direlist
+#     li
+#       dd
+#         a.bookname
+#
+# Niadd no longer has ul.direlist.
+#
+# Manga detail links are still stable:
+#
+#   /manga/<name>.html
+#
+# We therefore identify entries by their canonical /manga/ URL instead of
+# presentation CSS classes.
+#
+replace_exact(
+    "Niadd manga list parser",
+    '''        val doc = webClient.httpGet(url).parseHtml()
+        val root = doc.body().selectFirstOrThrow("ul.direlist")
+        val baseHost = root.baseUri().toHttpUrl().host
+        return root.select("li").map { node ->
+            val href = node.selectFirstOrThrow("a").attrAsAbsoluteUrl("href")
+            val relUrl = href.toRelativeUrl(baseHost)
+            val dd = node.selectFirst("dd")
+            Manga(
+                id = generateUid(relUrl),
+                url = relUrl,
+                publicUrl = href,
+                title = dd?.selectFirst("a.bookname")?.text()?.toCamelCase().orEmpty(),
+                altTitles = emptySet(),
+                coverUrl = node.selectFirst("img")?.src(),
+                rating = RATING_UNKNOWN,
+                authors = emptySet(),
+                contentRating = null,
+                tags = emptySet(),
+                state = null,
+                source = source,
+                description = dd?.selectFirst("p")?.html(),
+            )
+        }
+''',
+    '''        val doc = webClient.httpGet(url).parseHtml()
+
+        if (domain.endsWith("niadd.com")) {
+            val mangaLinks = doc.select("a[href]").filter { link ->
+                val href = link.attr("href")
+                href.startsWith("/manga/") ||
+                    href.startsWith("https://${domain}/manga/")
+            }
+
+            val linksByUrl = mangaLinks.groupBy { link ->
+                link.attrAsAbsoluteUrl("href")
+            }
+
+            return linksByUrl.mapNotNull { (href, links) ->
+                val relUrl = href.toRelativeUrl(domain)
+
+                val title = links.asSequence()
+                    .mapNotNull { link ->
+                        link.selectFirst(
+                            "h1, h2, h3, h4, h5, h6, strong, b"
+                        )?.textOrNull()
+                    }
+                    .map { it.trim() }
+                    .firstOrNull { it.isNotEmpty() }
+                    ?: links.asSequence()
+                        .map { it.ownText().trim() }
+                        .firstOrNull { it.isNotEmpty() }
+                    ?: links.asSequence()
+                        .mapNotNull { link ->
+                            link.selectFirst("img[alt]")
+                                ?.attr("alt")
+                                ?.trim()
+                                ?.takeIf { it.isNotEmpty() }
+                        }
+                        .firstOrNull()
+                    ?: return@mapNotNull null
+
+                val coverUrl = links.asSequence()
+                    .mapNotNull { link ->
+                        link.selectFirst("img")?.src()
+                            ?: link.parent()
+                                ?.selectFirst("img")
+                                ?.src()
+                            ?: link.parent()
+                                ?.parent()
+                                ?.selectFirst("img")
+                                ?.src()
+                    }
+                    .firstOrNull()
+
+                val description = links.asSequence()
+                    .mapNotNull { link ->
+                        link.select("p")
+                            .mapNotNull { it.textOrNull() }
+                            .maxByOrNull { it.length }
+                    }
+                    .firstOrNull()
+                    ?.takeIf { it.isNotBlank() }
+
+                Manga(
+                    id = generateUid(relUrl),
+                    url = relUrl,
+                    publicUrl = href,
+                    title = title.toCamelCase(),
+                    altTitles = emptySet(),
+                    coverUrl = coverUrl,
+                    rating = RATING_UNKNOWN,
+                    authors = emptySet(),
+                    contentRating = null,
+                    tags = emptySet(),
+                    state = null,
+                    source = source,
+                    description = description,
+                )
+            }
+        }
+
+        val root = doc.body().selectFirstOrThrow("ul.direlist")
+        val baseHost = root.baseUri().toHttpUrl().host
+
+        return root.select("li").map { node ->
+            val href = node.selectFirstOrThrow("a").attrAsAbsoluteUrl("href")
+            val relUrl = href.toRelativeUrl(baseHost)
+            val dd = node.selectFirst("dd")
+
+            Manga(
+                id = generateUid(relUrl),
+                url = relUrl,
+                publicUrl = href,
+                title = dd?.selectFirst("a.bookname")
+                    ?.text()
+                    ?.toCamelCase()
+                    .orEmpty(),
+                altTitles = emptySet(),
+                coverUrl = node.selectFirst("img")?.src(),
+                rating = RATING_UNKNOWN,
+                authors = emptySet(),
+                contentRating = null,
+                tags = emptySet(),
+                state = null,
+                source = source,
+                description = dd?.selectFirst("p")?.html(),
+            )
+        }
+''',
+)
+
+
+#
+# ---------------------------------------------------------------------------
+# 3. DETAILS + CHAPTERS
+# ---------------------------------------------------------------------------
+#
+# Old parser depends on:
+#
+#   div.manga
+#   div.bookintro
+#   div.chapterbox
+#   ul.sub_vol_ul
+#   a.chapter_list_a
+#
+# Niadd no longer has that hierarchy.
+#
+# Current Niadd however has stable semantic URLs:
+#
+# Manga:
+#   /manga/Foo.html
+#
+# Full chapter list:
+#   /manga/Foo/chapters.html
+#
+# Chapter:
+#   /chapter/3_81/5259607/
+#   /chapter/2_80_2/5205144.html
+#
+# So for Niadd we parse by semantic links, not CSS classes.
+#
+replace_exact(
+    "Niadd details and chapters parser",
+    '''    override suspend fun getDetails(manga: Manga): Manga {
+        val doc = webClient.httpGet(
+            manga.url.toAbsoluteUrl(domain) + "?waring=1",
+        ).parseHtml()
+        val root = doc.body().selectFirstOrThrow("div.manga")
+        val infoRoot = root.selectFirstOrThrow("div.bookintro")
+        val tagMap = getOrCreateTagMap()
+        val selectTag = infoRoot.getElementsByAttributeValue("itemprop", "genre").first()?.select("a")
+        val tags = selectTag?.mapNotNullToSet { tagMap[it.text()] }
+        val author = infoRoot.getElementsByAttributeValue("itemprop", "author").first()?.textOrNull()
+        return manga.copy(
+            title = root.selectFirst("h1[itemprop=name]")?.textOrNull()?.removeSuffix("Manga")?.trimEnd()
+                ?: manga.title,
+            tags = tags.orEmpty(),
+            authors = setOfNotNull(author),
+            state = parseStatus(infoRoot.select("li a.red").text()),
+            description = infoRoot.getElementsByAttributeValue("itemprop", "description").first()?.html()
+                ?.substringAfter("</b>"),
+            chapters = root.selectFirst("div.chapterbox")?.select("ul.sub_vol_ul > li")
+                ?.mapChapters(reversed = true) { i, li ->
+                    val a = li.selectFirstOrThrow("a.chapter_list_a")
+                    val href = a.attrAsRelativeUrl("href").replace("%20", " ")
+                    MangaChapter(
+                        id = generateUid(href),
+                        title = a.textOrNull(),
+                        number = i + 1f,
+                        volume = 0,
+                        url = href,
+                        uploadDate = parseChapterDateByLang(li.selectFirst("span")?.text().orEmpty()),
+                        source = source,
+                        scanlator = null,
+                        branch = null,
+                    )
+                },
+        )
+    }
+''',
+    '''    override suspend fun getDetails(manga: Manga): Manga {
+        if (domain.endsWith("niadd.com")) {
+            val mangaPath = manga.url
+                .substringBefore("?")
+                .removeSuffix("/")
+
+            val chaptersPath = if (mangaPath.endsWith(".html")) {
+                mangaPath.removeSuffix(".html") + "/chapters.html"
+            } else {
+                "$mangaPath/chapters.html"
+            }
+
+            val chaptersDoc = webClient.httpGet(
+                chaptersPath.toAbsoluteUrl(domain) + "?waring=1",
+            ).parseHtml()
+
+            /*
+             * Current Niadd chapters page already contains
+             * manga title, status, authors, genres and language,
+             * so avoid a second HTTP request to the details page.
+             */
+            val detailsDoc = chaptersDoc
+
+            val heading = detailsDoc.selectFirst("h1")
+                ?.textOrNull()
+                ?.trim()
+                .orEmpty()
+
+            val title = heading
+                .replace(
+                    Regex(
+                        """\\s*\\((?:постоянный|завершенный)\\)\\s*$""",
+                        RegexOption.IGNORE_CASE,
+                    ),
+                    "",
+                )
+                .trim()
+                .ifEmpty { manga.title }
+
+            val tagMap = getOrCreateTagMap()
+
+            val tags = detailsDoc
+                .select("a[href^=/category/]")
+                .mapNotNullToSet { element ->
+                    val tagTitle = element.text()
+                        .trim()
+                        .removePrefix(",")
+                        .trim()
+
+                    tagMap[tagTitle]
+                }
+
+            val authors = detailsDoc
+                .select("a[href*=author=]")
+                .mapNotNullToSet { element ->
+                    element.textOrNull()
+                        ?.trim()
+                        ?.takeIf { it.isNotEmpty() }
+                }
+
+            val pageDescription = detailsDoc
+                .selectFirst("meta[name=description]")
+                ?.attr("content")
+                ?.trim()
+                ?.takeIf { it.isNotEmpty() }
+
+            val chapterDateRegex = Regex(
+                """\\b[A-Z][a-z]{2}\\s+\\d{1,2},\\s+\\d{4}\\b"""
+            )
+
+            val chapterLinks = chaptersDoc
+                .select("a[href]")
+                .filter { element ->
+                    val href = element.attr("href")
+
+                    (
+                        href.startsWith("/chapter/") ||
+                            href.startsWith(
+                                "https://${domain}/chapter/"
+                            )
+                        ) &&
+                        !href.contains("/manga_for_adults/")
+                }
+                .distinctBy { element ->
+                    element.attrAsAbsoluteUrl("href")
+                }
+                .reversed()
+
+            val chapters = chapterLinks.mapIndexed { i, a ->
+                val href = a.attrAsRelativeUrl("href")
+                    .replace("%20", " ")
+
+                val anchorText = a.text().trim()
+                val rowText = a.parent()?.text().orEmpty().trim()
+
+                val rawTitle = a.ownText()
+                    .trim()
+                    .ifEmpty {
+                        anchorText
+                    }
+
+                val anchorDate = chapterDateRegex
+                    .find(anchorText)
+                    ?.value
+
+                val rowDate = chapterDateRegex
+                    .find(rowText)
+                    ?.value
+
+                val dateText = anchorDate ?: rowDate
+
+                val metadataText = when {
+                    anchorDate != null -> anchorText
+                    rowDate != null -> rowText
+                    else -> ""
+                }
+
+                val textBeforeDate = if (dateText != null) {
+                    metadataText
+                        .substringBefore(dateText)
+                        .trim()
+                } else {
+                    metadataText
+                }
+
+                /*
+                 * Niadd exposes the page count immediately before
+                 * the chapter date, for example:
+                 *
+                 *   Падение русала 1 - 42 41 Aug 02, 2024
+                 *                           ^^
+                 *
+                 * Store that number in a URL fragment. Fragments
+                 * are local metadata and are not sent to the server.
+                 */
+                val pageCount = Regex(
+                    """\\s+(\\d+)\\s*$"""
+                )
+                    .find(textBeforeDate)
+                    ?.groupValues
+                    ?.getOrNull(1)
+                    ?.toIntOrNull()
+
+                val chapterTitle = if (
+                    dateText != null &&
+                    rawTitle.contains(dateText)
+                ) {
+                    rawTitle
+                        .substringBefore(dateText)
+                        .replace(
+                            Regex("""\\s+\\d+\\s*$"""),
+                            "",
+                        )
+                        .trim()
+                } else {
+                    rawTitle
+                }
+
+                val chapterUrl = if (
+                    pageCount != null &&
+                    pageCount > 0
+                ) {
+                    "$href#niadd-pages=$pageCount"
+                } else {
+                    href
+                }
+
+                MangaChapter(
+                    id = generateUid(href),
+                    title = chapterTitle.takeIf {
+                        it.isNotEmpty()
+                    },
+                    number = i + 1f,
+                    volume = 0,
+                    url = chapterUrl,
+                    uploadDate = parseChapterDateByLang(
+                        dateText.orEmpty()
+                    ),
+                    source = source,
+                    scanlator = null,
+                    branch = null,
+                )
+            }
+
+            return manga.copy(
+                title = title,
+                tags = tags,
+                authors = authors,
+                state = parseStatus(
+                    detailsDoc.body().text()
+                ),
+                description = manga.description
+                    ?: pageDescription,
+                chapters = chapters,
+            )
+        }
+
+        val doc = webClient.httpGet(
+            manga.url.toAbsoluteUrl(domain) + "?waring=1",
+        ).parseHtml()
+
+        val root = doc.body().selectFirstOrThrow("div.manga")
+        val infoRoot = root.selectFirstOrThrow("div.bookintro")
+
+        val tagMap = getOrCreateTagMap()
+
+        val selectTag = infoRoot
+            .getElementsByAttributeValue(
+                "itemprop",
+                "genre",
+            )
+            .first()
+            ?.select("a")
+
+        val tags = selectTag
+            ?.mapNotNullToSet {
+                tagMap[it.text()]
+            }
+
+        val author = infoRoot
+            .getElementsByAttributeValue(
+                "itemprop",
+                "author",
+            )
+            .first()
+            ?.textOrNull()
+
+        return manga.copy(
+            title = root
+                .selectFirst("h1[itemprop=name]")
+                ?.textOrNull()
+                ?.removeSuffix("Manga")
+                ?.trimEnd()
+                ?: manga.title,
+
+            tags = tags.orEmpty(),
+
+            authors = setOfNotNull(author),
+
+            state = parseStatus(
+                infoRoot.select("li a.red").text()
+            ),
+
+            description = infoRoot
+                .getElementsByAttributeValue(
+                    "itemprop",
+                    "description",
+                )
+                .first()
+                ?.html()
+                ?.substringAfter("</b>"),
+
+            chapters = root
+                .selectFirst("div.chapterbox")
+                ?.select("ul.sub_vol_ul > li")
+                ?.mapChapters(
+                    reversed = true
+                ) { i, li ->
+                    val a = li.selectFirstOrThrow(
+                        "a.chapter_list_a"
+                    )
+
+                    val href = a
+                        .attrAsRelativeUrl("href")
+                        .replace("%20", " ")
+
+                    MangaChapter(
+                        id = generateUid(href),
+                        title = a.textOrNull(),
+                        number = i + 1f,
+                        volume = 0,
+                        url = href,
+                        uploadDate = parseChapterDateByLang(
+                            li.selectFirst("span")
+                                ?.text()
+                                .orEmpty()
+                        ),
+                        source = source,
+                        scanlator = null,
+                        branch = null,
+                    )
+                },
+        )
+    }
+''',
+)
+
+
+#
+# ---------------------------------------------------------------------------
+# 4. CHAPTER PAGE LIST
+# ---------------------------------------------------------------------------
+#
+# Old NineManga:
+#
+#   #page option
+#
+# Current Niadd has several <select>'s on the reader:
+#   chapter selector
+#   batch loading
+#   image scale
+#   page selector
+#
+# Instead of relying on id="page", select only options whose text is:
+#
+#   1/61
+#   2/61
+#   ...
+#
+replace_exact(
+    "Niadd page list parser",
+    '''    override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
+        val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml()
+        return doc.body().requireElementById("page").select("option").map { option ->
+            val url = option.attr("value")
+            MangaPage(
+                id = generateUid(url),
+                url = url,
+                preview = null,
+                source = source,
+            )
+        }
+    }
+''',
+    '''    override suspend fun getPages(
+        chapter: MangaChapter,
+    ): List<MangaPage> {
+        if (domain.endsWith("niadd.com")) {
+            val originalUrl = chapter.url
+                .substringBefore("#")
+                .substringBefore("?")
+
+            val totalPages = chapter.url
+                .substringAfter(
+                    "#niadd-pages=",
+                    "",
+                )
+                .substringBefore("&")
+                .toIntOrNull()
+                ?.takeIf { it > 0 }
+                ?: error(
+                    "Niadd page count is missing for ${chapter.url}. " +
+                        "Refresh manga details and try again."
+                )
+
+            val normalizedUrl = originalUrl
+                .removeSuffix("/")
+
+            val baseUrl = if (
+                normalizedUrl.endsWith(".html")
+            ) {
+                normalizedUrl.removeSuffix(".html")
+            } else {
+                normalizedUrl
+            }
+
+            return (1..totalPages).map { pageNumber ->
+                val url = if (pageNumber == 1) {
+                    originalUrl
+                } else {
+                    "$baseUrl-$pageNumber.html"
+                }
+
+                MangaPage(
+                    id = generateUid(url),
+                    url = url,
+                    preview = null,
+                    source = source,
+                )
+            }
+        }
+
+        val doc = webClient
+            .httpGet(
+                chapter.url.toAbsoluteUrl(domain)
+            )
+            .parseHtml()
+
+        return doc
+            .body()
+            .requireElementById("page")
+            .select("option")
+            .map { option ->
+                val url = option.attr("value")
+
+                MangaPage(
+                    id = generateUid(url),
+                    url = url,
+                    preview = null,
+                    source = source,
+                )
+            }
+    }
+''',
+)
+
+
+#
+# ---------------------------------------------------------------------------
+# 5. IMAGE URL
+# ---------------------------------------------------------------------------
+#
+# Old NineManga:
+#
+#   a.pic_download
+#
+# Current Niadd serves page images from an external image host.
+#
+# Example:
+#
+#   https://ruwebp.movietop.cc/comics/.../image.webp?acc=...&exp=...
+#
+# So find an image-looking URL instead of relying on CSS class.
+#
+replace_exact(
+    "Niadd image URL parser",
+    '''    override suspend fun getPageUrl(page: MangaPage): String {
+        val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml()
+        val root = doc.body()
+        return root.selectFirstOrThrow("a.pic_download").attrAsAbsoluteUrl("href")
+    }
+''',
+    '''    override suspend fun getPageUrl(
+        page: MangaPage,
+    ): String {
+        val doc = webClient
+            .httpGet(
+                page.url.toAbsoluteUrl(domain)
+            )
+            .parseHtml()
+
+        val root = doc.body()
+
+        if (domain.endsWith("niadd.com")) {
+            val readerImageLinkRegex = Regex(
+                """^\\s*\\d+\\s+of\\s+\\d+\\s*$""",
+                RegexOption.IGNORE_CASE,
+            )
+
+            val imageAnchor = root
+                .select("a[href]")
+                .firstOrNull { element ->
+                    readerImageLinkRegex.matches(
+                        element.text()
+                    )
+                }
+
+            if (imageAnchor != null) {
+                return imageAnchor.attrAsAbsoluteUrl(
+                    "href"
+                )
+            }
+
+            /*
+             * Fallback in case Niadd removes the "N of M"
+             * image link but leaves the actual image element.
+             */
+            val imageUrlRegex = Regex(
+                """(?i)\\.(?:jpe?g|png|gif|webp|avif)(?:\\?|$)"""
+            )
+
+            val image = root
+                .select("img[src]")
+                .firstOrNull { element ->
+                    val src = element.attr("src")
+
+                    imageUrlRegex.containsMatchIn(src) &&
+                        !src.contains("logo", ignoreCase = true) &&
+                        !src.contains("avatar", ignoreCase = true)
+                }
+
+            if (image != null) {
+                return image.attrAsAbsoluteUrl("src")
+            }
+
+            error(
+                "Cannot find Niadd reader image for ${page.url}"
+            )
+        }
+
+        return root
+            .selectFirstOrThrow("a.pic_download")
+            .attrAsAbsoluteUrl("href")
+    }
+''',
+)
+
+
+#
+# ---------------------------------------------------------------------------
+# 6. RUSSIAN STATUS
+# ---------------------------------------------------------------------------
+#
+# Niadd currently uses e.g.:
+#
+#   постоянный
+#   Завершенный
+#
 replace_exact(
     "Russian status parsing",
     '''        //ru
@@ -74,12 +819,28 @@ replace_exact(
         status.contains("завершенный") -> MangaState.FINISHED
 ''',
     '''        //ru
-        status.contains("постоянный", ignoreCase = true) -> MangaState.ONGOING
+        status.contains(
-        status.contains("завершенный", ignoreCase = true) -> MangaState.FINISHED
+            "постоянный",
+            ignoreCase = true,
+        ) -> MangaState.ONGOING
+
+        status.contains(
+            "завершенный",
+            ignoreCase = true,
+        ) -> MangaState.FINISHED
 ''',
 )
 
 
+#
+# ---------------------------------------------------------------------------
+# 7. ENABLE RUSSIAN NIADD
+# ---------------------------------------------------------------------------
+#
+# Keep NINEMANGA_RU unchanged intentionally.
+#
+# That preserves the existing source ID used by Futon database entries.
+#
 replace_exact(
     "NineManga RU to Niadd RU",
     '''    @Broken
@@ -91,7 +852,9 @@ replace_exact(
     )
 ''',
     '''    @MangaSourceParser("NINEMANGA_RU", "Niadd Русский", "ru")
-    class Russian(context: MangaLoaderContext) : NineMangaParser(
+    class Russian(
+        context: MangaLoaderContext,
+    ) : NineMangaParser(
         context,
         MangaParserSource.NINEMANGA_RU,
         "ru.niadd.com",
@@ -100,7 +863,21 @@ replace_exact(
 )
 
 
-parser_file.write_text(text, encoding="utf-8")
+parser_file.write_text(
+    text,
+    encoding="utf-8",
+)
+
 
 print()
 print(f"[OK] Patched: {parser_file}")
+print()
+print("[OK] Niadd domain")
+print("[OK] Niadd category pagination")
+print("[OK] Niadd manga list")
+print("[OK] Niadd details")
+print("[OK] Niadd chapter list")
+print("[OK] Niadd reader page list")
+print("[OK] Niadd image URL")
+print("[OK] Russian statuses")
+print("[OK] Russian source enabled")