fix ninemanga page structure parse

scripts/patch_ninemanga_niadd.py

@@ -5,7 +5,10 @@ import sys
 
 
 if len(sys.argv) != 2:
-    print(f"Usage: {sys.argv[0]} <kotatsu-parsers-redo-dir>", file=sys.stderr)
+    print(
+        f"Usage: {sys.argv[0]} <kotatsu-parsers-redo-dir>",
+        file=sys.stderr,
+    )
     sys.exit(2)
 
 
@@ -34,7 +37,8 @@ def replace_exact(name: str, old: str, new: str) -> None:
 
     if count != 1:
         print(
-            f"ERROR: patch '{name}' expected exactly 1 match, found {count}",
+            f"ERROR: patch '{name}' expected exactly 1 match, "
+            f"found {count}",
             file=sys.stderr,
         )
         print(
@@ -44,42 +48,25 @@ def replace_exact(name: str, old: str, new: str) -> None:
         sys.exit(1)
 
     text = text.replace(old, new, 1)
+
     print(f"[OK] {name}")
 
 
 #
-# Niadd search pagination:
+# ---------------------------------------------------------------------------
+# 1. CATEGORY PAGINATION
+# ---------------------------------------------------------------------------
 #
-# NineManga:
+# Old NineManga:
-#   ?page=2
 #
-# Niadd:
+#   /category/index_1
-#   ?page=2.html
+#   /category/index_2
 #
-replace_exact(
+# Current Niadd:
-    "Niadd search pagination",
-    '''                append("/search/")
-                append("?page=")
-                append(page.toString())
-''',
-    '''                append("/search/")
-                append("?page=")
-                append(page.toString())
-                if (domain.endsWith("niadd.com")) {
-                    append(".html")
-                }
-''',
-)
-
-
 #
-# Niadd category pagination:
-#
-# Page 1:
 #   /category/
-#
-# Page 2+:
 #   /category/index_2.html
+#   /category/index_3.html
 #
 replace_exact(
     "Niadd category pagination",
@@ -105,20 +92,25 @@ replace_exact(
 
 
 #
-# NineManga used:
+# ---------------------------------------------------------------------------
+# 2. MANGA LIST
+# ---------------------------------------------------------------------------
 #
-#   ul.direlist > li
+# Old NineManga expects:
+#
+#   ul.direlist
+#     li
+#       dd
+#         a.bookname
 #
 # Niadd no longer has ul.direlist.
 #
-# Instead, discover manga cards through their canonical /manga/ links.
+# Manga detail links are still stable:
 #
-# The same manga can have several links:
+#   /manga/<name>.html
-#   - cover
-#   - title
-#   - description
 #
-# Therefore links are grouped by absolute manga URL.
+# We therefore identify entries by their canonical /manga/ URL instead of
+# presentation CSS classes.
 #
 replace_exact(
     "Niadd manga list parser",
@@ -162,37 +154,54 @@ replace_exact(
             return linksByUrl.mapNotNull { (href, links) ->
                 val relUrl = href.toRelativeUrl(domain)
 
-                val texts = links
+                val title = links.asSequence()
-                    .map { it.text().trim() }
+                    .mapNotNull { link ->
-                    .filter { it.isNotEmpty() }
+                        link.selectFirst(
-                    .distinct()
+                            "h1, h2, h3, h4, h5, h6, strong, b"
-
+                        )?.textOrNull()
-                val rawTitle = texts.minByOrNull { it.length }
+                    }
+                    .map { it.trim() }
+                    .firstOrNull { it.isNotEmpty() }
+                    ?: links.asSequence()
+                        .map { it.ownText().trim() }
+                        .firstOrNull { it.isNotEmpty() }
+                    ?: links.asSequence()
+                        .mapNotNull { link ->
+                            link.selectFirst("img[alt]")
+                                ?.attr("alt")
+                                ?.trim()
+                                ?.takeIf { it.isNotEmpty() }
+                        }
+                        .firstOrNull()
                     ?: return@mapNotNull null
 
-                val description = texts
+                val coverUrl = links.asSequence()
-                    .asSequence()
-                    .filter { it != rawTitle }
-                    .filter { it.length > rawTitle.length }
-                    .maxByOrNull { it.length }
-                    ?.removePrefix(rawTitle)
-                    ?.trim()
-                    ?.takeIf { it.isNotEmpty() }
-
-                val coverUrl = links
-                    .asSequence()
                     .mapNotNull { link ->
                         link.selectFirst("img")?.src()
-                            ?: link.parent()?.selectFirst("img")?.src()
+                            ?: link.parent()
-                            ?: link.parent()?.parent()?.selectFirst("img")?.src()
+                                ?.selectFirst("img")
+                                ?.src()
+                            ?: link.parent()
+                                ?.parent()
+                                ?.selectFirst("img")
+                                ?.src()
                     }
                     .firstOrNull()
 
+                val description = links.asSequence()
+                    .mapNotNull { link ->
+                        link.select("p")
+                            .mapNotNull { it.textOrNull() }
+                            .maxByOrNull { it.length }
+                    }
+                    .firstOrNull()
+                    ?.takeIf { it.isNotBlank() }
+
                 Manga(
                     id = generateUid(relUrl),
                     url = relUrl,
                     publicUrl = href,
-                    title = rawTitle.toCamelCase(),
+                    title = title.toCamelCase(),
                     altTitles = emptySet(),
                     coverUrl = coverUrl,
                     rating = RATING_UNKNOWN,
@@ -218,7 +227,10 @@ replace_exact(
                 id = generateUid(relUrl),
                 url = relUrl,
                 publicUrl = href,
-                title = dd?.selectFirst("a.bookname")?.text()?.toCamelCase().orEmpty(),
+                title = dd?.selectFirst("a.bookname")
+                    ?.text()
+                    ?.toCamelCase()
+                    .orEmpty(),
                 altTitles = emptySet(),
                 coverUrl = node.selectFirst("img")?.src(),
                 rating = RATING_UNKNOWN,
@@ -235,7 +247,554 @@ replace_exact(
 
 
 #
-# Russian statuses on Niadd are not consistently cased:
+# ---------------------------------------------------------------------------
+# 3. DETAILS + CHAPTERS
+# ---------------------------------------------------------------------------
+#
+# Old parser depends on:
+#
+#   div.manga
+#   div.bookintro
+#   div.chapterbox
+#   ul.sub_vol_ul
+#   a.chapter_list_a
+#
+# Niadd no longer has that hierarchy.
+#
+# Current Niadd however has stable semantic URLs:
+#
+# Manga:
+#   /manga/Foo.html
+#
+# Full chapter list:
+#   /manga/Foo/chapters.html
+#
+# Chapter:
+#   /chapter/3_81/5259607/
+#   /chapter/2_80_2/5205144.html
+#
+# So for Niadd we parse by semantic links, not CSS classes.
+#
+replace_exact(
+    "Niadd details and chapters parser",
+    '''    override suspend fun getDetails(manga: Manga): Manga {
+        val doc = webClient.httpGet(
+            manga.url.toAbsoluteUrl(domain) + "?waring=1",
+        ).parseHtml()
+        val root = doc.body().selectFirstOrThrow("div.manga")
+        val infoRoot = root.selectFirstOrThrow("div.bookintro")
+        val tagMap = getOrCreateTagMap()
+        val selectTag = infoRoot.getElementsByAttributeValue("itemprop", "genre").first()?.select("a")
+        val tags = selectTag?.mapNotNullToSet { tagMap[it.text()] }
+        val author = infoRoot.getElementsByAttributeValue("itemprop", "author").first()?.textOrNull()
+        return manga.copy(
+            title = root.selectFirst("h1[itemprop=name]")?.textOrNull()?.removeSuffix("Manga")?.trimEnd()
+                ?: manga.title,
+            tags = tags.orEmpty(),
+            authors = setOfNotNull(author),
+            state = parseStatus(infoRoot.select("li a.red").text()),
+            description = infoRoot.getElementsByAttributeValue("itemprop", "description").first()?.html()
+                ?.substringAfter("</b>"),
+            chapters = root.selectFirst("div.chapterbox")?.select("ul.sub_vol_ul > li")
+                ?.mapChapters(reversed = true) { i, li ->
+                    val a = li.selectFirstOrThrow("a.chapter_list_a")
+                    val href = a.attrAsRelativeUrl("href").replace("%20", " ")
+                    MangaChapter(
+                        id = generateUid(href),
+                        title = a.textOrNull(),
+                        number = i + 1f,
+                        volume = 0,
+                        url = href,
+                        uploadDate = parseChapterDateByLang(li.selectFirst("span")?.text().orEmpty()),
+                        source = source,
+                        scanlator = null,
+                        branch = null,
+                    )
+                },
+        )
+    }
+''',
+    '''    override suspend fun getDetails(manga: Manga): Manga {
+        if (domain.endsWith("niadd.com")) {
+            val detailsUrl = manga.url.toAbsoluteUrl(domain)
+
+            val detailsDoc = webClient.httpGet(
+                "$detailsUrl?waring=1",
+            ).parseHtml()
+
+            val mangaPath = manga.url
+                .substringBefore("?")
+                .removeSuffix("/")
+
+            val chaptersPath = if (mangaPath.endsWith(".html")) {
+                mangaPath.removeSuffix(".html") + "/chapters.html"
+            } else {
+                "$mangaPath/chapters.html"
+            }
+
+            val chaptersDoc = webClient.httpGet(
+                chaptersPath.toAbsoluteUrl(domain) + "?waring=1",
+            ).parseHtml()
+
+            val heading = detailsDoc.selectFirst("h1")
+                ?.textOrNull()
+                ?.trim()
+                .orEmpty()
+
+            val title = heading
+                .replace(
+                    Regex(
+                        """\\s*\\((?:постоянный|завершенный)\\)\\s*$""",
+                        RegexOption.IGNORE_CASE,
+                    ),
+                    "",
+                )
+                .trim()
+                .ifEmpty { manga.title }
+
+            val tagMap = getOrCreateTagMap()
+
+            val tags = detailsDoc
+                .select("a[href^=/category/]")
+                .mapNotNullToSet { element ->
+                    val tagTitle = element.text()
+                        .trim()
+                        .removePrefix(",")
+                        .trim()
+
+                    tagMap[tagTitle]
+                }
+
+            val authors = detailsDoc
+                .select("a[href*=author=]")
+                .mapNotNullToSet { element ->
+                    element.textOrNull()
+                        ?.trim()
+                        ?.takeIf { it.isNotEmpty() }
+                }
+
+            val pageDescription = detailsDoc
+                .selectFirst("meta[name=description]")
+                ?.attr("content")
+                ?.trim()
+                ?.takeIf { it.isNotEmpty() }
+
+            val chapterDateRegex = Regex(
+                """\\b[A-Z][a-z]{2}\\s+\\d{1,2},\\s+\\d{4}\\b"""
+            )
+
+            val chapterLinks = chaptersDoc
+                .select("a[href]")
+                .filter { element ->
+                    val href = element.attr("href")
+
+                    (
+                        href.startsWith("/chapter/") ||
+                            href.startsWith(
+                                "https://${domain}/chapter/"
+                            )
+                        ) &&
+                        !href.contains("/manga_for_adults/")
+                }
+                .distinctBy { element ->
+                    element.attrAsAbsoluteUrl("href")
+                }
+                .reversed()
+
+            val chapters = chapterLinks.mapIndexed { i, a ->
+                val href = a.attrAsRelativeUrl("href")
+                    .replace("%20", " ")
+
+                val rawTitle = a.ownText()
+                    .trim()
+                    .ifEmpty {
+                        a.text().trim()
+                    }
+
+                val anchorDate = chapterDateRegex
+                    .find(a.text())
+                    ?.value
+
+                val rowDate = chapterDateRegex
+                    .find(a.parent()?.text().orEmpty())
+                    ?.value
+
+                val dateText = anchorDate ?: rowDate
+
+                val chapterTitle = if (
+                    dateText != null &&
+                    rawTitle.contains(dateText)
+                ) {
+                    rawTitle
+                        .substringBefore(dateText)
+                        .replace(
+                            Regex("""\\s+\\d+\\s*$"""),
+                            "",
+                        )
+                        .trim()
+                } else {
+                    rawTitle
+                }
+
+                MangaChapter(
+                    id = generateUid(href),
+                    title = chapterTitle.takeIf {
+                        it.isNotEmpty()
+                    },
+                    number = i + 1f,
+                    volume = 0,
+                    url = href,
+                    uploadDate = parseChapterDateByLang(
+                        dateText.orEmpty()
+                    ),
+                    source = source,
+                    scanlator = null,
+                    branch = null,
+                )
+            }
+
+            return manga.copy(
+                title = title,
+                tags = tags,
+                authors = authors,
+                state = parseStatus(
+                    detailsDoc.body().text()
+                ),
+                description = manga.description
+                    ?: pageDescription,
+                chapters = chapters,
+            )
+        }
+
+        val doc = webClient.httpGet(
+            manga.url.toAbsoluteUrl(domain) + "?waring=1",
+        ).parseHtml()
+
+        val root = doc.body().selectFirstOrThrow("div.manga")
+        val infoRoot = root.selectFirstOrThrow("div.bookintro")
+
+        val tagMap = getOrCreateTagMap()
+
+        val selectTag = infoRoot
+            .getElementsByAttributeValue(
+                "itemprop",
+                "genre",
+            )
+            .first()
+            ?.select("a")
+
+        val tags = selectTag
+            ?.mapNotNullToSet {
+                tagMap[it.text()]
+            }
+
+        val author = infoRoot
+            .getElementsByAttributeValue(
+                "itemprop",
+                "author",
+            )
+            .first()
+            ?.textOrNull()
+
+        return manga.copy(
+            title = root
+                .selectFirst("h1[itemprop=name]")
+                ?.textOrNull()
+                ?.removeSuffix("Manga")
+                ?.trimEnd()
+                ?: manga.title,
+
+            tags = tags.orEmpty(),
+
+            authors = setOfNotNull(author),
+
+            state = parseStatus(
+                infoRoot.select("li a.red").text()
+            ),
+
+            description = infoRoot
+                .getElementsByAttributeValue(
+                    "itemprop",
+                    "description",
+                )
+                .first()
+                ?.html()
+                ?.substringAfter("</b>"),
+
+            chapters = root
+                .selectFirst("div.chapterbox")
+                ?.select("ul.sub_vol_ul > li")
+                ?.mapChapters(
+                    reversed = true
+                ) { i, li ->
+                    val a = li.selectFirstOrThrow(
+                        "a.chapter_list_a"
+                    )
+
+                    val href = a
+                        .attrAsRelativeUrl("href")
+                        .replace("%20", " ")
+
+                    MangaChapter(
+                        id = generateUid(href),
+                        title = a.textOrNull(),
+                        number = i + 1f,
+                        volume = 0,
+                        url = href,
+                        uploadDate = parseChapterDateByLang(
+                            li.selectFirst("span")
+                                ?.text()
+                                .orEmpty()
+                        ),
+                        source = source,
+                        scanlator = null,
+                        branch = null,
+                    )
+                },
+        )
+    }
+''',
+)
+
+
+#
+# ---------------------------------------------------------------------------
+# 4. CHAPTER PAGE LIST
+# ---------------------------------------------------------------------------
+#
+# Old NineManga:
+#
+#   #page option
+#
+# Current Niadd has several <select>'s on the reader:
+#   chapter selector
+#   batch loading
+#   image scale
+#   page selector
+#
+# Instead of relying on id="page", select only options whose text is:
+#
+#   1/61
+#   2/61
+#   ...
+#
+replace_exact(
+    "Niadd page list parser",
+    '''    override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
+        val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml()
+        return doc.body().requireElementById("page").select("option").map { option ->
+            val url = option.attr("value")
+            MangaPage(
+                id = generateUid(url),
+                url = url,
+                preview = null,
+                source = source,
+            )
+        }
+    }
+''',
+    '''    override suspend fun getPages(
+        chapter: MangaChapter,
+    ): List<MangaPage> {
+        val doc = webClient
+            .httpGet(
+                chapter.url.toAbsoluteUrl(domain)
+            )
+            .parseHtml()
+
+        if (domain.endsWith("niadd.com")) {
+            val pageLabelRegex = Regex(
+                """^\\s*(\\d+)\\s*/\\s*(\\d+)\\s*$"""
+            )
+
+            val pageOptions = doc
+                .select("option[value]")
+                .filter { option ->
+                    pageLabelRegex.matches(
+                        option.text()
+                    ) &&
+                        option.attr("value").isNotBlank()
+                }
+                .distinctBy { option ->
+                    option.attr("value")
+                }
+
+            if (pageOptions.isNotEmpty()) {
+                return pageOptions.map { option ->
+                    val url = option.attr("value")
+
+                    MangaPage(
+                        id = generateUid(url),
+                        url = url,
+                        preview = null,
+                        source = source,
+                    )
+                }
+            }
+
+            /*
+             * Fallback for a future Niadd layout where page
+             * <option> elements stop containing values.
+             *
+             * Page URL structure currently is:
+             *
+             * page 1:
+             *   /chapter/3_81/5259607/
+             * or
+             *   /chapter/2_80_2/5205144.html
+             *
+             * page 2:
+             *   /chapter/3_81/5259607-2.html
+             */
+            val totalPages = doc
+                .select("option")
+                .mapNotNull { option ->
+                    pageLabelRegex
+                        .matchEntire(option.text())
+                        ?.groupValues
+                        ?.getOrNull(2)
+                        ?.toIntOrNull()
+                }
+                .maxOrNull()
+
+            if (totalPages != null && totalPages > 0) {
+                val originalUrl = chapter.url
+                    .substringBefore("?")
+
+                val baseUrl = originalUrl
+                    .replace(
+                        Regex("""-\\d+\\.html$"""),
+                        "",
+                    )
+                    .removeSuffix(".html")
+                    .removeSuffix("/")
+
+                return (1..totalPages).map { pageNumber ->
+                    val url = if (pageNumber == 1) {
+                        originalUrl
+                    } else {
+                        "$baseUrl-$pageNumber.html"
+                    }
+
+                    MangaPage(
+                        id = generateUid(url),
+                        url = url,
+                        preview = null,
+                        source = source,
+                    )
+                }
+            }
+
+            error(
+                "Cannot find Niadd page selector for ${chapter.url}"
+            )
+        }
+
+        return doc
+            .body()
+            .requireElementById("page")
+            .select("option")
+            .map { option ->
+                val url = option.attr("value")
+
+                MangaPage(
+                    id = generateUid(url),
+                    url = url,
+                    preview = null,
+                    source = source,
+                )
+            }
+    }
+''',
+)
+
+
+#
+# ---------------------------------------------------------------------------
+# 5. IMAGE URL
+# ---------------------------------------------------------------------------
+#
+# Old NineManga:
+#
+#   a.pic_download
+#
+# Current Niadd serves page images from an external image host.
+#
+# Example:
+#
+#   https://ruwebp.movietop.cc/comics/.../image.webp?acc=...&exp=...
+#
+# So find an image-looking URL instead of relying on CSS class.
+#
+replace_exact(
+    "Niadd image URL parser",
+    '''    override suspend fun getPageUrl(page: MangaPage): String {
+        val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml()
+        val root = doc.body()
+        return root.selectFirstOrThrow("a.pic_download").attrAsAbsoluteUrl("href")
+    }
+''',
+    '''    override suspend fun getPageUrl(
+        page: MangaPage,
+    ): String {
+        val doc = webClient
+            .httpGet(
+                page.url.toAbsoluteUrl(domain)
+            )
+            .parseHtml()
+
+        val root = doc.body()
+
+        if (domain.endsWith("niadd.com")) {
+            val imageUrlRegex = Regex(
+                """(?i)\\.(?:jpe?g|png|gif|webp|avif)(?:\\?|$)"""
+            )
+
+            val imageAnchor = root
+                .select("a[href]")
+                .firstOrNull { element ->
+                    val href = element.attr("href")
+
+                    imageUrlRegex.containsMatchIn(href)
+                }
+
+            if (imageAnchor != null) {
+                return imageAnchor.attrAsAbsoluteUrl(
+                    "href"
+                )
+            }
+
+            val image = root
+                .select("img[src]")
+                .firstOrNull { element ->
+                    val src = element.attr("src")
+
+                    imageUrlRegex.containsMatchIn(src)
+                }
+
+            if (image != null) {
+                return image.src()
+            }
+
+            error(
+                "Cannot find Niadd page image for ${page.url}"
+            )
+        }
+
+        return root
+            .selectFirstOrThrow("a.pic_download")
+            .attrAsAbsoluteUrl("href")
+    }
+''',
+)
+
+
+#
+# ---------------------------------------------------------------------------
+# 6. RUSSIAN STATUS
+# ---------------------------------------------------------------------------
+#
+# Niadd currently uses e.g.:
 #
 #   постоянный
 #   Завершенный
@@ -247,17 +806,27 @@ replace_exact(
         status.contains("завершенный") -> MangaState.FINISHED
 ''',
     '''        //ru
-        status.contains("постоянный", ignoreCase = true) -> MangaState.ONGOING
+        status.contains(
-        status.contains("завершенный", ignoreCase = true) -> MangaState.FINISHED
+            "постоянный",
+            ignoreCase = true,
+        ) -> MangaState.ONGOING
+
+        status.contains(
+            "завершенный",
+            ignoreCase = true,
+        ) -> MangaState.FINISHED
 ''',
 )
 
 
 #
-# Activate Russian source and migrate it to Niadd.
+# ---------------------------------------------------------------------------
+# 7. ENABLE RUSSIAN NIADD
+# ---------------------------------------------------------------------------
 #
-# Keep NINEMANGA_RU intentionally unchanged so existing Futon
+# Keep NINEMANGA_RU unchanged intentionally.
-# database/source references remain compatible.
+#
+# That preserves the existing source ID used by Futon database entries.
 #
 replace_exact(
     "NineManga RU to Niadd RU",
@@ -270,7 +839,9 @@ replace_exact(
     )
 ''',
     '''    @MangaSourceParser("NINEMANGA_RU", "Niadd Русский", "ru")
-    class Russian(context: MangaLoaderContext) : NineMangaParser(
+    class Russian(
+        context: MangaLoaderContext,
+    ) : NineMangaParser(
         context,
         MangaParserSource.NINEMANGA_RU,
         "ru.niadd.com",
@@ -290,7 +861,10 @@ print(f"[OK] Patched: {parser_file}")
 print()
 print("[OK] Niadd domain")
 print("[OK] Niadd category pagination")
-print("[OK] Niadd search pagination")
+print("[OK] Niadd manga list")
-print("[OK] Niadd manga list parser")
+print("[OK] Niadd details")
+print("[OK] Niadd chapter list")
+print("[OK] Niadd reader page list")
+print("[OK] Niadd image URL")
 print("[OK] Russian statuses")
 print("[OK] Russian source enabled")