Files
gandc 9ba7eed81a
Build Futon with Niadd patch / Build Futon Niadd debug APK (push) Successful in 5m24s
v5.1
2026-08-11 22:38:20 +03:00

855 lines
25 KiB
Python

#!/usr/bin/env python3
from pathlib import Path
import sys
if len(sys.argv) != 2:
print(
f"Usage: {sys.argv[0]} <kotatsu-parsers-redo-dir>",
file=sys.stderr,
)
sys.exit(2)
repo_dir = Path(sys.argv[1]).resolve()
parser_file = (
repo_dir
/ "src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/NineMangaParser.kt"
)
if not parser_file.is_file():
print(
f"ERROR: parser file not found: {parser_file}",
file=sys.stderr,
)
sys.exit(1)
text = parser_file.read_text(encoding="utf-8")
def replace_exact(name: str, old: str, new: str) -> None:
global text
count = text.count(old)
if count != 1:
print(
f"ERROR: patch '{name}' expected exactly 1 match, "
f"found {count}",
file=sys.stderr,
)
print(
"Upstream NineMangaParser.kt has probably changed.",
file=sys.stderr,
)
sys.exit(1)
text = text.replace(old, new, 1)
print(f"[OK] {name}")
#
# ---------------------------------------------------------------------------
# 1. CATEGORY PAGINATION
# ---------------------------------------------------------------------------
#
# Old NineManga:
#
# /category/index_1
# /category/index_2
#
# Current Niadd:
#
# /category/
# /category/index_2.html
# /category/index_3.html
#
replace_exact(
"Niadd category pagination",
''' } else {
append("/category/index_")
append(page.toString())
}
''',
''' } else if (domain.endsWith("niadd.com")) {
if (page == 1) {
append("/category/")
} else {
append("/category/index_")
append(page.toString())
append(".html")
}
} else {
append("/category/index_")
append(page.toString())
}
''',
)
#
# ---------------------------------------------------------------------------
# 2. MANGA LIST
# ---------------------------------------------------------------------------
#
# Old NineManga expects:
#
# ul.direlist
# li
# dd
# a.bookname
#
# Niadd no longer has ul.direlist.
#
# Manga detail links are still stable:
#
# /manga/<name>.html
#
# We therefore identify entries by their canonical /manga/ URL instead of
# presentation CSS classes.
#
replace_exact(
"Niadd manga list parser",
''' val doc = webClient.httpGet(url).parseHtml()
val root = doc.body().selectFirstOrThrow("ul.direlist")
val baseHost = root.baseUri().toHttpUrl().host
return root.select("li").map { node ->
val href = node.selectFirstOrThrow("a").attrAsAbsoluteUrl("href")
val relUrl = href.toRelativeUrl(baseHost)
val dd = node.selectFirst("dd")
Manga(
id = generateUid(relUrl),
url = relUrl,
publicUrl = href,
title = dd?.selectFirst("a.bookname")?.text()?.toCamelCase().orEmpty(),
altTitles = emptySet(),
coverUrl = node.selectFirst("img")?.src(),
rating = RATING_UNKNOWN,
authors = emptySet(),
contentRating = null,
tags = emptySet(),
state = null,
source = source,
description = dd?.selectFirst("p")?.html(),
)
}
''',
''' val doc = webClient.httpGet(url).parseHtml()
if (domain.endsWith("niadd.com")) {
val mangaLinks = doc.select("a[href]").filter { link ->
val href = link.attr("href")
href.startsWith("/manga/") ||
href.startsWith("https://${domain}/manga/")
}
val linksByUrl = mangaLinks.groupBy { link ->
link.attrAsAbsoluteUrl("href")
}
return linksByUrl.mapNotNull { (href, links) ->
val relUrl = href.toRelativeUrl(domain)
val title = links.asSequence()
.mapNotNull { link ->
link.selectFirst(
"h1, h2, h3, h4, h5, h6, strong, b"
)?.textOrNull()
}
.map { it.trim() }
.firstOrNull { it.isNotEmpty() }
?: links.asSequence()
.map { it.ownText().trim() }
.firstOrNull { it.isNotEmpty() }
?: links.asSequence()
.mapNotNull { link ->
link.selectFirst("img[alt]")
?.attr("alt")
?.trim()
?.takeIf { it.isNotEmpty() }
}
.firstOrNull()
?: return@mapNotNull null
val coverUrl = links.asSequence()
.mapNotNull { link ->
link.selectFirst("img")?.src()
?: link.parent()
?.selectFirst("img")
?.src()
?: link.parent()
?.parent()
?.selectFirst("img")
?.src()
}
.firstOrNull()
val description = links.asSequence()
.mapNotNull { link ->
link.select("p")
.mapNotNull { it.textOrNull() }
.maxByOrNull { it.length }
}
.firstOrNull()
?.takeIf { it.isNotBlank() }
Manga(
id = generateUid(relUrl),
url = relUrl,
publicUrl = href,
title = title.toCamelCase(),
altTitles = emptySet(),
coverUrl = coverUrl,
rating = RATING_UNKNOWN,
authors = emptySet(),
contentRating = null,
tags = emptySet(),
state = null,
source = source,
description = description,
)
}
}
val root = doc.body().selectFirstOrThrow("ul.direlist")
val baseHost = root.baseUri().toHttpUrl().host
return root.select("li").map { node ->
val href = node.selectFirstOrThrow("a").attrAsAbsoluteUrl("href")
val relUrl = href.toRelativeUrl(baseHost)
val dd = node.selectFirst("dd")
Manga(
id = generateUid(relUrl),
url = relUrl,
publicUrl = href,
title = dd?.selectFirst("a.bookname")
?.text()
?.toCamelCase()
.orEmpty(),
altTitles = emptySet(),
coverUrl = node.selectFirst("img")?.src(),
rating = RATING_UNKNOWN,
authors = emptySet(),
contentRating = null,
tags = emptySet(),
state = null,
source = source,
description = dd?.selectFirst("p")?.html(),
)
}
''',
)
#
# ---------------------------------------------------------------------------
# 3. DETAILS + CHAPTERS
# ---------------------------------------------------------------------------
#
# Old parser depends on:
#
# div.manga
# div.bookintro
# div.chapterbox
# ul.sub_vol_ul
# a.chapter_list_a
#
# Niadd no longer has that hierarchy.
#
# Current Niadd however has stable semantic URLs:
#
# Manga:
# /manga/Foo.html
#
# Full chapter list:
# /manga/Foo/chapters.html
#
# Chapter:
# /chapter/3_81/5259607/
# /chapter/2_80_2/5205144.html
#
# So for Niadd we parse by semantic links, not CSS classes.
#
replace_exact(
"Niadd details and chapters parser",
''' override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(
manga.url.toAbsoluteUrl(domain) + "?waring=1",
).parseHtml()
val root = doc.body().selectFirstOrThrow("div.manga")
val infoRoot = root.selectFirstOrThrow("div.bookintro")
val tagMap = getOrCreateTagMap()
val selectTag = infoRoot.getElementsByAttributeValue("itemprop", "genre").first()?.select("a")
val tags = selectTag?.mapNotNullToSet { tagMap[it.text()] }
val author = infoRoot.getElementsByAttributeValue("itemprop", "author").first()?.textOrNull()
return manga.copy(
title = root.selectFirst("h1[itemprop=name]")?.textOrNull()?.removeSuffix("Manga")?.trimEnd()
?: manga.title,
tags = tags.orEmpty(),
authors = setOfNotNull(author),
state = parseStatus(infoRoot.select("li a.red").text()),
description = infoRoot.getElementsByAttributeValue("itemprop", "description").first()?.html()
?.substringAfter("</b>"),
chapters = root.selectFirst("div.chapterbox")?.select("ul.sub_vol_ul > li")
?.mapChapters(reversed = true) { i, li ->
val a = li.selectFirstOrThrow("a.chapter_list_a")
val href = a.attrAsRelativeUrl("href").replace("%20", " ")
MangaChapter(
id = generateUid(href),
title = a.textOrNull(),
number = i + 1f,
volume = 0,
url = href,
uploadDate = parseChapterDateByLang(li.selectFirst("span")?.text().orEmpty()),
source = source,
scanlator = null,
branch = null,
)
},
)
}
''',
''' override suspend fun getDetails(manga: Manga): Manga {
if (domain.endsWith("niadd.com")) {
val mangaPath = manga.url
.substringBefore("?")
.removeSuffix("/")
val chaptersPath = if (mangaPath.endsWith(".html")) {
mangaPath.removeSuffix(".html") + "/chapters.html"
} else {
"$mangaPath/chapters.html"
}
val chaptersDoc = webClient.httpGet(
chaptersPath.toAbsoluteUrl(domain) + "?waring=1",
).parseHtml()
/*
* Current Niadd chapters page already contains
* manga title, status, authors, genres and language,
* so avoid a second HTTP request to the details page.
*/
val detailsDoc = chaptersDoc
val heading = detailsDoc.selectFirst("h1")
?.textOrNull()
?.trim()
.orEmpty()
val title = heading
.replace(
Regex(
"""\\s*\\((?:постоянный|завершенный)\\)\\s*$""",
RegexOption.IGNORE_CASE,
),
"",
)
.trim()
.ifEmpty { manga.title }
val tagMap = getOrCreateTagMap()
val tags = detailsDoc
.select("a[href^=/category/]")
.mapNotNullToSet { element ->
val tagTitle = element.text()
.trim()
.removePrefix(",")
.trim()
tagMap[tagTitle]
}
val authors = detailsDoc
.select("a[href*=author=]")
.mapNotNullToSet { element ->
element.textOrNull()
?.trim()
?.takeIf { it.isNotEmpty() }
}
val pageDescription = detailsDoc
.selectFirst("meta[name=description]")
?.attr("content")
?.trim()
?.takeIf { it.isNotEmpty() }
val chapterDateRegex = Regex(
"""\\b[A-Z][a-z]{2}\\s+\\d{1,2},\\s+\\d{4}\\b"""
)
val chapterLinks = chaptersDoc
.select("a[href]")
.filter { element ->
val href = element.attr("href")
(
href.startsWith("/chapter/") ||
href.startsWith(
"https://${domain}/chapter/"
)
) &&
!href.contains("/manga_for_adults/")
}
.distinctBy { element ->
element.attrAsAbsoluteUrl("href")
}
.reversed()
val chapters = chapterLinks.mapIndexed { i, a ->
val href = a.attrAsRelativeUrl("href")
.replace("%20", " ")
val anchorText = a.text().trim()
val rowText = a.parent()?.text().orEmpty().trim()
val rawTitle = a.ownText()
.trim()
.ifEmpty {
anchorText
}
val anchorDate = chapterDateRegex
.find(anchorText)
?.value
val rowDate = chapterDateRegex
.find(rowText)
?.value
val dateText = anchorDate ?: rowDate
val chapterTitle = if (
dateText != null &&
rawTitle.contains(dateText)
) {
rawTitle
.substringBefore(dateText)
.replace(
Regex("""\\s+\\d+\\s*$"""),
"",
)
.trim()
} else {
rawTitle
}
MangaChapter(
id = generateUid(href),
title = chapterTitle.takeIf {
it.isNotEmpty()
},
number = i + 1f,
volume = 0,
url = href,
uploadDate = parseChapterDateByLang(
dateText.orEmpty()
),
source = source,
scanlator = null,
branch = null,
)
}
return manga.copy(
title = title,
tags = tags,
authors = authors,
state = parseStatus(
detailsDoc.body().text()
),
description = manga.description
?: pageDescription,
chapters = chapters,
)
}
val doc = webClient.httpGet(
manga.url.toAbsoluteUrl(domain) + "?waring=1",
).parseHtml()
val root = doc.body().selectFirstOrThrow("div.manga")
val infoRoot = root.selectFirstOrThrow("div.bookintro")
val tagMap = getOrCreateTagMap()
val selectTag = infoRoot
.getElementsByAttributeValue(
"itemprop",
"genre",
)
.first()
?.select("a")
val tags = selectTag
?.mapNotNullToSet {
tagMap[it.text()]
}
val author = infoRoot
.getElementsByAttributeValue(
"itemprop",
"author",
)
.first()
?.textOrNull()
return manga.copy(
title = root
.selectFirst("h1[itemprop=name]")
?.textOrNull()
?.removeSuffix("Manga")
?.trimEnd()
?: manga.title,
tags = tags.orEmpty(),
authors = setOfNotNull(author),
state = parseStatus(
infoRoot.select("li a.red").text()
),
description = infoRoot
.getElementsByAttributeValue(
"itemprop",
"description",
)
.first()
?.html()
?.substringAfter("</b>"),
chapters = root
.selectFirst("div.chapterbox")
?.select("ul.sub_vol_ul > li")
?.mapChapters(
reversed = true
) { i, li ->
val a = li.selectFirstOrThrow(
"a.chapter_list_a"
)
val href = a
.attrAsRelativeUrl("href")
.replace("%20", " ")
MangaChapter(
id = generateUid(href),
title = a.textOrNull(),
number = i + 1f,
volume = 0,
url = href,
uploadDate = parseChapterDateByLang(
li.selectFirst("span")
?.text()
.orEmpty()
),
source = source,
scanlator = null,
branch = null,
)
},
)
}
''',
)
#
# ---------------------------------------------------------------------------
# 4. CHAPTER PAGE LIST
# ---------------------------------------------------------------------------
#
# Old NineManga:
#
# #page option
#
# Current Niadd has several <select>'s on the reader:
# chapter selector
# batch loading
# image scale
# page selector
#
# Instead of relying on id="page", select only options whose text is:
#
# 1/61
# 2/61
# ...
#
replace_exact(
"Niadd page list parser",
''' override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml()
return doc.body().requireElementById("page").select("option").map { option ->
val url = option.attr("value")
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = source,
)
}
}
''',
''' override suspend fun getPages(
chapter: MangaChapter,
): List<MangaPage> {
val chapterUrl = chapter.url
.substringBefore("#")
val doc = webClient
.httpGet(
chapterUrl.toAbsoluteUrl(domain)
)
.parseHtml()
if (domain.endsWith("niadd.com")) {
val pageLabelRegex = Regex(
"""^\\s*(\\d+)\\s*/\\s*(\\d+)\\s*$"""
)
val pageOptions = doc
.select("option[value]")
.mapNotNull { option ->
val match = pageLabelRegex.matchEntire(
option.text().trim()
) ?: return@mapNotNull null
val pageNumber = match
.groupValues
.getOrNull(1)
?.toIntOrNull()
?: return@mapNotNull null
val totalPages = match
.groupValues
.getOrNull(2)
?.toIntOrNull()
?: return@mapNotNull null
val url = option.attr("value").trim()
if (url.isEmpty()) {
return@mapNotNull null
}
Triple(pageNumber, totalPages, url)
}
.distinctBy { (pageNumber, _, _) ->
pageNumber
}
.sortedBy { (pageNumber, _, _) ->
pageNumber
}
val totalPages = pageOptions
.firstOrNull()
?.second
?: error(
"Cannot find Niadd page selector for $chapterUrl"
)
check(
pageOptions.all { (_, total, _) ->
total == totalPages
}
) {
"Inconsistent Niadd page count for $chapterUrl"
}
check(
pageOptions.size == totalPages &&
pageOptions.map { it.first } ==
(1..totalPages).toList()
) {
"Incomplete Niadd page selector for $chapterUrl: " +
"found ${pageOptions.size} of $totalPages pages"
}
return pageOptions.map { (_, _, url) ->
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = source,
)
}
}
return doc
.body()
.requireElementById("page")
.select("option")
.map { option ->
val url = option.attr("value")
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = source,
)
}
}
''',
)
#
# ---------------------------------------------------------------------------
# 5. IMAGE URL
# ---------------------------------------------------------------------------
#
# Old NineManga:
#
# a.pic_download
#
# Current Niadd serves page images from an external image host.
#
# Example:
#
# https://ruwebp.movietop.cc/comics/.../image.webp?acc=...&exp=...
#
# So find an image-looking URL instead of relying on CSS class.
#
replace_exact(
"Niadd image URL parser",
''' override suspend fun getPageUrl(page: MangaPage): String {
val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml()
val root = doc.body()
return root.selectFirstOrThrow("a.pic_download").attrAsAbsoluteUrl("href")
}
''',
''' override suspend fun getPageUrl(
page: MangaPage,
): String {
val doc = webClient
.httpGet(
page.url.toAbsoluteUrl(domain)
)
.parseHtml()
val root = doc.body()
if (domain.endsWith("niadd.com")) {
val readerImageLinkRegex = Regex(
"""^\\s*\\d+\\s+of\\s+\\d+\\s*$""",
RegexOption.IGNORE_CASE,
)
val imageAnchor = root
.select("a[href]")
.firstOrNull { element ->
readerImageLinkRegex.matches(
element.text()
)
}
if (imageAnchor != null) {
return imageAnchor.attrAsAbsoluteUrl(
"href"
)
}
// Do not fall back to an arbitrary <img>. Niadd may return a
// service/notice page with a perfectly valid static image, which
// would otherwise be cached as every manga page.
error(
"Cannot find Niadd reader image for ${page.url}"
)
}
return root
.selectFirstOrThrow("a.pic_download")
.attrAsAbsoluteUrl("href")
}
''',
)
#
# ---------------------------------------------------------------------------
# 6. RUSSIAN STATUS
# ---------------------------------------------------------------------------
#
# Niadd currently uses e.g.:
#
# постоянный
# Завершенный
#
replace_exact(
"Russian status parsing",
''' //ru
status.contains("постоянный") -> MangaState.ONGOING
status.contains("завершенный") -> MangaState.FINISHED
''',
''' //ru
status.contains(
"постоянный",
ignoreCase = true,
) -> MangaState.ONGOING
status.contains(
"завершенный",
ignoreCase = true,
) -> MangaState.FINISHED
''',
)
#
# ---------------------------------------------------------------------------
# 7. ENABLE RUSSIAN NIADD
# ---------------------------------------------------------------------------
#
# Keep NINEMANGA_RU unchanged intentionally.
#
# That preserves the existing source ID used by Futon database entries.
#
replace_exact(
"NineManga RU to Niadd RU",
''' @Broken
@MangaSourceParser("NINEMANGA_RU", "NineManga Русский", "ru")
class Russian(context: MangaLoaderContext) : NineMangaParser(
context,
MangaParserSource.NINEMANGA_RU,
"ru.ninemanga.com",
)
''',
''' @MangaSourceParser("NINEMANGA_RU", "Niadd Русский", "ru")
class Russian(
context: MangaLoaderContext,
) : NineMangaParser(
context,
MangaParserSource.NINEMANGA_RU,
"ru.niadd.com",
)
''',
)
parser_file.write_text(
text,
encoding="utf-8",
)
print()
print(f"[OK] Patched: {parser_file}")
print()
print("[OK] Niadd domain")
print("[OK] Niadd category pagination")
print("[OK] Niadd manga list")
print("[OK] Niadd details")
print("[OK] Niadd chapter list")
print("[OK] Niadd reader page list")
print("[OK] Niadd image URL")
print("[OK] Russian statuses")
print("[OK] Russian source enabled")