gandc/futon-niadd-gitea-ci
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

fix ninemanga page structure parse
Build Futon with Niadd patch / Build Futon Niadd debug APK (push) Failing after 10m18s
Details

Browse Source
This commit is contained in:
gandc
2026-08-09 23:07:20 +03:00
parent bdb8b68c3e
commit 9e6d724865
1 changed files with 640 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/patch_ninemanga_niadd.py
+640 -66
View File
@@ -5,7 +5,10 @@ import sys
if len(sys.argv) != 2:
print(f"Usage: {sys.argv[0]} <kotatsu-parsers-redo-dir>", file=sys.stderr)
print(
f"Usage: {sys.argv[0]} <kotatsu-parsers-redo-dir>",
file=sys.stderr,
)
sys.exit(2)
@@ -34,7 +37,8 @@ def replace_exact(name: str, old: str, new: str) -> None:
if count != 1:
print(
f"ERROR: patch '{name}' expected exactly 1 match, found {count}",
f"ERROR: patch '{name}' expected exactly 1 match, "
f"found {count}",
file=sys.stderr,
)
print(
@@ -44,42 +48,25 @@ def replace_exact(name: str, old: str, new: str) -> None:
sys.exit(1)
text = text.replace(old, new, 1)
print(f"[OK] {name}")
#
# Niadd search pagination:
# ---------------------------------------------------------------------------
# 1. CATEGORY PAGINATION
# ---------------------------------------------------------------------------
#
# NineManga:
# ?page=2
# Old NineManga:
#
# Niadd:
# ?page=2.html
# /category/index_1
# /category/index_2
#
replace_exact(
"Niadd search pagination",
''' append("/search/")
append("?page=")
append(page.toString())
''',
''' append("/search/")
append("?page=")
append(page.toString())
if (domain.endsWith("niadd.com")) {
append(".html")
}
''',
)
# Current Niadd:
#
# Niadd category pagination:
#
# Page 1:
# /category/
#
# Page 2+:
# /category/index_2.html
# /category/index_3.html
#
replace_exact(
"Niadd category pagination",
@@ -105,20 +92,25 @@ replace_exact(
#
# NineManga used:
# ---------------------------------------------------------------------------
# 2. MANGA LIST
# ---------------------------------------------------------------------------
#
# ul.direlist > li
# Old NineManga expects:
#
# ul.direlist
# li
# dd
# a.bookname
#
# Niadd no longer has ul.direlist.
#
# Instead, discover manga cards through their canonical /manga/ links.
# Manga detail links are still stable:
#
# The same manga can have several links:
# - cover
# - title
# - description
# /manga/<name>.html
#
# Therefore links are grouped by absolute manga URL.
# We therefore identify entries by their canonical /manga/ URL instead of
# presentation CSS classes.
#
replace_exact(
"Niadd manga list parser",
@@ -162,37 +154,54 @@ replace_exact(
return linksByUrl.mapNotNull { (href, links) ->
val relUrl = href.toRelativeUrl(domain)
val texts = links
.map { it.text().trim() }
.filter { it.isNotEmpty() }
.distinct()
val rawTitle = texts.minByOrNull { it.length }
val title = links.asSequence()
.mapNotNull { link ->
link.selectFirst(
"h1, h2, h3, h4, h5, h6, strong, b"
)?.textOrNull()
}
.map { it.trim() }
.firstOrNull { it.isNotEmpty() }
?: links.asSequence()
.map { it.ownText().trim() }
.firstOrNull { it.isNotEmpty() }
?: links.asSequence()
.mapNotNull { link ->
link.selectFirst("img[alt]")
?.attr("alt")
?.trim()
?.takeIf { it.isNotEmpty() }
}
.firstOrNull()
?: return@mapNotNull null
val description = texts
.asSequence()
.filter { it != rawTitle }
.filter { it.length > rawTitle.length }
.maxByOrNull { it.length }
?.removePrefix(rawTitle)
?.trim()
?.takeIf { it.isNotEmpty() }
val coverUrl = links
.asSequence()
val coverUrl = links.asSequence()
.mapNotNull { link ->
link.selectFirst("img")?.src()
?: link.parent()?.selectFirst("img")?.src()
?: link.parent()?.parent()?.selectFirst("img")?.src()
?: link.parent()
?.selectFirst("img")
?.src()
?: link.parent()
?.parent()
?.selectFirst("img")
?.src()
}
.firstOrNull()
val description = links.asSequence()
.mapNotNull { link ->
link.select("p")
.mapNotNull { it.textOrNull() }
.maxByOrNull { it.length }
}
.firstOrNull()
?.takeIf { it.isNotBlank() }
Manga(
id = generateUid(relUrl),
url = relUrl,
publicUrl = href,
title = rawTitle.toCamelCase(),
title = title.toCamelCase(),
altTitles = emptySet(),
coverUrl = coverUrl,
rating = RATING_UNKNOWN,
@@ -218,7 +227,10 @@ replace_exact(
id = generateUid(relUrl),
url = relUrl,
publicUrl = href,
title = dd?.selectFirst("a.bookname")?.text()?.toCamelCase().orEmpty(),
title = dd?.selectFirst("a.bookname")
?.text()
?.toCamelCase()
.orEmpty(),
altTitles = emptySet(),
coverUrl = node.selectFirst("img")?.src(),
rating = RATING_UNKNOWN,
@@ -235,7 +247,554 @@ replace_exact(
#
# Russian statuses on Niadd are not consistently cased:
# ---------------------------------------------------------------------------
# 3. DETAILS + CHAPTERS
# ---------------------------------------------------------------------------
#
# Old parser depends on:
#
# div.manga
# div.bookintro
# div.chapterbox
# ul.sub_vol_ul
# a.chapter_list_a
#
# Niadd no longer has that hierarchy.
#
# Current Niadd however has stable semantic URLs:
#
# Manga:
# /manga/Foo.html
#
# Full chapter list:
# /manga/Foo/chapters.html
#
# Chapter:
# /chapter/3_81/5259607/
# /chapter/2_80_2/5205144.html
#
# So for Niadd we parse by semantic links, not CSS classes.
#
replace_exact(
"Niadd details and chapters parser",
''' override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(
manga.url.toAbsoluteUrl(domain) + "?waring=1",
).parseHtml()
val root = doc.body().selectFirstOrThrow("div.manga")
val infoRoot = root.selectFirstOrThrow("div.bookintro")
val tagMap = getOrCreateTagMap()
val selectTag = infoRoot.getElementsByAttributeValue("itemprop", "genre").first()?.select("a")
val tags = selectTag?.mapNotNullToSet { tagMap[it.text()] }
val author = infoRoot.getElementsByAttributeValue("itemprop", "author").first()?.textOrNull()
return manga.copy(
title = root.selectFirst("h1[itemprop=name]")?.textOrNull()?.removeSuffix("Manga")?.trimEnd()
?: manga.title,
tags = tags.orEmpty(),
authors = setOfNotNull(author),
state = parseStatus(infoRoot.select("li a.red").text()),
description = infoRoot.getElementsByAttributeValue("itemprop", "description").first()?.html()
?.substringAfter("</b>"),
chapters = root.selectFirst("div.chapterbox")?.select("ul.sub_vol_ul > li")
?.mapChapters(reversed = true) { i, li ->
val a = li.selectFirstOrThrow("a.chapter_list_a")
val href = a.attrAsRelativeUrl("href").replace("%20", " ")
MangaChapter(
id = generateUid(href),
title = a.textOrNull(),
number = i + 1f,
volume = 0,
url = href,
uploadDate = parseChapterDateByLang(li.selectFirst("span")?.text().orEmpty()),
source = source,
scanlator = null,
branch = null,
)
},
)
}
''',
''' override suspend fun getDetails(manga: Manga): Manga {
if (domain.endsWith("niadd.com")) {
val detailsUrl = manga.url.toAbsoluteUrl(domain)
val detailsDoc = webClient.httpGet(
"$detailsUrl?waring=1",
).parseHtml()
val mangaPath = manga.url
.substringBefore("?")
.removeSuffix("/")
val chaptersPath = if (mangaPath.endsWith(".html")) {
mangaPath.removeSuffix(".html") + "/chapters.html"
} else {
"$mangaPath/chapters.html"
}
val chaptersDoc = webClient.httpGet(
chaptersPath.toAbsoluteUrl(domain) + "?waring=1",
).parseHtml()
val heading = detailsDoc.selectFirst("h1")
?.textOrNull()
?.trim()
.orEmpty()
val title = heading
.replace(
Regex(
"""\\s*\\((?:постоянный|завершенный)\\)\\s*$""",
RegexOption.IGNORE_CASE,
),
"",
)
.trim()
.ifEmpty { manga.title }
val tagMap = getOrCreateTagMap()
val tags = detailsDoc
.select("a[href^=/category/]")
.mapNotNullToSet { element ->
val tagTitle = element.text()
.trim()
.removePrefix(",")
.trim()
tagMap[tagTitle]
}
val authors = detailsDoc
.select("a[href*=author=]")
.mapNotNullToSet { element ->
element.textOrNull()
?.trim()
?.takeIf { it.isNotEmpty() }
}
val pageDescription = detailsDoc
.selectFirst("meta[name=description]")
?.attr("content")
?.trim()
?.takeIf { it.isNotEmpty() }
val chapterDateRegex = Regex(
"""\\b[A-Z][a-z]{2}\\s+\\d{1,2},\\s+\\d{4}\\b"""
)
val chapterLinks = chaptersDoc
.select("a[href]")
.filter { element ->
val href = element.attr("href")
(
href.startsWith("/chapter/") ||
href.startsWith(
"https://${domain}/chapter/"
)
) &&
!href.contains("/manga_for_adults/")
}
.distinctBy { element ->
element.attrAsAbsoluteUrl("href")
}
.reversed()
val chapters = chapterLinks.mapIndexed { i, a ->
val href = a.attrAsRelativeUrl("href")
.replace("%20", " ")
val rawTitle = a.ownText()
.trim()
.ifEmpty {
a.text().trim()
}
val anchorDate = chapterDateRegex
.find(a.text())
?.value
val rowDate = chapterDateRegex
.find(a.parent()?.text().orEmpty())
?.value
val dateText = anchorDate ?: rowDate
val chapterTitle = if (
dateText != null &&
rawTitle.contains(dateText)
) {
rawTitle
.substringBefore(dateText)
.replace(
Regex("""\\s+\\d+\\s*$"""),
"",
)
.trim()
} else {
rawTitle
}
MangaChapter(
id = generateUid(href),
title = chapterTitle.takeIf {
it.isNotEmpty()
},
number = i + 1f,
volume = 0,
url = href,
uploadDate = parseChapterDateByLang(
dateText.orEmpty()
),
source = source,
scanlator = null,
branch = null,
)
}
return manga.copy(
title = title,
tags = tags,
authors = authors,
state = parseStatus(
detailsDoc.body().text()
),
description = manga.description
?: pageDescription,
chapters = chapters,
)
}
val doc = webClient.httpGet(
manga.url.toAbsoluteUrl(domain) + "?waring=1",
).parseHtml()
val root = doc.body().selectFirstOrThrow("div.manga")
val infoRoot = root.selectFirstOrThrow("div.bookintro")
val tagMap = getOrCreateTagMap()
val selectTag = infoRoot
.getElementsByAttributeValue(
"itemprop",
"genre",
)
.first()
?.select("a")
val tags = selectTag
?.mapNotNullToSet {
tagMap[it.text()]
}
val author = infoRoot
.getElementsByAttributeValue(
"itemprop",
"author",
)
.first()
?.textOrNull()
return manga.copy(
title = root
.selectFirst("h1[itemprop=name]")
?.textOrNull()
?.removeSuffix("Manga")
?.trimEnd()
?: manga.title,
tags = tags.orEmpty(),
authors = setOfNotNull(author),
state = parseStatus(
infoRoot.select("li a.red").text()
),
description = infoRoot
.getElementsByAttributeValue(
"itemprop",
"description",
)
.first()
?.html()
?.substringAfter("</b>"),
chapters = root
.selectFirst("div.chapterbox")
?.select("ul.sub_vol_ul > li")
?.mapChapters(
reversed = true
) { i, li ->
val a = li.selectFirstOrThrow(
"a.chapter_list_a"
)
val href = a
.attrAsRelativeUrl("href")
.replace("%20", " ")
MangaChapter(
id = generateUid(href),
title = a.textOrNull(),
number = i + 1f,
volume = 0,
url = href,
uploadDate = parseChapterDateByLang(
li.selectFirst("span")
?.text()
.orEmpty()
),
source = source,
scanlator = null,
branch = null,
)
},
)
}
''',
)
#
# ---------------------------------------------------------------------------
# 4. CHAPTER PAGE LIST
# ---------------------------------------------------------------------------
#
# Old NineManga:
#
# #page option
#
# Current Niadd has several <select>'s on the reader:
# chapter selector
# batch loading
# image scale
# page selector
#
# Instead of relying on id="page", select only options whose text is:
#
# 1/61
# 2/61
# ...
#
replace_exact(
"Niadd page list parser",
''' override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml()
return doc.body().requireElementById("page").select("option").map { option ->
val url = option.attr("value")
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = source,
)
}
}
''',
''' override suspend fun getPages(
chapter: MangaChapter,
): List<MangaPage> {
val doc = webClient
.httpGet(
chapter.url.toAbsoluteUrl(domain)
)
.parseHtml()
if (domain.endsWith("niadd.com")) {
val pageLabelRegex = Regex(
"""^\\s*(\\d+)\\s*/\\s*(\\d+)\\s*$"""
)
val pageOptions = doc
.select("option[value]")
.filter { option ->
pageLabelRegex.matches(
option.text()
) &&
option.attr("value").isNotBlank()
}
.distinctBy { option ->
option.attr("value")
}
if (pageOptions.isNotEmpty()) {
return pageOptions.map { option ->
val url = option.attr("value")
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = source,
)
}
}
/*
* Fallback for a future Niadd layout where page
* <option> elements stop containing values.
*
* Page URL structure currently is:
*
* page 1:
* /chapter/3_81/5259607/
* or
* /chapter/2_80_2/5205144.html
*
* page 2:
* /chapter/3_81/5259607-2.html
*/
val totalPages = doc
.select("option")
.mapNotNull { option ->
pageLabelRegex
.matchEntire(option.text())
?.groupValues
?.getOrNull(2)
?.toIntOrNull()
}
.maxOrNull()
if (totalPages != null && totalPages > 0) {
val originalUrl = chapter.url
.substringBefore("?")
val baseUrl = originalUrl
.replace(
Regex("""-\\d+\\.html$"""),
"",
)
.removeSuffix(".html")
.removeSuffix("/")
return (1..totalPages).map { pageNumber ->
val url = if (pageNumber == 1) {
originalUrl
} else {
"$baseUrl-$pageNumber.html"
}
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = source,
)
}
}
error(
"Cannot find Niadd page selector for ${chapter.url}"
)
}
return doc
.body()
.requireElementById("page")
.select("option")
.map { option ->
val url = option.attr("value")
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = source,
)
}
}
''',
)
#
# ---------------------------------------------------------------------------
# 5. IMAGE URL
# ---------------------------------------------------------------------------
#
# Old NineManga:
#
# a.pic_download
#
# Current Niadd serves page images from an external image host.
#
# Example:
#
# https://ruwebp.movietop.cc/comics/.../image.webp?acc=...&exp=...
#
# So find an image-looking URL instead of relying on CSS class.
#
replace_exact(
"Niadd image URL parser",
''' override suspend fun getPageUrl(page: MangaPage): String {
val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml()
val root = doc.body()
return root.selectFirstOrThrow("a.pic_download").attrAsAbsoluteUrl("href")
}
''',
''' override suspend fun getPageUrl(
page: MangaPage,
): String {
val doc = webClient
.httpGet(
page.url.toAbsoluteUrl(domain)
)
.parseHtml()
val root = doc.body()
if (domain.endsWith("niadd.com")) {
val imageUrlRegex = Regex(
"""(?i)\\.(?:jpe?g|png|gif|webp|avif)(?:\\?|$)"""
)
val imageAnchor = root
.select("a[href]")
.firstOrNull { element ->
val href = element.attr("href")
imageUrlRegex.containsMatchIn(href)
}
if (imageAnchor != null) {
return imageAnchor.attrAsAbsoluteUrl(
"href"
)
}
val image = root
.select("img[src]")
.firstOrNull { element ->
val src = element.attr("src")
imageUrlRegex.containsMatchIn(src)
}
if (image != null) {
return image.src()
}
error(
"Cannot find Niadd page image for ${page.url}"
)
}
return root
.selectFirstOrThrow("a.pic_download")
.attrAsAbsoluteUrl("href")
}
''',
)
#
# ---------------------------------------------------------------------------
# 6. RUSSIAN STATUS
# ---------------------------------------------------------------------------
#
# Niadd currently uses e.g.:
#
# постоянный
# Завершенный
@@ -247,17 +806,27 @@ replace_exact(
status.contains("завершенный") -> MangaState.FINISHED
''',
''' //ru
status.contains("постоянный", ignoreCase = true) -> MangaState.ONGOING
status.contains("завершенный", ignoreCase = true) -> MangaState.FINISHED
status.contains(
"постоянный",
ignoreCase = true,
) -> MangaState.ONGOING
status.contains(
"завершенный",
ignoreCase = true,
) -> MangaState.FINISHED
''',
)
#
# Activate Russian source and migrate it to Niadd.
# ---------------------------------------------------------------------------
# 7. ENABLE RUSSIAN NIADD
# ---------------------------------------------------------------------------
#
# Keep NINEMANGA_RU intentionally unchanged so existing Futon
# database/source references remain compatible.
# Keep NINEMANGA_RU unchanged intentionally.
#
# That preserves the existing source ID used by Futon database entries.
#
replace_exact(
"NineManga RU to Niadd RU",
@@ -270,7 +839,9 @@ replace_exact(
)
''',
''' @MangaSourceParser("NINEMANGA_RU", "Niadd Русский", "ru")
class Russian(context: MangaLoaderContext) : NineMangaParser(
class Russian(
context: MangaLoaderContext,
) : NineMangaParser(
context,
MangaParserSource.NINEMANGA_RU,
"ru.niadd.com",
@@ -290,7 +861,10 @@ print(f"[OK] Patched: {parser_file}")
print()
print("[OK] Niadd domain")
print("[OK] Niadd category pagination")
print("[OK] Niadd search pagination")
print("[OK] Niadd manga list parser")
print("[OK] Niadd manga list")
print("[OK] Niadd details")
print("[OK] Niadd chapter list")
print("[OK] Niadd reader page list")
print("[OK] Niadd image URL")
print("[OK] Russian statuses")
print("[OK] Russian source enabled")