From db6cc5f2c1e7825109a641e1f0603b826cd785c7 Mon Sep 17 00:00:00 2001 From: tmfahad39 Date: Sat, 24 Oct 2026 01:22:07 +0600 Subject: [PATCH 1/4] feat(bangla): add Golper City (golpercity.blogspot.com) plugin Scrapes the Bangla novel blog Golper City using the Blogger JSON feed: labels are novels, posts are chapters in publish order. Co-Authored-By: opencode --- plugins/bangla/golpercity.ts | 214 +++++++++++++++++++++++ public/static/src/bn/golpercity/icon.png | Bin 0 -> 266 bytes 2 files changed, 214 insertions(+) create mode 100644 plugins/bangla/golpercity.ts create mode 100644 public/static/src/bn/golpercity/icon.png diff --git a/plugins/bangla/golpercity.ts b/plugins/bangla/golpercity.ts new file mode 100644 index 000000000..f8c08ee1a --- /dev/null +++ b/plugins/bangla/golpercity.ts @@ -0,0 +1,214 @@ +import { fetchApi } from '@libs/fetch'; +import { Plugin } from '@/types/plugin'; +import { Filters } from '@libs/filterInputs'; +import { load as loadCheerio } from 'cheerio'; +import { defaultCover } from '@libs/defaultCover'; + +type FeedLink = { rel: string; href: string }; +type FeedEntry = { + title: { $t: string }; + link: FeedLink[]; + published?: { $t: string }; + media$thumbnail?: { url: string }; +}; +type FeedResponse = { + feed?: { + openSearch$totalResults?: { $t: string }; + category?: { term: string }[]; + entry?: FeedEntry[]; + }; +}; + +const PAGE_SIZE = 20; +const FEED_PAGE_SIZE = 150; +const COVER_SIZE = 's320'; +const CHAPTER_PATH = /^\/\d{4}\/\d{2}\/[^/]+\.html$/; +const NOVEL_PATH = /^\/search\/label\/(.+)$/; +// Labels that are series names double as novels. The ones below group posts by +// genre or are not stories at all. +const EXCLUDED_LABELS = new Set([ + 'ফ্যান্টাসি গল্প', + 'রোমান্টিক গল্প', + 'সামাজিক গল্প', + 'এসাইনমেন্ট', + 'Web Novel Link', +]); + +const postHref = (entry: FeedEntry): string | undefined => + entry.link?.find(link => link.rel === 'alternate')?.href; + +class GolperCity implements Plugin.PluginBase { + id = 'golpercity'; + name = 'Golper City'; + icon = 'src/bn/golpercity/icon.png'; + site = 'https://golpercity.blogspot.com'; + version = '1.0.0'; + filters: Filters | undefined = undefined; + + private async request(url: string) { + const response = await fetchApi(url); + if (!response.ok) { + throw Object.assign(new Error(`Golper City: HTTP ${response.status}.`), { + status: response.status, + }); + } + return response; + } + + private async getFeed( + params: Record, + label?: string, + ): Promise { + const feed = label ? `summary/-/${encodeURIComponent(label)}` : 'summary'; + const query = new URLSearchParams({ alt: 'json', ...params }).toString(); + const data: FeedResponse = await ( + await this.request(`${this.site}/feeds/posts/${feed}?${query}`) + ).json(); + return data.feed; + } + + /** Blogger's label list is the blog's own grouping of novels. */ + private async novelLabels(): Promise { + const feed = await this.getFeed({ 'max-results': '1' }); + return (feed?.category || []) + .map(category => category.term) + .filter(label => !EXCLUDED_LABELS.has(label)); + } + + private novelPath(label: string): string { + return `/search/label/${encodeURIComponent(label)}`; + } + + private labelFromPath(novelPath: string): string { + const match = NOVEL_PATH.exec(new URL(novelPath, this.site).pathname); + if (!match) throw new Error('Golper City: invalid novel address.'); + return decodeURIComponent(match[1]); + } + + private coverFrom(entry?: FeedEntry): string { + const thumbnail = entry?.media$thumbnail?.url; + if (!thumbnail) return defaultCover; + return thumbnail.replace(/\/s\d+(-c)?\//, `/${COVER_SIZE}/`); + } + + /** Each novel's cover is the thumbnail of its newest post, if any. */ + private async toNovelItems(labels: string[]): Promise { + return Promise.all( + labels.map(async label => { + const feed = await this.getFeed({ 'max-results': '1' }, label).catch( + () => undefined, + ); + return { + name: label, + path: this.novelPath(label), + cover: this.coverFrom(feed?.entry?.[0]), + }; + }), + ); + } + + async popularNovels(pageNo: number): Promise { + const labels = (await this.novelLabels()).slice( + (pageNo - 1) * PAGE_SIZE, + pageNo * PAGE_SIZE, + ); + return this.toNovelItems(labels); + } + + /** Every post of a label, newest first, with duplicates across pages removed. */ + private async labelPosts(label: string): Promise { + const posts: FeedEntry[] = []; + const seen = new Set(); + let start = 1; + let total = 1; + while (start <= total) { + const feed = await this.getFeed( + { + 'max-results': String(FEED_PAGE_SIZE), + 'start-index': String(start), + }, + label, + ); + const entries = feed?.entry || []; + if (!entries.length) break; + for (const entry of entries) { + const href = postHref(entry); + if (!href || seen.has(href)) continue; + seen.add(href); + posts.push(entry); + } + // Blogger returns fewer posts than asked per page, so step by what came back. + start += entries.length; + total = Number(feed?.openSearch$totalResults?.$t || 0); + } + return posts; + } + + async parseNovel(novelPath: string): Promise { + const label = this.labelFromPath(novelPath); + const posts = await this.labelPosts(label); + + const chapters: Plugin.ChapterItem[] = []; + for (const entry of posts) { + const href = postHref(entry); + const name = entry.title?.$t?.trim(); + if (!href || !name) continue; + const path = new URL(href).pathname; + if (!CHAPTER_PATH.test(path)) continue; + chapters.push({ + name, + path, + releaseTime: entry.published?.$t, + }); + } + if (!chapters.length) + throw new Error('Golper City: no chapters were found.'); + + // Blogger lists newest first; readers expect the series in publish order. + chapters.reverse(); + + return { + name: label, + path: this.novelPath(label), + cover: this.coverFrom(posts[0]), + chapters, + }; + } + + async parseChapter(chapterPath: string): Promise { + const url = this.resolveUrl(chapterPath); + const $ = loadCheerio(await (await this.request(url)).text()); + const content = $('div.post-body').first(); + content + .find( + 'script,style,iframe,noscript,form,.sharethis-inline-reaction-buttons', + ) + .remove(); + return content.html() || ''; + } + + async searchNovels( + searchTerm: string, + pageNo: number, + ): Promise { + const terms = searchTerm.toLowerCase().split(/\s+/).filter(Boolean); + if (!terms.length) return []; + // Searching the label catalogue finds novels by name. Blogger's post + // search would return chapter posts instead, which crowd novels out. + const labels = (await this.novelLabels()).filter(label => + terms.every(term => label.toLowerCase().includes(term)), + ); + return this.toNovelItems( + labels.slice((pageNo - 1) * PAGE_SIZE, pageNo * PAGE_SIZE), + ); + } + + resolveUrl(path: string): string { + const url = new URL(path, this.site); + if (url.origin !== this.site) + throw new Error('Golper City: invalid page address.'); + return url.href; + } +} + +export default new GolperCity(); diff --git a/public/static/src/bn/golpercity/icon.png b/public/static/src/bn/golpercity/icon.png new file mode 100644 index 0000000000000000000000000000000000000000..6fb098fb6a4ffcadceedfb4d984579db74f8b043 GIT binary patch literal 266 zcmeAS@N?(olHy`uVBq!ia0vp^2_VeD1|%QND7OGo7d>4ZLn>~)z2zv#ARyuz$hbtn zfo-G9)i$AH$LAcqJG*>ZywqC%qf>hr7#bKDE~GN3hUfAC^$`XQw;2}M>$~=D{#=W! zxFS0NnTx`1Xy!)YBXK@3H!v^=FfcGNI5049FfcGOC@?UvfZQa|u%Dqsje&tdf{B5F bbt}Wt7fI_9-W_=W6k+gm^>bP0l+XkKbKx)} literal 0 HcmV?d00001 From 642c8dda0e36bd75d409d91a46859282719600e3 Mon Sep 17 00:00:00 2001 From: tmfahad39 Date: Sat, 24 Oct 2026 01:29:10 +0600 Subject: [PATCH 2/4] chore(bangla): register Bangla language in plugin manifest Needed so plugins/bangla plugins are picked up by build:manifest. Co-Authored-By: opencode --- scripts/languages.js | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/languages.js b/scripts/languages.js index 2892b5935..168a7b512 100644 --- a/scripts/languages.js +++ b/scripts/languages.js @@ -1,5 +1,6 @@ export default { Arabic: '‎العربية', + Bangla: 'বাংলা', Chinese: '中文, 汉语, 漢語', English: 'English', French: 'Français', From 7f594cfc8a12998da7fbfb56a6ebd89bb2db4a99 Mon Sep 17 00:00:00 2001 From: tmfahad39 Date: Sat, 24 Oct 2026 03:21:15 +0600 Subject: [PATCH 3/4] fix(english/lncrawler): keep chapter heading and add search filters - Keep the API's leading

in chapter bodies so readers see the chapter title (including its number) in the chapter header. - Add filter support to popularNovels/searchNovels via the search API: languages, status, sort_by, sort_order and adult content, with popular = popularity desc and Latest = last_updated desc. - Bump version to 1.1.0. Co-Authored-By: opencode --- plugins/english/lncrawler.ts | 160 ++++++++++++++++++++++++++++++++--- 1 file changed, 149 insertions(+), 11 deletions(-) diff --git a/plugins/english/lncrawler.ts b/plugins/english/lncrawler.ts index b40a2012f..0ee127315 100644 --- a/plugins/english/lncrawler.ts +++ b/plugins/english/lncrawler.ts @@ -1,6 +1,7 @@ import { fetchApi } from '@libs/fetch'; import { Plugin } from '@/types/plugin'; import { NovelStatus } from '@libs/novelStatus'; +import { Filters, FilterTypes } from '@libs/filterInputs'; // LNCrawler — https://lncrawler.monster // The web frontend is a React SPA; all data comes from its public JSON @@ -304,32 +305,168 @@ function parseChaptersPage(data: unknown): ChaptersPage { /** * Chapter body HTML. The API prefixes the body with an

of the - * chapter title — the app already shows the title, so it is dropped. + * chapter title — which includes the chapter number on LNCrawler. + * Keeping it gives in-page headings showing title and number, so it + * is left in place (only executable content is stripped below). */ function parseChapterBody(data: unknown): string { const res = data as { body?: string }; if (typeof res.body !== 'string') { throw new Error('Chapter response had an unexpected shape'); } - const withoutTitle = res.body - .replace(/^\s*]*>[\s\S]*?<\/h1>\s*/i, '') - .trim(); - return sanitizeChapterHtml(withoutTitle); + return sanitizeChapterHtml(res.body.trim()); } +/** + * Languages the LNCrawler search API understands (from the site's own + * frontend), used for the Languages filter. Responses repeat the + * `languages` param, which the API aliases to `language`. + */ +const LANGUAGES: { label: string; value: string }[] = [ + { label: 'English', value: 'en' }, + { label: 'French', value: 'fr' }, + { label: 'Spanish', value: 'es' }, + { label: 'German', value: 'de' }, + { label: 'Italian', value: 'it' }, + { label: 'Japanese', value: 'ja' }, + { label: 'Korean', value: 'ko' }, + { label: 'Chinese', value: 'zh' }, + { label: 'Portuguese', value: 'pt' }, + { label: 'Russian', value: 'ru' }, + { label: 'Arabic', value: 'ar' }, + { label: 'Hindi', value: 'hi' }, + { label: 'Thai', value: 'th' }, + { label: 'Vietnamese', value: 'vi' }, + { label: 'Indonesian', value: 'id' }, + { label: 'Turkish', value: 'tr' }, + { label: 'Polish', value: 'pl' }, + { label: 'Dutch', value: 'nl' }, + { label: 'Swedish', value: 'sv' }, + { label: 'Danish', value: 'da' }, +]; + +const SORT_OPTIONS: { label: string; value: string }[] = [ + { label: 'Title', value: 'title' }, + { label: 'Rating', value: 'rating' }, + { label: 'Date added', value: 'date_added' }, + { label: 'All-time views', value: 'popularity' }, + { label: 'Trending', value: 'trending' }, + { label: 'Last updated', value: 'last_updated' }, +]; + +const STATUS_OPTIONS: { label: string; value: string }[] = [ + { label: 'Ongoing', value: 'Ongoing' }, + { label: 'Completed', value: 'Completed' }, + { label: 'On Hiatus', value: 'On Hiatus' }, + { label: 'Cancelled', value: 'Cancelled' }, + { label: 'Unknown', value: 'Unknown' }, +]; + +const ADULT_OPTIONS: { label: string; value: string }[] = [ + { label: 'Hide adult content', value: 'hide' }, + { label: 'Show everything', value: 'show' }, + { label: 'Adult content only', value: 'only' }, +]; + class LNCrawler implements Plugin.PluginBase { id = 'lncrawler'; name = 'LNCrawler'; icon = 'src/en/lncrawler/icon.png'; site = 'https://lncrawler.monster'; - version = '1.0.1'; + version = '1.1.0'; + + filters = { + languages: { + type: FilterTypes.CheckboxGroup, + label: 'Languages', + value: ['en'], + options: LANGUAGES, + }, + status: { + type: FilterTypes.Picker, + label: 'Status', + value: '', + options: STATUS_OPTIONS, + }, + sort_by: { + type: FilterTypes.Picker, + label: 'Sort by', + value: '', + options: SORT_OPTIONS, + }, + sort_order: { + type: FilterTypes.Picker, + label: 'Order', + value: '', + options: [ + { label: 'Ascending', value: 'asc' }, + { label: 'Descending', value: 'desc' }, + ], + }, + adult: { + type: FilterTypes.Picker, + label: 'Adult content', + value: '', + options: ADULT_OPTIONS, + }, + } satisfies Filters; + + /** + * Build the /novels/search/ query from filters. Catalog defaults: + * popular = all-time views descending; the Latest toggle switches to + * last-updated descending. An empty filter value means "server + * default" and is simply omitted. + */ + private buildSearchQuery( + options: Plugin.PopularNovelsOptions, + pageNo: number, + query?: string, + ): URLSearchParams { + const params = new URLSearchParams({ + page: String(pageNo), + page_size: String(PAGE_SIZE), + }); + if (query) params.set('query', query); + const opts: Plugin.PopularNovelsOptions = options || { + showLatestNovels: false, + filters: undefined, + }; + const filters = opts.filters; + if (!filters && !options) { + // Interface callers pass no options: catalog default = English. + params.append('languages', 'en'); + } else { + for (const language of filters?.languages?.value || []) { + if (language) params.append('languages', language); + } + } + if (filters?.status?.value) params.set('status', filters.status.value); + if (filters?.sort_by?.value) { + params.set('sort_by', filters.sort_by.value); + } else { + params.set( + 'sort_by', + opts.showLatestNovels ? 'last_updated' : 'popularity', + ); + } + if (filters?.sort_order?.value) { + params.set('sort_order', filters.sort_order.value); + } else { + params.set('sort_order', 'desc'); + } + if (filters?.adult?.value) params.set('adult', filters.adult.value); + return params; + } - async popularNovels(pageNo: number): Promise { + async popularNovels( + pageNo: number, + options: Plugin.PopularNovelsOptions, + ): Promise { // The search endpoint is the only listing with a language filter; - // with no query it serves as the catalog. The API exposes no sort - // options, so the Latest toggle lands here too. + // with no query it serves as the catalog. Popular = all-time views + // descending, the Latest toggle = last-updated descending. const data = await fetchJson( - `${API}/novels/search/?language=en&page=${pageNo}&page_size=${PAGE_SIZE}`, + `${API}/novels/search/?${this.buildSearchQuery(options, pageNo)}`, 'Failed to load popular novels', ); const page = parseNovelListPage(data); @@ -417,9 +554,10 @@ class LNCrawler implements Plugin.PluginBase { async searchNovels( searchTerm: string, pageNo: number, + options?: Plugin.PopularNovelsOptions, ): Promise { const data = await fetchJson( - `${API}/novels/search/?query=${encodeURIComponent(searchTerm)}&language=en&page=${pageNo}&page_size=${PAGE_SIZE}`, + `${API}/novels/search/?${this.buildSearchQuery(options, pageNo, searchTerm)}`, 'Failed to search novels', ); const page = parseNovelListPage(data); From 209af16a0952a68ae6bc5f2e7e2cf5b338f7cb5f Mon Sep 17 00:00:00 2001 From: tmfahad39 Date: Sat, 24 Oct 2026 03:28:13 +0600 Subject: [PATCH 4/4] Revert "fix(english/lncrawler): keep chapter heading and add search filters" This reverts commit 7f594cfc8a12998da7fbfb56a6ebd89bb2db4a99. --- plugins/english/lncrawler.ts | 160 +++-------------------------------- 1 file changed, 11 insertions(+), 149 deletions(-) diff --git a/plugins/english/lncrawler.ts b/plugins/english/lncrawler.ts index 0ee127315..b40a2012f 100644 --- a/plugins/english/lncrawler.ts +++ b/plugins/english/lncrawler.ts @@ -1,7 +1,6 @@ import { fetchApi } from '@libs/fetch'; import { Plugin } from '@/types/plugin'; import { NovelStatus } from '@libs/novelStatus'; -import { Filters, FilterTypes } from '@libs/filterInputs'; // LNCrawler — https://lncrawler.monster // The web frontend is a React SPA; all data comes from its public JSON @@ -305,168 +304,32 @@ function parseChaptersPage(data: unknown): ChaptersPage { /** * Chapter body HTML. The API prefixes the body with an

of the - * chapter title — which includes the chapter number on LNCrawler. - * Keeping it gives in-page headings showing title and number, so it - * is left in place (only executable content is stripped below). + * chapter title — the app already shows the title, so it is dropped. */ function parseChapterBody(data: unknown): string { const res = data as { body?: string }; if (typeof res.body !== 'string') { throw new Error('Chapter response had an unexpected shape'); } - return sanitizeChapterHtml(res.body.trim()); + const withoutTitle = res.body + .replace(/^\s*]*>[\s\S]*?<\/h1>\s*/i, '') + .trim(); + return sanitizeChapterHtml(withoutTitle); } -/** - * Languages the LNCrawler search API understands (from the site's own - * frontend), used for the Languages filter. Responses repeat the - * `languages` param, which the API aliases to `language`. - */ -const LANGUAGES: { label: string; value: string }[] = [ - { label: 'English', value: 'en' }, - { label: 'French', value: 'fr' }, - { label: 'Spanish', value: 'es' }, - { label: 'German', value: 'de' }, - { label: 'Italian', value: 'it' }, - { label: 'Japanese', value: 'ja' }, - { label: 'Korean', value: 'ko' }, - { label: 'Chinese', value: 'zh' }, - { label: 'Portuguese', value: 'pt' }, - { label: 'Russian', value: 'ru' }, - { label: 'Arabic', value: 'ar' }, - { label: 'Hindi', value: 'hi' }, - { label: 'Thai', value: 'th' }, - { label: 'Vietnamese', value: 'vi' }, - { label: 'Indonesian', value: 'id' }, - { label: 'Turkish', value: 'tr' }, - { label: 'Polish', value: 'pl' }, - { label: 'Dutch', value: 'nl' }, - { label: 'Swedish', value: 'sv' }, - { label: 'Danish', value: 'da' }, -]; - -const SORT_OPTIONS: { label: string; value: string }[] = [ - { label: 'Title', value: 'title' }, - { label: 'Rating', value: 'rating' }, - { label: 'Date added', value: 'date_added' }, - { label: 'All-time views', value: 'popularity' }, - { label: 'Trending', value: 'trending' }, - { label: 'Last updated', value: 'last_updated' }, -]; - -const STATUS_OPTIONS: { label: string; value: string }[] = [ - { label: 'Ongoing', value: 'Ongoing' }, - { label: 'Completed', value: 'Completed' }, - { label: 'On Hiatus', value: 'On Hiatus' }, - { label: 'Cancelled', value: 'Cancelled' }, - { label: 'Unknown', value: 'Unknown' }, -]; - -const ADULT_OPTIONS: { label: string; value: string }[] = [ - { label: 'Hide adult content', value: 'hide' }, - { label: 'Show everything', value: 'show' }, - { label: 'Adult content only', value: 'only' }, -]; - class LNCrawler implements Plugin.PluginBase { id = 'lncrawler'; name = 'LNCrawler'; icon = 'src/en/lncrawler/icon.png'; site = 'https://lncrawler.monster'; - version = '1.1.0'; - - filters = { - languages: { - type: FilterTypes.CheckboxGroup, - label: 'Languages', - value: ['en'], - options: LANGUAGES, - }, - status: { - type: FilterTypes.Picker, - label: 'Status', - value: '', - options: STATUS_OPTIONS, - }, - sort_by: { - type: FilterTypes.Picker, - label: 'Sort by', - value: '', - options: SORT_OPTIONS, - }, - sort_order: { - type: FilterTypes.Picker, - label: 'Order', - value: '', - options: [ - { label: 'Ascending', value: 'asc' }, - { label: 'Descending', value: 'desc' }, - ], - }, - adult: { - type: FilterTypes.Picker, - label: 'Adult content', - value: '', - options: ADULT_OPTIONS, - }, - } satisfies Filters; - - /** - * Build the /novels/search/ query from filters. Catalog defaults: - * popular = all-time views descending; the Latest toggle switches to - * last-updated descending. An empty filter value means "server - * default" and is simply omitted. - */ - private buildSearchQuery( - options: Plugin.PopularNovelsOptions, - pageNo: number, - query?: string, - ): URLSearchParams { - const params = new URLSearchParams({ - page: String(pageNo), - page_size: String(PAGE_SIZE), - }); - if (query) params.set('query', query); - const opts: Plugin.PopularNovelsOptions = options || { - showLatestNovels: false, - filters: undefined, - }; - const filters = opts.filters; - if (!filters && !options) { - // Interface callers pass no options: catalog default = English. - params.append('languages', 'en'); - } else { - for (const language of filters?.languages?.value || []) { - if (language) params.append('languages', language); - } - } - if (filters?.status?.value) params.set('status', filters.status.value); - if (filters?.sort_by?.value) { - params.set('sort_by', filters.sort_by.value); - } else { - params.set( - 'sort_by', - opts.showLatestNovels ? 'last_updated' : 'popularity', - ); - } - if (filters?.sort_order?.value) { - params.set('sort_order', filters.sort_order.value); - } else { - params.set('sort_order', 'desc'); - } - if (filters?.adult?.value) params.set('adult', filters.adult.value); - return params; - } + version = '1.0.1'; - async popularNovels( - pageNo: number, - options: Plugin.PopularNovelsOptions, - ): Promise { + async popularNovels(pageNo: number): Promise { // The search endpoint is the only listing with a language filter; - // with no query it serves as the catalog. Popular = all-time views - // descending, the Latest toggle = last-updated descending. + // with no query it serves as the catalog. The API exposes no sort + // options, so the Latest toggle lands here too. const data = await fetchJson( - `${API}/novels/search/?${this.buildSearchQuery(options, pageNo)}`, + `${API}/novels/search/?language=en&page=${pageNo}&page_size=${PAGE_SIZE}`, 'Failed to load popular novels', ); const page = parseNovelListPage(data); @@ -554,10 +417,9 @@ class LNCrawler implements Plugin.PluginBase { async searchNovels( searchTerm: string, pageNo: number, - options?: Plugin.PopularNovelsOptions, ): Promise { const data = await fetchJson( - `${API}/novels/search/?${this.buildSearchQuery(options, pageNo, searchTerm)}`, + `${API}/novels/search/?query=${encodeURIComponent(searchTerm)}&language=en&page=${pageNo}&page_size=${PAGE_SIZE}`, 'Failed to search novels', ); const page = parseNovelListPage(data);