diff --git a/plugins/english/onlinereadfreenovel.ts b/plugins/english/onlinereadfreenovel.ts new file mode 100644 index 000000000..1bc5c0937 --- /dev/null +++ b/plugins/english/onlinereadfreenovel.ts @@ -0,0 +1,330 @@ +import { fetchApi } from '@libs/fetch'; +import { Plugin } from '@/types/plugin'; +import { Filters, FilterTypes } from '@libs/filterInputs'; +import { CheerioAPI, load as loadCheerio } from 'cheerio'; +import { defaultCover } from '@libs/defaultCover'; + +/** + * Notes on the site (onlinereadfreenovel.com): + * - Listing: `/allbooks/` and `/gr//` list books as `article.box` + * items (`h2.title a` + preview image); further pages are `/p//`. + * - Search: `/search_by_author_title_series/q/` lists `ul.books li a`. + * Its links are cut to a fixed length, sometimes inside the slug, so + * `bookPath` rebuilds them. + * - Book: `//-_read.html` is page 1 of the text. The text + * is split across numbered pages, `//p,,-_read.html`. + * Each page is a plugin "chapter". The site has no chapter list, so the + * pages are walked until the last one (a page with text but no next link). + * - Some pages are served as an empty 200 response, in runs of up to six seen + * so far, with text continuing after them. Those pages are skipped rather + * than listed as empty chapters, and do not end the walk. + */ + +// Cloudflare on this site challenges requests from datacenter networks (seen +// from GitHub's runners) that send a browser User-Agent or a generic one such +// as node or curl, but lets a client that names itself through. +const HEADERS = { 'User-Agent': 'Mozilla/5.0 (compatible; LNReader)' }; + +// Upper bound on text pages walked per book, so a single book cannot issue an +// unbounded number of requests. Books longer than this are truncated, and +// parsePages logs a warning when it hits the cap. +const MAX_PAGES = 400; + +// Pages fetched at once while walking a book. Fetching them one at a time +// took about 90 seconds for a 240-page book. +const PAGE_BATCH = 6; + +// Retries of a chapter-list page after a rate limit, server error or network +// error, waiting 1s, 2s, 4s between them. +const PAGE_RETRIES = 3; + +// Empty pages in a row after a page that links to a next one before the walk +// gives up. It only ends the walk when the book's real last page is empty. +const MAX_EMPTY_RUN = 12; + +// A paragraph that opens a chapter, e.g. "Chapter 1 The Boy Who Lived", +// "Chapter One", "Prologue". Sentences (with end punctuation) are excluded. +const HEADING = /^(?:chapter\s+[\w-]+|prologue|epilogue)\b[^.!?]{0,60}$/i; + +// Text pages hold at most a few chapter openings (two seen so far); a page +// with more is a table of contents (18 and 21 seen) and is not named after +// its first entry. +const MAX_HEADINGS = 4; + +class OnlineReadFreeNovelPlugin implements Plugin.PluginBase { + id = 'onlinereadfreenovel'; + name = 'Online Read Free Novel'; + icon = 'src/en/onlinereadfreenovel/icon.png'; + site = 'https://onlinereadfreenovel.com'; + version = '1.0.0'; + + filters = { + genre: { + label: 'Genre', + value: 'all', + options: [ + { label: 'All', value: 'all' }, + { label: 'Action & Adventure', value: 'actions_adventure' }, + { label: 'Fantasy', value: 'fantasy' }, + { label: 'History & Fiction', value: 'historical_fiction' }, + { label: 'Horror', value: 'horror' }, + { label: 'Humor', value: 'humor' }, + { label: 'Mystery & Detective', value: 'mystery_detective' }, + { label: 'Romance & Love', value: 'romance_love' }, + { label: 'Science Fiction', value: 'science_fiction' }, + { label: 'Thrillers & Crime', value: 'thrillers_crime' }, + { label: 'Western', value: 'western' }, + { label: 'Young Adult', value: 'young_adult' }, + ], + type: FilterTypes.Picker, + }, + } satisfies Filters; + + async popularNovels( + pageNo: number, + { filters }: Plugin.PopularNovelsOptions, + ): Promise { + const genre = filters?.genre?.value || 'all'; + const base = genre === 'all' ? '/allbooks/' : '/gr/' + genre + '/'; + const path = pageNo > 1 ? base + 'p/' + pageNo + '/' : base; + const $ = await this.fetchDom(path); + return this.parseBookList($); + } + + async parseNovel(novelPath: string): Promise { + const $ = await this.fetchDom(novelPath); + + // A book page that failed on the site comes back as an empty 200 + // response; throw rather than overwrite the library entry with a blank + // title and no chapters. + const name = $('h1').first().text().trim(); + if (!name) throw new Error(`No book found at ${novelPath}`); + + const novel: Plugin.SourceNovel = { + path: novelPath, + name, + author: $('.box_in > b > a').first().text().trim(), + }; + + const cover = $('#textToRead img').first().attr('src'); + novel.cover = cover ? this.toAbsolute(cover) : defaultCover; + + novel.chapters = await this.parsePages(novelPath, $); + return novel; + } + + async parseChapter(chapterPath: string): Promise { + const paragraphs = this.paragraphs(await this.fetchDom(chapterPath)); + if (paragraphs.length === 0) { + throw new Error(`The site returned no text for ${chapterPath}`); + } + return paragraphs.map(part => '

' + part + '

').join(''); + } + + async searchNovels( + searchTerm: string, + pageNo: number, + ): Promise { + const base = + '/search_by_author_title_series/q/' + encodeURIComponent(searchTerm); + const path = pageNo > 1 ? base + '/p/' + pageNo : base; + const $ = await this.fetchDom(path); + + return $('ul.books li a') + .toArray() + .flatMap(el => { + const item = $(el); + const href = item.attr('href'); + if (!href) return []; + const style = item.find('.i1').attr('style') || ''; + const cover = style.match(/url\(["']?([^"')]+)["']?\)/)?.[1]; + const path = this.bookPath(href, cover); + if (!path) return []; + return [ + { + name: item.find('.h2').attr('title') || item.text().trim(), + path, + cover: cover ? this.toAbsolute(cover) : defaultCover, + }, + ]; + }); + } + + resolveUrl = (path: string) => this.site + path; + + parseBookList($: CheerioAPI): Plugin.NovelItem[] { + return $('article.box') + .toArray() + .flatMap(el => { + const link = $(el).find('h2.title a').first(); + const href = link.attr('href'); + if (!href) return []; + const src = $(el).find('div.text img').first().attr('src'); + return [ + { + name: link.text().trim(), + path: this.toPath(href), + cover: src ? this.toAbsolute(src) : defaultCover, + }, + ]; + }); + } + + /** + * Returns the book's `_read.html` path for a search link. Search links + * point at `//-.html` but are cut to a fixed length, e.g. + * `.../1301-harry_potter_and_the_philosophers_stone.ht` or + * `.../458266-the_sociology_of_harry_potter_22_enchanting_`. Book pages need + * the exact slug, so when the cut falls inside it, the slug is taken from + * the cover file name (`[_]_preview.jpg`). Returns undefined when + * the slug cannot be recovered. + */ + bookPath(href: string, cover?: string): string | undefined { + const path = this.toPath(href); + if (/\/\d+-[^/]+_read\.html$/.test(path)) return path; + const match = path.match(/^(\/[^/]+\/\d+-)([^/]*)$/); + if (!match) return undefined; + const [, prefix, tail] = match; + + const dot = tail.indexOf('.'); + if (dot > 0) return prefix + tail.slice(0, dot) + '_read.html'; + + const file = cover?.match(/\/([^/]+)_preview\.\w+$/)?.[1]; + if (!file) return undefined; + const slug = [file, file.replace(/^\d+_/, '')].find(name => + name.startsWith(tail), + ); + return slug ? prefix + slug + '_read.html' : undefined; + } + + /** + * Walks the text pages of a book. Page 1 is the novel path itself; page `n` + * is `p,,` inserted before the id. Pages are fetched PAGE_BATCH at a time + * and handled in order. The walk ends at the first page with text and no + * next link, or after MAX_EMPTY_RUN empty pages in a row. A page answered + * with no text or a 404 counts as empty and is skipped. Any other failure + * throws after retries, so a rate limit never shortens the list silently. + */ + async parsePages( + novelPath: string, + firstPage: CheerioAPI, + ): Promise { + const chapters: Plugin.ChapterItem[] = []; + // A path not in the `_read.html` form cannot be paged; list page 1 only. + const lastPage = this.pagePath(novelPath, 2) === novelPath ? 1 : MAX_PAGES; + let emptyRun = 0; + let hasNext = false; + + for (let start = 1; start <= lastPage; start += PAGE_BATCH) { + const pages: number[] = []; + for (let page = start; page < start + PAGE_BATCH; page++) { + if (page <= lastPage) pages.push(page); + } + const doms = await Promise.all( + pages.map(page => + page === 1 + ? firstPage + : this.fetchPageDom(this.pagePath(novelPath, page)).catch( + (error: Error) => error, + ), + ), + ); + + for (let i = 0; i < pages.length; i++) { + const $ = doms[i]; + if ($ instanceof Error) throw $; + const paragraphs = $ ? this.paragraphs($) : []; + if (!$ || paragraphs.length === 0) { + hasNext = false; + if (++emptyRun >= MAX_EMPTY_RUN) return chapters; + continue; + } + emptyRun = 0; + const headings = paragraphs + .map(part => part.replace(/<[^>]*>/g, '').trim()) + .filter(text => HEADING.test(text)); + // Many headings on one page is a table of contents, not chapters. + const heading = + headings.length <= MAX_HEADINGS ? headings[0] : undefined; + chapters.push({ + name: 'Page ' + pages[i] + (heading ? ': ' + heading : ''), + path: this.pagePath(novelPath, pages[i]), + chapterNumber: chapters.length + 1, + }); + hasNext = $('.page_next a').length > 0; + if (!hasNext) return chapters; + } + } + + if (lastPage === MAX_PAGES && hasNext) { + console.warn( + `onlinereadfreenovel: stopped at MAX_PAGES (${MAX_PAGES}) for ${novelPath}; remaining pages were not listed`, + ); + } + return chapters; + } + + /** + * The text of a page as paragraph HTML. The text is one block of + * `
`-separated lines; a run of two or more line breaks marks a + * paragraph boundary. Images, scripts, comments, empty paragraphs and a + * stray `-->` paragraph left by the site's markup are dropped. + */ + paragraphs($: CheerioAPI): string[] { + const text = $('#textToRead'); + text.find('img, a.highslide, script, style').remove(); + const html = (text.html() || '').replace(/<\/?p[^>]*>/gi, ''); + return html + .split(/(?:\s*){2,}/i) + .map(part => + part + .replace(//g, '') + .replace(/^(?:| |\u00a0|\s)+/i, '') + .replace(/(?:| |\u00a0|\s)+$/i, ''), + ) + .filter(part => part.length > 0 && part !== '-->'); + } + + pagePath(novelPath: string, page: number): string { + if (page <= 1) return novelPath; + return novelPath.replace(/\/(\d+-[^/]+_read\.html)$/, '/p,' + page + ',$1'); + } + + async fetchDom(path: string): Promise { + const res = await fetchApi(this.site + path, { headers: HEADERS }); + if (!res.ok) { + throw Object.assign(new Error(`HTTP error for ${path}`), { + status: res.status, + }); + } + return loadCheerio(await res.text()); + } + + async fetchPageDom(path: string): Promise { + for (let attempt = 0; ; attempt++) { + try { + const res = await fetchApi(this.site + path, { headers: HEADERS }); + if (res.status === 404) return null; + if (!res.ok) { + throw Object.assign(new Error(`HTTP error for ${path}`), { + status: res.status, + }); + } + return loadCheerio(await res.text()); + } catch (error) { + if (attempt >= PAGE_RETRIES) throw error; + await new Promise(resolve => setTimeout(resolve, 1000 * 2 ** attempt)); + } + } + } + + toPath(href: string): string { + return new URL(href, this.site).pathname; + } + + toAbsolute(src: string): string { + return new URL(src, this.site).href; + } +} + +export default new OnlineReadFreeNovelPlugin(); diff --git a/public/static/src/en/onlinereadfreenovel/icon.png b/public/static/src/en/onlinereadfreenovel/icon.png new file mode 100644 index 000000000..430e4725b Binary files /dev/null and b/public/static/src/en/onlinereadfreenovel/icon.png differ