fea: webtoons (close #3164) (#3178)

docs/anime.md

@@ -133,6 +133,14 @@ pageClass: routes
 
 </Route>
 
+## Webtoons
+
+### 漫画更新
+
+<Route author="machsix" path="/webtoons/:lang/:category/:name/:id" example="/webtoons/zh-hant/drama/gongzhuweimian/894" :paramsDesc="['语言','类别','名称','ID']"/>
+
+比如漫画公主彻夜未眠的网址为https://www.webtoons.com/zh-hant/drama/gongzhuweimian/list?title_no=894, 则`lang=zh-hant`,`category=drama`,`name=gongzhucheyeweimian`,`id=894`.
+
 ## 嘀哩嘀哩 - dilidili
 
 ### 嘀哩嘀哩番剧更新
@@ -159,7 +167,7 @@ pageClass: routes
 
 ### 漫画更新
 
-<Route author="Machsix" path="/dongmanmanhua/comic/:category/:name/:id" example="/dongmanmanhua/comic/COMEDY/xin-xinlingdeshengyin/381" :paramsDesc="['类别','名称','ID']"/>
+<Route author="machsix" path="/dongmanmanhua/:category/:name/:id" example="/dongmanmanhua/COMEDY/xin-xinlingdeshengyin/381" :paramsDesc="['类别','名称','ID']"/>
 
 ## 動漫狂
 

lib/router.js

@@ -883,7 +883,9 @@ router.get('/cartoonmad/comic/:id', require('./routes/cartoonmad/comic'));
 // Vol
 router.get('/vol/:mode?', require('./routes/vol/lastupdate'));
 // 咚漫
-router.get('/dongmanmanhua/comic/:category/:name/:id', require('./routes/dongmanmanhua/comic'));
+router.get('/dongmanmanhua/:category/:name/:id', require('./routes/dongmanmanhua/comic'));
+// webtoons
+router.get('/webtoons/:lang/:category/:name/:id', require('./routes/webtoons/comic'));
 
 // Tits Guru
 router.get('/tits-guru/home', require('./routes/titsguru/home'));

lib/routes/dongmanmanhua/comic.js

@@ -1,45 +1,45 @@
-const cheerio = require('cheerio');
+const parser = require('@/utils/rss-parser');
 const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const dateParser = require('@/utils/dateParser');
 const domain = 'https://www.dongmanmanhua.cn';
 
 module.exports = async (ctx) => {
-    const category = ctx.params.category;
+    const { category, name, id } = ctx.params;
-    const name = ctx.params.name;
-    const id = ctx.params.id;
-
     const comicLink = `${domain}/${category}/${name}/list?title_no=${id}`;
-    const { data } = await got.get(comicLink);
+    const rssLink = `${domain}/${category}/${name}/rss?title_no=${id}`;
-    const $ = cheerio.load(data);
 
-    const bookName = $('.detail_header .info .subj').text();
+    let rss;
-    const title = $('#_listUl span.subj')
+    try {
-        .map(function() {
+        const body = await parser.parseURL(rssLink);
-            return $(this).text();
+        rss = {
-        })
+            title: `咚漫 - ${body.title}`,
-        .get();
+            link: comicLink,
-    const date = $('#_listUl span.date')
+            description: body.description,
-        .map(function() {
+            item: body.items.map((x) => ({
-            return $(this)
+                title: x.title,
-                .text()
+                pubDate: dateParser(x.pubDate, 'DD MMMM YYYY HH:mm:ss', 'zh-cn'),
-                .replace(/\n|\r|\t/g, '');
+                link: x.link,
-        })
+                description: `<a href=${x.link} target="_blank">${x.title}</a>`,
-        .get();
+            })),
-    const link = $('#_listUl > li > a')
+        };
-        .map(function() {
+    } catch (error) {
-            return 'https:' + $(this).attr('href');
+        const { body } = await got.get(comicLink);
-        })
+        const $ = cheerio.load(body);
-        .get();
+        rss = {
-    const resultItem = title.map((t, i) => ({
+            title: `咚漫 - ${$('.detail_header .info .subj').text()}`,
-        title: t,
+            link: comicLink,
-        pubDate: new Date(date[i]).toUTCString(),
+            description: $('p.summary').text(),
-        link: link[i],
+            item: $('#_listUl > li > a')
-        description: `<a href=${link[i]} target="_blank">${t}</a>`,
+                .toArray()
-    }));
+                .map((ep) => ({
-
+                    title: $('.subj > span', ep).text(),
-    ctx.state.data = {
+                    pubDate: new Date($('.date', ep).text()).toUTCString(),
-        title: `咚漫 ${bookName}`,
+                    link: $(ep).attr('href'),
-        link: comicLink,
+                    description: `<a href=${$(ep).attr('href')} target="_blank">${$('.subj > span', ep).text()}</a>`,
-        description: `咚漫 ${bookName}`,
+                })),
-        item: resultItem,
+        };
-    };
+    }
+    rss.item = rss.item.sort((a, b) => (new Date(a.pubDate) > new Date(b.pubDate) ? -1 : 1));
+    ctx.state.data = rss;
 };

lib/routes/webtoons/comic.js

@@ -0,0 +1,54 @@
+const parser = require('@/utils/rss-parser');
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const dateParser = require('@/utils/dateParser');
+const domain = 'https://www.webtoons.com';
+
+module.exports = async (ctx) => {
+    const { lang, category, name, id } = ctx.params;
+    const comicLink = `${domain}/${lang}/${category}/${name}/list?title_no=${id}`;
+    const rssLink = `${domain}/${lang}/${category}/${name}/rss?title_no=${id}`;
+    const dP = (html, lang) => {
+        if (lang === 'zh-cn' || lang === 'zh-hant') {
+            return dateParser(html, 'DD MMMM YYYY HH:mm:ss', lang);
+        } else if (lang === 'en') {
+            return dateParser(html, 'DD MMM YYYY HH:mm:ss');
+        } else {
+            return html;
+        }
+    };
+
+    let rss;
+    try {
+        const body = await parser.parseURL(rssLink);
+        rss = {
+            title: `Webtoons - ${body.title}`,
+            link: comicLink,
+            description: body.description,
+            item: body.items.map((x) => ({
+                title: x.title,
+                pubDate: dP(x.pubDate, lang),
+                link: x.link,
+                description: `<a href=${x.link} target="_blank">${x.title}</a>`,
+            })),
+        };
+    } catch (error) {
+        const { body } = await got.get(comicLink);
+        const $ = cheerio.load(body);
+        rss = {
+            title: `Webtoons - ${$('.detail_header .info .subj').text()}`,
+            link: comicLink,
+            description: $('p.summary').text(),
+            item: $('#_listUl > li > a')
+                .toArray()
+                .map((ep) => ({
+                    title: $('.subj > span', ep).text(),
+                    pubDate: new Date($('.date', ep).text()).toUTCString(),
+                    link: $(ep).attr('href'),
+                    description: `<a href=${$(ep).attr('href')} target="_blank">${$('.subj > span', ep).text()}</a>`,
+                })),
+        };
+    }
+    rss.item = rss.item.sort((a, b) => (new Date(a.pubDate) > new Date(b.pubDate) ? -1 : 1));
+    ctx.state.data = rss;
+};

lib/utils/dateParser.js

@@ -0,0 +1,92 @@
+const date = require('./date');
+const dayjs = require('dayjs');
+const customParseFormat = require('dayjs/plugin/customParseFormat');
+const logger = require('./logger');
+const utc = require('dayjs/plugin/utc');
+dayjs.extend(utc);
+dayjs.extend(customParseFormat);
+
+/**
+ * Convert unconventional i8n to the one supported by dayjs https://bit.ly/2psVwIJ
+ * @param {String} x i8n string
+ */
+const i8nconv = (x) => {
+    const c = {
+        'zh-hans': 'zh-cn',
+        'zh-chs': 'zh-cn',
+        'zh-sg': 'zh-cn',
+        'zh-hant': 'zh-hk',
+        'zh-cht': 'zh-hk',
+        'zh-mo': 'zh-hk',
+    };
+    for (const prop in c) {
+        if (RegExp(`^${prop}$`, 'i').test(x)) {
+            x = c[prop];
+            break;
+        }
+    }
+    return x;
+};
+
+/**
+ * A function to convert a string of time based on specified format
+ * @param {string} [html]                    A string of time to convert.
+ * @param {string} [customFormat=undefined]  Format to parse html by dayjs.
+ * @param {string} [lang=en]                 Language (must be supported by dayjs).
+ * @param {int}    [htmlOffset=0]            UTC offset of html. It will be neglected if html contains timezone indicated by strings like "+0800".
+ */
+const tStringParser = (html, customFormat = undefined, lang = 'en', htmlOffset = 0) => {
+    lang = i8nconv(lang);
+
+    // Remove weekdays and comma from the string
+    // dayjs v1.8.16 is not able to parse weekdays
+    // https://github.com/iamkun/dayjs/blob/dev/docs/en/Plugin.md#list-of-all-available-format-tokens
+    // We don't remove weekdayMini since the month may contains weekdayMini, like "六" in "六月"
+    let removeStr = [];
+    if (lang !== 'en') {
+        try {
+            require(`dayjs/locale/${lang}`);
+            if (/^zh/.test(lang)) {
+                removeStr = removeStr.concat(['，']);
+            }
+            // Add locale
+            dayjs.locale(lang);
+        } catch (error) {
+            logger.error(`Locale "${lang}" passed to dateParser is not supported by dayjs`);
+            return date(html);
+        }
+    }
+    Object.values(dayjs.Ls).forEach((k) => {
+        ['weekdays', 'weekdaysShort'].forEach((x) => {
+            if (k.hasOwnProperty(x)) {
+                const a = k[x].map((z) => `${z}`);
+                removeStr = removeStr.concat(...a);
+            }
+        });
+    });
+    removeStr = removeStr.concat([',', 'Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat']);
+    let htmlP = html;
+    removeStr.forEach((x) => {
+        // Order matters
+        htmlP = htmlP.replace(RegExp(x, 'gi'), '');
+    });
+
+    const d = dayjs.utc(htmlP, customFormat);
+    // console.log(htmlP,d)
+    if (d.isValid()) {
+        if (/[+-](\d{2}:?\d{2})/.test(html)) {
+            return d.toDate().toUTCString();
+        } else {
+            return d
+                .add(htmlOffset, 'h')
+                .toDate()
+                .toUTCString();
+        }
+    } else {
+        return date(html);
+    }
+};
+
+tStringParser.i8nconv = i8nconv;
+
+module.exports = tStringParser;

test/utils/dateParser.js

@@ -0,0 +1,103 @@
+const dateParser = require('../../lib/utils/dateParser');
+const dayjs = require('dayjs');
+const utc = require('dayjs/plugin/utc');
+const MockDate = require('mockdate');
+dayjs.extend(utc);
+
+describe('dateParser', () => {
+    MockDate.set('2019-01-01');
+    const now = new Date();
+    const serverOffset = now.getTimezoneOffset() / 60;
+    require('dayjs/locale/zh-cn');
+    require('dayjs/locale/zh-hk');
+
+    // ['en', 'zh-cn', 'zh-hant'].forEach((lang0) => {
+    //     const lang = dateParser.i8nconv(lang0);
+    //     dayjs.locale(lang);
+
+    // Test of input as a string of UTC Time
+    test(`UTCString`, () => {
+        expect(
+            dateParser(
+                dayjs
+                    .utc(now.toUTCString())
+                    .locale('en')
+                    .format('YYYY-MM-DD HH:mm:ss')
+            )
+        ).toBe(now.toUTCString());
+    });
+
+    // Test of input as a string of local time with timezone in ISO 8601
+    test(`ISO 8601`, () => {
+        expect(
+            dateParser(
+                dayjs(now.toUTCString())
+                    .locale('en')
+                    .format('YYYY-MM-DDTHH:mm:ssZ')
+            )
+        ).toBe(now.toUTCString());
+    });
+
+    // Test of input as a string of local time with timezone set by htmlOffset
+    test(`htmlOffset`, () => {
+        expect(
+            dateParser(
+                dayjs(now.toUTCString())
+                    .locale('en')
+                    .format('YYYY-MM-DDTHH:mm:ss'),
+                null,
+                'en',
+                serverOffset
+            )
+        ).toBe(now.toUTCString());
+    });
+
+    // Test of input as a string of UTC Time with week
+    test(`en UTCString with week`, () => {
+        expect(
+            dateParser(
+                dayjs
+                    .utc(now.toUTCString())
+                    .locale('en')
+                    .format('dddd, DD MMMM YYYY HH:mm:ss'),
+                'DD MMMM YYYY HH:mm:ss'
+            )
+        ).toBe(now.toUTCString());
+    });
+
+    test(`zh-cn UTCString with week`, () => {
+        expect(
+            dateParser(
+                dayjs
+                    .utc(now.toUTCString())
+                    .locale('zh-cn')
+                    .format('dddd, DD MMMM YYYY HH:mm:ss'),
+                'DD MMMM YYYY HH:mm:ss',
+                'zh-cn'
+            )
+        ).toBe(now.toUTCString());
+    });
+
+    test(`zh-hant UTCString with week`, () => {
+        expect(
+            dateParser(
+                dayjs
+                    .utc(now.toUTCString())
+                    .locale(dateParser.i8nconv('zh-hant'))
+                    .format('dddd, DD MMMM YYYY HH:mm:ss'),
+                'DD MMMM YYYY HH:mm:ss',
+                'zh-hant'
+            )
+        ).toBe(now.toUTCString());
+    });
+
+    // fallback
+    test('fallback', () => {
+        expect(+new Date(dateParser('10分钟前'))).toBe(+now - 10 * 60 * 1000);
+    });
+
+    // error handling
+    test('error handling', () => {
+        expect(+new Date(dateParser('10分钟前', null, 'Klingon'))).toBe(+now - 10 * 60 * 1000);
+    });
+});