From 68e063e2358f9d2d84bacb8e207d8bc87941c314 Mon Sep 17 00:00:00 2001 From: Henry Wang Date: Sat, 1 Feb 2020 09:25:01 +0000 Subject: [PATCH] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E5=A4=9A=E7=BB=B4?= =?UTF-8?q?=E6=A6=9C=E5=8D=95=20(#3842)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: 添加多维经济 * feat: 修复多维榜单 --- docs/traditional-media.md | 10 +-- lib/router.js | 2 +- lib/routes/dwnews/rank.js | 60 +++++----------- lib/routes/dwnews/utils.js | 140 ++++++++++++++---------------------- lib/routes/dwnews/yaowen.js | 29 ++++---- 5 files changed, 89 insertions(+), 152 deletions(-) diff --git a/docs/traditional-media.md b/docs/traditional-media.md index 5e5871de10..77b2b439cc 100644 --- a/docs/traditional-media.md +++ b/docs/traditional-media.md @@ -178,15 +178,15 @@ Category 列表: -| 全部 | 国际 | 中国 | 香港 | 台湾 | -| ------ | ------ | ----- | -------- | ------ | -| yaowen | global | china | hongkong | taiwan | +| 全部 | 国际 | 中国 | 香港 | 台湾 | 经济 | +| ------ | ------ | ----- | -------- | ------ | ------ | +| yaowen | global | china | hongkong | taiwan | jingji | -### 新闻排行榜 +### 24 小时新闻排行榜 - + ## 华尔街见闻 diff --git a/lib/router.js b/lib/router.js index 8ae5f2d482..af89f28484 100644 --- a/lib/router.js +++ b/lib/router.js @@ -823,7 +823,7 @@ router.get('/guanzhi', require('./routes/guanzhi/guanzhi')); // 多维新闻网 router.get('/dwnews/yaowen/:region?', require('./routes/dwnews/yaowen')); -router.get('/dwnews/rank/:type/:range', require('./routes/dwnews/rank')); +router.get('/dwnews/rank/:type?/:range?', require('./routes/dwnews/rank')); // 知晓程序 router.get('/miniapp/article/:category', require('./routes/miniapp/article')); diff --git a/lib/routes/dwnews/rank.js b/lib/routes/dwnews/rank.js index fc9d6d3eff..a16ac18e1e 100644 --- a/lib/routes/dwnews/rank.js +++ b/lib/routes/dwnews/rank.js @@ -3,60 +3,38 @@ const cheerio = require('cheerio'); const utils = require('./utils'); module.exports = async (ctx) => { - const response = await got.get('http://www.dwnews.com/html/share/24-7topnews.js'); + const link = 'https://www.dwnews.com/'; + const api = 'https://prod-site-api.dwnews.com/v2/articles'; + const response = await got.get(link); - const data = JSON.parse(response.data.replace('var rankingData = ', '')); - - let list, title; - switch (ctx.params.type) { - case 'news': - if (ctx.params.range === '1') { - list = data.newsRanklist['24H']; - title = '新闻 24 小时排行'; - } else { - list = data.newsRanklist['7D']; - title = '新闻 7 日排行'; - } - break; - case 'photo': - if (ctx.params.range === '1') { - list = data.photoRanklist['24H']; - title = '图集 24 小时排行'; - } else { - list = data.photoRanklist['7D']; - title = '图集 7 日排行'; - } - - break; - default: - break; - } + const $ = cheerio.load(response.data, { xmlMode: true }); + const list = JSON.parse($('#__NEXT_DATA__')[0].children[0].data).props.pageProps.initialState.home.page.sections.find((s) => s.name === '新闻排行榜 24h').items; const out = await Promise.all( - list.slice(0, 10).map(async (item) => { - const cache = await ctx.cache.get(item.url); + list.map(async (item) => { + const url = `${api}/${item.id}`; + const cache = await ctx.cache.get(url); if (cache) { return Promise.resolve(JSON.parse(cache)); } - const response = await got.get(item.url.replace('.html', '_all.html')); - const $ = cheerio.load(response.data); + const response = await got.get(url); - const result = utils.ProcessRank($, ctx.params.type); + const single = utils.ProcessFeed(response.data); - const single = { - title: item.title, - link: item.url, - author: result.author, - description: result.description, - pubDate: result.pubDate, - }; - ctx.cache.set(item.url, JSON.stringify(single)); + ctx.cache.set(url, JSON.stringify(single)); return Promise.resolve(single); }) ); + if (ctx.params.type || ctx.params.range) { + out.push({ + title: '由于源站改版，该源地址及参数已更新，请参考文档。', + pubDate: new Date(), + }); + } + ctx.state.data = { - title: `多维新闻网 - ${title}`, + title: `多维新闻网 - 24 小时新闻排行榜`, description: '多维新闻网—记住世界的轨迹 更需多维的视线，海外华人首选的中文门户新闻网站，及时全面的向全球海外华人更新世界各地时事政治、经济、科技、人文历史、图片、视频等新闻内容，是海外华人必上的新闻门户网站。', link: 'http://www.dwnews.com/', item: out, diff --git a/lib/routes/dwnews/utils.js b/lib/routes/dwnews/utils.js index f9a66a554c..aeb8097683 100644 --- a/lib/routes/dwnews/utils.js +++ b/lib/routes/dwnews/utils.js @@ -1,101 +1,65 @@ -const cheerio = require('cheerio'); +const ProcessFeed = (data) => { + let description = ''; -const ProcessFeed = ($, item) => { - let content; - if (item.filetype === 'TJ') { - const images = $('div#slider_component_4_0 > img'); + const genFigure = (source) => `
+ ${source.caption} +
${source.caption}
+

`; - $('div.captions > p').each((i, e) => { - $(images[i]).insertBefore(e); - }); + let hasBold = false; - content = $('div.captions'); - } else { - content = $('div.dia-lead-one'); - } + data.blocks.forEach((b) => { + switch (b.blockType) { + case 'summary': + description += `
${b.summary.join('
')}
`; + break; + case 'text': + b.htmlTokens.forEach((t) => { + t.forEach((tt) => { + switch (tt.type) { + case 'text': + if (hasBold) { + hasBold = false; + description += `${tt.content}

`; + } else { + description += `

${tt.content}

`; + } + break; + case 'boldText': + description = description.slice(0, -4); + description += `${tt.content}`; + hasBold = true; + break; - ProcessQuote($, content); - ProcessImage($, content); - Clean($, content); - - return content.html(); -}; - -const ProcessRank = ($, type) => { - let content, author; - - const pubDate = new Date($('div.time').text()); - pubDate.setHours(pubDate.getHours() - 8); - - content = $('div.dia-lead-one'); - if (type === 'news') { - if (content.length === 0) { - content = $('div.container'); - - author = $('div.author').text(); - } else { - author = $('div.nw').text(); - } - - ProcessQuote($, content); - ProcessImage($, content); - Clean($, content); - } else { - let images = $('div#slider_component_4_0 > img'); - if ($('div#slider_component_4_0 > img').length > 0) { - $('div.captions > p').each((i, e) => { - $(images[i]).insertBefore(e); - }); - content = $('div.captions'); - } else { - images = $('div#component_4_0 .sd-page > img'); - - content = cheerio.load('
'); - content = content('div'); - - $('.sqbtnbox > .tzbox') - .slice(1) - .each((i, e) => { - $(images[i]).appendTo(content); - $(e).appendTo(content); + default: + break; + } + }); }); - } - } + break; + case 'image': + description += genFigure(b.image); + break; + case 'gallery': + b.images.forEach((t) => { + description += genFigure(t); + }); + break; + + default: + break; + } + }); return { - description: content.html(), - author, - pubDate, + title: data.title, + link: data.publishUrl, + author: data.authors[0].publishName, + description, + pubDate: new Date(data.publishTime).toUTCString(), }; }; -const ProcessQuote = ($, content) => { - content.find('div.ed-top').each((i, e) => { - const quote = $(e).siblings('p'); - if (quote.length > 0) { - $(`
${quote.text()}
`).insertBefore(e); - - $(quote).remove(); - $(e).remove(); - } - }); -}; - -const ProcessImage = ($, content) => { - content.find('a[href="javascript:;"]').each((i, e) => { - const img = $(e).find('img'); - $(img).insertBefore(e); - $(e).remove(); - }); -}; - -const Clean = ($, content) => { - content.find('div.clear, div.badoo, div.hu-bqsm, div.sign, div.xyy, div#component_14_0').each((i, e) => { - $(e).remove(); - }); -}; - module.exports = { ProcessFeed, - ProcessRank, }; diff --git a/lib/routes/dwnews/yaowen.js b/lib/routes/dwnews/yaowen.js index c78b32d8c3..2f6090d477 100644 --- a/lib/routes/dwnews/yaowen.js +++ b/lib/routes/dwnews/yaowen.js @@ -1,8 +1,9 @@ const got = require('@/utils/got'); -const cheerio = require('cheerio'); +const utils = require('./utils'); module.exports = async (ctx) => { let link = 'https://www.dwnews.com'; + const api = 'https://prod-site-api.dwnews.com/v2/articles'; let region, regionid, host, @@ -27,6 +28,10 @@ module.exports = async (ctx) => { title = `台湾${title}`; regionid = '10000119'; break; + case 'jingji': + title = `经济${title}`; + regionid = '10000123'; + break; default: break; } @@ -40,27 +45,17 @@ module.exports = async (ctx) => { const out = await Promise.all( list.data.items.map(async (item) => { - const cache = await ctx.cache.get(item.data.publishUrl); + const url = `${api}/${item.id}`; + const cache = await ctx.cache.get(url); if (cache) { return Promise.resolve(JSON.parse(cache)); } - const response = await got.get(encodeURI(item.data.publishUrl)); - const $ = cheerio.load(response.data); + const response = await got.get(url); - const pubDate = new Date(item.data.publishTime * 1000); - pubDate.setHours(pubDate.getHours() - 8); + const single = utils.ProcessFeed(response.data); - const description = $('article').html() || item.data.description; - - const single = { - title: item.data.title, - link: item.data.publishUrl, - author: item.data.authors[0].publishName, - description, - pubDate: pubDate.toUTCString(), - }; - ctx.cache.set(item.data.publishUrl, JSON.stringify(single)); + ctx.cache.set(url, JSON.stringify(single)); return Promise.resolve(single); }) ); @@ -68,7 +63,7 @@ module.exports = async (ctx) => { ctx.state.data = { title: `多维新闻网 - ${title}`, description: '多维新闻网—记住世界的轨迹 更需多维的视线，海外华人首选的中文门户新闻网站，及时全面的向全球海外华人更新世界各地时事政治、经济、科技、人文历史、图片、视频等新闻内容，是海外华人必上的新闻门户网站。', - link: link, + link, item: out, }; };