From b4183a2510f71799b671fac571f8566f3fc19682 Mon Sep 17 00:00:00 2001 From: hillerliao Date: Tue, 3 Sep 2019 12:05:08 +0800 Subject: [PATCH] feat: crawl 36kr newsflashes (#2984) * crawl 36kr newsflashes * xueqiu hot statuses --- docs/other.md | 4 +++ docs/social-media.md | 4 +++ lib/router.js | 2 ++ lib/routes/36kr/newsflashes.js | 40 ++++++++++++++++++++++++ lib/routes/cninfo/fund_announcement.js | 2 +- lib/routes/xueqiu/hots.js | 42 ++++++++++++++++++++++++++ lib/routes/xueqiu/user_stock.js | 7 ++--- 7 files changed, 95 insertions(+), 6 deletions(-) create mode 100644 lib/routes/36kr/newsflashes.js create mode 100644 lib/routes/xueqiu/hots.js diff --git a/docs/other.md b/docs/other.md index d0f78644d7..5a67a34721 100644 --- a/docs/other.md +++ b/docs/other.md @@ -6,6 +6,10 @@ pageClass: routes ## 36kr +### 快讯 + + + ### 搜索文章 diff --git a/docs/social-media.md b/docs/social-media.md index 75f8ac1dd0..8dac72da83 100644 --- a/docs/social-media.md +++ b/docs/social-media.md @@ -751,6 +751,10 @@ pageClass: routes +### 热帖 + + + ## 一亩三分地 ### 主题帖 diff --git a/lib/router.js b/lib/router.js index 982c9e661e..4a956ae91f 100644 --- a/lib/router.js +++ b/lib/router.js @@ -379,6 +379,7 @@ router.get('/xueqiu/user_stock/:id', require('./routes/xueqiu/user_stock')); router.get('/xueqiu/fund/:id', require('./routes/xueqiu/fund')); router.get('/xueqiu/stock_info/:id/:type?', require('./routes/xueqiu/stock_info')); router.get('/xueqiu/snb/:id', require('./routes/xueqiu/snb')); +router.get('/xueqiu/hots', require('./routes/xueqiu/hots')); // Greasy Fork router.get('/greasyfork/:language/:domain?', require('./routes/greasyfork/scripts')); @@ -973,6 +974,7 @@ router.get('/tingdiantz/nanjing', require('./routes/tingdiantz/nanjing')); // 36kr router.get('/36kr/search/article/:keyword', require('./routes/36kr/search/article')); +router.get('/36kr/newsflashes', require('./routes/36kr/newsflashes')); // icourse163 router.get('/icourse163/newest', require('./routes/icourse163/newest')); diff --git a/lib/routes/36kr/newsflashes.js b/lib/routes/36kr/newsflashes.js new file mode 100644 index 0000000000..a8cddcd671 --- /dev/null +++ b/lib/routes/36kr/newsflashes.js @@ -0,0 +1,40 @@ +const got = require('@/utils/got'); + +module.exports = async (ctx) => { + const response = await got({ + method: 'get', + url: `https://36kr.com/api/newsflash?per_page=30`, + headers: { + 'Content-Type': 'application/json; charset=utf-8', + }, + }); + + const newsflashes = response.data.data.items; + + let newsflashesList = []; + for (let i = 0; i < newsflashes.length; i++) { + newsflashesList = newsflashesList.concat(newsflashes[i]); + } + + const out = newsflashesList.map((item) => { + const date = item.published_at; + const link = item.news_url; + const title = item.title; + const description = item.description; + + const single = { + title, + link, + pubDate: new Date(date).toUTCString(), + description, + }; + + return single; + }); + + ctx.state.data = { + title: `快讯 - 36氪`, + link: `https://36kr.com/newsflashes`, + item: out, + }; +}; diff --git a/lib/routes/cninfo/fund_announcement.js b/lib/routes/cninfo/fund_announcement.js index c1edf3fe55..88367748a9 100644 --- a/lib/routes/cninfo/fund_announcement.js +++ b/lib/routes/cninfo/fund_announcement.js @@ -30,7 +30,7 @@ module.exports = async (ctx) => { let name = ''; const out = announcementsList.map((item) => { const date = item.announcementTime; - const link = `http://www.cninfo.com.cn/new/disclosure/detail?announcementId=${item.announcementId}`; + const link = `http://static.cninfo.com.cn/${item.adjunctUrl}`; name = item.secName; const title = name + ': ' + item.announcementTitle; diff --git a/lib/routes/xueqiu/hots.js b/lib/routes/xueqiu/hots.js new file mode 100644 index 0000000000..a392bd6227 --- /dev/null +++ b/lib/routes/xueqiu/hots.js @@ -0,0 +1,42 @@ +const got = require('@/utils/got'); + +module.exports = async (ctx) => { + const res1 = await got({ + method: 'get', + url: 'https://xueqiu.com/', + }); + const token = res1.headers['set-cookie'].find((s) => s.startsWith('xq_a_token=')).split(';')[0]; + + const res2 = await got({ + method: 'get', + url: 'https://xueqiu.com/statuses/hots.json', + params: { + a: '1', + count: '10', + page: '1', + scope: 'day', + type: 'status', + meigu: '0', + }, + headers: { + Cookie: token, + Referer: `https://xueqiu.com/`, + }, + }); + const data = res2.data; + + ctx.state.data = { + title: `热帖 - 雪球`, + link: `https://xueqiu.com/`, + description: `雪球热门帖子`, + item: data.map((item) => { + const description = item.text; + return { + title: item.title ? item.title : description, + description: item.text, + pubDate: new Date(item.created_at).toUTCString(), + link: `https://xueqiu.com${item.target}`, + }; + }), + }; +}; diff --git a/lib/routes/xueqiu/user_stock.js b/lib/routes/xueqiu/user_stock.js index 82245a09bf..eeb575fac1 100644 --- a/lib/routes/xueqiu/user_stock.js +++ b/lib/routes/xueqiu/user_stock.js @@ -11,16 +11,13 @@ module.exports = async (ctx) => { const res2 = await got({ method: 'get', - url: 'https://stock.xueqiu.com/v5/stock/portfolio/stock/list.json?category=1&size=1000', - params: { - uid: id, - }, + url: `https://stock.xueqiu.com/v5/stock/portfolio/stock/list.json?category=1&size=1000&uid=${id}`, headers: { Cookie: token, Referer: `https://xueqiu.com/u/${id}`, }, }); - const data = res2.data.stocks; + const data = res2.data.data.stocks; const res3 = await got({ method: 'get',