diff --git a/docs/.vuepress/config.js b/docs/.vuepress/config.js index c742de3bdb..691eb539e9 100644 --- a/docs/.vuepress/config.js +++ b/docs/.vuepress/config.js @@ -129,6 +129,7 @@ module.exports = { 'reading', 'government', 'study', + 'journal', 'finance', 'other', ], @@ -192,6 +193,7 @@ module.exports = { 'reading', 'government', 'study', + 'journal', 'finance', 'other', ], diff --git a/docs/en/journal.md b/docs/en/journal.md new file mode 100644 index 0000000000..66b5b3cdd5 --- /dev/null +++ b/docs/en/journal.md @@ -0,0 +1,101 @@ +--- +pageClass: routes +--- + +# Sciences Journal + +## Online papers + +### eLife - Latest Research-ALL + + + +### eLife - Latest Research-Research by Subject + + + +### Nature Journal - Latest Research + + + +### Nature Journal - News + + + +### Nature Journal - Research Highlight + + + +### Nature Genetics (ng) - Latest Research + + + +### Nature Methods (nmeth) - Latest Research + + + +### Nature Biotechnology (nbt) - Latest Research + + + +### Nature Neuroscience (neuro) - Latest Research + + + +### Nature Machine Intelligence (natmachintell) - Latest Research + + + +### Proceedings of The National Academy of Sciences (PNAS) - Latest Articles - ALL + + + +### Proceedings of The National Academy of Sciences (PNAS) - Latest Articles-Articles by Topic + + + +### Science Journal - Current Issue + + + +### Science Journal - First Release + + + +### Science Advances - Current Issue + + + +## Search Engine + +### X-MOL Platform - News + + + +### X-MOL Platform - Journal + + + +### Google Scholar - Keywords Monitoring + + + +::: warning + +Google Scholar has strict anti-crawling mechanism implemented, the demo below doesn't guarantee availability. Please deploy your own instance as it might increase the stability. + +::: + +1. Basic mode, sample query is the keywords desired, eg.「data visualization」, [https://rsshub.app/google/scholar/data+visualization](https://rsshub.app/google/scholar/data+visualization). + +2. Advanced mode, visit [Google Scholar](https://scholar.google.com/schhp?hl=en&as_sdt=0,5), click the top left corner and select「Advanced Search」, fill in your conditions and submit the search. The URL should look like this: [https://scholar.google.com/scholar?as_q=data+visualization&as_epq=&as_oq=&as_eq=&as_occt=any&as_sauthors=&as_publication=&as_ylo=2018&as_yhi=&hl=en&as_sdt=0%2C5](https://scholar.google.com/scholar?as_q=data+visualization&as_epq=&as_oq=&as_eq=&as_occt=any&as_sauthors=&as_publication=&as_ylo=2018&as_yhi=&hl=en&as_sdt=0%2C5), copy everything after `https://scholar.google.com/scholar?` from the URL and use it as the query for this route. The complete URL for the above example should look like this: [https://rsshub.app/google/scholar/as_q=data+visualization&as_epq=&as_oq=&as_eq=&as_occt=any&as_sauthors=&as_publication=&as_ylo=2018&as_yhi=&hl=en&as_sdt=0%2C5](https://rsshub.app/google/scholar/as_q=data+visualization&as_epq=&as_oq=&as_eq=&as_occt=any&as_sauthors=&as_publication=&as_ylo=2018&as_yhi=&hl=en&as_sdt=0%2C5). + + + +### Google Scholar - Author Citations + + + +The parameter id in the route is the id in the URL of the user ’s Google Scholar reference page，for example `https://scholar.google.com/citations?hl=zh-CN&user=mlmE4JMAAAAJ` to `mlmE4JMAAAAJ` + + diff --git a/docs/en/study.md b/docs/en/study.md index e1cfd391ac..d018b2f7d8 100644 --- a/docs/en/study.md +++ b/docs/en/study.md @@ -4,42 +4,6 @@ pageClass: routes # Study -## eLife - -### Latest Research-ALL - - - -### Latest Research-Research by Subject - - - -## Google Scholar - -### Keywords Monitoring - - - -::: warning - -Google Scholar has strict anti-crawling mechanism implemented, the demo below doesn't guarantee availability. Please deploy your own instance as it might increase the stability. - -::: - -1. Basic mode, sample query is the keywords desired, eg.「data visualization」, [https://rsshub.app/google/scholar/data+visualization](https://rsshub.app/google/scholar/data+visualization). - -2. Advanced mode, visit [Google Scholar](https://scholar.google.com/schhp?hl=en&as_sdt=0,5), click the top left corner and select「Advanced Search」, fill in your conditions and submit the search. The URL should look like this: [https://scholar.google.com/scholar?as_q=data+visualization&as_epq=&as_oq=&as_eq=&as_occt=any&as_sauthors=&as_publication=&as_ylo=2018&as_yhi=&hl=en&as_sdt=0%2C5](https://scholar.google.com/scholar?as_q=data+visualization&as_epq=&as_oq=&as_eq=&as_occt=any&as_sauthors=&as_publication=&as_ylo=2018&as_yhi=&hl=en&as_sdt=0%2C5), copy everything after `https://scholar.google.com/scholar?` from the URL and use it as the query for this route. The complete URL for the above example should look like this: [https://rsshub.app/google/scholar/as_q=data+visualization&as_epq=&as_oq=&as_eq=&as_occt=any&as_sauthors=&as_publication=&as_ylo=2018&as_yhi=&hl=en&as_sdt=0%2C5](https://rsshub.app/google/scholar/as_q=data+visualization&as_epq=&as_oq=&as_eq=&as_occt=any&as_sauthors=&as_publication=&as_ylo=2018&as_yhi=&hl=en&as_sdt=0%2C5). - - - -### Author citations - - - -The parameter id in the route is the id in the URL of the user ’s Google Scholar reference page，for example `https://scholar.google.com/citations?hl=zh-CN&user=mlmE4JMAAAAJ` to `mlmE4JMAAAAJ` - - - ## gradCafe ### gradCafe result @@ -56,26 +20,39 @@ The parameter id in the route is the id in the URL of the user ’s Google Schol -## Nature +## ZhiShiFenZi -### Nature +### News - + -### Nature Machine Intelligence +| `:type` | type name | +| --------- | --------- | +| biology | Biology | +| medicine | Medicine | +| ai | AI | +| physics | physics | +| chymistry | Chymistry | +| astronomy | Astronomy | +| others | Others | - +> leave it blank（`/zhishifenzi/news`）to get all -### Nature Neuroscience +### depth - + -## Proceedings of The National Academy of Sciences +### 创新 -### Latest Articles-ALL + - +| `:type` | type name | +| ------------- | ------------- | +| ~~multiple~~ | ~~Multiple~~ | +| company | Company | +| product | Product | +| technology | Technology | +| ~~character~~ | ~~Character~~ | +| policy | Policy | -### Latest Articles-Articles by Topic - - +> leave it blank（`/zhishifenzi/innovation`）to get all diff --git a/docs/journal.md b/docs/journal.md new file mode 100644 index 0000000000..9cb5f88337 --- /dev/null +++ b/docs/journal.md @@ -0,0 +1,94 @@ +--- +pageClass: routes +--- + +# 科学期刊 + +## 聚合平台 + +### X-MOL 平台-新闻 + + + +### X-MOL 平台-期刊 + + + +### 谷歌学术-关键词更新 + + + +1. 简单模式, 例如「data visualization」, . +2. 高级模式, 前往 [Google Scholar](https://scholar.google.com/schhp?hl=zh-cn&as_sdt=0,5), 点击左上角, 选择高级搜索并提交查询. 此时 URL 应为: , 复制`https://scholar.google.com/scholar?`后的所有语句作为本路由的查询参数. 例子所对应的完整路由为. + + + +### 谷歌学术-作者引用更新 + + + +路由中的参数 id，即用户谷歌学术引用页面 url 中的 id，如 https://scholar.google.com/citations?hl=zh-CN&user=mlmE4JMAAAAJ 中 user= 后的 mlmE4JMAAAAJ。 + + + +## 期刊网站 + +### eLife-最新成果-综合 + + + +### elife-最新成果-细分领域 + + + +### Nature 主刊-最新成果 + + + +### Nature 主刊-新闻动态 + + + +### Nature 主刊-精彩研究 + + + +### Nature Genetics (ng)-最新成果 + + + +### Nature Methods (nmeth)-最新成果 + + + +### Nature Biotechnology (nbt)-最新成果 + + + +### Nature Neuroscience (neuro)-最新成果 + + + +### Nature Machine Intelligence (natmachintell)-最新成果 + + + +### PNAS-最新文章(全部) + + + +### PNAS-最新文章(根据领域分类) + + + +### Science 主刊-本期刊物 + + + +### Science 主刊-在线发表 + + + +### Science Advances-本期刊物 + + diff --git a/docs/study.md b/docs/study.md index 761a6126ac..c6f1c91ae5 100644 --- a/docs/study.md +++ b/docs/study.md @@ -4,16 +4,6 @@ pageClass: routes # 学习 -## eLife - -### 最新成果-综合 - - - -### 最新成果-细分领域 - - - ## gradCafe ### gradCafe result @@ -24,59 +14,6 @@ pageClass: routes -## Nature - -### Nature - - - -### Nature Machine Intelligence - - - -### Nature Neuroscience - - - -## Proceedings of The National Academy of Sciences - -### 最新文章-全部 - - - -### 最新文章-根据领域分类 - - - -## X-MOL 平台 - -### 新闻 - - - -### 期刊 - - - -## 谷歌学术 - -### 谷歌学术关键词更新 - - - -1. 简单模式, 例如「data visualization」, . -2. 高级模式, 前往 [Google Scholar](https://scholar.google.com/schhp?hl=zh-cn&as_sdt=0,5), 点击左上角, 选择高级搜索并提交查询. 此时 URL 应为: , 复制`https://scholar.google.com/scholar?`后的所有语句作为本路由的查询参数. 例子所对应的完整路由为. - - - -### 谷歌学术作者引用更新 - - - -路由中的参数 id，即用户谷歌学术引用页面 url 中的 id，如 https://scholar.google.com/citations?hl=zh-CN&user=mlmE4JMAAAAJ 中 user= 后的 mlmE4JMAAAAJ。 - - - ## 码农周刊 ### issues @@ -195,6 +132,43 @@ pageClass: routes +## 知識分子 + +### 新聞 + + + +| `:type` | 类别名称 | +| --------- | -------- | +| biology | 生物 | +| medicine | 医药 | +| ai | 人工智能 | +| physics | 物理 | +| chymistry | 化学 | +| astronomy | 天文 | +| others | 其他 | + +> 参数置空（`/zhishifenzi/news`）获取所有类别 + +### 深度 + + + +### 创新 + + + +| `:type` | 类别名称 | +| ------------- | -------- | +| ~~multiple~~ | ~~综合~~ | +| company | 公司 | +| product | 产品 | +| technology | 技术 | +| ~~character~~ | ~~人物~~ | +| policy | 政策 | + +> 参数置空（`/zhishifenzi/innovation`）获取所有类别 + ## 中国大学 MOOC(慕课) ### 最新 diff --git a/lib/router.js b/lib/router.js index 82266bc3c5..15024695e2 100644 --- a/lib/router.js +++ b/lib/router.js @@ -1962,6 +1962,11 @@ router.get('/factcheck', require('./routes/tencent/factcheck')); router.get('/x-mol/news/:tag?', require('./routes/x-mol/news.js')); router.get('/x-mol/paper/:type/:magazine', require('./routes/x-mol/paper')); +// 知识分子 +router.get('/zhishifenzi/news/:type?', require('./routes/zhishifenzi/news')); +router.get('/zhishifenzi/depth', require('./routes/zhishifenzi/depth')); +router.get('/zhishifenzi/innovation/:type?', require('./routes/zhishifenzi/innovation')); + // 電撃Online router.get('/dengekionline/:type?', require('./routes/dengekionline/new')); @@ -1982,18 +1987,30 @@ router.get('/hatena/anonymous_diary/archive', require('./routes/hatena/anonymous router.get('/kaggle/discussion/:forumId/:sort?', require('./routes/kaggle/discussion')); router.get('/kaggle/competitions/:category?', require('./routes/kaggle/competitions')); -// eLife +// eLife [Sci Journal] router.get('/elife/latest', require('./routes/elife/latest')); router.get('/elife/:tid', require('./routes/elife/subject')); -// PNAS +// PNAS [Sci Journal] router.get('/pnas/latest', require('./routes/pnas/latest')); router.get('/pnas/:tid', require('./routes/pnas/topic')); -// nature +// nature [Sci Journal] +router.get('/nature/nature/research', require('./routes/nature/nature/research')); +router.get('/nature/nature/news', require('./routes/nature/nature/news')); +router.get('/nature/nature/highlight', require('./routes/nature/nature/highlight')); +// nature 子刊 [Sci Journal] +router.get('/nature/nmeth/research', require('./routes/nature/nmeth/research')); +router.get('/nature/ng/research', require('./routes/nature/ng/research')); +router.get('/nature/nbt/research', require('./routes/nature/nbt/research')); +router.get('/nature/neuro/research', require('./routes/nature/neuro/research')); router.get('/nature/natmachintell/research', require('./routes/nature/natmachintell/research')); -router.get('/nature/neuroscience/research', require('./routes/nature/neuroscience/research')); -router.get('/nature/research', require('./routes/nature/research')); + +// science [Sci Journal] +router.get('/sciencemag/science/current', require('./routes/sciencemag/science/current')); +router.get('/sciencemag/science/early', require('./routes/sciencemag/science/early')); +// science 子刊 [Sci Journal] +router.get('/sciencemag/advances/current', require('./routes/sciencemag/advances/current')); // dlsite router.get('/dlsite/new/:type', require('./routes/dlsite/new')); diff --git a/lib/routes/nature/nature/highlight.js b/lib/routes/nature/nature/highlight.js new file mode 100644 index 0000000000..35c5335aa3 --- /dev/null +++ b/lib/routes/nature/nature/highlight.js @@ -0,0 +1,60 @@ +const cheerio = require('cheerio'); +const got = require('@/utils/got'); + +module.exports = async (ctx) => { + const base = `https://www.nature.com`; + const url = `${base}/nature/articles?type=research-highlight`; + + const res = await got.get(url); + const $ = cheerio.load(res.data); + const list = $('.border-bottom-1.pb20').get(); + + const out = await Promise.all( + list.slice(0, 2).map(async (item) => { + const $ = cheerio.load(item); + const title = $('h3 > a').text(); + const partial = $('h3 > a').attr('href'); + const address = `${base}${partial}`; + const time = $('time').text(); + let author; + if ($('.js-list-authors-3 li').length > 3) { + author = + $('.js-list-authors-3 li') + .slice(0, 1) + .text() + ' et al.'; + } else { + author = $('.js-list-authors-3 li').text(); + } + const cache = await ctx.cache.get(address); + if (cache) { + return Promise.resolve(JSON.parse(cache)); + } + const res = await got.get(address); + const capture = cheerio.load(res.data); + let figure = capture('figure .figure--bleed').html(); + if (figure === null) { + figure = ''; + } + let contents = capture('.article-item--open .article-item__body').html(); + if (contents === null) { + contents = ''; + } + const single = { + title, + author: author, + description: figure + contents, + link: address, + guid: address, + pubDate: new Date(time).toUTCString(), + }; + ctx.cache.set(address, JSON.stringify(single)); + return Promise.resolve(single); + }) + ); + ctx.state.data = { + title: `Nature | Research Highlight`, + description: `Nature, a nature research journal`, + link: url, + item: out, + }; +}; diff --git a/lib/routes/nature/nature/news.js b/lib/routes/nature/nature/news.js new file mode 100644 index 0000000000..ab81becbc5 --- /dev/null +++ b/lib/routes/nature/nature/news.js @@ -0,0 +1,54 @@ +const cheerio = require('cheerio'); +const got = require('@/utils/got'); + +module.exports = async (ctx) => { + const base = `https://www.nature.com`; + const url = `${base}/nature/articles?type=news`; + + const res = await got.get(url); + const $ = cheerio.load(res.data); + const list = $('.border-bottom-1.pb20').get(); + + const out = await Promise.all( + list.slice(0, 2).map(async (item) => { + const $ = cheerio.load(item); + const title = $('h3 > a').text(); + const partial = $('h3 > a').attr('href'); + const address = `${base}${partial}`; + const time = $('time').text(); + let author; + if ($('.js-list-authors-3 li').length > 3) { + author = + $('.js-list-authors-3 li') + .slice(0, 1) + .text() + ' et al.'; + } else { + author = $('.js-list-authors-3 li').text(); + } + const cache = await ctx.cache.get(address); + if (cache) { + return Promise.resolve(JSON.parse(cache)); + } + const res = await got.get(address); + const capture = cheerio.load(res.data); + const contents = capture('.content .article__body').html(); + + const single = { + title, + author: author, + description: contents, + link: address, + guid: address, + pubDate: new Date(time).toUTCString(), + }; + ctx.cache.set(address, JSON.stringify(single)); + return Promise.resolve(single); + }) + ); + ctx.state.data = { + title: `Nature | Latest News`, + description: `Nature, a nature research journal`, + link: url, + item: out, + }; +}; diff --git a/lib/routes/nature/research.js b/lib/routes/nature/nature/research.js similarity index 66% rename from lib/routes/nature/research.js rename to lib/routes/nature/nature/research.js index f1fe91f77b..d50f03904a 100644 --- a/lib/routes/nature/research.js +++ b/lib/routes/nature/nature/research.js @@ -2,7 +2,8 @@ const cheerio = require('cheerio'); const got = require('@/utils/got'); module.exports = async (ctx) => { - const url = `https://www.nature.com/nature/research`; + const base = `https://www.nature.com`; + const url = `${base}/nature/research`; const res = await got.get(url); const $ = cheerio.load(res.data); @@ -13,8 +14,8 @@ module.exports = async (ctx) => { const $ = cheerio.load(item); const title = $('h3 > a').text(); const partial = $('h3 > a').attr('href'); - const address = `https://www.nature.com${partial}`; - const description = $('.hide-overflow.inline').text(); + const address = `${base}${partial}`; + const brief = $('.hide-overflow.inline').text(); const time = $('time').text(); let author; if ($('.js-list-authors-3 li').length > 3) { @@ -31,18 +32,27 @@ module.exports = async (ctx) => { } const res = await got.get(address); const capture = cheerio.load(res.data); - const abs = capture('div#Abs1-content.c-article-section__content').html(); - let contents; - if (description !== '' && abs !== null) { - contents = `${description}

Abstraction

${abs}

`; - } else { - if (abs !== null) { - contents = `
Abstraction
` + abs; - } - if (description !== '') { - contents = description; - } + const abs = capture('div#Abs1-content.c-article-section__content > p').html(); + + let briefContents = ''; + if (brief !== '') { + briefContents = ` +
+

Brief

+

${brief}

+
+ `; } + let absContents = ''; + if (abs !== null) { + absContents = ` +
+

Abstract

+

${abs}

+
+ `; + } + const contents = briefContents + absContents; const single = { title, @@ -58,6 +68,7 @@ module.exports = async (ctx) => { ); ctx.state.data = { title: `Nature | Latest Research`, + description: `Nature, a nature research journal`, link: url, item: out, }; diff --git a/lib/routes/nature/nbt/research.js b/lib/routes/nature/nbt/research.js new file mode 100644 index 0000000000..59cb6398d7 --- /dev/null +++ b/lib/routes/nature/nbt/research.js @@ -0,0 +1,75 @@ +const cheerio = require('cheerio'); +const got = require('@/utils/got'); + +module.exports = async (ctx) => { + const base = `https://www.nature.com`; + const url = `${base}/nbt/research`; + + const res = await got.get(url); + const $ = cheerio.load(res.data); + const list = $('.border-bottom-1.pb20').get(); + + const out = await Promise.all( + list.map(async (item) => { + const $ = cheerio.load(item); + const title = $('h3 > a').text(); + const partial = $('h3 > a').attr('href'); + const address = `${base}${partial}`; + const brief = $('.hide-overflow.inline').text(); + const time = $('time').text(); + let author; + if ($('.js-list-authors-3 li').length > 3) { + author = + $('.js-list-authors-3 li') + .slice(0, 1) + .text() + ' et al.'; + } else { + author = $('.js-list-authors-3 li').text(); + } + const cache = await ctx.cache.get(address); + if (cache) { + return Promise.resolve(JSON.parse(cache)); + } + const res = await got.get(address); + const capture = cheerio.load(res.data); + const abs = capture('div#Abs1-content.c-article-section__content > p').html(); + + let briefContents = ''; + if (brief !== '') { + briefContents = ` +
+

Brief

+

${brief}

+
+ `; + } + let absContents = ''; + if (abs !== null) { + absContents = ` +
+

Abstract

+

${abs}

+
+ `; + } + const contents = briefContents + absContents; + + const single = { + title, + author: author, + description: contents, + link: address, + guid: address, + pubDate: new Date(time).toUTCString(), + }; + ctx.cache.set(address, JSON.stringify(single)); + return Promise.resolve(single); + }) + ); + ctx.state.data = { + title: `Nature Biotechnology | Latest Research`, + description: `Nature Biotechnology, a nature research journal`, + link: url, + item: out, + }; +}; diff --git a/lib/routes/nature/neuroscience/research.js b/lib/routes/nature/neuro/research.js similarity index 67% rename from lib/routes/nature/neuroscience/research.js rename to lib/routes/nature/neuro/research.js index c125f6188a..8daca578e1 100644 --- a/lib/routes/nature/neuroscience/research.js +++ b/lib/routes/nature/neuro/research.js @@ -2,19 +2,20 @@ const cheerio = require('cheerio'); const got = require('@/utils/got'); module.exports = async (ctx) => { - const url = `https://www.nature.com/neuro/research`; + const base = `https://www.nature.com`; + const url = `${base}/neuro/research`; const res = await got.get(url); const $ = cheerio.load(res.data); const list = $('.border-bottom-1.pb20').get(); const out = await Promise.all( - list.map(async (item) => { + list.slice(0, 3).map(async (item) => { const $ = cheerio.load(item); const title = $('h3 > a').text(); const partial = $('h3 > a').attr('href'); - const address = `https://www.nature.com${partial}`; - const description = $('.hide-overflow.inline').text(); + const address = `${base}${partial}`; + const brief = $('.hide-overflow.inline').text(); const time = $('time').text(); let author; if ($('.js-list-authors-3 li').length > 3) { @@ -31,18 +32,27 @@ module.exports = async (ctx) => { } const res = await got.get(address); const capture = cheerio.load(res.data); - const abs = capture('div#Abs1-content.c-article-section__content').html(); - let contents; - if (description !== '' && abs !== null) { - contents = `${description}

Abstraction

${abs}

`; - } else { - if (abs !== null) { - contents = `
Abstraction
` + abs; - } - if (description !== '') { - contents = description; - } + const abs = capture('div#Abs1-content.c-article-section__content > p').html(); + + let briefContents = ''; + if (brief !== '') { + briefContents = ` +
+

Brief

+

${brief}

+
+ `; } + let absContents = ''; + if (abs !== null) { + absContents = ` +
+

Abstract

+

${abs}

+
+ `; + } + const contents = briefContents + absContents; const single = { title, diff --git a/lib/routes/nature/ng/research.js b/lib/routes/nature/ng/research.js new file mode 100644 index 0000000000..c1d843dc8b --- /dev/null +++ b/lib/routes/nature/ng/research.js @@ -0,0 +1,75 @@ +const cheerio = require('cheerio'); +const got = require('@/utils/got'); + +module.exports = async (ctx) => { + const base = `https://www.nature.com`; + const url = `${base}/ng/research`; + + const res = await got.get(url); + const $ = cheerio.load(res.data); + const list = $('.border-bottom-1.pb20').get(); + + const out = await Promise.all( + list.map(async (item) => { + const $ = cheerio.load(item); + const title = $('h3 > a').text(); + const partial = $('h3 > a').attr('href'); + const address = `${base}${partial}`; + const brief = $('.hide-overflow.inline').text(); + const time = $('time').text(); + let author; + if ($('.js-list-authors-3 li').length > 3) { + author = + $('.js-list-authors-3 li') + .slice(0, 1) + .text() + ' et al.'; + } else { + author = $('.js-list-authors-3 li').text(); + } + const cache = await ctx.cache.get(address); + if (cache) { + return Promise.resolve(JSON.parse(cache)); + } + const res = await got.get(address); + const capture = cheerio.load(res.data); + const abs = capture('div#Abs1-content.c-article-section__content > p').html(); + + let briefContents = ''; + if (brief !== '') { + briefContents = ` +
+

Brief

+

${brief}

+
+ `; + } + let absContents = ''; + if (abs !== null) { + absContents = ` +
+

Abstract

+

${abs}

+
+ `; + } + const contents = briefContents + absContents; + + const single = { + title, + author: author, + description: contents, + link: address, + guid: address, + pubDate: new Date(time).toUTCString(), + }; + ctx.cache.set(address, JSON.stringify(single)); + return Promise.resolve(single); + }) + ); + ctx.state.data = { + title: `Nature Genetics | Latest Research`, + description: `Nature Genetics, a nature research journal`, + link: url, + item: out, + }; +}; diff --git a/lib/routes/nature/nmeth/research.js b/lib/routes/nature/nmeth/research.js new file mode 100644 index 0000000000..8f49a97bbc --- /dev/null +++ b/lib/routes/nature/nmeth/research.js @@ -0,0 +1,75 @@ +const cheerio = require('cheerio'); +const got = require('@/utils/got'); + +module.exports = async (ctx) => { + const base = `https://www.nature.com`; + const url = `${base}/nmeth/research`; + + const res = await got.get(url); + const $ = cheerio.load(res.data); + const list = $('.border-bottom-1.pb20').get(); + + const out = await Promise.all( + list.map(async (item) => { + const $ = cheerio.load(item); + const title = $('h3 > a').text(); + const partial = $('h3 > a').attr('href'); + const address = `${base}${partial}`; + const brief = $('.hide-overflow.inline').text(); + const time = $('time').text(); + let author; + if ($('.js-list-authors-3 li').length > 3) { + author = + $('.js-list-authors-3 li') + .slice(0, 1) + .text() + ' et al.'; + } else { + author = $('.js-list-authors-3 li').text(); + } + const cache = await ctx.cache.get(address); + if (cache) { + return Promise.resolve(JSON.parse(cache)); + } + const res = await got.get(address); + const capture = cheerio.load(res.data); + const abs = capture('div#Abs1-content.c-article-section__content > p').html(); + + let briefContents = ''; + if (brief !== '') { + briefContents = ` +
+

Brief

+

${brief}

+
+ `; + } + let absContents = ''; + if (abs !== null) { + absContents = ` +
+

Abstract

+

${abs}

+
+ `; + } + const contents = briefContents + absContents; + + const single = { + title, + author: author, + description: contents, + link: address, + guid: address, + pubDate: new Date(time).toUTCString(), + }; + ctx.cache.set(address, JSON.stringify(single)); + return Promise.resolve(single); + }) + ); + ctx.state.data = { + title: `Nature Method | Latest Research`, + description: `Nature Method, a nature research journal`, + link: url, + item: out, + }; +}; diff --git a/lib/routes/rsshub/rss.js b/lib/routes/rsshub/rss.js index b293f4af12..62a98ca0c4 100644 --- a/lib/routes/rsshub/rss.js +++ b/lib/routes/rsshub/rss.js @@ -23,6 +23,7 @@ module.exports = async (ctx) => { 'reading', 'government', 'study', + 'journal', 'finance', 'other', ]; diff --git a/lib/routes/sciencemag/advances/current.js b/lib/routes/sciencemag/advances/current.js new file mode 100644 index 0000000000..8ae093c916 --- /dev/null +++ b/lib/routes/sciencemag/advances/current.js @@ -0,0 +1,98 @@ +const cheerio = require('cheerio'); +const got = require('@/utils/got'); + +module.exports = async (ctx) => { + const base = `https://advances.sciencemag.org`; + + const res = await got.get(base); + const pageCapture = cheerio.load(res.data); + + const list = pageCapture('ul > li > div > div > article > div').get(); + + const out = await Promise.all( + list.map(async (item) => { + const $ = cheerio.load(item); + + const title = $('h3').text(); + const partial = $('h3 > a').attr('href'); + // TODO: .full.txt is a way for getting text preview + const address = `${base}${partial}`; + + let author; + const authorList = $('span.highwire-citation-authors > span.highwire-citation-author') + .map(function(i, el) { + return $(el).text(); + }) + .get(); + if (authorList.length > 5) { + author = authorList.slice(0, 5).join(', ') + ' et al.'; + } else { + author = authorList.join(', '); + } + + const time = new Date($('p.highwire-cite-metadata > time').text()).toUTCString(); + + const cache = await ctx.cache.get(address); + if (cache) { + return Promise.resolve(JSON.parse(cache)); + } + + // contents + // brief content + const brief = $('div.highwire-cite-snippet > div > div > p').text(); + let briefContents = ''; + if (brief !== '') { + briefContents = ` +
+

Brief

+

${brief}

+
+ `; + } + + const itemPage = await got.get(address); + const itemCapture = cheerio.load(itemPage.data); + // section and subject content + const section = itemCapture('header > div.overline > span.overline__section').text(); + const subject = itemCapture('header > div.overline > span.overline__subject').text(); + let sectionContents = ''; + if (section !== '' || subject !== '') { + sectionContents = ` +
+ ${section} + ${subject} +
+ `; + } + // abs content + const abs = itemCapture('div.section.abstract > p').text(); + let absContents = ''; + if (abs !== '') { + absContents = ` +
+

Abstract

+ ${abs} +
+ `; + } + const contents = sectionContents + briefContents + absContents; + + const single = { + title: title, + author: author, + description: contents, + link: address, + guid: address, + pubDate: time, + }; + ctx.cache.set(address, JSON.stringify(single)); + return Promise.resolve(single); + }) + ); + ctx.state.data = { + title: `Science | First Release`, + description: `Science, a research journal. For papers that published online.`, + link: base, + item: out, + }; +}; diff --git a/lib/routes/sciencemag/science/current.js b/lib/routes/sciencemag/science/current.js new file mode 100644 index 0000000000..466be0227c --- /dev/null +++ b/lib/routes/sciencemag/science/current.js @@ -0,0 +1,110 @@ +const cheerio = require('cheerio'); +const got = require('@/utils/got'); + +module.exports = async (ctx) => { + const base = `https://science.sciencemag.org`; + + const res = await got.get(base); + const pageCapture = cheerio.load(res.data); + + // just select paper relative sections + const sectionList = ['issue-toc-section-research-articles', 'issue-toc-section-review', 'issue-toc-section-reports']; + const list = [].concat.apply( + [], + sectionList.map((section) => { + const sec = cheerio.load(pageCapture(`ul > li .issue-toc-section.${section}`).html()); + const sectionName = sec('h2').text(); + const sectionList = sec('ul > li > div > div > article > div') + .append(`
${sectionName}
`) + .get(); + return sectionList; + }) + ); + + const out = await Promise.all( + list.map(async (item) => { + const $ = cheerio.load(item); + + const title = $('h3').text(); + const partial = $('h3 > a').attr('href'); + const address = `${base}${partial}`; + const section = $('div .toc-section-type').text(); + + let author; + const authorList = $('span.highwire-citation-authors > span.highwire-citation-author') + .map(function(i, el) { + return $(el).text(); + }) + .get(); + if (authorList.length > 5) { + author = authorList.slice(0, 5).join(', ') + ' et al.'; + } else { + author = authorList.join(', '); + } + + const time = new Date($('p.highwire-cite-metadata > time').text()).toUTCString(); + + const cache = await ctx.cache.get(address); + if (cache) { + return Promise.resolve(JSON.parse(cache)); + } + + // contents + // section content + let sectionContents = ''; + if (section !== '') { + sectionContents = ` +
+ ${section} +
+ `; + } + // brief content + const brief = $('div.highwire-cite-snippet > div > div > p').text(); + let briefContents = ''; + if (brief !== '') { + briefContents = ` +
+

${brief}

+
+ `; + } + const itemPage = await got.get(address); + const itemCapture = cheerio.load(itemPage.data); + const abs = itemCapture('div > div.abstract-view > div.section') + .map(function(i, el) { + return $(el).html(); + }) + .get() + .join('
'); + + // abs content + let absContents = ''; + if (abs !== null) { + absContents = ` +
+ ${abs} +
+ `; + } + const contents = sectionContents + briefContents + absContents; + + const single = { + title: title, + author: author, + description: contents, + link: address, + guid: address, + pubDate: time, + }; + ctx.cache.set(address, JSON.stringify(single)); + return Promise.resolve(single); + }) + ); + ctx.state.data = { + title: `Science | Current Table of Contents`, + description: `Science, a research journal`, + link: base, + item: out, + }; +}; diff --git a/lib/routes/sciencemag/science/early.js b/lib/routes/sciencemag/science/early.js new file mode 100644 index 0000000000..dd07c1a681 --- /dev/null +++ b/lib/routes/sciencemag/science/early.js @@ -0,0 +1,99 @@ +const cheerio = require('cheerio'); +const got = require('@/utils/got'); + +module.exports = async (ctx) => { + const base = `https://science.sciencemag.org/content/early/recent`; + + const res = await got.get(base); + const pageCapture = cheerio.load(res.data); + + const list = pageCapture('ul > li > div > article > div').get(); + + const out = await Promise.all( + list.map(async (item) => { + const $ = cheerio.load(item); + + const title = $('h3').text(); + const partial = $('h3 > a').attr('href'); + const address = `${base}${partial}`; + + let author; + const authorList = $('span.highwire-citation-authors > span.highwire-citation-author') + .map(function(i, el) { + return $(el).text(); + }) + .get(); + if (authorList.length > 5) { + author = authorList.slice(0, 5).join(', ') + ' et al.'; + } else { + author = authorList.join(', '); + } + + const time = new Date($('p.highwire-cite-metadata > time').text()).toUTCString(); + + const cache = await ctx.cache.get(address); + if (cache) { + return Promise.resolve(JSON.parse(cache)); + } + + // contents + // brief content + const brief = $('div.highwire-cite-snippet > div > div > p').text(); + let briefContents = ''; + if (brief !== '') { + briefContents = ` +
+

${brief}

+
+ `; + } + + const itemPage = await got.get(address); + const itemCapture = cheerio.load(itemPage.data); + // section content + const section = itemCapture('header > div.overline').text(); + let sectionContents = ''; + if (section !== '') { + sectionContents = ` +
+ ${section} + [Published Online] +
+ `; + } + // abs content + const abs = itemCapture('div > div.abstract-view > div.section') + .map(function(i, el) { + return $(el).html(); + }) + .get() + .join('
'); + let absContents = ''; + if (abs !== null) { + absContents = ` +
+ ${abs} +
+ `; + } + const contents = sectionContents + briefContents + absContents; + + const single = { + title: title, + author: author, + description: contents, + link: address, + guid: address, + pubDate: time, + }; + ctx.cache.set(address, JSON.stringify(single)); + return Promise.resolve(single); + }) + ); + ctx.state.data = { + title: `Science | First Release`, + description: `Science, a research journal. For papers that published online.`, + link: base, + item: out, + }; +}; diff --git a/lib/routes/zhishifenzi/depth.js b/lib/routes/zhishifenzi/depth.js new file mode 100644 index 0000000000..a09e854292 --- /dev/null +++ b/lib/routes/zhishifenzi/depth.js @@ -0,0 +1,60 @@ +const got = require('@/utils/got'); +const cheerio = require('cheerio'); +const url = require('url'); +const date = require('@/utils/date'); + +const base = 'http://zhishifenzi.com'; + +module.exports = async (ctx) => { + const link = `${base}/depth/`; + const response = await got.get(encodeURI(link)); + const pageCapture = cheerio.load(response.data); + + const title = '深度'; + + const list = pageCapture('div.inner_depth_list > ul > li').get(); + const out = await Promise.all( + list.slice(0, 3).map(async (elem) => { + const $ = cheerio.load(elem); + const title = $('h5 > a').text(); + const partial = $('h5 > a').attr('href'); + const address = url.resolve(base, partial); + const pubDate = date( + $('div.inner_depth_list_tags > p') + .first() + .text() + ); + + // const keywords = $('div.inner_depth_list_tags > p') + // .last() + // .html(); + const item = { + title, + pubDate, + link: encodeURI(address), + }; + + const value = await ctx.cache.get(address); + if (value) { + item.description = value; + } else { + const detail = await got.get(item.link); + const detailCapture = cheerio.load(detail.data); + + item.description = detailCapture('.main div.inner_content') + .html() + .replace(/src="\//g, `src="${base}/`); + ctx.cache.set(address, item.description); + } + + return Promise.resolve(item); + }) + ); + + ctx.state.data = { + title: `知識分子 | ${title}`, + description: `知識分子 | 科學 文明 智慧`, + link: link, + item: out, + }; +}; diff --git a/lib/routes/zhishifenzi/innovation.js b/lib/routes/zhishifenzi/innovation.js new file mode 100644 index 0000000000..9c647ec10e --- /dev/null +++ b/lib/routes/zhishifenzi/innovation.js @@ -0,0 +1,74 @@ +const got = require('@/utils/got'); +const cheerio = require('cheerio'); +const url = require('url'); +const date = require('@/utils/date'); + +const base = 'http://zhishifenzi.com'; + +module.exports = async (ctx) => { + let link = `${base}/innovation`; + + const type = ctx.params.type; + // type can be on of: + // ["multiple", "company", "product", "technology", "character", "policy"] + if (typeof type !== 'undefined') { + link = `${link}/${type}`; + } + const response = await got.get(encodeURI(link)); + const pageCapture = cheerio.load(response.data); + + let title = pageCapture('div.innovation_nav > ul > li.sel') + .text() + .split('：') + .pop(); + if (title === '') { + title = '全部創新'; + } else { + title = `創新「${title}」`; + } + const list = pageCapture('div.inner_news_list > ul > li.clearfix').get(); + const out = await Promise.all( + list.slice(0, 4).map(async (elem) => { + const $ = cheerio.load(elem); + const title = $('div > h5 > a').text(); + const partial = $('div > h5 > a').attr('href'); + const address = url.resolve(base, partial); + const pubDate = date( + $('span') + .first() + .text() + ); + + // const keywords = $('span') + // .last() + // .html(); + const item = { + title, + pubDate, + link: encodeURI(address), + }; + + const value = await ctx.cache.get(address); + if (value) { + item.description = value; + } else { + const detail = await got.get(item.link); + const detailCapture = cheerio.load(detail.data); + + item.description = detailCapture('.main div.inner_content') + .html() + .replace(/src="\//g, `src="${base}/`); + ctx.cache.set(address, item.description); + } + + return Promise.resolve(item); + }) + ); + + ctx.state.data = { + title: `知識分子 | ${title}`, + description: `知識分子 | 科學 文明 智慧`, + link: link, + item: out, + }; +}; diff --git a/lib/routes/zhishifenzi/news.js b/lib/routes/zhishifenzi/news.js new file mode 100644 index 0000000000..67e8755de0 --- /dev/null +++ b/lib/routes/zhishifenzi/news.js @@ -0,0 +1,74 @@ +const got = require('@/utils/got'); +const cheerio = require('cheerio'); +const url = require('url'); +const date = require('@/utils/date'); + +const base = 'http://zhishifenzi.com'; + +module.exports = async (ctx) => { + let link = `${base}/news/`; + + const type = ctx.params.type; + // type can be on of: + // ["biology", "medicine", "ai", "physics", "chymistry", "astronomy", "others"] + if (typeof type !== 'undefined') { + link = `${base}/news/${type}`; + } + const response = await got.get(encodeURI(link)); + const pageCapture = cheerio.load(response.data); + + let title = pageCapture('div.all_categories > ul > li.category') + .text() + .split('：') + .pop(); + if (title === '') { + title = '全部資訊'; + } else { + title = `資訊「${title}」`; + } + const list = pageCapture('div.inner_news_list > ul > li.clearfix').get(); + const out = await Promise.all( + list.map(async (elem) => { + const $ = cheerio.load(elem); + const title = $('div > h5 > a').text(); + const partial = $('div > h5 > a').attr('href'); + const address = url.resolve(base, partial); + const pubDate = date( + $('span') + .first() + .text() + ); + + // const keywords = $('span') + // .last() + // .html(); + const item = { + title, + pubDate, + link: encodeURI(address), + }; + + const value = await ctx.cache.get(address); + if (value) { + item.description = value; + } else { + const detail = await got.get(item.link); + const detailCapture = cheerio.load(detail.data); + + item.description = detailCapture('.main div.inner_content') + .html() + .replace(/src="\//g, `src="${base}/`); + ctx.cache.set(address, item.description); + } + + return Promise.resolve(item); + }) + ); + + ctx.state.data = { + title: `知識分子 | ${title}`, + description: `知識分子 | 科學 文明 智慧`, + link: link, + item: out, + }; +};