diff --git a/docs/en/install/README.md b/docs/en/install/README.md index dd3bdd4bce..8ee68e03a1 100644 --- a/docs/en/install/README.md +++ b/docs/en/install/README.md @@ -564,7 +564,7 @@ It is also valid to contain route parameters, e.g. `/weibo/user/2612249974`. ::: tip Experimental features -Configs in this sections are in beta stage, and are turn off by default. Please read corresponded description and turn on if necessary. +Configs in this sections are in beta stage, and **are turn off by default**. Please read corresponded description and turn on if necessary. ::: @@ -572,6 +572,8 @@ Configs in this sections are in beta stage, and are turn off by default. Please `FILTER_REGEX_ENGINE`: Define Regex engine used in [Parameters->filtering](/en/parameter.html#filtering). Valid value are `[re2, regexp]`. Default value is `re2`. We suggest public instance should leave this value to default, and this option right now is mainly for backward compatibility. +`ALLOW_USER_SUPPLY_UNSAFE_DOMAIN`: allow users to provide a domain as a parameter to routes that are not in their allow list, respectively. Public instances are suggested to leave this value default, as it may lead to [Server-Side Request Forgery (SSRF)](https://owasp.org/www-community/attacks/Server_Side_Request_Forgery) + ### Other Application Configurations `DISALLOW_ROBOT`: prevent indexing by search engine, default to enable, set false or 0 to disable diff --git a/docs/install/README.md b/docs/install/README.md index eeb03460b9..e23bc21373 100644 --- a/docs/install/README.md +++ b/docs/install/README.md @@ -571,7 +571,7 @@ RSSHub 支持使用访问密钥 / 码,白名单和黑名单三种方式进行 ::: tip 测试特性 -这个板块控制的是一些新特性的选项,默认他们都是关闭的。如果有需要请阅读对应说明后按需开启 +这个板块控制的是一些新特性的选项,他们都是**默认关闭**的。如果有需要请阅读对应说明后按需开启 ::: @@ -579,6 +579,8 @@ RSSHub 支持使用访问密钥 / 码,白名单和黑名单三种方式进行 `FILTER_REGEX_ENGINE`: 控制 [通用参数 -> 内容过滤](/parameter.html#nei-rong-guo-lu) 使用的正则引擎。可选`[re2, regexp]`,默认`re2`。我们推荐公开实例不要调整这个选项,这个选项目前主要用于向后兼容。 +`ALLOW_USER_SUPPLY_UNSAFE_DOMAIN`: 允许用户为路由提供域名作为参数。建议公共实例不要调整此选项,开启后可能会导致 [服务端请求伪造(SSRF)](https://owasp.org/www-community/attacks/Server_Side_Request_Forgery) + ### 其他应用配置 `DISALLOW_ROBOT`: 阻止搜索引擎收录,默认开启,设置 false 或 0 关闭 diff --git a/lib/config.js b/lib/config.js index a48b5198f3..c48078091b 100644 --- a/lib/config.js +++ b/lib/config.js @@ -97,6 +97,7 @@ const calculateValue = () => { feature: { allow_user_hotlink_template: envs.ALLOW_USER_HOTLINK_TEMPLATE === 'true', filter_regex_engine: envs.FILTER_REGEX_ENGINE || 're2', + allow_user_supply_unsafe_domain: envs.ALLOW_USER_SUPPLY_UNSAFE_DOMAIN === 'true', }, suffix: envs.SUFFIX, titleLengthLimit: parseInt(envs.TITLE_LENGTH_LIMIT) || 150, diff --git a/lib/routes/bandisoft/index.js b/lib/routes/bandisoft/index.js index 7ce535e3a9..fe07e3aa3e 100644 --- a/lib/routes/bandisoft/index.js +++ b/lib/routes/bandisoft/index.js @@ -1,9 +1,13 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); +const { isValidHost } = require('@/utils/valid-host'); module.exports = async (ctx) => { const lang = ctx.params.lang || 'en'; const id = ctx.params.id || 'bandizip'; + if (!isValidHost(lang)) { + throw Error('Invalid language code'); + } const rootUrl = `https://${lang}.bandisoft.com`; const currentUrl = `${rootUrl}/${id}/history/`; diff --git a/lib/routes/biobio/others.js b/lib/routes/biobio/others.js index 8286407fc5..826ec344d7 100644 --- a/lib/routes/biobio/others.js +++ b/lib/routes/biobio/others.js @@ -1,7 +1,12 @@ const cheerio = require('cheerio'); const got = require('@/utils/got'); +const { isValidHost } = require('@/utils/valid-host'); module.exports = async (ctx) => { + if (!isValidHost(ctx.params.column)) { + throw Error('Invalid column'); + } + const url = `http://${ctx.params.column}.bio1000.com/${ctx.params.id}`; const res = await got.get(url); const $ = cheerio.load(res.data); diff --git a/lib/routes/blogs/hedwig.js b/lib/routes/blogs/hedwig.js index 028650e356..afa559d244 100644 --- a/lib/routes/blogs/hedwig.js +++ b/lib/routes/blogs/hedwig.js @@ -4,9 +4,13 @@ const md = require('markdown-it')({ html: true, }); const dayjs = require('dayjs'); +const { isValidHost } = require('@/utils/valid-host'); module.exports = async (ctx) => { const type = ctx.params.type; + if (!isValidHost(type)) { + throw Error('Invalid type'); + } const url = `https://${type}.hedwig.pub`; const res = await got({ diff --git a/lib/routes/blogs/wordpress.js b/lib/routes/blogs/wordpress.js index 2e5154f2a8..7c414e6f4f 100644 --- a/lib/routes/blogs/wordpress.js +++ b/lib/routes/blogs/wordpress.js @@ -1,7 +1,12 @@ const parser = require('@/utils/rss-parser'); const config = require('@/config').value; +const allowDomain = ['lawrence.code.blog']; module.exports = async (ctx) => { + if (!config.feature.allow_user_supply_unsafe_domain && !allowDomain.includes(ctx.params.domain)) { + ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); + } + const scheme = ctx.params.https || 'https'; const cdn = config.wordpress.cdnUrl; diff --git a/lib/routes/booth-pm/shop.js b/lib/routes/booth-pm/shop.js index 45bc0ad008..22ae198399 100644 --- a/lib/routes/booth-pm/shop.js +++ b/lib/routes/booth-pm/shop.js @@ -1,10 +1,13 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); - +const { isValidHost } = require('@/utils/valid-host'); const maxPages = 5; module.exports = async (ctx) => { const { subdomain } = ctx.params; + if (!isValidHost(subdomain)) { + throw Error('Invalid subdomain'); + } const shopUrl = `https://${subdomain}.booth.pm`; let shopName; diff --git a/lib/routes/caixin/blog.js b/lib/routes/caixin/blog.js index 3993c9f0e7..39722b5188 100644 --- a/lib/routes/caixin/blog.js +++ b/lib/routes/caixin/blog.js @@ -1,6 +1,7 @@ const got = require('@/utils/got'); const parser = require('@/utils/rss-parser'); const cheerio = require('cheerio'); +const { isValidHost } = require('@/utils/valid-host'); async function load(link, need_feed_description) { const response = await got.get(link); @@ -29,6 +30,9 @@ async function load(link, need_feed_description) { module.exports = async (ctx) => { const { column } = ctx.params; + if (!isValidHost(column)) { + throw Error('Invalid column'); + } const link = `http://${column}.blog.caixin.com`; const feed_url = `${link}/feed`; const feed = await parser.parseURL(feed_url); diff --git a/lib/routes/caixin/category.js b/lib/routes/caixin/category.js index 3630633444..a5fdaf7657 100644 --- a/lib/routes/caixin/category.js +++ b/lib/routes/caixin/category.js @@ -1,10 +1,14 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); +const { isValidHost } = require('@/utils/valid-host'); module.exports = async (ctx) => { const category = ctx.params.category; const column = ctx.params.column; const url = `http://${column}.caixin.com/${category}`; + if (!isValidHost(column)) { + throw Error('Invalid column'); + } const response = await got({ method: 'get', diff --git a/lib/routes/craigslist/search.js b/lib/routes/craigslist/search.js index 691f79f040..0327154bdd 100644 --- a/lib/routes/craigslist/search.js +++ b/lib/routes/craigslist/search.js @@ -1,7 +1,12 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); +const { isValidHost } = require('@/utils/valid-host'); module.exports = async (ctx) => { + if (!isValidHost(ctx.params.location)) { + throw Error('Invalid location'); + } + const queryParams = ctx.request.querystring; const queryUrl = `https://${ctx.params.location}.craigslist.org/search/${ctx.params.type}?${queryParams}`; const { data } = await got.get(queryUrl); diff --git a/lib/routes/engadget/home.js b/lib/routes/engadget/home.js index 4aa39030e2..197d2222d9 100644 --- a/lib/routes/engadget/home.js +++ b/lib/routes/engadget/home.js @@ -1,9 +1,13 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); const parser = require('@/utils/rss-parser'); +const allowLang = ['chinese', 'cn', 'us', 'japanese', 'www']; module.exports = async (ctx) => { const lang = ctx.params.lang === 'us' ? 'www' : ctx.params.lang || 'cn'; + if (!allowLang.includes(lang)) { + throw Error('Invalid lang'); + } const rssUrl = `https://${lang}.engadget.com/rss.xml`; const feed = await parser.parseURL(rssUrl); diff --git a/lib/routes/fanbox/main.js b/lib/routes/fanbox/main.js index 863e296f20..c173d98deb 100644 --- a/lib/routes/fanbox/main.js +++ b/lib/routes/fanbox/main.js @@ -4,12 +4,15 @@ // user?: fanbox domain name const got = require('@/utils/got'); - +const { isValidHost } = require('@/utils/valid-host'); const conv_item = require('./conv'); const get_header = require('./header'); module.exports = async (ctx) => { const user = ctx.params.user || 'official'; // if no user specified, just go to official page + if (!isValidHost(user)) { + throw Error('Invalid user'); + } const box_url = `https://${user}.fanbox.cc`; // get user info diff --git a/lib/routes/fashionnetwork/headline.js b/lib/routes/fashionnetwork/headline.js index 922ed9dad1..e0e38bf33d 100644 --- a/lib/routes/fashionnetwork/headline.js +++ b/lib/routes/fashionnetwork/headline.js @@ -1,8 +1,12 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); +const { isValidHost } = require('@/utils/valid-host'); module.exports = async (ctx) => { const country = ctx.params.country || 'ww'; + if (!isValidHost(country)) { + throw Error('Invalid country'); + } const rootUrl = `https://${country}.fashionnetwork.com`; const response = await got({ diff --git a/lib/routes/fashionnetwork/news.js b/lib/routes/fashionnetwork/news.js index c26bc54c9a..105334e7b6 100644 --- a/lib/routes/fashionnetwork/news.js +++ b/lib/routes/fashionnetwork/news.js @@ -1,5 +1,6 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); +const { isValidHost } = require('@/utils/valid-host'); module.exports = async (ctx) => { const country = ctx.params.country || 'ww'; @@ -12,6 +13,10 @@ module.exports = async (ctx) => { const sectorsUrl = sectors ? 'sectors%5B%5D=' + sectors.split(',').join('§ors%5B%5D=') : ''; const categoriesUrl = categories ? 'categs%5B%5D=' + categories.split(',').join('&categs%5B%5D=') : ''; + if (!isValidHost(country)) { + throw Error('Invalid country'); + } + const rootUrl = `https://${country}.fashionnetwork.com`; const currentUrl = `${rootUrl}/news/s.jsonp?${sectorsUrl}&${categoriesUrl}`; const response = await got({ diff --git a/lib/routes/gitlab/common.js b/lib/routes/gitlab/common.js new file mode 100644 index 0000000000..5ad06ad4eb --- /dev/null +++ b/lib/routes/gitlab/common.js @@ -0,0 +1,5 @@ +const allowHost = ['gitlab.com']; + +module.exports = { + allowHost, +}; diff --git a/lib/routes/gitlab/explore.js b/lib/routes/gitlab/explore.js index f77939b756..9d4971fb66 100644 --- a/lib/routes/gitlab/explore.js +++ b/lib/routes/gitlab/explore.js @@ -1,5 +1,7 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); +const config = require('@/config').value; +const { allowHost } = require('./common'); module.exports = async (ctx) => { let { type, host } = ctx.params; @@ -10,6 +12,9 @@ module.exports = async (ctx) => { starred: 'Most stars', all: 'All', }; + if (!config.feature.allow_user_supply_unsafe_domain && !allowHost.includes(new URL(host).hostname)) { + ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); + } const res = await got({ method: 'get', diff --git a/lib/routes/gitlab/release.js b/lib/routes/gitlab/release.js index ee734df265..f179a71f27 100644 --- a/lib/routes/gitlab/release.js +++ b/lib/routes/gitlab/release.js @@ -1,8 +1,13 @@ const got = require('@/utils/got'); const { parseDate } = require('@/utils/parse-date'); +const config = require('@/config').value; +const { allowHost } = require('./common'); module.exports = async (ctx) => { const { namespace, project, host } = ctx.params; + if (!config.feature.allow_user_supply_unsafe_domain && !allowHost.includes(host)) { + ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); + } const host_ = host ? host : 'gitlab.com'; const namespace_ = encodeURIComponent(namespace); diff --git a/lib/routes/gitlab/tag.js b/lib/routes/gitlab/tag.js index 6e267be525..c81d756da6 100644 --- a/lib/routes/gitlab/tag.js +++ b/lib/routes/gitlab/tag.js @@ -1,8 +1,13 @@ const got = require('@/utils/got'); const { parseDate } = require('@/utils/parse-date'); +const config = require('@/config').value; +const { allowHost } = require('./common'); module.exports = async (ctx) => { const { namespace, project, host } = ctx.params; + if (!config.feature.allow_user_supply_unsafe_domain && !allowHost.includes(host)) { + ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); + } const host_ = host ? host : 'gitlab.com'; const namespace_ = encodeURIComponent(namespace); diff --git a/lib/routes/hexo/fluid.js b/lib/routes/hexo/fluid.js index 31242be9fc..d13e8cb02d 100644 --- a/lib/routes/hexo/fluid.js +++ b/lib/routes/hexo/fluid.js @@ -1,7 +1,11 @@ const cheerio = require('cheerio'); const got = require('@/utils/got'); +const config = require('@/config').value; module.exports = async (ctx) => { + if (!config.feature.allow_user_supply_unsafe_domain) { + ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); + } const url = `http://${ctx.params.url}`; const res = await got.get(`${url}/archives/`); const $ = cheerio.load(res.data); diff --git a/lib/routes/hexo/next.js b/lib/routes/hexo/next.js index 389cc5d11e..a4ff209ca7 100644 --- a/lib/routes/hexo/next.js +++ b/lib/routes/hexo/next.js @@ -1,7 +1,11 @@ const cheerio = require('cheerio'); const got = require('@/utils/got'); +const config = require('@/config').value; module.exports = async (ctx) => { + if (!config.feature.allow_user_supply_unsafe_domain) { + ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); + } const url = `http://${ctx.params.url}`; const res = await got.get(`${url}/archives/`); const $ = cheerio.load(res.data); diff --git a/lib/routes/hexo/yilia.js b/lib/routes/hexo/yilia.js index 90f4489798..0c3043f17b 100644 --- a/lib/routes/hexo/yilia.js +++ b/lib/routes/hexo/yilia.js @@ -1,7 +1,11 @@ const cheerio = require('cheerio'); const got = require('@/utils/got'); +const config = require('@/config').value; module.exports = async (ctx) => { + if (!config.feature.allow_user_supply_unsafe_domain) { + ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); + } const url = `http://${ctx.params.url}`; const res = await got.get(url); const $ = cheerio.load(res.data); diff --git a/lib/routes/mastodon/account_id.js b/lib/routes/mastodon/account_id.js index 23212555c3..5c197b8700 100644 --- a/lib/routes/mastodon/account_id.js +++ b/lib/routes/mastodon/account_id.js @@ -1,9 +1,13 @@ const utils = require('./utils'); +const config = require('@/config').value; module.exports = async (ctx) => { const site = ctx.params.site; const account_id = ctx.params.account_id; const only_media = ctx.params.only_media ? 'true' : 'false'; + if (!config.feature.allow_user_supply_unsafe_domain && !utils.allowSiteList.includes(site)) { + ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); + } const { account_data, data } = await utils.getAccountStatuses(site, account_id, only_media); diff --git a/lib/routes/mastodon/timeline_local.js b/lib/routes/mastodon/timeline_local.js index f22fbae15c..7a8b9ed2fc 100644 --- a/lib/routes/mastodon/timeline_local.js +++ b/lib/routes/mastodon/timeline_local.js @@ -1,9 +1,13 @@ const got = require('@/utils/got'); const utils = require('./utils'); +const config = require('@/config').value; module.exports = async (ctx) => { const site = ctx.params.site; const only_media = ctx.params.only_media ? 'true' : 'false'; + if (!config.feature.allow_user_supply_unsafe_domain && !utils.allowSiteList.includes(site)) { + ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); + } const url = `http://${site}/api/v1/timelines/public?local=true&only_media=${only_media}`; diff --git a/lib/routes/mastodon/timeline_remote.js b/lib/routes/mastodon/timeline_remote.js index ce60b0e6af..96d431fd2b 100644 --- a/lib/routes/mastodon/timeline_remote.js +++ b/lib/routes/mastodon/timeline_remote.js @@ -1,9 +1,13 @@ const got = require('@/utils/got'); const utils = require('./utils'); +const config = require('@/config').value; module.exports = async (ctx) => { const site = ctx.params.site; const only_media = ctx.params.only_media ? 'true' : 'false'; + if (!config.feature.allow_user_supply_unsafe_domain && !utils.allowSiteList.includes(site)) { + ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); + } const url = `http://${site}/api/v1/timelines/public?remote=true&only_media=${only_media}`; diff --git a/lib/routes/mastodon/utils.js b/lib/routes/mastodon/utils.js index 18022e3b34..4983a103ca 100644 --- a/lib/routes/mastodon/utils.js +++ b/lib/routes/mastodon/utils.js @@ -1,6 +1,8 @@ const got = require('@/utils/got'); const { parseDate } = require('@/utils/parse-date'); +const allowSiteList = ['mastodon.social', 'pawoo.net']; + const parseStatuses = (data) => data.map((item) => { // docs on: https://docs.joinmastodon.org/entities/status/ @@ -125,4 +127,5 @@ module.exports = { parseStatuses, getAccountStatuses, getAccountIdByAcct, + allowSiteList, }; diff --git a/lib/routes/pornhub/category_url.js b/lib/routes/pornhub/category_url.js index 58ba2a0a1e..f6a4e5ce91 100644 --- a/lib/routes/pornhub/category_url.js +++ b/lib/routes/pornhub/category_url.js @@ -1,10 +1,14 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); +const { isValidHost } = require('@/utils/valid-host'); module.exports = async (ctx) => { const language = ctx.params.language || 'www'; const url = ctx.params.url || 'video'; const link = `https://${language}.pornhub.com/${url}`; + if (!isValidHost(language)) { + throw Error('Invalid language'); + } const response = await got.get(link); const $ = cheerio.load(response.data); diff --git a/lib/routes/pornhub/model.js b/lib/routes/pornhub/model.js index 82971ab3c0..468db7cf1e 100644 --- a/lib/routes/pornhub/model.js +++ b/lib/routes/pornhub/model.js @@ -1,11 +1,15 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); +const { isValidHost } = require('@/utils/valid-host'); module.exports = async (ctx) => { const language = ctx.params.language || 'www'; const username = ctx.params.username; const sort = ctx.params.sort || 'mr'; const link = `https://${language}.pornhub.com/model/${username}/videos?o=${sort}`; + if (!isValidHost(language)) { + throw Error('Invalid language'); + } const response = await got.get(link); const $ = cheerio.load(response.data); diff --git a/lib/routes/pornhub/pornstar.js b/lib/routes/pornhub/pornstar.js index 26491bd96b..771ae27767 100644 --- a/lib/routes/pornhub/pornstar.js +++ b/lib/routes/pornhub/pornstar.js @@ -1,11 +1,15 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); +const { isValidHost } = require('@/utils/valid-host'); module.exports = async (ctx) => { const language = ctx.params.language || 'www'; const username = ctx.params.username; const sort = ctx.params.sort || 'mr'; const link = `https://${language}.pornhub.com/pornstar/${username}/videos?o=${sort}`; + if (!isValidHost(language)) { + throw Error('Invalid language'); + } const response = await got.get(link); const $ = cheerio.load(response.data); diff --git a/lib/routes/pornhub/users.js b/lib/routes/pornhub/users.js index 0971b06f4e..a632b16637 100644 --- a/lib/routes/pornhub/users.js +++ b/lib/routes/pornhub/users.js @@ -1,10 +1,14 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); +const { isValidHost } = require('@/utils/valid-host'); module.exports = async (ctx) => { const language = ctx.params.language || 'www'; const username = ctx.params.username; const link = `https://${language}.pornhub.com/users/${username}/videos`; + if (!isValidHost(language)) { + throw Error('Invalid language'); + } const response = await got.get(link); const $ = cheerio.load(response.data); diff --git a/lib/routes/touhougarakuta/index.js b/lib/routes/touhougarakuta/index.js index e895490c39..3a742bbf76 100644 --- a/lib/routes/touhougarakuta/index.js +++ b/lib/routes/touhougarakuta/index.js @@ -11,6 +11,9 @@ const getBaseUrl = (language) => (language === 'ja' ? 'https://touhougarakuta.co module.exports = async (ctx) => { const { language, type } = ctx.params; + if (!Object.keys(languageCodes).includes(language)) { + throw Error('Invalid language'); + } const baseUrl = getBaseUrl(language); diff --git a/lib/routes/weforum/report.js b/lib/routes/weforum/report.js index cd9033eee0..26ef70d2ac 100644 --- a/lib/routes/weforum/report.js +++ b/lib/routes/weforum/report.js @@ -1,13 +1,17 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); +const { isValidHost } = require('@/utils/valid-host'); module.exports = async (ctx) => { - ctx.params.lang = ctx.params.lang || 'www'; - ctx.params.platform = ctx.params.platform || ''; - ctx.params.year = ctx.params.year || ''; + const lang = ctx.params.lang || 'www'; + const platform = ctx.params.platform || ''; + const year = ctx.params.year || ''; + if (!isValidHost(lang)) { + throw Error('Invalid lang'); + } - const rootUrl = `https://${ctx.params.lang === 'en' ? 'www' : ctx.params.lang}.weforum.org`; - const currentUrl = `${rootUrl}/reports?platform=${ctx.params.platform}&year=${ctx.params.year}`; + const rootUrl = `https://${lang === 'en' ? 'www' : lang}.weforum.org`; + const currentUrl = `${rootUrl}/reports?platform=${platform}&year=${year}`; const response = await got({ method: 'get', url: currentUrl, diff --git a/lib/routes/yahoo-news/index.js b/lib/routes/yahoo-news/index.js index f91f6aaef6..e25837c942 100644 --- a/lib/routes/yahoo-news/index.js +++ b/lib/routes/yahoo-news/index.js @@ -1,9 +1,13 @@ const got = require('@/utils/got'); const parser = require('@/utils/rss-parser'); const cheerio = require('cheerio'); +const { isValidHost } = require('@/utils/valid-host'); module.exports = async (ctx) => { const region = ctx.params.region === 'en' ? '' : ctx.params.region.toLowerCase() + '.'; + if (!isValidHost(region)) { + throw Error('Invalid region'); + } const category = ctx.params.category ? ctx.params.category.toLowerCase() : ''; const rssUrl = `https://${region}news.yahoo.com/rss/${category}`; const feed = await parser.parseURL(rssUrl); diff --git a/lib/routes/ziroom/room.js b/lib/routes/ziroom/room.js index 37e3536fa6..6a71d290c1 100644 --- a/lib/routes/ziroom/room.js +++ b/lib/routes/ziroom/room.js @@ -1,4 +1,5 @@ const got = require('@/utils/got'); +const { isValidHost } = require('@/utils/valid-host'); module.exports = async (ctx) => { const city = ctx.params.city || 'sh'; @@ -7,6 +8,10 @@ module.exports = async (ctx) => { const room = ctx.params.room || '1'; const domain = `${city === 'bj' ? '' : city + '.'}m.ziroom.com`; + if (!isValidHost(city)) { + throw Error('Invalid city'); + } + const response = await got({ method: 'post', url: `http://${domain}/list/ajax-get-data`, diff --git a/lib/utils/valid-host.js b/lib/utils/valid-host.js new file mode 100644 index 0000000000..9181684f4d --- /dev/null +++ b/lib/utils/valid-host.js @@ -0,0 +1,16 @@ +/** + * Check if a sub-domain is valid + * @param {String} hostname sub-domain + * @returns {Boolean} true if valid + */ +const isValidHost = (hostname) => { + if (typeof hostname !== 'string') { + return false; + } + const regex = /^[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?$/; + return regex.test(hostname); +}; + +module.exports = { + isValidHost, +}; diff --git a/lib/v2/19lou/index.js b/lib/v2/19lou/index.js index d8e89c6a56..cb18d7f633 100644 --- a/lib/v2/19lou/index.js +++ b/lib/v2/19lou/index.js @@ -3,6 +3,7 @@ const cheerio = require('cheerio'); const timezone = require('@/utils/timezone'); const { parseDate } = require('@/utils/parse-date'); const iconv = require('iconv-lite'); +const { isValidHost } = require('@/utils/valid-host'); const setCookie = function (cookieName, cookieValue, seconds, path, domain, secure) { let expires = null; @@ -15,6 +16,9 @@ const setCookie = function (cookieName, cookieValue, seconds, path, domain, secu module.exports = async (ctx) => { const city = ctx.params.city ?? 'www'; + if (!isValidHost(city)) { + throw Error('Invalid city'); + } const rootUrl = `https://${city}.19lou.com`; diff --git a/lib/v2/91porn/author.js b/lib/v2/91porn/author.js index 3586214d72..8ea49ba299 100644 --- a/lib/v2/91porn/author.js +++ b/lib/v2/91porn/author.js @@ -3,11 +3,13 @@ const cheerio = require('cheerio'); const { parseDate } = require('@/utils/parse-date'); const { art } = require('@/utils/render'); const path = require('path'); +const { domainValidation } = require('./utils'); module.exports = async (ctx) => { const { domain = '91porn.com' } = ctx.query; const { uid, lang = 'en_US' } = ctx.params; const siteUrl = `https://${domain}/uvideos.php?UID=${uid}&type=public`; + domainValidation(domain, ctx); const response = await got.post(siteUrl, { form: { diff --git a/lib/v2/91porn/index.js b/lib/v2/91porn/index.js index 592e6a26e1..fdfd440b2e 100644 --- a/lib/v2/91porn/index.js +++ b/lib/v2/91porn/index.js @@ -3,11 +3,13 @@ const cheerio = require('cheerio'); const { parseDate } = require('@/utils/parse-date'); const { art } = require('@/utils/render'); const path = require('path'); +const { domainValidation } = require('./utils'); module.exports = async (ctx) => { const { domain = '91porn.com' } = ctx.query; const siteUrl = `https://${domain}/index.php`; const { lang = 'en_US' } = ctx.params; + domainValidation(domain, ctx); const response = await got.post(siteUrl, { form: { diff --git a/lib/v2/91porn/utils.js b/lib/v2/91porn/utils.js new file mode 100644 index 0000000000..339460b490 --- /dev/null +++ b/lib/v2/91porn/utils.js @@ -0,0 +1,12 @@ +const config = require('@/config').value; +const allowDomain = ['91porn.com', 'www.91porn.com', '0122.91p30.com', 'www.91zuixindizhi.com', 'w1218.91p46.com']; + +const domainValidation = (domain, ctx) => { + if (!config.feature.allow_user_supply_unsafe_domain && !allowDomain.includes(domain)) { + ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); + } +}; + +module.exports = { + domainValidation, +}; diff --git a/lib/v2/bendibao/news.js b/lib/v2/bendibao/news.js index 1fe7a93f83..7e57bb9442 100644 --- a/lib/v2/bendibao/news.js +++ b/lib/v2/bendibao/news.js @@ -2,9 +2,13 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); const timezone = require('@/utils/timezone'); const { parseDate } = require('@/utils/parse-date'); +const { isValidHost } = require('@/utils/valid-host'); module.exports = async (ctx) => { const city = ctx.params.city; + if (!isValidHost(city)) { + throw Error('Invalid city'); + } const rootUrl = `http://${city}.bendibao.com`; diff --git a/lib/v2/biquge/index.js b/lib/v2/biquge/index.js index 8c210cc4c8..97b5534b5e 100644 --- a/lib/v2/biquge/index.js +++ b/lib/v2/biquge/index.js @@ -3,10 +3,31 @@ const cheerio = require('cheerio'); const iconv = require('iconv-lite'); const timezone = require('@/utils/timezone'); const { parseDate } = require('@/utils/parse-date'); +const config = require('@/config').value; +const allowHost = [ + 'www.xbiquwx.la', + 'www.biqu5200.net', + 'www.xbiquge.so', + 'www.biqugeu.net', + 'www.b520.cc', + 'www.ahfgb.com', + 'www.ibiquge.la', + 'www.biquge.tv', + 'www.bswtan.com', + 'www.biquge.co', + 'www.bqzhh.com', + 'www.biqugse.com', + 'www.ibiquge.info', + 'www.ishuquge.com', + 'www.mayiwxw.com', +]; module.exports = async (ctx) => { const rootUrl = ctx.path.split('/').slice(1, 4).join('/'); const currentUrl = ctx.path.slice(1); + if (!config.feature.allow_user_supply_unsafe_domain && !allowHost.includes(new URL(rootUrl).hostname)) { + ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); + } const response = await got({ method: 'get', diff --git a/lib/v2/btzj/index.js b/lib/v2/btzj/index.js index 4c83694334..9134e484b4 100644 --- a/lib/v2/btzj/index.js +++ b/lib/v2/btzj/index.js @@ -4,10 +4,15 @@ const timezone = require('@/utils/timezone'); const { parseDate } = require('@/utils/parse-date'); const { art } = require('@/utils/render'); const path = require('path'); +const config = require('@/config').value; +const allowDomain = ['btbtt15.com']; module.exports = async (ctx) => { let category = ctx.params.category ?? ''; - let domain = ctx.query.domain ?? 'btbtt20.com'; + let domain = ctx.query.domain ?? 'btbtt15.com'; + if (!config.feature.allow_user_supply_unsafe_domain && !allowDomain.includes(new URL(domain).hostname)) { + ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); + } if (category === 'base') { category = ''; diff --git a/lib/v2/cnjxol/index.js b/lib/v2/cnjxol/index.js index ec13c01204..a5351ca81e 100644 --- a/lib/v2/cnjxol/index.js +++ b/lib/v2/cnjxol/index.js @@ -12,6 +12,9 @@ const categories = { module.exports = async (ctx) => { const category = ctx.params.category ?? 'jxrb'; const id = ctx.params.id; + if (!Object.keys(categories).includes(category)) { + throw Error('Invalid category'); + } const rootUrl = `https://${category}.cnjxol.com`; const currentUrl = `${rootUrl}/${category}Paper/pc/layout`; diff --git a/lib/v2/dut/index.js b/lib/v2/dut/index.js index cdba4a91db..6efb5a4e2c 100644 --- a/lib/v2/dut/index.js +++ b/lib/v2/dut/index.js @@ -3,9 +3,13 @@ const cheerio = require('cheerio'); const { parseDate } = require('@/utils/parse-date'); const defaults = require('./defaults'); const shortcuts = require('./shortcuts'); +const { isValidHost } = require('@/utils/valid-host'); module.exports = async (ctx) => { const site = ctx.params[0] ?? 'news'; + if (!isValidHost(site)) { + throw Error('Invalid site'); + } let items; let category = ctx.params[1] ?? (defaults.hasOwnProperty(site) ? defaults[site] : ''); diff --git a/lib/v2/eagle/blog.js b/lib/v2/eagle/blog.js index 8b407f445f..7391507f07 100644 --- a/lib/v2/eagle/blog.js +++ b/lib/v2/eagle/blog.js @@ -1,16 +1,20 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); const { parseDate } = require('@/utils/parse-date'); - +const { isValidHost } = require('@/utils/valid-host'); const cateList = ['all', 'design-resources', 'learn-design', 'inside-eagle']; module.exports = async (ctx) => { let cate = ctx.params.cate ?? 'all'; let language = ctx.params.language ?? 'cn'; - if (cateList.indexOf(cate) === -1) { + if (!isValidHost(cate) || !isValidHost(language)) { + throw Error('Invalid host'); + } + if (!cateList.includes(cate)) { language = cate; cate = 'all'; } + const host = `https://${language}.eagle.cool`; const url = `${host}/blog/${cate === 'all' ? '' : cate}`; diff --git a/lib/v2/eprice/rss.js b/lib/v2/eprice/rss.js index e6988c0b4b..b12988a918 100644 --- a/lib/v2/eprice/rss.js +++ b/lib/v2/eprice/rss.js @@ -4,9 +4,13 @@ const cheerio = require('cheerio'); const { parseDate } = require('@/utils/parse-date'); const { art } = require('@/utils/render'); const path = require('path'); +const allowRegion = ['tw', 'hk']; module.exports = async (ctx) => { const region = ctx.params.region ?? 'tw'; + if (!allowRegion.includes(region)) { + throw Error('Invalid region'); + } const feed = await parser.parseURL(`https://www.eprice.com.${region}/news/rss.xml`); diff --git a/lib/v2/ff14/ff14_global.js b/lib/v2/ff14/ff14_global.js index 9d9a86d5b1..0a3dabd05c 100644 --- a/lib/v2/ff14/ff14_global.js +++ b/lib/v2/ff14/ff14_global.js @@ -2,11 +2,16 @@ const got = require('@/utils/got'); const { parseDate } = require('@/utils/parse-date'); const { art } = require('@/utils/render'); const path = require('path'); +const { isValidHost } = require('@/utils/valid-host'); module.exports = async (ctx) => { const lang = ctx.params.lang; const type = ctx.params.type ?? 'all'; + if (!isValidHost(lang)) { + throw Error('Invalid lang'); + } + const response = await got({ method: 'get', url: `https://lodestonenews.com/news/${type}?locale=${lang}`, diff --git a/lib/v2/gamme/category.js b/lib/v2/gamme/category.js index fba6fe5ddb..50e762fb03 100644 --- a/lib/v2/gamme/category.js +++ b/lib/v2/gamme/category.js @@ -1,9 +1,13 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); const parser = require('@/utils/rss-parser'); +const { isValidHost } = require('@/utils/valid-host'); module.exports = async (ctx) => { const { domain = 'news', category } = ctx.params; + if (!isValidHost(domain)) { + throw Error('Invalid domain'); + } const baseUrl = `https://${domain}.gamme.com.tw`; const feed = await parser.parseURL(`${baseUrl + (category ? `/category/${category}` : '')}/feed`); diff --git a/lib/v2/gamme/tag.js b/lib/v2/gamme/tag.js index 35e5f71673..eaf7134ba8 100644 --- a/lib/v2/gamme/tag.js +++ b/lib/v2/gamme/tag.js @@ -1,9 +1,13 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); const { parseDate } = require('@/utils/parse-date'); +const { isValidHost } = require('@/utils/valid-host'); module.exports = async (ctx) => { const { domain = 'news', tag } = ctx.params; + if (!isValidHost(domain)) { + throw Error('Invalid domain'); + } const baseUrl = `https://${domain}.gamme.com.tw`; const pageUrl = `${baseUrl}/tag/${tag}`; diff --git a/lib/v2/gumroad/index.js b/lib/v2/gumroad/index.js index b2d03c24cc..0ce86d685a 100644 --- a/lib/v2/gumroad/index.js +++ b/lib/v2/gumroad/index.js @@ -2,10 +2,14 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); const { art } = require('@/utils/render'); const path = require('path'); +const { isValidHost } = require('@/utils/valid-host'); module.exports = async (ctx) => { const username = ctx.params.username; const products = ctx.params.products; + if (!isValidHost(username)) { + throw Error('Invalid username'); + } const url = `https://${username}.gumroad.com/l/${products}`; const response = await got(url); diff --git a/lib/v2/huanqiu/index.js b/lib/v2/huanqiu/index.js index 17b8a672a8..363afdfe4c 100644 --- a/lib/v2/huanqiu/index.js +++ b/lib/v2/huanqiu/index.js @@ -1,6 +1,7 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); const { parseDate } = require('@/utils/parse-date'); +const { isValidHost } = require('@/utils/valid-host'); function getKeysRecursive(dic, key, attr, array) { Object.values(dic).forEach((v) => { @@ -15,6 +16,10 @@ function getKeysRecursive(dic, key, attr, array) { module.exports = async (ctx) => { const category = ctx.params.category ?? 'china'; + if (!isValidHost(category)) { + throw Error('Invalid category'); + } + const host = 'https://' + category + '.huanqiu.com'; const resp = await got({ diff --git a/lib/v2/itch/devlog.js b/lib/v2/itch/devlog.js index 1d9e28d0f1..974b4e6691 100644 --- a/lib/v2/itch/devlog.js +++ b/lib/v2/itch/devlog.js @@ -4,10 +4,14 @@ const timezone = require('@/utils/timezone'); const { parseDate } = require('@/utils/parse-date'); const { art } = require('@/utils/render'); const path = require('path'); +const { isValidHost } = require('@/utils/valid-host'); module.exports = async (ctx) => { const user = ctx.params.user ?? ''; const id = ctx.params.id ?? ''; + if (!isValidHost(user)) { + throw Error('Invalid user'); + } const rootUrl = `https://${user}.itch.io/${id}/devlog`; diff --git a/lib/v2/javbus/index.js b/lib/v2/javbus/index.js index 786d3e770e..b1dfe76791 100644 --- a/lib/v2/javbus/index.js +++ b/lib/v2/javbus/index.js @@ -3,17 +3,27 @@ const cheerio = require('cheerio'); const { parseDate } = require('@/utils/parse-date'); const { art } = require('@/utils/render'); const path = require('path'); +const config = require('@/config').value; const toSize = (raw) => { const matches = raw.match(/(\d+(\.\d+)?)(\w+)/); return matches[3] === 'GB' ? matches[1] * 1024 : matches[1]; }; +const allowDomain = ['javbus.com', 'javbus.org', 'javsee.icu', 'javsee.one']; + module.exports = async (ctx) => { const isWestern = /^\/western/.test(ctx.path); + const domain = ctx.query.domain ?? 'javbus.com'; + const westernDomain = ctx.query.western_domain ?? 'javbus.org'; + + const rootUrl = `https://www.${domain}`; + const westernUrl = `https://www.${westernDomain}`; + + if (!config.feature.allow_user_supply_unsafe_domain && (!allowDomain.includes(new URL(domain).hostname) || !allowDomain.includes(new URL(westernDomain).hostname))) { + ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); + } - const rootUrl = `https://www.${ctx.query.domain ?? 'javbus.com'}`; - const westernUrl = `https://www.${ctx.query.western_domain ?? 'javbus.org'}`; const currentUrl = `${isWestern ? westernUrl : rootUrl}${ctx.path.replace(/^\/western/, '').replace(/\/home/, '')}`; const response = await got({ diff --git a/lib/v2/javdb/utils.js b/lib/v2/javdb/utils.js index d849d7932e..b1e8f981d5 100644 --- a/lib/v2/javdb/utils.js +++ b/lib/v2/javdb/utils.js @@ -1,10 +1,16 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); const { parseDate } = require('@/utils/parse-date'); +const config = require('@/config').value; +const allowDomain = ['javdb.com', 'javdb36.com', 'javdb007.com']; module.exports = { ProcessItems: async (ctx, currentUrl, title) => { const domain = ctx.query.domain ?? 'javdb.com'; + if (!config.feature.allow_user_supply_unsafe_domain && !allowDomain.includes(new URL(domain).hostname)) { + ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); + } + const rootUrl = `https://${domain}`; const response = await got({ diff --git a/lib/v2/lofter/user.js b/lib/v2/lofter/user.js index 120dd95608..f3a7466f26 100644 --- a/lib/v2/lofter/user.js +++ b/lib/v2/lofter/user.js @@ -1,9 +1,13 @@ const got = require('@/utils/got'); const { parseDate } = require('@/utils/parse-date'); +const { isValidHost } = require('@/utils/valid-host'); module.exports = async (ctx) => { const name = ctx.params.name ?? 'i'; const limit = ctx.query.limit ? parseInt(ctx.query.limit) : '50'; + if (!isValidHost(name)) { + throw Error('Invalid name'); + } const rootUrl = `${name}.lofter.com`; diff --git a/lib/v2/mirror/index.js b/lib/v2/mirror/index.js index 7b5476e10f..c24d127d3a 100644 --- a/lib/v2/mirror/index.js +++ b/lib/v2/mirror/index.js @@ -4,10 +4,13 @@ const md = require('markdown-it')({ html: true, linkify: true, }); +const { isValidHost } = require('@/utils/valid-host'); module.exports = async (ctx) => { const id = ctx.params.id; - + if (!id.endsWith('.eth') && !isValidHost(id)) { + throw Error('Invalid id'); + } const rootUrl = 'https://mirror.xyz'; const currentUrl = id.endsWith('.eth') ? `${rootUrl}/${id}` : `https://${id}.mirror.xyz`; diff --git a/lib/v2/myfigurecollection/activity.js b/lib/v2/myfigurecollection/activity.js index bac5b4762e..a2e86a5532 100644 --- a/lib/v2/myfigurecollection/activity.js +++ b/lib/v2/myfigurecollection/activity.js @@ -4,6 +4,7 @@ const timezone = require('@/utils/timezone'); const { parseDate } = require('@/utils/parse-date'); const { art } = require('@/utils/render'); const path = require('path'); +const { isValidHost } = require('@/utils/valid-host'); module.exports = async (ctx) => { const category = ctx.params.category ?? '-1'; @@ -13,6 +14,10 @@ module.exports = async (ctx) => { const latestAlerts = ctx.params.latestAlerts ?? '1'; const latestPictures = ctx.params.latestPictures ?? '1'; + if (language && !isValidHost(language)) { + throw Error('Invalid language'); + } + const rootUrl = `https://${language === 'en' || language === '' ? '' : `${language}.`}myfigurecollection.net`; const currentUrl = `${rootUrl}/browse.v4.php?mode=activity&latestAdditions=${latestAdditions}&latestEdits=${latestEdits}&latestAlerts=${latestAlerts}&latestPictures=${latestPictures}&rootId=${category}`; diff --git a/lib/v2/myfigurecollection/index.js b/lib/v2/myfigurecollection/index.js index f8486599da..768bf85cb9 100644 --- a/lib/v2/myfigurecollection/index.js +++ b/lib/v2/myfigurecollection/index.js @@ -2,6 +2,7 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); const { art } = require('@/utils/render'); const path = require('path'); +const { isValidHost } = require('@/utils/valid-host'); const shortcuts = { potd: 'picture/browse/potd/', @@ -12,6 +13,9 @@ const shortcuts = { module.exports = async (ctx) => { const language = ctx.params.language ?? ''; const category = ctx.params.category ?? 'figure'; + if (language && !isValidHost(language)) { + throw Error('Invalid language'); + } const rootUrl = `https://${language === 'en' || language === '' ? '' : `${language}.`}myfigurecollection.net`; const currentUrl = `${rootUrl}/${shortcuts.hasOwnProperty(category) ? shortcuts[category] : category}`; diff --git a/lib/v2/nikkei-cn/index.js b/lib/v2/nikkei-cn/index.js index 412b9f48c0..863570ec21 100644 --- a/lib/v2/nikkei-cn/index.js +++ b/lib/v2/nikkei-cn/index.js @@ -2,6 +2,7 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); const timezone = require('@/utils/timezone'); const { parseDate } = require('@/utils/parse-date'); +const { isValidHost } = require('@/utils/valid-host'); const cleanContent = (language, content) => { switch (language) { @@ -23,6 +24,9 @@ module.exports = async (ctx) => { const language = ctx.params.language ?? 'cn'; const category = ctx.params.category ?? ''; const type = ctx.params.type ?? ''; + if (!isValidHost(language)) { + throw Error('Invalid language'); + } const rootUrl = `https://${language === 'zh' ? `zh.cn` : language}.nikkei.com`; const currentUrl = `${rootUrl}/${category ? (category === 'rss' ? 'rss.html' : `${category}${type ? `/${type}` : ''}.html`) : ''}`; diff --git a/lib/v2/people/index.js b/lib/v2/people/index.js index acaaa75815..931b3aacf9 100644 --- a/lib/v2/people/index.js +++ b/lib/v2/people/index.js @@ -3,12 +3,16 @@ const cheerio = require('cheerio'); const iconv = require('iconv-lite'); const timezone = require('@/utils/timezone'); const { parseDate } = require('@/utils/parse-date'); +const { isValidHost } = require('@/utils/valid-host'); module.exports = async (ctx) => { const site = ctx.params[0] ?? 'www'; let category = ctx.params[1] ?? (site === 'www' ? '59476' : ''); category = site === 'cpc' && category === '24h' ? '87228' : category; + if (!isValidHost(site)) { + throw Error('Invalid site'); + } const rootUrl = `http://${site}.people.com.cn`; const currentUrl = `${rootUrl}/GB/${category}/index.html`; diff --git a/lib/v2/scitation/journal.js b/lib/v2/scitation/journal.js index 24430c9373..3c437e4ec7 100644 --- a/lib/v2/scitation/journal.js +++ b/lib/v2/scitation/journal.js @@ -1,12 +1,16 @@ const cheerio = require('cheerio'); const { puppeteerGet, renderDesc } = require('./utils'); const config = require('@/config').value; +const { isValidHost } = require('@/utils/valid-host'); module.exports = async (ctx) => { const pub = ctx.params.pub; const jrn = ctx.params.jrn; const host = `https://${pub}.scitation.org`; const jrnlUrl = `${host}/toc/${jrn}/current?size=all`; + if (!isValidHost(pub)) { + throw Error('Invalid pub'); + } // use Puppeteer due to the obstacle by cloudflare challenge const browser = await require('@/utils/puppeteer')(); diff --git a/lib/v2/scitation/section.js b/lib/v2/scitation/section.js index 75779f484a..ae307f61cf 100644 --- a/lib/v2/scitation/section.js +++ b/lib/v2/scitation/section.js @@ -1,6 +1,7 @@ const cheerio = require('cheerio'); const { puppeteerGet, renderDesc } = require('./utils'); const config = require('@/config').value; +const { isValidHost } = require('@/utils/valid-host'); module.exports = async (ctx) => { const pub = ctx.params.pub; @@ -8,6 +9,9 @@ module.exports = async (ctx) => { const sec = ctx.params.sec.split('+').join(' '); const host = `https://${pub}.scitation.org`; const jrnlUrl = `${host}/toc/${jrn}/current?size=all`; + if (!isValidHost(pub)) { + throw Error('Invalid pub'); + } // use Puppeteer due to the obstacle by cloudflare challenge const browser = await require('@/utils/puppeteer')(); diff --git a/lib/v2/solidot/main.js b/lib/v2/solidot/main.js index ca02950f6e..4154bdd8aa 100644 --- a/lib/v2/solidot/main.js +++ b/lib/v2/solidot/main.js @@ -6,9 +6,14 @@ const got = require('@/utils/got'); // get web content const cheerio = require('cheerio'); // html parser const get_article = require('./_article'); +const { isValidHost } = require('@/utils/valid-host'); module.exports = async (ctx) => { const type = ctx.params.type ?? 'www'; + if (!isValidHost(type)) { + throw Error('Invalid type'); + } + const base_url = `https://${type}.solidot.org`; const response = await got({ method: 'get', diff --git a/lib/v2/zcool/user.js b/lib/v2/zcool/user.js index cfacc9d194..d7838227a7 100644 --- a/lib/v2/zcool/user.js +++ b/lib/v2/zcool/user.js @@ -2,11 +2,15 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); const { parseDate } = require('@/utils/parse-date'); const { extractArticle, extractWork } = require('./utils'); +const { isValidHost } = require('@/utils/valid-host'); module.exports = async (ctx) => { const { uid } = ctx.params; let pageUrl = `https://www.zcool.com.cn/u/${uid}`; if (isNaN(uid)) { + if (!isValidHost(uid)) { + throw Error('Invalid uid'); + } pageUrl = `https://${uid}.zcool.com.cn`; } const { data: response } = await got(pageUrl); diff --git a/lib/v2/zhubai/index.js b/lib/v2/zhubai/index.js index 4827b85c88..eb01711cdd 100644 --- a/lib/v2/zhubai/index.js +++ b/lib/v2/zhubai/index.js @@ -1,9 +1,13 @@ const got = require('@/utils/got'); const { parseDate } = require('@/utils/parse-date'); +const { isValidHost } = require('@/utils/valid-host'); module.exports = async (ctx) => { const { id } = ctx.params; const limit = ctx.query.limit ? parseInt(ctx.query.limit) : 20; + if (!isValidHost(id)) { + throw Error('Invalid id'); + } const response = await got({ method: 'get', diff --git a/test/utils/valid-host.js b/test/utils/valid-host.js new file mode 100644 index 0000000000..1a0205efb7 --- /dev/null +++ b/test/utils/valid-host.js @@ -0,0 +1,21 @@ +const { isValidHost } = require('../../lib/utils/valid-host'); + +describe('valid-host', () => { + it('validate hostname', () => { + expect(isValidHost()).toBe(false); + expect(isValidHost(123)).toBe(false); + expect(isValidHost('')).toBe(false); + expect(isValidHost('subd0main')).toBe(true); + expect(isValidHost('-subd0main')).toBe(false); + expect(isValidHost('sub-d0main')).toBe(true); + expect(isValidHost('subd0main-')).toBe(false); + expect(isValidHost('sub.d0main')).toBe(false); + expect(isValidHost('sub-.d0main')).toBe(false); + expect(isValidHost('s')).toBe(true); + expect(isValidHost('-')).toBe(false); + expect(isValidHost('0')).toBe(true); + expect(isValidHost('s-')).toBe(false); + expect(isValidHost('s-u')).toBe(true); + expect(isValidHost('su')).toBe(true); + }); +});