diff --git a/lib/config.js b/lib/config.js index 0ecdd68a21..bdba654855 100644 --- a/lib/config.js +++ b/lib/config.js @@ -1,4 +1,5 @@ require('dotenv').config(); +const randUserAgent = require('@/utils/rand-user-agent'); let envs = process.env; let value; @@ -39,7 +40,7 @@ const calculateValue = () => { listenInaddrAny: envs.LISTEN_INADDR_ANY || 1, // 是否允许公网连接,取值 0 1 requestRetry: parseInt(envs.REQUEST_RETRY) || 2, // 请求失败重试次数 requestTimeout: parseInt(envs.REQUEST_TIMEOUT) || 30000, // Milliseconds to wait for the server to end the response before aborting the request - ua: envs.UA || 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36', + ua: envs.UA || randUserAgent({ browser: 'chrome', os: 'mac os', device: 'desktop' }), // cors request allowOrigin: envs.ALLOW_ORIGIN, // cache diff --git a/lib/utils/rand-user-agent.js b/lib/utils/rand-user-agent.js new file mode 100644 index 0000000000..c1b915256d --- /dev/null +++ b/lib/utils/rand-user-agent.js @@ -0,0 +1,29 @@ +const randUserAgent = require('rand-user-agent'); + +/** + * A handy function to help generate a legit useragent. + * + * @param {Object} randUserAgent + * @param {string} randUserAgent.browser Name of a browser, case-insensitive. `chrome`, `edge`, `firefox`, `mobile safari`(ios only) or `safari`. + * @param {string} randUserAgent.os Name of an OS, case-insensitive. `android`, `ios`, `mac os`, `linux` or `windows`. + * @param {string} randUserAgent.device Name of a device, case-insensitive. `desktop`, `mobile` or `tablet`. + * @returns A random useragent for the given specifications. + */ +module.exports = ({ browser = 'chrome', os = 'mac os', device = 'desktop' }) => { + device = device.toLowerCase(); + browser = browser.toLowerCase(); + os = os.toLowerCase(); + let UA = randUserAgent(device, browser, os); + + if (browser === 'chrome') { + while (UA.includes('Chrome-Lighthouse') || UA.includes('Gener8') || UA.includes('HeadlessChrome') || UA.includes('SMTBot')) { + UA = randUserAgent(device, browser, os); + } + } + if (browser === 'safari') { + while (UA.includes('Applebot')) { + UA = randUserAgent(device, browser, os); + } + } + return UA; +}; diff --git a/package.json b/package.json index 87fd5b5181..a9b0ea7e87 100644 --- a/package.json +++ b/package.json @@ -119,6 +119,7 @@ "plist": "3.0.5", "puppeteer": "13.5.2", "query-string": "7.1.1", + "rand-user-agent": "1.0.58", "require-all": "3.0.0", "rss-parser": "3.12.0", "showdown": "2.0.3", diff --git a/test/utils/rand-user-agent.js b/test/utils/rand-user-agent.js new file mode 100644 index 0000000000..8efbc71c3d --- /dev/null +++ b/test/utils/rand-user-agent.js @@ -0,0 +1,40 @@ +const got = require('@/utils/got'); +const config = require('@/config').value; +const nock = require('nock'); +const randUserAgent = require('@/utils/rand-user-agent'); +const mobileUa = require('@/utils/rand-user-agent')({ browser: 'mobile safari', os: 'ios', device: 'mobile' }); + +describe('rand-user-agent', () => { + it('chrome should not include headlesschrome', () => { + const uaArr = Array(100) + .fill() + .map(() => randUserAgent({ browser: 'chrome', os: 'windows' })); + const match = uaArr.find((e) => (e.includes('Chrome-Lighthouse') || e.includes('HeadlessChrome') ? true : false)); + expect(match).toBeFalsy(); + }); + + it('should has default random ua', async () => { + nock('https://rsshub.test') + .get('/test') + .reply(function () { + expect(this.req.headers['user-agent']).toBe(config.ua); + expect(this.req.headers['user-agent']).not.toBe(mobileUa); + expect(this.req.headers['user-agent']).not.toBe('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36'); + return [200, '']; + }); + await got('https://rsshub.test/test'); + }); + + it('should match ua configurated', async () => { + const response1 = await got('https://www.whatsmyua.info/api/v1/ua'); + expect(response1.data[0].ua.rawUa).toBe(config.ua); + + const response2 = await got('https://www.whatsmyua.info/api/v1/ua', { + headers: { + 'user-agent': mobileUa, + }, + }); + expect(response2.data[0].ua.rawUa).toBe(mobileUa); + expect(response2.data[0].ua.rawUa).not.toBe(config.ua); + }); +}); diff --git a/yarn.lock b/yarn.lock index 8427190341..d53873390c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3566,9 +3566,9 @@ caniuse-api@^3.0.0: lodash.uniq "^4.5.0" caniuse-lite@^1.0.0, caniuse-lite@^1.0.30001109, caniuse-lite@^1.0.30001317: - version "1.0.30001327" - resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001327.tgz#c1546d7d7bb66506f0ccdad6a7d07fc6d668c858" - integrity sha512-1/Cg4jlD9qjZzhbzkzEaAC2JHsP0WrOc8Rd/3a3LuajGzGWR/hD7TVyvq99VqmTy99eVh8Zkmdq213OgvgXx7w== + version "1.0.30001328" + resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001328.tgz#0ed7a2ca65ec45872c613630201644237ba1e329" + integrity sha512-Ue55jHkR/s4r00FLNiX+hGMMuwml/QGqqzVeMQ5thUewznU2EdULFvI3JR7JJid6OrjJNfFvHY2G2dIjmRaDDQ== caseless@~0.12.0: version "0.12.0" @@ -5057,9 +5057,9 @@ ee-first@1.1.1: integrity sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0= electron-to-chromium@^1.4.84: - version "1.4.106" - resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.106.tgz#e7a3bfa9d745dd9b9e597616cb17283cc349781a" - integrity sha512-ZYfpVLULm67K7CaaGP7DmjyeMY4naxsbTy+syVVxT6QHI1Ww8XbJjmr9fDckrhq44WzCrcC5kH3zGpdusxwwqg== + version "1.4.107" + resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.107.tgz#564257014ab14033b4403a309c813123c58a3fb9" + integrity sha512-Huen6taaVrUrSy8o7mGStByba8PfOWWluHNxSHGBrCgEdFVLtvdQDBr9LBCF9Uci8SYxh28QNNMO0oC17wbGAg== ellipsize@0.1.0: version "0.1.0" @@ -5255,9 +5255,9 @@ error-inject@^1.0.0: integrity sha1-4rPZG1Su1nLzCdlQ0VSFD6EdTzc= es-abstract@^1.17.2, es-abstract@^1.19.1: - version "1.19.2" - resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.19.2.tgz#8f7b696d8f15b167ae3640b4060670f3d054143f" - integrity sha512-gfSBJoZdlL2xRiOCy0g8gLMryhoe1TlimjzU99L/31Z8QEGIhVQI+EWwt5lT+AuU9SnorVupXFqqOGqGfsyO6w== + version "1.19.3" + resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.19.3.tgz#4dd9da55868192756c83c5c30c7878d04e77125d" + integrity sha512-4axXLNovnMYf0+csS5rVnS5hLmV1ek+ecx9MuCjByL1E5Nn54avf6CHQxIjgQIHBnfX9AMxTRIy0q+Yu5J/fXA== dependencies: call-bind "^1.0.2" es-to-primitive "^1.2.1" @@ -5270,7 +5270,7 @@ es-abstract@^1.17.2, es-abstract@^1.19.1: is-callable "^1.2.4" is-negative-zero "^2.0.2" is-regex "^1.1.4" - is-shared-array-buffer "^1.0.1" + is-shared-array-buffer "^1.0.2" is-string "^1.0.7" is-weakref "^1.0.2" object-inspect "^1.12.0" @@ -5916,9 +5916,9 @@ fd-slicer@~1.1.0: pend "~1.2.0" fecha@^4.2.0: - version "4.2.1" - resolved "https://registry.yarnpkg.com/fecha/-/fecha-4.2.1.tgz#0a83ad8f86ef62a091e22bb5a039cd03d23eecce" - integrity sha512-MMMQ0ludy/nBs1/o0zVOiKTpG7qMbonKUzjJgQFEuvq6INZ1OraKPRAWkBq5vlKLOUMpmNYG1JoN3oDPUQ9m3Q== + version "4.2.2" + resolved "https://registry.yarnpkg.com/fecha/-/fecha-4.2.2.tgz#902c69d332b93c69be48992b9a11e41d36c03581" + integrity sha512-5rOQWkBVz3FnYWTi/ELZmq4CoK1Pb+xKNZWuJRsOwo0+8DrP43CrWJtyLVvb5U7z7ggE5llahfDbLjaVNzXVJQ== figgy-pudding@^3.5.1: version "3.5.2" @@ -7771,7 +7771,7 @@ is-resolvable@^1.0.0: resolved "https://registry.yarnpkg.com/is-resolvable/-/is-resolvable-1.1.0.tgz#fb18f87ce1feb925169c9a407c19318a3206ed88" integrity sha512-qgDYXFSR5WvEfuS5dMj6oTMEbrrSaM0CrFk2Yiq/gXnBvD9pMa2jGXxyhGLfvhZpuMZe18CJpFxAt3CRs42NMg== -is-shared-array-buffer@^1.0.1: +is-shared-array-buffer@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/is-shared-array-buffer/-/is-shared-array-buffer-1.0.2.tgz#8f259c573b60b6a32d4058a1a07430c0a7344c79" integrity sha512-sqN2UDu1/0y6uvXyStCOzyhAjCSlHceFoMKJW8W9EU9cvic/QdsZ0kEU93HEy3IUEFZIiH/3w+AH/UQbPHNdhA== @@ -11353,6 +11353,11 @@ ramda@^0.26.1: resolved "https://registry.yarnpkg.com/ramda/-/ramda-0.26.1.tgz#8d41351eb8111c55353617fc3bbffad8e4d35d06" integrity sha512-hLWjpy7EnsDBb0p+Z3B7rPi3GDeRG5ZtiI33kJhTt+ORCd38AbAIjB/9zRIUoeTbE/AVX5ZkU7m6bznsvrf8eQ== +rand-user-agent@1.0.58: + version "1.0.58" + resolved "https://registry.yarnpkg.com/rand-user-agent/-/rand-user-agent-1.0.58.tgz#b98eab16708f168a42b144070bc9820b0e69d877" + integrity sha512-i+E4FZXDBuj/SgB6YeOJm1ZKwtceoxOm96yI8el13zyecQhHNrf/9ThxjcyH6GqczGVguR+VjovAPDLPQnJTXA== + randexp@0.4.6: version "0.4.6" resolved "https://registry.yarnpkg.com/randexp/-/randexp-0.4.6.tgz#e986ad5e5e31dae13ddd6f7b3019aa7c87f60ca3"