diff --git a/lib/index.js b/lib/index.js
index 4f56c17b0a..4746d6cdcb 100644
--- a/lib/index.js
+++ b/lib/index.js
@@ -4,6 +4,7 @@ const config = require('./config');
const Koa = require('koa');
const fs = require('fs');
const logger = require('./utils/logger');
+require('./utils/request-wrapper');
const onerror = require('./middleware/onerror');
const header = require('./middleware/header');
diff --git a/lib/utils/got.js b/lib/utils/got.js
index bb184991dc..7f3ac1e3c0 100644
--- a/lib/utils/got.js
+++ b/lib/utils/got.js
@@ -1,52 +1,8 @@
const logger = require('./logger');
const config = require('@/config');
-const SocksProxyAgent = require('socks-proxy-agent');
-const tunnel = require('tunnel');
const got = require('got');
const queryString = require('query-string');
-let agent = null;
-if (config.proxy && config.proxy.protocol && config.proxy.host && config.proxy.port) {
- agent = {};
- const proxyUrl = `${config.proxy.protocol}://${config.proxy.host}:${config.proxy.port}`;
-
- switch (config.proxy.protocol) {
- case 'socks':
- agent.http = new SocksProxyAgent(proxyUrl);
- agent.https = new SocksProxyAgent(proxyUrl);
- break;
- case 'http':
- process.env.NODE_TLS_REJECT_UNAUTHORIZED = 0;
- agent.http = tunnel.httpOverHttp({
- proxy: {
- host: config.proxy.host,
- port: parseInt(config.proxy.port),
- },
- });
- agent.https = tunnel.httpsOverHttp({
- proxy: {
- host: config.proxy.host,
- port: parseInt(config.proxy.port),
- },
- });
- break;
- case 'https':
- agent.http = tunnel.httpOverHttps({
- proxy: {
- host: config.proxy.host,
- port: parseInt(config.proxy.port),
- },
- });
- agent.https = tunnel.httpsOverHttps({
- proxy: {
- host: config.proxy.host,
- port: parseInt(config.proxy.port),
- },
- });
- break;
- }
-}
-
const custom = got.extend({
retry: {
retries: config.requestRetry,
@@ -85,20 +41,11 @@ const custom = got.extend({
options.query = options.query.replace(/([\u4e00-\u9fa5])/g, (str) => encodeURIComponent(str));
options.searchParams = options.query; // for Got v11 after
}
- if (agent && new RegExp(config.proxy.url_regex).test(options.href)) {
- options.agent = agent[options.protocol.slice(0, -1)];
-
- if (config.proxy.auth) {
- options.headers['Proxy-Authorization'] = `Basic ${config.proxy.auth}`;
- }
- logger.info(`Proxy for ${options.href}`);
- }
},
],
},
headers: {
'user-agent': config.ua,
- 'x-app': 'RSSHub',
},
});
custom.all = (list) => Promise.all(list);
diff --git a/lib/utils/request-wrapper.js b/lib/utils/request-wrapper.js
new file mode 100644
index 0000000000..01d02a8ccf
--- /dev/null
+++ b/lib/utils/request-wrapper.js
@@ -0,0 +1,99 @@
+const config = require('@/config');
+const SocksProxyAgent = require('socks-proxy-agent');
+const tunnel = require('tunnel');
+const logger = require('./logger');
+const http = require('http');
+const https = require('https');
+
+let agent = null;
+if (config.proxy && config.proxy.protocol && config.proxy.host && config.proxy.port) {
+ agent = {};
+ const proxyUrl = `${config.proxy.protocol}://${config.proxy.host}:${config.proxy.port}`;
+
+ switch (config.proxy.protocol) {
+ case 'socks':
+ agent.http = new SocksProxyAgent(proxyUrl);
+ agent.https = new SocksProxyAgent(proxyUrl);
+ break;
+ case 'http':
+ process.env.NODE_TLS_REJECT_UNAUTHORIZED = 0;
+ agent.http = tunnel.httpOverHttp({
+ proxy: {
+ host: config.proxy.host,
+ port: parseInt(config.proxy.port),
+ },
+ });
+ agent.https = tunnel.httpsOverHttp({
+ proxy: {
+ host: config.proxy.host,
+ port: parseInt(config.proxy.port),
+ },
+ });
+ break;
+ case 'https':
+ agent.http = tunnel.httpOverHttps({
+ proxy: {
+ host: config.proxy.host,
+ port: parseInt(config.proxy.port),
+ },
+ });
+ agent.https = tunnel.httpsOverHttps({
+ proxy: {
+ host: config.proxy.host,
+ port: parseInt(config.proxy.port),
+ },
+ });
+ break;
+ }
+}
+
+const requestWrapper = (url, options) => {
+ if (agent && new RegExp(config.proxy.url_regex).test(url)) {
+ let agentResult;
+ try {
+ agentResult = agent[(options.protocol || url.match(/(https?:)/)[1]).slice(0, -1)];
+ } catch (error) {
+ agentResult = null;
+ }
+ options.agent = agentResult;
+
+ if (config.proxy.auth) {
+ if (!options.headers) {
+ options.headers = {};
+ }
+ options.headers['Proxy-Authorization'] = `Basic ${config.proxy.auth}`;
+ }
+ logger.info(`Proxy for ${url}`);
+ }
+ let hasUA = false;
+ for (const header in options.headers) {
+ if (header.toLowerCase() === 'user-agent') {
+ hasUA = true;
+ }
+ }
+ if (!hasUA) {
+ if (!options.headers) {
+ options.headers = {};
+ }
+ options.headers['user-agent'] = config.ua;
+ }
+ options.headers['x-app'] = 'RSSHub';
+};
+
+const httpWrap = (func) => {
+ const origin = func;
+ return function(url, request) {
+ if (typeof url === 'object') {
+ const req = url;
+ requestWrapper(req.url || req.href || `${req.protocol}//${req.hostname}${req.path}`, req);
+ } else {
+ requestWrapper(url, request);
+ }
+ return origin.apply(this, arguments);
+ };
+};
+
+http.get = httpWrap(http.get);
+https.get = httpWrap(https.get);
+http.request = httpWrap(http.request);
+https.request = httpWrap(https.request);
diff --git a/lib/utils/rss-parser.js b/lib/utils/rss-parser.js
index 4768d56745..28c36e7faa 100644
--- a/lib/utils/rss-parser.js
+++ b/lib/utils/rss-parser.js
@@ -7,7 +7,6 @@ const parser = new Parser({
},
headers: {
'User-Agent': config.ua,
- 'X-APP': 'RSSHub',
},
});
diff --git a/test/utils/got.js b/test/utils/got.js
index 317116cefc..1e1fd3a671 100644
--- a/test/utils/got.js
+++ b/test/utils/got.js
@@ -1,22 +1,13 @@
-let got = require('../../lib/utils/got');
+const got = require('../../lib/utils/got');
const config = require('../../lib/config');
const nock = require('nock');
-afterEach(() => {
- delete process.env.PROXY_PROTOCOL;
- delete process.env.PROXY_HOST;
- delete process.env.PROXY_PORT;
- delete process.env.PROXY_AUTH;
- delete process.env.PROXY_URL_REGEX;
-});
-
describe('got', () => {
it('headers', async () => {
nock('http://rsshub.test')
.get('/test')
.reply(function() {
expect(this.req.headers['user-agent']).toBe(config.ua);
- expect(this.req.headers['x-app']).toBe('RSSHub');
return [200, ''];
});
@@ -81,127 +72,4 @@ describe('got', () => {
},
});
});
-
- it('proxy socks', async () => {
- process.env.PROXY_PROTOCOL = 'socks';
- process.env.PROXY_HOST = 'rsshub.proxy';
- process.env.PROXY_PORT = '2333';
- jest.resetModules();
- got = require('../../lib/utils/got');
- nock('http://rsshub.test')
- .get('/proxy')
- .reply(() => [200, '']);
-
- await got.get('http://rsshub.test/proxy', {
- hooks: {
- beforeRequest: [
- (options) => {
- expect(options.agent.constructor.name).toBe('SocksProxyAgent');
- expect(options.agent.options.href).toBe('socks://rsshub.proxy:2333');
- },
- ],
- },
- });
- });
-
- it('proxy http', async () => {
- process.env.PROXY_PROTOCOL = 'http';
- process.env.PROXY_HOST = 'rsshub.proxy';
- process.env.PROXY_PORT = '2333';
- jest.resetModules();
- got = require('../../lib/utils/got');
- nock('http://rsshub.test')
- .get('/proxy')
- .reply(() => [200, '']);
-
- await got.get('http://rsshub.test/proxy', {
- hooks: {
- beforeRequest: [
- (options) => {
- expect(options.agent.constructor.name).toBe('TunnelingAgent');
- expect(options.agent.options.proxy.host).toBe('rsshub.proxy');
- expect(options.agent.options.proxy.port).toBe(2333);
- },
- ],
- },
- });
- });
-
- it('proxy https', async () => {
- process.env.PROXY_PROTOCOL = 'https';
- process.env.PROXY_HOST = 'rsshub.proxy';
- process.env.PROXY_PORT = '2333';
- jest.resetModules();
- got = require('../../lib/utils/got');
- nock('http://rsshub.test')
- .get('/proxy')
- .reply(() => [200, '']);
-
- await got.get('http://rsshub.test/proxy', {
- hooks: {
- beforeRequest: [
- (options) => {
- expect(options.agent.constructor.name).toBe('TunnelingAgent');
- expect(options.agent.options.proxy.host).toBe('rsshub.proxy');
- expect(options.agent.options.proxy.port).toBe(2333);
- },
- ],
- },
- });
- });
-
- it('auth', async () => {
- process.env.PROXY_AUTH = 'testtest';
- process.env.PROXY_PROTOCOL = 'socks';
- process.env.PROXY_HOST = 'rsshub.proxy';
- process.env.PROXY_PORT = '2333';
- jest.resetModules();
- got = require('../../lib/utils/got');
- nock('http://rsshub.test')
- .get('/auth')
- .reply(function() {
- expect(this.req.headers['user-agent']).toBe(config.ua);
- expect(this.req.headers['proxy-authorization']).toBe('Basic testtest');
- return [200, ''];
- });
-
- await got.get('http://rsshub.test/auth');
- });
-
- it('url_regex', async () => {
- process.env.PROXY_URL_REGEX = 'url_regex';
- process.env.PROXY_PROTOCOL = 'socks';
- process.env.PROXY_HOST = 'rsshub.proxy';
- process.env.PROXY_PORT = '2333';
- jest.resetModules();
- got = require('../../lib/utils/got');
-
- nock('http://rsshub.test')
- .get('/url_regex')
- .reply(() => [200, '']);
- nock('http://rsshub.test')
- .get('/proxy')
- .reply(() => [200, '']);
-
- await got.get('http://rsshub.test/url_regex', {
- hooks: {
- beforeRequest: [
- (options) => {
- expect(options.agent.constructor.name).toBe('SocksProxyAgent');
- expect(options.agent.options.href).toBe('socks://rsshub.proxy:2333');
- },
- ],
- },
- });
-
- await got.get('http://rsshub.test/proxy', {
- hooks: {
- beforeRequest: [
- (options) => {
- expect(options.agent).toBe(undefined);
- },
- ],
- },
- });
- });
});
diff --git a/test/utils/request-wrapper.js b/test/utils/request-wrapper.js
new file mode 100644
index 0000000000..1577f79930
--- /dev/null
+++ b/test/utils/request-wrapper.js
@@ -0,0 +1,167 @@
+const got = require('../../lib/utils/got');
+const parser = require('../../lib/utils/rss-parser');
+const nock = require('nock');
+require('../../lib/utils/request-wrapper');
+let check = () => {};
+const simpleResponse = '