From 08878e1d5233b6ae36f87fe2d0b16611ac36b87a Mon Sep 17 00:00:00 2001 From: DIYgod Date: Sat, 3 Aug 2019 20:38:37 +0800 Subject: [PATCH 1/3] fix: set proxy for http module, close #2647 #2094 --- lib/index.js | 1 + lib/utils/agent-wrapper.js | 83 ++++++++++++++++++++++++++++++++++++++ lib/utils/got.js | 53 ------------------------ 3 files changed, 84 insertions(+), 53 deletions(-) create mode 100644 lib/utils/agent-wrapper.js diff --git a/lib/index.js b/lib/index.js index 4f56c17b0a..fe20c40973 100644 --- a/lib/index.js +++ b/lib/index.js @@ -4,6 +4,7 @@ const config = require('./config'); const Koa = require('koa'); const fs = require('fs'); const logger = require('./utils/logger'); +require('./utils/agent-wrapper'); const onerror = require('./middleware/onerror'); const header = require('./middleware/header'); diff --git a/lib/utils/agent-wrapper.js b/lib/utils/agent-wrapper.js new file mode 100644 index 0000000000..687126b8d8 --- /dev/null +++ b/lib/utils/agent-wrapper.js @@ -0,0 +1,83 @@ +const config = require('@/config'); +const SocksProxyAgent = require('socks-proxy-agent'); +const tunnel = require('tunnel'); +const logger = require('./logger'); +const http = require('http'); +const https = require('https'); + +let agent = null; +if (config.proxy && config.proxy.protocol && config.proxy.host && config.proxy.port) { + agent = {}; + const proxyUrl = `${config.proxy.protocol}://${config.proxy.host}:${config.proxy.port}`; + + switch (config.proxy.protocol) { + case 'socks': + agent.http = new SocksProxyAgent(proxyUrl); + agent.https = new SocksProxyAgent(proxyUrl); + break; + case 'http': + process.env.NODE_TLS_REJECT_UNAUTHORIZED = 0; + agent.http = tunnel.httpOverHttp({ + proxy: { + host: config.proxy.host, + port: parseInt(config.proxy.port), + }, + }); + agent.https = tunnel.httpsOverHttp({ + proxy: { + host: config.proxy.host, + port: parseInt(config.proxy.port), + }, + }); + break; + case 'https': + agent.http = tunnel.httpOverHttps({ + proxy: { + host: config.proxy.host, + port: parseInt(config.proxy.port), + }, + }); + agent.https = tunnel.httpsOverHttps({ + proxy: { + host: config.proxy.host, + port: parseInt(config.proxy.port), + }, + }); + break; + } +} + +const agentWrapper = (url, options) => { + if (agent && new RegExp(config.proxy.url_regex).test(url)) { + let agentResult; + try { + agentResult = agent[(options.protocol || url.match(/(https?:)/)[1]).slice(0, -1)]; + } catch (error) { + agentResult = null; + } + options.agent = agentResult; + + if (config.proxy.auth) { + options.headers['Proxy-Authorization'] = `Basic ${config.proxy.auth}`; + } + logger.info(`Proxy for ${url}`); + } +}; + +const httpWrap = (func) => { + const origin = func; + return function(url, request) { + if (typeof url === 'object') { + const req = url; + agentWrapper(req.url || req.href || `${req.protocol}//${req.hostname}${req.path}`, req); + } else { + agentWrapper(url, request); + } + return origin.apply(this, arguments); + }; +}; + +http.get = httpWrap(http.get); +https.get = httpWrap(https.get); +http.request = httpWrap(http.request); +https.request = httpWrap(https.request); diff --git a/lib/utils/got.js b/lib/utils/got.js index 525b75557c..39ef12c303 100644 --- a/lib/utils/got.js +++ b/lib/utils/got.js @@ -1,52 +1,8 @@ const logger = require('./logger'); const config = require('@/config'); -const SocksProxyAgent = require('socks-proxy-agent'); -const tunnel = require('tunnel'); const got = require('got'); const queryString = require('query-string'); -let agent = null; -if (config.proxy && config.proxy.protocol && config.proxy.host && config.proxy.port) { - agent = {}; - const proxyUrl = `${config.proxy.protocol}://${config.proxy.host}:${config.proxy.port}`; - - switch (config.proxy.protocol) { - case 'socks': - agent.http = new SocksProxyAgent(proxyUrl); - agent.https = new SocksProxyAgent(proxyUrl); - break; - case 'http': - process.env.NODE_TLS_REJECT_UNAUTHORIZED = 0; - agent.http = tunnel.httpOverHttp({ - proxy: { - host: config.proxy.host, - port: parseInt(config.proxy.port), - }, - }); - agent.https = tunnel.httpsOverHttp({ - proxy: { - host: config.proxy.host, - port: parseInt(config.proxy.port), - }, - }); - break; - case 'https': - agent.http = tunnel.httpOverHttps({ - proxy: { - host: config.proxy.host, - port: parseInt(config.proxy.port), - }, - }); - agent.https = tunnel.httpsOverHttps({ - proxy: { - host: config.proxy.host, - port: parseInt(config.proxy.port), - }, - }); - break; - } -} - const custom = got.extend({ retry: { retries: config.requestRetry, @@ -82,15 +38,6 @@ const custom = got.extend({ options.query = options.query || queryString.stringify(options.params); options.searchParams = options.query; // for Got v11 after } - - if (agent && new RegExp(config.proxy.url_regex).test(options.href)) { - options.agent = agent[options.protocol.slice(0, -1)]; - - if (config.proxy.auth) { - options.headers['Proxy-Authorization'] = `Basic ${config.proxy.auth}`; - } - logger.info(`Proxy for ${options.href}`); - } }, ], }, From 6f3a7836f29642366244432679c73ed4935e4fda Mon Sep 17 00:00:00 2001 From: DIYgod Date: Tue, 6 Aug 2019 13:00:22 +0800 Subject: [PATCH 2/3] feat: request wrapper --- lib/index.js | 2 +- lib/utils/got.js | 1 - .../{agent-wrapper.js => request-wrapper.js} | 16 +++++++++++++--- lib/utils/rss-parser.js | 1 - 4 files changed, 14 insertions(+), 6 deletions(-) rename lib/utils/{agent-wrapper.js => request-wrapper.js} (84%) diff --git a/lib/index.js b/lib/index.js index fe20c40973..4746d6cdcb 100644 --- a/lib/index.js +++ b/lib/index.js @@ -4,7 +4,7 @@ const config = require('./config'); const Koa = require('koa'); const fs = require('fs'); const logger = require('./utils/logger'); -require('./utils/agent-wrapper'); +require('./utils/request-wrapper'); const onerror = require('./middleware/onerror'); const header = require('./middleware/header'); diff --git a/lib/utils/got.js b/lib/utils/got.js index 39ef12c303..668f006154 100644 --- a/lib/utils/got.js +++ b/lib/utils/got.js @@ -43,7 +43,6 @@ const custom = got.extend({ }, headers: { 'user-agent': config.ua, - 'x-app': 'RSSHub', }, }); custom.all = (list) => Promise.all(list); diff --git a/lib/utils/agent-wrapper.js b/lib/utils/request-wrapper.js similarity index 84% rename from lib/utils/agent-wrapper.js rename to lib/utils/request-wrapper.js index 687126b8d8..79a7097662 100644 --- a/lib/utils/agent-wrapper.js +++ b/lib/utils/request-wrapper.js @@ -47,7 +47,7 @@ if (config.proxy && config.proxy.protocol && config.proxy.host && config.proxy.p } } -const agentWrapper = (url, options) => { +const requestWrapper = (url, options) => { if (agent && new RegExp(config.proxy.url_regex).test(url)) { let agentResult; try { @@ -62,6 +62,16 @@ const agentWrapper = (url, options) => { } logger.info(`Proxy for ${url}`); } + let hasUA = false; + for (const header in options.headers) { + if (header.toLowerCase() === 'user-agent') { + hasUA = true; + } + } + if (!hasUA) { + options.headers['user-agent'] = config.ua; + } + options.headers['x-app'] = 'RSSHub'; }; const httpWrap = (func) => { @@ -69,9 +79,9 @@ const httpWrap = (func) => { return function(url, request) { if (typeof url === 'object') { const req = url; - agentWrapper(req.url || req.href || `${req.protocol}//${req.hostname}${req.path}`, req); + requestWrapper(req.url || req.href || `${req.protocol}//${req.hostname}${req.path}`, req); } else { - agentWrapper(url, request); + requestWrapper(url, request); } return origin.apply(this, arguments); }; diff --git a/lib/utils/rss-parser.js b/lib/utils/rss-parser.js index 4768d56745..28c36e7faa 100644 --- a/lib/utils/rss-parser.js +++ b/lib/utils/rss-parser.js @@ -7,7 +7,6 @@ const parser = new Parser({ }, headers: { 'User-Agent': config.ua, - 'X-APP': 'RSSHub', }, }); From e5d34eda6b0fff0eab72004331152765998d3200 Mon Sep 17 00:00:00 2001 From: DIYgod Date: Sun, 11 Aug 2019 00:31:20 +0800 Subject: [PATCH 3/3] test: request-wrapper --- lib/utils/request-wrapper.js | 6 ++ test/utils/got.js | 134 +-------------------------- test/utils/request-wrapper.js | 167 ++++++++++++++++++++++++++++++++++ test/utils/rss-parser.js | 16 ++++ 4 files changed, 190 insertions(+), 133 deletions(-) create mode 100644 test/utils/request-wrapper.js create mode 100644 test/utils/rss-parser.js diff --git a/lib/utils/request-wrapper.js b/lib/utils/request-wrapper.js index 79a7097662..01d02a8ccf 100644 --- a/lib/utils/request-wrapper.js +++ b/lib/utils/request-wrapper.js @@ -58,6 +58,9 @@ const requestWrapper = (url, options) => { options.agent = agentResult; if (config.proxy.auth) { + if (!options.headers) { + options.headers = {}; + } options.headers['Proxy-Authorization'] = `Basic ${config.proxy.auth}`; } logger.info(`Proxy for ${url}`); @@ -69,6 +72,9 @@ const requestWrapper = (url, options) => { } } if (!hasUA) { + if (!options.headers) { + options.headers = {}; + } options.headers['user-agent'] = config.ua; } options.headers['x-app'] = 'RSSHub'; diff --git a/test/utils/got.js b/test/utils/got.js index 317116cefc..1e1fd3a671 100644 --- a/test/utils/got.js +++ b/test/utils/got.js @@ -1,22 +1,13 @@ -let got = require('../../lib/utils/got'); +const got = require('../../lib/utils/got'); const config = require('../../lib/config'); const nock = require('nock'); -afterEach(() => { - delete process.env.PROXY_PROTOCOL; - delete process.env.PROXY_HOST; - delete process.env.PROXY_PORT; - delete process.env.PROXY_AUTH; - delete process.env.PROXY_URL_REGEX; -}); - describe('got', () => { it('headers', async () => { nock('http://rsshub.test') .get('/test') .reply(function() { expect(this.req.headers['user-agent']).toBe(config.ua); - expect(this.req.headers['x-app']).toBe('RSSHub'); return [200, '']; }); @@ -81,127 +72,4 @@ describe('got', () => { }, }); }); - - it('proxy socks', async () => { - process.env.PROXY_PROTOCOL = 'socks'; - process.env.PROXY_HOST = 'rsshub.proxy'; - process.env.PROXY_PORT = '2333'; - jest.resetModules(); - got = require('../../lib/utils/got'); - nock('http://rsshub.test') - .get('/proxy') - .reply(() => [200, '']); - - await got.get('http://rsshub.test/proxy', { - hooks: { - beforeRequest: [ - (options) => { - expect(options.agent.constructor.name).toBe('SocksProxyAgent'); - expect(options.agent.options.href).toBe('socks://rsshub.proxy:2333'); - }, - ], - }, - }); - }); - - it('proxy http', async () => { - process.env.PROXY_PROTOCOL = 'http'; - process.env.PROXY_HOST = 'rsshub.proxy'; - process.env.PROXY_PORT = '2333'; - jest.resetModules(); - got = require('../../lib/utils/got'); - nock('http://rsshub.test') - .get('/proxy') - .reply(() => [200, '']); - - await got.get('http://rsshub.test/proxy', { - hooks: { - beforeRequest: [ - (options) => { - expect(options.agent.constructor.name).toBe('TunnelingAgent'); - expect(options.agent.options.proxy.host).toBe('rsshub.proxy'); - expect(options.agent.options.proxy.port).toBe(2333); - }, - ], - }, - }); - }); - - it('proxy https', async () => { - process.env.PROXY_PROTOCOL = 'https'; - process.env.PROXY_HOST = 'rsshub.proxy'; - process.env.PROXY_PORT = '2333'; - jest.resetModules(); - got = require('../../lib/utils/got'); - nock('http://rsshub.test') - .get('/proxy') - .reply(() => [200, '']); - - await got.get('http://rsshub.test/proxy', { - hooks: { - beforeRequest: [ - (options) => { - expect(options.agent.constructor.name).toBe('TunnelingAgent'); - expect(options.agent.options.proxy.host).toBe('rsshub.proxy'); - expect(options.agent.options.proxy.port).toBe(2333); - }, - ], - }, - }); - }); - - it('auth', async () => { - process.env.PROXY_AUTH = 'testtest'; - process.env.PROXY_PROTOCOL = 'socks'; - process.env.PROXY_HOST = 'rsshub.proxy'; - process.env.PROXY_PORT = '2333'; - jest.resetModules(); - got = require('../../lib/utils/got'); - nock('http://rsshub.test') - .get('/auth') - .reply(function() { - expect(this.req.headers['user-agent']).toBe(config.ua); - expect(this.req.headers['proxy-authorization']).toBe('Basic testtest'); - return [200, '']; - }); - - await got.get('http://rsshub.test/auth'); - }); - - it('url_regex', async () => { - process.env.PROXY_URL_REGEX = 'url_regex'; - process.env.PROXY_PROTOCOL = 'socks'; - process.env.PROXY_HOST = 'rsshub.proxy'; - process.env.PROXY_PORT = '2333'; - jest.resetModules(); - got = require('../../lib/utils/got'); - - nock('http://rsshub.test') - .get('/url_regex') - .reply(() => [200, '']); - nock('http://rsshub.test') - .get('/proxy') - .reply(() => [200, '']); - - await got.get('http://rsshub.test/url_regex', { - hooks: { - beforeRequest: [ - (options) => { - expect(options.agent.constructor.name).toBe('SocksProxyAgent'); - expect(options.agent.options.href).toBe('socks://rsshub.proxy:2333'); - }, - ], - }, - }); - - await got.get('http://rsshub.test/proxy', { - hooks: { - beforeRequest: [ - (options) => { - expect(options.agent).toBe(undefined); - }, - ], - }, - }); - }); }); diff --git a/test/utils/request-wrapper.js b/test/utils/request-wrapper.js new file mode 100644 index 0000000000..1577f79930 --- /dev/null +++ b/test/utils/request-wrapper.js @@ -0,0 +1,167 @@ +const got = require('../../lib/utils/got'); +const parser = require('../../lib/utils/rss-parser'); +const nock = require('nock'); +require('../../lib/utils/request-wrapper'); +let check = () => {}; +const simpleResponse = ''; + +afterEach(() => { + delete process.env.PROXY_PROTOCOL; + delete process.env.PROXY_HOST; + delete process.env.PROXY_PORT; + delete process.env.PROXY_AUTH; + delete process.env.PROXY_URL_REGEX; + + nock.restore(); + nock.activate(); + check = () => {}; + + const http = require('http'); + const httpWrap = (func) => { + const origin = func; + return function(url, request) { + if (typeof url === 'object') { + check(url); + } else { + check(request); + } + return origin.apply(this, arguments); + }; + }; + http.get = httpWrap(http.get); + http.request = httpWrap(http.request); +}); + +describe('got', () => { + it('headers', async () => { + nock('http://rsshub.test') + .get('/test') + .times(2) + .reply(function() { + expect(this.req.headers['x-app']).toBe('RSSHub'); + return [200, simpleResponse]; + }); + + await got.get('http://rsshub.test/test'); + await parser.parseURL('http://rsshub.test/test'); + }); + + it('proxy socks', async () => { + process.env.PROXY_PROTOCOL = 'socks'; + process.env.PROXY_HOST = 'rsshub.proxy'; + process.env.PROXY_PORT = '2333'; + + jest.resetModules(); + require('../../lib/utils/request-wrapper'); + check = (request) => { + expect(request.agent.constructor.name).toBe('SocksProxyAgent'); + expect(request.agent.options.href).toBe('socks://rsshub.proxy:2333'); + }; + + nock('http://rsshub.test') + .get('/proxy') + .times(2) + .reply(200, simpleResponse); + + await got.get('http://rsshub.test/proxy'); + await parser.parseURL('http://rsshub.test/proxy'); + }); + + it('proxy http', async () => { + process.env.PROXY_PROTOCOL = 'http'; + process.env.PROXY_HOST = 'rsshub.proxy'; + process.env.PROXY_PORT = '2333'; + + jest.resetModules(); + require('../../lib/utils/request-wrapper'); + check = (request) => { + expect(request.agent.constructor.name).toBe('TunnelingAgent'); + expect(request.agent.options.proxy.host).toBe('rsshub.proxy'); + expect(request.agent.options.proxy.port).toBe(2333); + }; + + nock('http://rsshub.test') + .get('/proxy') + .times(2) + .reply(200, simpleResponse); + + await got.get('http://rsshub.test/proxy'); + await parser.parseURL('http://rsshub.test/proxy'); + }); + + it('proxy https', async () => { + process.env.PROXY_PROTOCOL = 'https'; + process.env.PROXY_HOST = 'rsshub.proxy'; + process.env.PROXY_PORT = '2333'; + + jest.resetModules(); + require('../../lib/utils/request-wrapper'); + check = (request) => { + expect(request.agent.constructor.name).toBe('TunnelingAgent'); + expect(request.agent.options.proxy.host).toBe('rsshub.proxy'); + expect(request.agent.options.proxy.port).toBe(2333); + }; + + nock('http://rsshub.test') + .get('/proxy') + .times(2) + .reply(200, simpleResponse); + + await got.get('http://rsshub.test/proxy'); + await parser.parseURL('http://rsshub.test/proxy'); + }); + + it('auth', async () => { + process.env.PROXY_AUTH = 'testtest'; + process.env.PROXY_PROTOCOL = 'socks'; + process.env.PROXY_HOST = 'rsshub.proxy'; + process.env.PROXY_PORT = '2333'; + + jest.resetModules(); + require('../../lib/utils/request-wrapper'); + + nock('http://rsshub.test') + .get('/auth') + .times(2) + .reply(function() { + expect(this.req.headers['proxy-authorization']).toBe('Basic testtest'); + return [200, simpleResponse]; + }); + + await got.get('http://rsshub.test/auth'); + await parser.parseURL('http://rsshub.test/auth'); + }); + + it('url_regex', async () => { + process.env.PROXY_URL_REGEX = 'url_regex'; + process.env.PROXY_PROTOCOL = 'socks'; + process.env.PROXY_HOST = 'rsshub.proxy'; + process.env.PROXY_PORT = '2333'; + + jest.resetModules(); + require('../../lib/utils/request-wrapper'); + check = (request) => { + if (request.path === '/url_regex') { + expect(request.agent.constructor.name).toBe('SocksProxyAgent'); + expect(request.agent.options.href).toBe('socks://rsshub.proxy:2333'); + } else if (request.path === '/proxy') { + expect(request.agent).toBe(undefined); + } + }; + + nock('http://rsshub.test') + .get('/url_regex') + .times(2) + .reply(() => [200, simpleResponse]); + nock('http://rsshub.test') + .get('/proxy') + .times(2) + .reply(() => [200, simpleResponse]); + + await got.get('http://rsshub.test/url_regex'); + await parser.parseURL('http://rsshub.test/url_regex'); + + await got.get('http://rsshub.test/proxy'); + await parser.parseURL('http://rsshub.test/proxy'); + }); +}); diff --git a/test/utils/rss-parser.js b/test/utils/rss-parser.js new file mode 100644 index 0000000000..b72763c5d1 --- /dev/null +++ b/test/utils/rss-parser.js @@ -0,0 +1,16 @@ +const parser = require('../../lib/utils/rss-parser'); +const config = require('../../lib/config'); +const nock = require('nock'); + +describe('got', () => { + it('headers', async () => { + nock('http://rsshub.test') + .get('/test') + .reply(function() { + expect(this.req.headers['user-agent']).toBe(config.ua); + return [200, '']; + }); + + await parser.parseURL('http://rsshub.test/test'); + }); +});