feat(core): unify proxy; optimize request-wrapper (#11472)

* feat(core): unify proxy; optimize request-wrapper

Signed-off-by: Rongrong <i@rong.moe>

* Fix DeepScan issue

Signed-off-by: Rongrong <i@rong.moe>

* Simplify protocol existence judgment

Signed-off-by: Rongrong <i@rong.moe>

* chore: regenerate yarn.lock

Signed-off-by: Rongrong <i@rong.moe>

lib/utils/puppeteer.js

@@ -1,4 +1,5 @@
 const config = require('@/config').value;
+const { proxyUri, proxyUrlHandler } = require('./unify-proxy');
 let puppeteer = require('puppeteer');
 const proxyChain = require('proxy-chain');
 const logger = require('./logger');
@@ -9,13 +10,6 @@ const options = {
     ignoreHTTPSErrors: true,
 };
 
-let proxyUri;
-if (config.proxyUri && typeof config.proxyUri === 'string') {
-    proxyUri = config.proxyUri;
-} else if (config.proxy && config.proxy.protocol && config.proxy.host && config.proxy.port) {
-    proxyUri = `${config.proxy.protocol}://${config.proxy.host}:${config.proxy.port}`;
-}
-
 /**
  * @param {Object} extraOptions
  * @param {boolean} extraOptions.stealth - Use puppeteer-extra-plugin-stealth
@@ -51,9 +45,9 @@ module.exports = async (extraOptions = {}) => {
     }
     let browser;
     if (proxyUri) {
-        if (proxyUri.includes('@')) {
+        if (proxyUrlHandler.username || proxyUrlHandler.password) {
             // only proxies with authentication need to be anonymized
-            if (proxyUri.startsWith('http:')) {
+            if (proxyUrlHandler.protocol === 'http:') {
                 options.args.push(`--proxy-server=${await proxyChain.anonymizeProxy(proxyUri)}`);
             } else {
                 logger.warn('SOCKS/HTTPS proxy with authentication is not supported by puppeteer, continue without proxy');

lib/utils/request-wrapper.js

@@ -1,127 +1,65 @@
 const config = require('@/config').value;
+const { proxyUri, proxyObj, proxyUrlHandler } = require('./unify-proxy');
 const logger = require('./logger');
 const http = require('http');
 const https = require('https');
 
-let tunnel;
-let HttpsProxyAgent;
-let SocksProxyAgent;
-
 let agent = null;
-if (config.proxyUri && typeof config.proxyUri === 'string') {
+if (proxyUri) {
-    let proxy = null;
+    if (proxyUri.startsWith('http')) {
-    if (config.proxyUri.startsWith('http')) {
+        const HttpsProxyAgent = require('https-proxy-agent');
-        HttpsProxyAgent = HttpsProxyAgent || require('https-proxy-agent');
+        agent = new HttpsProxyAgent(proxyUri);
-        proxy = new HttpsProxyAgent(config.proxyUri);
+    } else if (proxyUri.startsWith('socks')) {
-    } else if (config.proxyUri.startsWith('socks')) {
+        const SocksProxyAgent = require('socks-proxy-agent').SocksProxyAgent;
-        SocksProxyAgent = SocksProxyAgent || require('socks-proxy-agent').SocksProxyAgent;
+        agent = new SocksProxyAgent(proxyUri);
-        proxy = new SocksProxyAgent(config.proxyUri);
+    }
-    } else {
-        throw 'Unknown proxy-uri format';
 }
 
-    agent = {
+let proxyWrapper = () => false;
-        http: proxy,
+if (agent) {
-        https: proxy,
+    const proxyRegex = new RegExp(proxyObj.url_regex);
+    const protocolMatch = (protocolLike) => protocolLike && protocolLike.toLowerCase().startsWith('http');
+
+    proxyWrapper = (url, options, urlHandler) => {
+        if (proxyRegex.test(url)) {
+            if ((protocolMatch(options.protocol) || protocolMatch(url)) && (!urlHandler || urlHandler.host !== proxyUrlHandler.host)) {
+                options.agent = agent;
+                if (proxyObj.auth) {
+                    options.headers['Proxy-Authorization'] = `Basic ${proxyObj.auth}`;
+                }
+                return true;
+            }
+        }
+        return false;
     };
-} else if (config.proxy && config.proxy.protocol && config.proxy.host && config.proxy.port) {
-    agent = {};
-    const proxyUrl = `${config.proxy.protocol}://${config.proxy.host}:${config.proxy.port}`;
-
-    switch (config.proxy.protocol) {
-        case 'socks':
-            SocksProxyAgent = SocksProxyAgent || require('socks-proxy-agent').SocksProxyAgent;
-            agent.http = new SocksProxyAgent(proxyUrl);
-            agent.https = new SocksProxyAgent(proxyUrl);
-            break;
-        case 'http':
-            tunnel = tunnel || require('tunnel');
-            process.env.NODE_TLS_REJECT_UNAUTHORIZED = 0;
-            agent.http = tunnel.httpOverHttp({
-                proxy: {
-                    host: config.proxy.host,
-                    port: parseInt(config.proxy.port),
-                },
-            });
-            agent.https = tunnel.httpsOverHttp({
-                proxy: {
-                    host: config.proxy.host,
-                    port: parseInt(config.proxy.port),
-                },
-            });
-            break;
-        case 'https':
-            tunnel = tunnel || require('tunnel');
-            agent.http = tunnel.httpOverHttps({
-                proxy: {
-                    host: config.proxy.host,
-                    port: parseInt(config.proxy.port),
-                },
-            });
-            agent.https = tunnel.httpsOverHttps({
-                proxy: {
-                    host: config.proxy.host,
-                    port: parseInt(config.proxy.port),
-                },
-            });
-            break;
-    }
 }
 
 const requestWrapper = (url, options) => {
-    // agent
+    let urlHandler;
-    if (agent && new RegExp(config.proxy.url_regex).test(url)) {
-        let agentResult;
     try {
-            agentResult = agent[(options.protocol || url.match(/(^https?:)/)[1]).slice(0, -1)];
+        urlHandler = new URL(url);
     } catch (error) {
-            agentResult = null;
+        // ignore
-        }
-        try {
-            if (new URL(url).host !== new URL(config.proxyUri).host) {
-                options.agent = agentResult;
-            }
-        } catch (error) {
-            options.agent = agentResult;
     }
+    options.headers = options.headers || {};
+    const headersLowerCaseKeys = Object.keys(options.headers).map((key) => key.toLowerCase());
 
-        if (config.proxy.auth) {
+    proxyWrapper(url, options, urlHandler) ? logger.info(`Proxy for ${url}`) : logger.debug(`Requesting ${url}`);
-            if (!options.headers) {
-                options.headers = {};
-            }
-            options.headers['Proxy-Authorization'] = `Basic ${config.proxy.auth}`;
-        }
-        logger.info(`Proxy for ${url}`);
-    } else {
-        logger.debug(`Requesting ${url}`);
-    }
 
     // ua
-    let hasUA = false;
+    if (!headersLowerCaseKeys.includes('user-agent')) {
-    for (const header in options.headers) {
-        if (header.toLowerCase() === 'user-agent') {
-            hasUA = true;
-        }
-    }
-    if (!hasUA) {
-        if (!options.headers) {
-            options.headers = {};
-        }
         options.headers['user-agent'] = config.ua;
     }
 
-    try {
+    if (urlHandler) {
-        const urlHandler = new URL(url);
         // referer
-        if (!options.headers.referer && !options.headers.Referer) {
+        if (!headersLowerCaseKeys.includes('referer')) {
             options.headers.referer = urlHandler.origin;
         }
         // host
-        if (!options.headers.host && !options.headers.Host) {
+        if (!headersLowerCaseKeys.includes('host')) {
             options.headers.host = urlHandler.host;
         }
-    } catch (e) {
-        // do nothing
     }
 };
 

lib/utils/unify-proxy.js

@@ -0,0 +1,116 @@
+const config = require('@/config').value;
+const logger = require('./logger');
+
+const defaultProtocol = 'http';
+const possibleProtocol = ['http', 'https', 'socks', 'socks4', 'socks4a', 'socks5', 'socks5h'];
+
+const unifyProxy = (proxyUri, proxyObj) => {
+    proxyObj = proxyObj || {};
+    const [oriProxyUri, oriProxyObj] = [proxyUri, proxyObj];
+    proxyObj = { ...proxyObj };
+
+    let proxyUrlHandler;
+
+    // PROXY_URI
+    if (proxyUri && typeof proxyUri === 'string') {
+        if (!proxyUri.includes('://')) {
+            logger.warn(`PROXY_URI contains no protocol, assuming ${defaultProtocol}`);
+            proxyUri = `${defaultProtocol}://${proxyUri}`;
+        }
+        try {
+            proxyUrlHandler = new URL(proxyUri);
+        } catch (e) {
+            logger.error(`Parse PROXY_URI error: ${e.stack}`);
+        }
+    }
+
+    // PROXY_{PROTOCOL,HOST,PORT}
+    if (proxyObj.protocol || proxyObj.host || proxyObj.port) {
+        if (proxyUrlHandler) {
+            logger.warn('PROXY_URI is set, ignoring PROXY_{PROTOCOL,HOST,PORT}');
+        } else if (proxyObj.host) {
+            let tempProxyStr = proxyObj.host;
+
+            if (tempProxyStr.includes('://')) {
+                logger.warn('PROXY_HOST contains protocol, ignoring PROXY_PROTOCOL');
+            } else if (!proxyObj.protocol) {
+                logger.warn(`PROXY_PROTOCOL is not set, assuming '${defaultProtocol}'`);
+                tempProxyStr = `${defaultProtocol}://${tempProxyStr}`;
+            } else {
+                tempProxyStr = `${proxyObj.protocol}://${tempProxyStr}`;
+            }
+            try {
+                proxyUrlHandler = new URL(tempProxyStr);
+                if (proxyUrlHandler.port && proxyObj.port) {
+                    logger.warn('PROXY_HOST contains port, ignoring PROXY_PORT');
+                } else if (proxyObj.port) {
+                    if (!parseInt(proxyObj.port)) {
+                        logger.warn(`PROXY_PORT is not a number, ignoring`);
+                    } else {
+                        proxyUrlHandler.port = proxyObj.port;
+                    }
+                } else {
+                    logger.warn('PROXY_PORT is not set, leaving proxy agent to determine');
+                }
+            } catch (e) {
+                logger.error(`Parse PROXY_HOST error: ${e.stack}`);
+            }
+        } else {
+            logger.warn('Either PROXY_{PROTOCOL,PORT} is set, but PROXY_HOST is missing, ignoring');
+        }
+    }
+
+    // PROXY_AUTH
+    if (proxyObj.auth && proxyUrlHandler) {
+        let promptProxyUri = false;
+        if (proxyUrlHandler.username || proxyUrlHandler.password) {
+            logger.warn('PROXY_URI contains username and/or password, ignoring PROXY_AUTH');
+            proxyObj.auth = undefined;
+        } else if (!['http:', 'https:'].includes(proxyUrlHandler.protocol)) {
+            logger.warn(`PROXY_AUTH is only supported by HTTP(S) proxies, but got ${proxyUrlHandler.protocol}, ignoring`);
+            proxyObj.auth = undefined;
+            promptProxyUri = true;
+        } else {
+            logger.info('PROXY_AUTH is set and will be used for requests from Node.js. However, requests from puppeteer will not use it');
+            promptProxyUri = true;
+        }
+        if (promptProxyUri) {
+            logger.info('To get rid of this, set PROXY_URI like protocol://username:password@host:port and clear PROXY_{AUTH,PROTOCOL,HOST,PORT}');
+        }
+    }
+
+    // is proxy enabled and valid?
+    let isProxyValid = false;
+    if (proxyUrlHandler) {
+        const protocol = proxyUrlHandler.protocol.replace(':', '');
+        if (possibleProtocol.includes(protocol)) {
+            if (protocol !== 'http' && (proxyUrlHandler.username || proxyUrlHandler.password)) {
+                logger.warn("PROXY_URI is an HTTPS/SOCKS proxy with authentication, which is not supported by puppeteer (ignore if you don't need it)");
+                logger.info('To get rid of this, consider using an HTTP proxy instead');
+            }
+            proxyObj.protocol = protocol;
+            proxyObj.host = proxyUrlHandler.hostname;
+            proxyObj.port = parseInt(proxyUrlHandler.port) || undefined;
+            // trailing slash will cause puppeteer to throw net::ERR_NO_SUPPORTED_PROXIES, trim it
+            proxyUri = proxyUrlHandler.href.endsWith('/') ? proxyUrlHandler.href.slice(0, -1) : proxyUrlHandler.href;
+            isProxyValid = true;
+        } else {
+            logger.error(`Unsupported proxy protocol: ${protocol}, expect one of ${possibleProtocol.join(', ')}`);
+        }
+    }
+    if (!isProxyValid) {
+        if ((oriProxyUri && typeof oriProxyUri === 'string') || oriProxyObj.protocol || oriProxyObj.host || oriProxyObj.port || oriProxyObj.auth) {
+            logger.error('Proxy is disabled due to misconfiguration');
+        }
+        proxyObj.protocol = proxyObj.host = proxyObj.port = proxyObj.auth = undefined;
+        proxyUri = null;
+        proxyUrlHandler = null;
+    }
+
+    return { proxyUri, proxyObj, proxyUrlHandler };
+};
+
+module.exports = {
+    unifyProxy,
+    ...unifyProxy(config.proxyUri, config.proxy),
+};

package.json

@@ -146,7 +146,6 @@
     "tiny-async-pool": "2.1.0",
     "torrent-search-api": "2.1.4",
     "tough-cookie": "4.1.2",
-    "tunnel": "0.0.6",
     "twitter-api-v2": "1.12.10",
     "winston": "3.8.2",
     "xml2js": "0.4.23"

test/utils/request-wrapper.js

@@ -145,9 +145,9 @@ describe('got', () => {
         jest.resetModules();
         require('../../lib/utils/request-wrapper');
         check = (request) => {
-            expect(request.agent.constructor.name).toBe('TunnelingAgent');
+            expect(request.agent.constructor.name).toBe('HttpsProxyAgent');
-            expect(request.agent.options.proxy.host).toBe('rsshub.proxy');
+            expect(request.agent.proxy.host).toBe('rsshub.proxy');
-            expect(request.agent.options.proxy.port).toBe(2333);
+            expect(request.agent.proxy.port).toBe(2333);
         };
 
         nock(/rsshub\.test/)
@@ -167,9 +167,9 @@ describe('got', () => {
         jest.resetModules();
         require('../../lib/utils/request-wrapper');
         check = (request) => {
-            expect(request.agent.constructor.name).toBe('TunnelingAgent');
+            expect(request.agent.constructor.name).toBe('HttpsProxyAgent');
-            expect(request.agent.options.proxy.host).toBe('rsshub.proxy');
+            expect(request.agent.proxy.host).toBe('rsshub.proxy');
-            expect(request.agent.options.proxy.port).toBe(2333);
+            expect(request.agent.proxy.port).toBe(2333);
         };
 
         nock(/rsshub\.test/)
@@ -183,7 +183,7 @@ describe('got', () => {
 
     it('auth', async () => {
         process.env.PROXY_AUTH = 'testtest';
-        process.env.PROXY_PROTOCOL = 'socks';
+        process.env.PROXY_PROTOCOL = 'http'; // only http(s) proxies extract auth from Headers
         process.env.PROXY_HOST = 'rsshub.proxy';
         process.env.PROXY_PORT = '2333';
 

test/utils/unify-proxy.js

@@ -0,0 +1,140 @@
+const { unifyProxy } = require('../../lib/utils/unify-proxy');
+
+const emptyProxyObj = {
+    protocol: undefined,
+    host: undefined,
+    port: undefined,
+    auth: undefined,
+    url_regex: '.*',
+};
+
+const effectiveExpect = ({ proxyUri, proxyObj }, expectUri, expectObj) => {
+    expect(proxyUri).toBe(expectUri);
+    expect(proxyObj).toEqual(expectObj);
+};
+
+describe('unify-proxy', () => {
+    const nullExpect = (unified) => effectiveExpect(unified, null, emptyProxyObj);
+    it('proxy empty', () => {
+        nullExpect(unifyProxy(null, emptyProxyObj));
+    });
+    it('proxy-uri invalid', () => {
+        nullExpect(unifyProxy('http://inv lid.test', emptyProxyObj));
+    });
+    it('proxy-uri invalid protocol', () => {
+        nullExpect(unifyProxy('ftp://rsshub.proxy', emptyProxyObj));
+    });
+    it('proxy-obj no host', () => {
+        nullExpect(unifyProxy(null, { ...emptyProxyObj, protocol: 'http', port: 2333 }));
+    });
+    it('proxy-obj invalid host', () => {
+        nullExpect(unifyProxy(null, { ...emptyProxyObj, protocol: 'http', host: 'inv lid.test', port: 2333 }));
+    });
+    it('proxy-obj invalid protocol', () => {
+        nullExpect(unifyProxy(null, { ...emptyProxyObj, protocol: 'ftp', host: 'rsshub.proxy', port: 2333 }));
+    });
+
+    const httpNoPortUri = 'http://rsshub.proxy';
+    const httpNoPortObj = { ...emptyProxyObj, protocol: 'http', host: 'rsshub.proxy' };
+    const httpNoPortExpect = (unified) => effectiveExpect(unified, httpNoPortUri, httpNoPortObj);
+    it('proxy-uri http no port', () => {
+        httpNoPortExpect(unifyProxy(httpNoPortUri, emptyProxyObj));
+    });
+    it('proxy-obj http no port', () => {
+        httpNoPortExpect(unifyProxy(null, httpNoPortObj));
+    });
+
+    const httpUri = 'http://rsshub.proxy:2333';
+    const httpObj = { ...httpNoPortObj, port: 2333 };
+    const httpExpect = (unified) => effectiveExpect(unified, httpUri, httpObj);
+    it('proxy-uri http', () => {
+        httpExpect(unifyProxy(httpUri, emptyProxyObj));
+    });
+    it('proxy-obj http', () => {
+        httpExpect(unifyProxy(null, httpObj));
+    });
+
+    const httpsUri = 'https://rsshub.proxy:2333';
+    const httpsObj = { ...httpObj, protocol: 'https' };
+    const httpsExpect = (unified) => effectiveExpect(unified, httpsUri, httpsObj);
+    it('proxy-uri https', () => {
+        httpsExpect(unifyProxy(httpsUri, emptyProxyObj));
+    });
+    it('proxy-obj https', () => {
+        httpsExpect(unifyProxy(null, httpsObj));
+    });
+
+    const socks5Uri = 'socks5://rsshub.proxy:2333';
+    const socks5Obj = { ...httpObj, protocol: 'socks5' };
+    const socks5Expect = (unified) => effectiveExpect(unified, socks5Uri, socks5Obj);
+    it('proxy-uri socks5', () => {
+        socks5Expect(unifyProxy(socks5Uri, emptyProxyObj));
+    });
+    it('proxy-obj socks5', () => {
+        socks5Expect(unifyProxy(null, socks5Obj));
+    });
+
+    const overrideObj = { ...emptyProxyObj, protocol: 'http', host: 'over.ride', port: 6666 };
+    it('proxy-uri override proxy-obj {PROTOCAL,HOST,PORT}', () => {
+        socks5Expect(unifyProxy(socks5Uri, overrideObj));
+    });
+
+    const noProtocolUri = 'rsshub.proxy:2333';
+    it('proxy-uri no protocol', () => {
+        httpExpect(unifyProxy(noProtocolUri, emptyProxyObj));
+    });
+
+    const noProtocolObj = { ...httpObj, protocol: undefined };
+    it('proxy-obj no protocol', () => {
+        httpExpect(unifyProxy(null, noProtocolObj));
+    });
+
+    const protocolInHostObj = { ...httpObj, host: httpNoPortUri, protocol: undefined };
+    it('proxy-obj protocol in host', () => {
+        httpExpect(unifyProxy(null, protocolInHostObj));
+    });
+
+    const portInHostObj = { ...httpNoPortObj, host: httpUri };
+    it('proxy-obj port in host', () => {
+        httpExpect(unifyProxy(null, portInHostObj));
+    });
+
+    const everythingInHostObj = { ...emptyProxyObj, host: httpUri };
+    it('proxy-obj everything in host', () => {
+        httpExpect(unifyProxy(null, everythingInHostObj));
+    });
+
+    const portBothObj = { ...portInHostObj, port: 6666 };
+    it('proxy-obj port in host override proxy-obj port', () => {
+        httpExpect(unifyProxy(null, portBothObj));
+    });
+
+    const PortNaNObj = { ...httpNoPortObj, port: 'test' };
+    it('proxy-obj port NaN', () => {
+        httpNoPortExpect(unifyProxy(null, PortNaNObj));
+    });
+
+    const httpsAuthUri = 'https://user:pass@rsshub.proxy:2333';
+    it('proxy-uri https auth', () => {
+        effectiveExpect(unifyProxy(httpsAuthUri, emptyProxyObj), httpsAuthUri, httpsObj);
+    });
+
+    const httpsAuthObj = { ...httpsObj, auth: 'testtest' };
+    it('proxy-obj https auth', () => {
+        effectiveExpect(unifyProxy(null, httpsAuthObj), httpsUri, httpsAuthObj);
+    });
+
+    const socks5AuthUri = 'socks5://user:pass@rsshub.proxy:2333';
+    it('proxy-uri socks5 auth', () => {
+        effectiveExpect(unifyProxy(socks5AuthUri, emptyProxyObj), socks5AuthUri, socks5Obj);
+    });
+
+    const socks5AuthObj = { ...socks5Obj, auth: 'testtest' };
+    it('proxy-obj socks5 auth (invalid)', () => {
+        socks5Expect(unifyProxy(null, socks5AuthObj));
+    });
+
+    it('proxy-uri user@pass override proxy-obj auth', () => {
+        effectiveExpect(unifyProxy(httpsAuthUri, httpsAuthObj), httpsAuthUri, httpsObj);
+    });
+});