Merge branch 'request-wrapper'

lib/index.js

@@ -4,6 +4,7 @@ const config = require('./config');
 const Koa = require('koa');
 const fs = require('fs');
 const logger = require('./utils/logger');
+require('./utils/request-wrapper');
 
 const onerror = require('./middleware/onerror');
 const header = require('./middleware/header');

lib/utils/got.js

@@ -1,52 +1,8 @@
 const logger = require('./logger');
 const config = require('@/config');
-const SocksProxyAgent = require('socks-proxy-agent');
-const tunnel = require('tunnel');
 const got = require('got');
 const queryString = require('query-string');
 
-let agent = null;
-if (config.proxy && config.proxy.protocol && config.proxy.host && config.proxy.port) {
-    agent = {};
-    const proxyUrl = `${config.proxy.protocol}://${config.proxy.host}:${config.proxy.port}`;
-
-    switch (config.proxy.protocol) {
-        case 'socks':
-            agent.http = new SocksProxyAgent(proxyUrl);
-            agent.https = new SocksProxyAgent(proxyUrl);
-            break;
-        case 'http':
-            process.env.NODE_TLS_REJECT_UNAUTHORIZED = 0;
-            agent.http = tunnel.httpOverHttp({
-                proxy: {
-                    host: config.proxy.host,
-                    port: parseInt(config.proxy.port),
-                },
-            });
-            agent.https = tunnel.httpsOverHttp({
-                proxy: {
-                    host: config.proxy.host,
-                    port: parseInt(config.proxy.port),
-                },
-            });
-            break;
-        case 'https':
-            agent.http = tunnel.httpOverHttps({
-                proxy: {
-                    host: config.proxy.host,
-                    port: parseInt(config.proxy.port),
-                },
-            });
-            agent.https = tunnel.httpsOverHttps({
-                proxy: {
-                    host: config.proxy.host,
-                    port: parseInt(config.proxy.port),
-                },
-            });
-            break;
-    }
-}
-
 const custom = got.extend({
     retry: {
         retries: config.requestRetry,
@@ -85,20 +41,11 @@ const custom = got.extend({
                     options.query = options.query.replace(/([\u4e00-\u9fa5])/g, (str) => encodeURIComponent(str));
                     options.searchParams = options.query; // for Got v11 after
                 }
-                if (agent && new RegExp(config.proxy.url_regex).test(options.href)) {
-                    options.agent = agent[options.protocol.slice(0, -1)];
-
-                    if (config.proxy.auth) {
-                        options.headers['Proxy-Authorization'] = `Basic ${config.proxy.auth}`;
-                    }
-                    logger.info(`Proxy for ${options.href}`);
-                }
             },
         ],
     },
     headers: {
         'user-agent': config.ua,
-        'x-app': 'RSSHub',
     },
 });
 custom.all = (list) => Promise.all(list);

lib/utils/request-wrapper.js

@@ -0,0 +1,99 @@
+const config = require('@/config');
+const SocksProxyAgent = require('socks-proxy-agent');
+const tunnel = require('tunnel');
+const logger = require('./logger');
+const http = require('http');
+const https = require('https');
+
+let agent = null;
+if (config.proxy && config.proxy.protocol && config.proxy.host && config.proxy.port) {
+    agent = {};
+    const proxyUrl = `${config.proxy.protocol}://${config.proxy.host}:${config.proxy.port}`;
+
+    switch (config.proxy.protocol) {
+        case 'socks':
+            agent.http = new SocksProxyAgent(proxyUrl);
+            agent.https = new SocksProxyAgent(proxyUrl);
+            break;
+        case 'http':
+            process.env.NODE_TLS_REJECT_UNAUTHORIZED = 0;
+            agent.http = tunnel.httpOverHttp({
+                proxy: {
+                    host: config.proxy.host,
+                    port: parseInt(config.proxy.port),
+                },
+            });
+            agent.https = tunnel.httpsOverHttp({
+                proxy: {
+                    host: config.proxy.host,
+                    port: parseInt(config.proxy.port),
+                },
+            });
+            break;
+        case 'https':
+            agent.http = tunnel.httpOverHttps({
+                proxy: {
+                    host: config.proxy.host,
+                    port: parseInt(config.proxy.port),
+                },
+            });
+            agent.https = tunnel.httpsOverHttps({
+                proxy: {
+                    host: config.proxy.host,
+                    port: parseInt(config.proxy.port),
+                },
+            });
+            break;
+    }
+}
+
+const requestWrapper = (url, options) => {
+    if (agent && new RegExp(config.proxy.url_regex).test(url)) {
+        let agentResult;
+        try {
+            agentResult = agent[(options.protocol || url.match(/(https?:)/)[1]).slice(0, -1)];
+        } catch (error) {
+            agentResult = null;
+        }
+        options.agent = agentResult;
+
+        if (config.proxy.auth) {
+            if (!options.headers) {
+                options.headers = {};
+            }
+            options.headers['Proxy-Authorization'] = `Basic ${config.proxy.auth}`;
+        }
+        logger.info(`Proxy for ${url}`);
+    }
+    let hasUA = false;
+    for (const header in options.headers) {
+        if (header.toLowerCase() === 'user-agent') {
+            hasUA = true;
+        }
+    }
+    if (!hasUA) {
+        if (!options.headers) {
+            options.headers = {};
+        }
+        options.headers['user-agent'] = config.ua;
+    }
+    options.headers['x-app'] = 'RSSHub';
+};
+
+const httpWrap = (func) => {
+    const origin = func;
+    return function(url, request) {
+        if (typeof url === 'object') {
+            const req = url;
+            requestWrapper(req.url || req.href || `${req.protocol}//${req.hostname}${req.path}`, req);
+        } else {
+            requestWrapper(url, request);
+        }
+        return origin.apply(this, arguments);
+    };
+};
+
+http.get = httpWrap(http.get);
+https.get = httpWrap(https.get);
+http.request = httpWrap(http.request);
+https.request = httpWrap(https.request);

lib/utils/rss-parser.js

@@ -7,7 +7,6 @@ const parser = new Parser({
     },
     headers: {
         'User-Agent': config.ua,
-        'X-APP': 'RSSHub',
     },
 });
 

test/utils/got.js

@@ -1,22 +1,13 @@
-let got = require('../../lib/utils/got');
+const got = require('../../lib/utils/got');
 const config = require('../../lib/config');
 const nock = require('nock');
 
-afterEach(() => {
-    delete process.env.PROXY_PROTOCOL;
-    delete process.env.PROXY_HOST;
-    delete process.env.PROXY_PORT;
-    delete process.env.PROXY_AUTH;
-    delete process.env.PROXY_URL_REGEX;
-});
-
 describe('got', () => {
     it('headers', async () => {
         nock('http://rsshub.test')
             .get('/test')
             .reply(function() {
                 expect(this.req.headers['user-agent']).toBe(config.ua);
-                expect(this.req.headers['x-app']).toBe('RSSHub');
                 return [200, ''];
             });
 
@@ -81,127 +72,4 @@ describe('got', () => {
             },
         });
     });
-
-    it('proxy socks', async () => {
-        process.env.PROXY_PROTOCOL = 'socks';
-        process.env.PROXY_HOST = 'rsshub.proxy';
-        process.env.PROXY_PORT = '2333';
-        jest.resetModules();
-        got = require('../../lib/utils/got');
-        nock('http://rsshub.test')
-            .get('/proxy')
-            .reply(() => [200, '']);
-
-        await got.get('http://rsshub.test/proxy', {
-            hooks: {
-                beforeRequest: [
-                    (options) => {
-                        expect(options.agent.constructor.name).toBe('SocksProxyAgent');
-                        expect(options.agent.options.href).toBe('socks://rsshub.proxy:2333');
-                    },
-                ],
-            },
-        });
-    });
-
-    it('proxy http', async () => {
-        process.env.PROXY_PROTOCOL = 'http';
-        process.env.PROXY_HOST = 'rsshub.proxy';
-        process.env.PROXY_PORT = '2333';
-        jest.resetModules();
-        got = require('../../lib/utils/got');
-        nock('http://rsshub.test')
-            .get('/proxy')
-            .reply(() => [200, '']);
-
-        await got.get('http://rsshub.test/proxy', {
-            hooks: {
-                beforeRequest: [
-                    (options) => {
-                        expect(options.agent.constructor.name).toBe('TunnelingAgent');
-                        expect(options.agent.options.proxy.host).toBe('rsshub.proxy');
-                        expect(options.agent.options.proxy.port).toBe(2333);
-                    },
-                ],
-            },
-        });
-    });
-
-    it('proxy https', async () => {
-        process.env.PROXY_PROTOCOL = 'https';
-        process.env.PROXY_HOST = 'rsshub.proxy';
-        process.env.PROXY_PORT = '2333';
-        jest.resetModules();
-        got = require('../../lib/utils/got');
-        nock('http://rsshub.test')
-            .get('/proxy')
-            .reply(() => [200, '']);
-
-        await got.get('http://rsshub.test/proxy', {
-            hooks: {
-                beforeRequest: [
-                    (options) => {
-                        expect(options.agent.constructor.name).toBe('TunnelingAgent');
-                        expect(options.agent.options.proxy.host).toBe('rsshub.proxy');
-                        expect(options.agent.options.proxy.port).toBe(2333);
-                    },
-                ],
-            },
-        });
-    });
-
-    it('auth', async () => {
-        process.env.PROXY_AUTH = 'testtest';
-        process.env.PROXY_PROTOCOL = 'socks';
-        process.env.PROXY_HOST = 'rsshub.proxy';
-        process.env.PROXY_PORT = '2333';
-        jest.resetModules();
-        got = require('../../lib/utils/got');
-        nock('http://rsshub.test')
-            .get('/auth')
-            .reply(function() {
-                expect(this.req.headers['user-agent']).toBe(config.ua);
-                expect(this.req.headers['proxy-authorization']).toBe('Basic testtest');
-                return [200, ''];
-            });
-
-        await got.get('http://rsshub.test/auth');
-    });
-
-    it('url_regex', async () => {
-        process.env.PROXY_URL_REGEX = 'url_regex';
-        process.env.PROXY_PROTOCOL = 'socks';
-        process.env.PROXY_HOST = 'rsshub.proxy';
-        process.env.PROXY_PORT = '2333';
-        jest.resetModules();
-        got = require('../../lib/utils/got');
-
-        nock('http://rsshub.test')
-            .get('/url_regex')
-            .reply(() => [200, '']);
-        nock('http://rsshub.test')
-            .get('/proxy')
-            .reply(() => [200, '']);
-
-        await got.get('http://rsshub.test/url_regex', {
-            hooks: {
-                beforeRequest: [
-                    (options) => {
-                        expect(options.agent.constructor.name).toBe('SocksProxyAgent');
-                        expect(options.agent.options.href).toBe('socks://rsshub.proxy:2333');
-                    },
-                ],
-            },
-        });
-
-        await got.get('http://rsshub.test/proxy', {
-            hooks: {
-                beforeRequest: [
-                    (options) => {
-                        expect(options.agent).toBe(undefined);
-                    },
-                ],
-            },
-        });
-    });
 });

test/utils/request-wrapper.js

@@ -0,0 +1,167 @@
+const got = require('../../lib/utils/got');
+const parser = require('../../lib/utils/rss-parser');
+const nock = require('nock');
+require('../../lib/utils/request-wrapper');
+let check = () => {};
+const simpleResponse = '<rss version="2.0"><channel><item></item></channel></rss>';
+
+afterEach(() => {
+    delete process.env.PROXY_PROTOCOL;
+    delete process.env.PROXY_HOST;
+    delete process.env.PROXY_PORT;
+    delete process.env.PROXY_AUTH;
+    delete process.env.PROXY_URL_REGEX;
+
+    nock.restore();
+    nock.activate();
+    check = () => {};
+
+    const http = require('http');
+    const httpWrap = (func) => {
+        const origin = func;
+        return function(url, request) {
+            if (typeof url === 'object') {
+                check(url);
+            } else {
+                check(request);
+            }
+            return origin.apply(this, arguments);
+        };
+    };
+    http.get = httpWrap(http.get);
+    http.request = httpWrap(http.request);
+});
+
+describe('got', () => {
+    it('headers', async () => {
+        nock('http://rsshub.test')
+            .get('/test')
+            .times(2)
+            .reply(function() {
+                expect(this.req.headers['x-app']).toBe('RSSHub');
+                return [200, simpleResponse];
+            });
+
+        await got.get('http://rsshub.test/test');
+        await parser.parseURL('http://rsshub.test/test');
+    });
+
+    it('proxy socks', async () => {
+        process.env.PROXY_PROTOCOL = 'socks';
+        process.env.PROXY_HOST = 'rsshub.proxy';
+        process.env.PROXY_PORT = '2333';
+
+        jest.resetModules();
+        require('../../lib/utils/request-wrapper');
+        check = (request) => {
+            expect(request.agent.constructor.name).toBe('SocksProxyAgent');
+            expect(request.agent.options.href).toBe('socks://rsshub.proxy:2333');
+        };
+
+        nock('http://rsshub.test')
+            .get('/proxy')
+            .times(2)
+            .reply(200, simpleResponse);
+
+        await got.get('http://rsshub.test/proxy');
+        await parser.parseURL('http://rsshub.test/proxy');
+    });
+
+    it('proxy http', async () => {
+        process.env.PROXY_PROTOCOL = 'http';
+        process.env.PROXY_HOST = 'rsshub.proxy';
+        process.env.PROXY_PORT = '2333';
+
+        jest.resetModules();
+        require('../../lib/utils/request-wrapper');
+        check = (request) => {
+            expect(request.agent.constructor.name).toBe('TunnelingAgent');
+            expect(request.agent.options.proxy.host).toBe('rsshub.proxy');
+            expect(request.agent.options.proxy.port).toBe(2333);
+        };
+
+        nock('http://rsshub.test')
+            .get('/proxy')
+            .times(2)
+            .reply(200, simpleResponse);
+
+        await got.get('http://rsshub.test/proxy');
+        await parser.parseURL('http://rsshub.test/proxy');
+    });
+
+    it('proxy https', async () => {
+        process.env.PROXY_PROTOCOL = 'https';
+        process.env.PROXY_HOST = 'rsshub.proxy';
+        process.env.PROXY_PORT = '2333';
+
+        jest.resetModules();
+        require('../../lib/utils/request-wrapper');
+        check = (request) => {
+            expect(request.agent.constructor.name).toBe('TunnelingAgent');
+            expect(request.agent.options.proxy.host).toBe('rsshub.proxy');
+            expect(request.agent.options.proxy.port).toBe(2333);
+        };
+
+        nock('http://rsshub.test')
+            .get('/proxy')
+            .times(2)
+            .reply(200, simpleResponse);
+
+        await got.get('http://rsshub.test/proxy');
+        await parser.parseURL('http://rsshub.test/proxy');
+    });
+
+    it('auth', async () => {
+        process.env.PROXY_AUTH = 'testtest';
+        process.env.PROXY_PROTOCOL = 'socks';
+        process.env.PROXY_HOST = 'rsshub.proxy';
+        process.env.PROXY_PORT = '2333';
+
+        jest.resetModules();
+        require('../../lib/utils/request-wrapper');
+
+        nock('http://rsshub.test')
+            .get('/auth')
+            .times(2)
+            .reply(function() {
+                expect(this.req.headers['proxy-authorization']).toBe('Basic testtest');
+                return [200, simpleResponse];
+            });
+
+        await got.get('http://rsshub.test/auth');
+        await parser.parseURL('http://rsshub.test/auth');
+    });
+
+    it('url_regex', async () => {
+        process.env.PROXY_URL_REGEX = 'url_regex';
+        process.env.PROXY_PROTOCOL = 'socks';
+        process.env.PROXY_HOST = 'rsshub.proxy';
+        process.env.PROXY_PORT = '2333';
+
+        jest.resetModules();
+        require('../../lib/utils/request-wrapper');
+        check = (request) => {
+            if (request.path === '/url_regex') {
+                expect(request.agent.constructor.name).toBe('SocksProxyAgent');
+                expect(request.agent.options.href).toBe('socks://rsshub.proxy:2333');
+            } else if (request.path === '/proxy') {
+                expect(request.agent).toBe(undefined);
+            }
+        };
+
+        nock('http://rsshub.test')
+            .get('/url_regex')
+            .times(2)
+            .reply(() => [200, simpleResponse]);
+        nock('http://rsshub.test')
+            .get('/proxy')
+            .times(2)
+            .reply(() => [200, simpleResponse]);
+
+        await got.get('http://rsshub.test/url_regex');
+        await parser.parseURL('http://rsshub.test/url_regex');
+
+        await got.get('http://rsshub.test/proxy');
+        await parser.parseURL('http://rsshub.test/proxy');
+    });
+});

test/utils/rss-parser.js

@@ -0,0 +1,16 @@
+const parser = require('../../lib/utils/rss-parser');
+const config = require('../../lib/config');
+const nock = require('nock');
+
+describe('got', () => {
+    it('headers', async () => {
+        nock('http://rsshub.test')
+            .get('/test')
+            .reply(function() {
+                expect(this.req.headers['user-agent']).toBe(config.ua);
+                return [200, '<rss version="2.0"><channel><item></item></channel></rss>'];
+            });
+
+        await parser.parseURL('http://rsshub.test/test');
+    });
+});