support http and https proxy

lib/config.js

@@ -51,13 +51,12 @@ module.exports = {
     },
     puppeteerWSEndpoint: process.env.PUPPETEER_WS_ENDPOINT,
     loggerLevel: process.env.LOGGER_LEVEL || 'info',
-    // Something like socks://{{host}}:{{port}}
     proxy: {
-        // Supported socks protocols see:
+        protocol: process.env.PROXY_PROTOCOL,
-        // https://github.com/TooTallNate/node-socks-proxy-agent/blob/5867dd04e92b51af0a35d5b3cb6a82f8f5590b6f/index.js#L53
+        host: process.env.PROXY_HOST,
-        protocol: process.env.PROXY_PROTOCOL || null,
+        port: process.env.PROXY_PORT,
-        host: process.env.PROXY_HOST || null,
+        auth: process.env.PROXY_AUTH,
-        port: process.env.PROXY_PORT || null,
+        url_regex: process.env.PROXY_URL_REGEX || '.*',
     },
     blacklist: process.env.BLACKLIST && process.env.BLACKLIST.split(','),
     whitelist: process.env.WHITELIST && process.env.WHITELIST.split(','),

lib/utils/axios.js

@@ -2,28 +2,59 @@ const logger = require('./logger');
 const config = require('../config');
 const SocksProxyAgent = require('socks-proxy-agent');
 const axiosRetry = require('axios-retry');
+const axios = require('axios');
+const tunnel = require('tunnel');
 
-let axios = require('axios');
+if (config.proxy && config.proxy.protocol && config.proxy.host && config.proxy.port) {
-if (config.proxy && config.proxy.protocol && typeof config.proxy.protocol === 'string' && config.proxy.protocol.slice(0, 5) === 'socks' && config.proxy.host && config.proxy.port) {
-    // axios closure lead to recursive invokation on create
     const proxyUrl = `${config.proxy.protocol}://${config.proxy.host}:${config.proxy.port}`;
-    const axiosCpy = axios;
+    axios.interceptors.request.use((options) => {
-    // When used directly
+        if (new RegExp(config.proxy.url_regex).test(options.url)) {
-    const dump = axios.create({
+            switch (config.proxy.protocol) {
-        httpAgent: new SocksProxyAgent(proxyUrl),
+                case 'socks':
-        httpsAgent: new SocksProxyAgent(proxyUrl),
+                    options.httpAgent = new SocksProxyAgent(proxyUrl);
+                    options.httpsAgent = new SocksProxyAgent(proxyUrl);
+                    break;
+                case 'http':
+                    process.env.NODE_TLS_REJECT_UNAUTHORIZED = 0;
+                    options.httpAgent = tunnel.httpOverHttp({
+                        proxy: {
+                            host: config.proxy.host,
+                            port: parseInt(config.proxy.port),
+                        },
+                    });
+                    options.httpsAgent = tunnel.httpsOverHttp({
+                        proxy: {
+                            host: config.proxy.host,
+                            port: parseInt(config.proxy.port),
+                        },
+                    });
+                    break;
+                case 'https':
+                    options.httpAgent = tunnel.httpOverHttps({
+                        proxy: {
+                            host: config.proxy.host,
+                            port: parseInt(config.proxy.port),
+                            proxyAuth: `${config.proxy.auth.username}:${config.proxy.auth.password}`,
+                        },
+                    });
+                    options.httpsAgent = tunnel.httpsOverHttps({
+                        proxy: {
+                            host: config.proxy.host,
+                            port: parseInt(config.proxy.port),
+                            proxyAuth: `${config.proxy.auth.username}:${config.proxy.auth.password}`,
+                        },
+                    });
+                    break;
+            }
+            if (config.proxy.auth) {
+                options.headers['Proxy-Authorization'] = `Basic ${config.proxy.auth}`;
+            }
+            logger.info(`Proxy for ${options.url}`);
+        }
+        return options;
     });
-    dump.create = function(option, ...args) {
-        option = option || {};
-        option = {
-            httpAgent: new SocksProxyAgent(proxyUrl),
-            httpsAgent: new SocksProxyAgent(proxyUrl),
-            ...option,
-        };
-        return axiosCpy.create(option, ...args);
-    };
-    axios = dump;
 }
+
 axiosRetry(axios, {
     retries: config.requestRetry,
     retryCondition: () => true,

package.json

@@ -81,6 +81,7 @@
     "sanitize-html": "^1.20.0",
     "sharp": "^0.22.0",
     "socks-proxy-agent": "^4.0.1",
+    "tunnel": "^0.0.6",
     "twit": "2.2.11",
     "winston": "3.2.1"
   },

yarn.lock

@@ -10774,6 +10774,11 @@ tunnel-agent@^0.6.0:
   dependencies:
     safe-buffer "^5.0.1"
 
+tunnel@^0.0.6:
+  version "0.0.6"
+  resolved "https://registry.yarnpkg.com/tunnel/-/tunnel-0.0.6.tgz#72f1314b34a5b192db012324df2cc587ca47f92c"
+  integrity sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==
+
 tweetnacl@^0.14.3, tweetnacl@~0.14.0:
   version "0.14.5"
   resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64"