diff --git a/lib/config.js b/lib/config.js index 963551dd32..832e5e1acf 100644 --- a/lib/config.js +++ b/lib/config.js @@ -51,13 +51,12 @@ module.exports = { }, puppeteerWSEndpoint: process.env.PUPPETEER_WS_ENDPOINT, loggerLevel: process.env.LOGGER_LEVEL || 'info', - // Something like socks://{{host}}:{{port}} proxy: { - // Supported socks protocols see: - // https://github.com/TooTallNate/node-socks-proxy-agent/blob/5867dd04e92b51af0a35d5b3cb6a82f8f5590b6f/index.js#L53 - protocol: process.env.PROXY_PROTOCOL || null, - host: process.env.PROXY_HOST || null, - port: process.env.PROXY_PORT || null, + protocol: process.env.PROXY_PROTOCOL, + host: process.env.PROXY_HOST, + port: process.env.PROXY_PORT, + auth: process.env.PROXY_AUTH, + url_regex: process.env.PROXY_URL_REGEX || '.*', }, blacklist: process.env.BLACKLIST && process.env.BLACKLIST.split(','), whitelist: process.env.WHITELIST && process.env.WHITELIST.split(','), diff --git a/lib/utils/axios.js b/lib/utils/axios.js index b65a0142de..86c4db11ec 100644 --- a/lib/utils/axios.js +++ b/lib/utils/axios.js @@ -2,28 +2,59 @@ const logger = require('./logger'); const config = require('../config'); const SocksProxyAgent = require('socks-proxy-agent'); const axiosRetry = require('axios-retry'); +const axios = require('axios'); +const tunnel = require('tunnel'); -let axios = require('axios'); -if (config.proxy && config.proxy.protocol && typeof config.proxy.protocol === 'string' && config.proxy.protocol.slice(0, 5) === 'socks' && config.proxy.host && config.proxy.port) { - // axios closure lead to recursive invokation on create +if (config.proxy && config.proxy.protocol && config.proxy.host && config.proxy.port) { const proxyUrl = `${config.proxy.protocol}://${config.proxy.host}:${config.proxy.port}`; - const axiosCpy = axios; - // When used directly - const dump = axios.create({ - httpAgent: new SocksProxyAgent(proxyUrl), - httpsAgent: new SocksProxyAgent(proxyUrl), + axios.interceptors.request.use((options) => { + if (new RegExp(config.proxy.url_regex).test(options.url)) { + switch (config.proxy.protocol) { + case 'socks': + options.httpAgent = new SocksProxyAgent(proxyUrl); + options.httpsAgent = new SocksProxyAgent(proxyUrl); + break; + case 'http': + process.env.NODE_TLS_REJECT_UNAUTHORIZED = 0; + options.httpAgent = tunnel.httpOverHttp({ + proxy: { + host: config.proxy.host, + port: parseInt(config.proxy.port), + }, + }); + options.httpsAgent = tunnel.httpsOverHttp({ + proxy: { + host: config.proxy.host, + port: parseInt(config.proxy.port), + }, + }); + break; + case 'https': + options.httpAgent = tunnel.httpOverHttps({ + proxy: { + host: config.proxy.host, + port: parseInt(config.proxy.port), + proxyAuth: `${config.proxy.auth.username}:${config.proxy.auth.password}`, + }, + }); + options.httpsAgent = tunnel.httpsOverHttps({ + proxy: { + host: config.proxy.host, + port: parseInt(config.proxy.port), + proxyAuth: `${config.proxy.auth.username}:${config.proxy.auth.password}`, + }, + }); + break; + } + if (config.proxy.auth) { + options.headers['Proxy-Authorization'] = `Basic ${config.proxy.auth}`; + } + logger.info(`Proxy for ${options.url}`); + } + return options; }); - dump.create = function(option, ...args) { - option = option || {}; - option = { - httpAgent: new SocksProxyAgent(proxyUrl), - httpsAgent: new SocksProxyAgent(proxyUrl), - ...option, - }; - return axiosCpy.create(option, ...args); - }; - axios = dump; } + axiosRetry(axios, { retries: config.requestRetry, retryCondition: () => true, diff --git a/package.json b/package.json index f38b504515..1e7711fc50 100644 --- a/package.json +++ b/package.json @@ -81,6 +81,7 @@ "sanitize-html": "^1.20.0", "sharp": "^0.22.0", "socks-proxy-agent": "^4.0.1", + "tunnel": "^0.0.6", "twit": "2.2.11", "winston": "3.2.1" }, diff --git a/yarn.lock b/yarn.lock index dec05a2007..9a9bdf1927 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10774,6 +10774,11 @@ tunnel-agent@^0.6.0: dependencies: safe-buffer "^5.0.1" +tunnel@^0.0.6: + version "0.0.6" + resolved "https://registry.yarnpkg.com/tunnel/-/tunnel-0.0.6.tgz#72f1314b34a5b192db012324df2cc587ca47f92c" + integrity sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg== + tweetnacl@^0.14.3, tweetnacl@~0.14.0: version "0.14.5" resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64"