opensource/RSSHub
1
0
Fork 0
mirror of https://github.com/DIYgod/RSSHub.git synced 2025-12-04 02:58:08 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(core): unify proxy; optimize request-wrapper (#11472)

Browse Source 
* feat(core): unify proxy; optimize request-wrapper

Signed-off-by: Rongrong <i@rong.moe>

* Fix DeepScan issue

Signed-off-by: Rongrong <i@rong.moe>

* Simplify protocol existence judgment

Signed-off-by: Rongrong <i@rong.moe>

* chore: regenerate yarn.lock

Signed-off-by: Rongrong <i@rong.moe>
This commit is contained in:
Rongrong
2022-12-23 01:13:01 +08:00
committed by GitHub
parent 58d00a6917
commit d60d89ea87
7 changed files with 835 additions and 640 deletions
Download Patch File Download Diff File Expand all files Collapse all files

138
lib/utils/request-wrapper.js
View File

@@ -1,127 +1,65 @@
const config = require('@/config').value;
const { proxyUri, proxyObj, proxyUrlHandler } = require('./unify-proxy');
const logger = require('./logger');
const http = require('http');
const https = require('https');
let tunnel;
let HttpsProxyAgent;
let SocksProxyAgent;
let agent = null;
if (config.proxyUri && typeof config.proxyUri === 'string') {
let proxy = null;
if (config.proxyUri.startsWith('http')) {
HttpsProxyAgent = HttpsProxyAgent || require('https-proxy-agent');
proxy = new HttpsProxyAgent(config.proxyUri);
} else if (config.proxyUri.startsWith('socks')) {
SocksProxyAgent = SocksProxyAgent || require('socks-proxy-agent').SocksProxyAgent;
proxy = new SocksProxyAgent(config.proxyUri);
} else {
throw 'Unknown proxy-uri format';
}
agent = {
http: proxy,
https: proxy,
};
} else if (config.proxy && config.proxy.protocol && config.proxy.host && config.proxy.port) {
agent = {};
const proxyUrl = `${config.proxy.protocol}://${config.proxy.host}:${config.proxy.port}`;
switch (config.proxy.protocol) {
case 'socks':
SocksProxyAgent = SocksProxyAgent || require('socks-proxy-agent').SocksProxyAgent;
agent.http = new SocksProxyAgent(proxyUrl);
agent.https = new SocksProxyAgent(proxyUrl);
break;
case 'http':
tunnel = tunnel || require('tunnel');
process.env.NODE_TLS_REJECT_UNAUTHORIZED = 0;
agent.http = tunnel.httpOverHttp({
proxy: {
host: config.proxy.host,
port: parseInt(config.proxy.port),
},
});
agent.https = tunnel.httpsOverHttp({
proxy: {
host: config.proxy.host,
port: parseInt(config.proxy.port),
},
});
break;
case 'https':
tunnel = tunnel || require('tunnel');
agent.http = tunnel.httpOverHttps({
proxy: {
host: config.proxy.host,
port: parseInt(config.proxy.port),
},
});
agent.https = tunnel.httpsOverHttps({
proxy: {
host: config.proxy.host,
port: parseInt(config.proxy.port),
},
});
break;
if (proxyUri) {
if (proxyUri.startsWith('http')) {
const HttpsProxyAgent = require('https-proxy-agent');
agent = new HttpsProxyAgent(proxyUri);
} else if (proxyUri.startsWith('socks')) {
const SocksProxyAgent = require('socks-proxy-agent').SocksProxyAgent;
agent = new SocksProxyAgent(proxyUri);
}
}
const requestWrapper = (url, options) => {
// agent
if (agent && new RegExp(config.proxy.url_regex).test(url)) {
let agentResult;
try {
agentResult = agent[(options.protocol || url.match(/(^https?:)/)[1]).slice(0, -1)];
} catch (error) {
agentResult = null;
}
try {
if (new URL(url).host !== new URL(config.proxyUri).host) {
options.agent = agentResult;
}
} catch (error) {
options.agent = agentResult;
}
let proxyWrapper = () => false;
if (agent) {
const proxyRegex = new RegExp(proxyObj.url_regex);
const protocolMatch = (protocolLike) => protocolLike && protocolLike.toLowerCase().startsWith('http');
if (config.proxy.auth) {
if (!options.headers) {
options.headers = {};
proxyWrapper = (url, options, urlHandler) => {
if (proxyRegex.test(url)) {
if ((protocolMatch(options.protocol) || protocolMatch(url)) && (!urlHandler || urlHandler.host !== proxyUrlHandler.host)) {
options.agent = agent;
if (proxyObj.auth) {
options.headers['Proxy-Authorization'] = `Basic ${proxyObj.auth}`;
}
return true;
}
options.headers['Proxy-Authorization'] = `Basic ${config.proxy.auth}`;
}
logger.info(`Proxy for ${url}`);
} else {
logger.debug(`Requesting ${url}`);
return false;
};
}
const requestWrapper = (url, options) => {
let urlHandler;
try {
urlHandler = new URL(url);
} catch (error) {
// ignore
}
options.headers = options.headers || {};
const headersLowerCaseKeys = Object.keys(options.headers).map((key) => key.toLowerCase());
proxyWrapper(url, options, urlHandler) ? logger.info(`Proxy for ${url}`) : logger.debug(`Requesting ${url}`);
// ua
let hasUA = false;
for (const header in options.headers) {
if (header.toLowerCase() === 'user-agent') {
hasUA = true;
}
}
if (!hasUA) {
if (!options.headers) {
options.headers = {};
}
if (!headersLowerCaseKeys.includes('user-agent')) {
options.headers['user-agent'] = config.ua;
}
try {
const urlHandler = new URL(url);
if (urlHandler) {
// referer
if (!options.headers.referer && !options.headers.Referer) {
if (!headersLowerCaseKeys.includes('referer')) {
options.headers.referer = urlHandler.origin;
}
// host
if (!options.headers.host && !options.headers.Host) {
if (!headersLowerCaseKeys.includes('host')) {
options.headers.host = urlHandler.host;
}
} catch (e) {
// do nothing
}
};