diff --git a/.husky/pre-commit b/.husky/pre-commit deleted file mode 100755 index d24fdfc601..0000000000 --- a/.husky/pre-commit +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env sh -. "$(dirname -- "$0")/_/husky.sh" - -npx lint-staged diff --git a/lib/config.js b/lib/config.ts similarity index 51% rename from lib/config.js rename to lib/config.ts index 13f7812367..fc5ac8b25b 100644 --- a/lib/config.js +++ b/lib/config.ts @@ -1,16 +1,321 @@ -require('dotenv').config(); -const randUserAgent = require('./utils/rand-user-agent'); -const got = require('got'); +import 'dotenv/config' +import randUserAgent from '@/utils/rand-user-agent'; +import got from 'got'; + let envs = process.env; -let value; + +type Config = { + disallowRobot: boolean; + enableCluster?: string; + isPackage: boolean; + nodeName?: string; + puppeteerWSEndpoint?: string; + chromiumExecutablePath?: string; + connect: { + port: number; + socket: string | null; + }; + listenInaddrAny: boolean; + requestRetry: number; + requestTimeout: number; + ua: string; + trueUA: string; + allowOrigin?: string; + cache: { + type: string; + requestTimeout: number; + routeExpire: number; + contentExpire: number; + }; + memory: { + max: number; + }; + redis: { + url: string; + }; + proxyUri?: string; + proxy: { + protocol?: string; + host?: string; + port?: string; + auth?: string; + url_regex: string; + }; + proxyStrategy: string; + reverseProxyUrl?: string; + pacUri?: string; + pacScript?: string; + authentication: { + name: string; + pass: string; + }; + denylist?: string[]; + allowlist?: string[]; + allowLocalhost: boolean; + accessKey?: string; + debugInfo: string; + loggerLevel: string; + noLogfiles?: boolean; + showLoggerTimestamp?: boolean; + sentry: { + dsn?: string; + routeTimeout: number; + }; + hotlink: { + template?: string; + includePaths?: string[]; + excludePaths?: string[]; + }; + feature: { + allow_user_hotlink_template: boolean; + filter_regex_engine: string; + allow_user_supply_unsafe_domain: boolean; + }; + suffix?: string; + titleLengthLimit: number; + openai: { + apiKey?: string; + model?: string; + temperature?: number; + maxTokens?: number; + endpoint?: string; + prompt?: string; + }; + bilibili: { + cookies: Record; + dmImgList?: string; + }; + bitbucket: { + username?: string; + password?: string; + }; + btbyr: { + host?: string; + cookies?: string; + }; + bupt: { + portal_cookie?: string; + }; + civitai: { + cookie?: string; + }; + dida365: { + username?: string; + password?: string; + }; + discord: { + authorization?: string; + }; + discourse: { + config: Record; + }; + discuz: { + cookies: Record; + }; + disqus: { + api_key?: string; + }; + douban: { + cookie?: string; + }; + ehentai: { + ipb_member_id?: string; + ipb_pass_hash?: string; + sk?: string; + igneous?: string; + star?: string; + img_proxy?: string; + }; + email: { + config: Record; + }; + fanbox: { + session?: string; + }; + fanfou: { + consumer_key?: string; + consumer_secret?: string; + username?: string; + password?: string; + }; + fantia: { + cookies?: string; + }; + game4399: { + cookie?: string; + }; + github: { + access_token?: string; + }; + gitee: { + access_token?: string; + }; + google: { + fontsApiKey?: string; + }; + hefeng: { + key?: string; + }; + infzm: { + cookie?: string; + }; + initium: { + username?: string; + password?: string; + bearertoken?: string; + iap_receipt?: string; + }; + instagram: { + username?: string; + password?: string; + proxy?: string; + cookie?: string; + }; + iwara: { + username?: string; + password?: string; + }; + lastfm: { + api_key?: string; + }; + lightnovel: { + cookie?: string; + }; + manhuagui: { + cookie?: string; + }; + mastodon: { + apiHost?: string; + accessToken?: string; + acctDomain?: string; + }; + medium: { + cookies: Record; + articleCookie?: string; + }; + miniflux: { + instance?: string; + token?: string; + }; + ncm: { + cookies?: string; + }; + newrank: { + cookie?: string; + }; + nga: { + uid?: string; + cid?: string; + }; + nhentai: { + username?: string; + password?: string; + }; + notion: { + key?: string; + }; + pianyuan: { + cookie?: string; + }; + pixabay: { + key?: string; + }; + pixiv: { + refreshToken?: string; + bypassCdn?: boolean; + bypassCdnHostname?: string; + bypassCdnDoh?: string; + imgProxy?: string; + }; + pkubbs: { + cookie?: string; + }; + saraba1st: { + cookie?: string; + }; + sehuatang: { + cookie?: string; + }; + scboy: { + token?: string; + }; + scihub: { + host?: string; + }; + spotify: { + clientId?: string; + clientSecret?: string; + refreshToken?: string; + }; + telegram: { + token?: string; + }; + tophub: { + cookie?: string; + }; + twitter: { + oauthTokens?: string[]; + oauthTokenSecrets?: string[]; + }; + weibo: { + app_key?: string; + app_secret?: string; + cookies?: string; + redirect_url?: string; + }; + wenku8: { + cookie?: string; + }; + wordpress: { + cdnUrl?: string; + }; + xiaoyuzhou: { + device_id?: string; + refresh_token?: string; + }; + ximalaya: { + token?: string; + }; + youtube: { + key?: string; + clientId?: string; + clientSecret?: string; + refreshToken?: string; + }; + zhihu: { + cookies?: string; + }; + zodgame: { + cookie?: string; + }; +}; + +let _value: Config | undefined = undefined; + const TRUE_UA = 'RSSHub/1.0 (+http://github.com/DIYgod/RSSHub; like FeedFetcher-Google)'; +const toBoolean = (value: string | undefined, defaultValue: boolean) => { + if (value !== undefined) { + return (value === '' || value === '0' || value === 'false') ? false : !!value; + } else { + return defaultValue; + } +} + +const toInt = (value: string | undefined, defaultValue: number) => { + if (value !== undefined) { + return Number.parseInt(value); + } else { + return defaultValue; + } +} + const calculateValue = () => { - const bilibili_cookies = {}; - const email_config = {}; - const discuz_cookies = {}; - const medium_cookies = {}; - const discourse_config = {}; + const bilibili_cookies: Record = {}; + const email_config: Record = {}; + const discuz_cookies: Record = {}; + const medium_cookies: Record = {}; + const discourse_config: Record = {}; for (const name in envs) { if (name.startsWith('BILIBILI_COOKIE_')) { @@ -27,39 +332,39 @@ const calculateValue = () => { medium_cookies[username] = envs[name]; } else if (name.startsWith('DISCOURSE_CONFIG_')) { const id = name.slice('DISCOURSE_CONFIG_'.length); - discourse_config[id] = JSON.parse(envs[name]); + discourse_config[id] = JSON.parse(envs[name] || '{}'); } } - value = { + _value = { // app config disallowRobot: envs.DISALLOW_ROBOT !== '0' && envs.DISALLOW_ROBOT !== 'false', enableCluster: envs.ENABLE_CLUSTER, - isPackage: envs.IS_PACKAGE, + isPackage: !!envs.IS_PACKAGE, nodeName: envs.NODE_NAME, puppeteerWSEndpoint: envs.PUPPETEER_WS_ENDPOINT, chromiumExecutablePath: envs.CHROMIUM_EXECUTABLE_PATH, // network connect: { - port: envs.PORT || 1200, // 监听端口 + port: toInt(envs.PORT, 1200), // 监听端口 socket: envs.SOCKET || null, // 监听 Unix Socket, null 为禁用 }, - listenInaddrAny: envs.LISTEN_INADDR_ANY || 1, // 是否允许公网连接，取值 0 1 - requestRetry: Number.parseInt(envs.REQUEST_RETRY) || 2, // 请求失败重试次数 - requestTimeout: Number.parseInt(envs.REQUEST_TIMEOUT) || 30000, // Milliseconds to wait for the server to end the response before aborting the request - ua: envs.UA ?? (envs.NO_RANDOM_UA === 'true' || envs.NO_RANDOM_UA === '1' ? TRUE_UA : randUserAgent({ browser: 'chrome', os: 'mac os', device: 'desktop' })), + listenInaddrAny: toBoolean(envs.LISTEN_INADDR_ANY, true), // 是否允许公网连接，取值 0 1 + requestRetry: toInt(envs.REQUEST_RETRY, 2), // 请求失败重试次数 + requestTimeout: toInt(envs.REQUEST_TIMEOUT, 30000), // Milliseconds to wait for the server to end the response before aborting the request + ua: envs.UA ?? (toBoolean(envs.NO_RANDOM_UA, false) ? TRUE_UA : randUserAgent({ browser: 'chrome', os: 'mac os', device: 'desktop' })), trueUA: TRUE_UA, // cors request allowOrigin: envs.ALLOW_ORIGIN, // cache cache: { - type: envs.CACHE_TYPE === undefined ? 'memory' : envs.CACHE_TYPE, // 缓存类型，支持 'memory' 和 'redis'，设为空可以禁止缓存 - requestTimeout: Number.parseInt(envs.CACHE_REQUEST_TIMEOUT) || 60, - routeExpire: Number.parseInt(envs.CACHE_EXPIRE) || 5 * 60, // 路由缓存时间，单位为秒 - contentExpire: Number.parseInt(envs.CACHE_CONTENT_EXPIRE) || 1 * 60 * 60, // 不变内容缓存时间，单位为秒 + type: envs.CACHE_TYPE || 'memory', // 缓存类型，支持 'memory' 和 'redis'，设为空可以禁止缓存 + requestTimeout: toInt(envs.CACHE_REQUEST_TIMEOUT, 60), + routeExpire: toInt(envs.CACHE_EXPIRE, 5 * 60), // 路由缓存时间，单位为秒 + contentExpire: toInt(envs.CACHE_CONTENT_EXPIRE, 1 * 60 * 60), // 不变内容缓存时间，单位为秒 }, memory: { - max: Number.parseInt(envs.MEMORY_MAX) || Math.pow(2, 8), // The maximum number of items that remain in the cache. This must be a positive finite intger. + max: toInt(envs.MEMORY_MAX, Math.pow(2, 8)), // The maximum number of items that remain in the cache. This must be a positive finite intger. // https://github.com/isaacs/node-lru-cache#options }, redis: { @@ -84,38 +389,38 @@ const calculateValue = () => { pass: envs.HTTP_BASIC_AUTH_PASS || 'passw0rd', }, // access control - denylist: envs.DENYLIST && envs.DENYLIST.split(','), - allowlist: envs.ALLOWLIST && envs.ALLOWLIST.split(','), - allowLocalhost: envs.ALLOW_LOCALHOST, + denylist: envs.DENYLIST ? envs.DENYLIST.split(',') : undefined, + allowlist: envs.ALLOWLIST ? envs.ALLOWLIST.split(',') : undefined, + allowLocalhost: toBoolean(envs.ALLOW_LOCALHOST, false), accessKey: envs.ACCESS_KEY, // logging // 是否显示 Debug 信息，取值 'true' 'false' 'some_string' ，取值为 'true' 时永久显示，取值为 'false' 时永远隐藏，取值为 'some_string' 时请求带上 '?debug=some_string' 显示 debugInfo: envs.DEBUG_INFO || 'true', loggerLevel: envs.LOGGER_LEVEL || 'info', - noLogfiles: envs.NO_LOGFILES, - showLoggerTimestamp: envs.SHOW_LOGGER_TIMESTAMP, + noLogfiles: toBoolean(envs.NO_LOGFILES, false), + showLoggerTimestamp: toBoolean(envs.SHOW_LOGGER_TIMESTAMP, false), sentry: { dsn: envs.SENTRY, - routeTimeout: Number.parseInt(envs.SENTRY_ROUTE_TIMEOUT) || 30000, + routeTimeout: toInt(envs.SENTRY_ROUTE_TIMEOUT, 30000), }, // feed config hotlink: { template: envs.HOTLINK_TEMPLATE, - includePaths: envs.HOTLINK_INCLUDE_PATHS && envs.HOTLINK_INCLUDE_PATHS.split(','), - excludePaths: envs.HOTLINK_EXCLUDE_PATHS && envs.HOTLINK_EXCLUDE_PATHS.split(','), + includePaths: envs.HOTLINK_INCLUDE_PATHS ? envs.HOTLINK_INCLUDE_PATHS.split(',') : undefined, + excludePaths: envs.HOTLINK_EXCLUDE_PATHS ? envs.HOTLINK_EXCLUDE_PATHS.split(',') : undefined, }, feature: { - allow_user_hotlink_template: envs.ALLOW_USER_HOTLINK_TEMPLATE === 'true', + allow_user_hotlink_template: toBoolean(envs.ALLOW_USER_HOTLINK_TEMPLATE, false), filter_regex_engine: envs.FILTER_REGEX_ENGINE || 're2', - allow_user_supply_unsafe_domain: envs.ALLOW_USER_SUPPLY_UNSAFE_DOMAIN === 'true', + allow_user_supply_unsafe_domain: toBoolean(envs.ALLOW_USER_SUPPLY_UNSAFE_DOMAIN, false), }, suffix: envs.SUFFIX, - titleLengthLimit: Number.parseInt(envs.TITLE_LENGTH_LIMIT) || 150, + titleLengthLimit: toInt(envs.TITLE_LENGTH_LIMIT, 150), openai: { apiKey: envs.OPENAI_API_KEY, model: envs.OPENAI_MODEL || 'gpt-3.5-turbo-16k', - temperature: envs.OPENAI_TEMPERATURE || 0.2, - maxTokens: envs.OPENAI_MAX_TOKENS || null, + temperature: toInt(envs.OPENAI_TEMPERATURE, 0.2), + maxTokens: toInt(envs.OPENAI_MAX_TOKENS, 0) || undefined, endpoint: envs.OPENAI_API_ENDPOINT || 'https://api.openai.com/v1', prompt: envs.OPENAI_PROMPT || 'Please summarize the following article and reply with markdown format.', }, @@ -263,7 +568,7 @@ const calculateValue = () => { }, pixiv: { refreshToken: envs.PIXIV_REFRESHTOKEN, - bypassCdn: envs.PIXIV_BYPASS_CDN && envs.PIXIV_BYPASS_CDN !== '0' && envs.PIXIV_BYPASS_CDN !== 'false', + bypassCdn: toBoolean(envs.PIXIV_BYPASS_CDN, false), bypassCdnHostname: envs.PIXIV_BYPASS_HOSTNAME || 'public-api.secure.pixiv.net', bypassCdnDoh: envs.PIXIV_BYPASS_DOH || 'https://1.1.1.1/dns-query', imgProxy: envs.PIXIV_IMG_PROXY || 'https://i.pixiv.re', @@ -340,20 +645,17 @@ if (envs.REMOTE_CONFIG) { if (data) { envs = Object.assign(envs, data); calculateValue(); - require('@/utils/logger').info('Remote config loaded.'); + require('@/utils/logger').default.info('Remote config loaded.'); } }) .catch((error) => { - require('@/utils/logger').error('Remote config load failed.', error); + require('@/utils/logger').default.error('Remote config load failed.', error); }); } -module.exports = { - set: (env) => { - envs = Object.assign(process.env, env); - calculateValue(); - }, - get value() { - return value; - }, -}; +export const config: Config = _value!; + +export const setConfig = (env: Partial) => { + envs = Object.assign(process.env, env); + calculateValue(); +} diff --git a/lib/errors/RequestInProgress.js b/lib/errors/RequestInProgress.ts similarity index 53% rename from lib/errors/RequestInProgress.js rename to lib/errors/RequestInProgress.ts index 222be7e91d..99118977e8 100644 --- a/lib/errors/RequestInProgress.js +++ b/lib/errors/RequestInProgress.ts @@ -1,3 +1,3 @@ class RequestInProgressError extends Error {} -module.exports = RequestInProgressError; +export default RequestInProgressError; diff --git a/lib/errors/index.js b/lib/errors/index.js deleted file mode 100644 index 63fe1f2820..0000000000 --- a/lib/errors/index.js +++ /dev/null @@ -1,3 +0,0 @@ -module.exports = { - RequestInProgressError: require('./RequestInProgress'), -}; diff --git a/lib/errors/index.ts b/lib/errors/index.ts new file mode 100644 index 0000000000..97027ed4db --- /dev/null +++ b/lib/errors/index.ts @@ -0,0 +1,3 @@ +import _RequestInProgressError from './RequestInProgress'; + +export const RequestInProgressError = _RequestInProgressError; diff --git a/lib/index.ts b/lib/index.ts new file mode 100644 index 0000000000..ff31db74ba --- /dev/null +++ b/lib/index.ts @@ -0,0 +1,51 @@ +import { serve } from '@hono/node-server' +import { Handler, Hono } from 'hono' + +import cache from '@/middleware/cache' +import template from '@/middleware/template' +import onerror from '@/middleware/onerror' +import accessControl from '@/middleware/access-control' +import debug from '@/middleware/debug' +import header from '@/middleware/header' +import antiHotlink from '@/middleware/anti-hotlink' +import parameter from '@/middleware/parameter' + +import routes from '@/routes' +import { config } from '@/config' + +const app = new Hono() + +app.use('*', onerror); +app.use('*', accessControl); +app.use('*', debug); +app.use('*', header); +app.use('*', template); +app.use('*', antiHotlink); +app.use('*', parameter); +app.use('*', cache); + +for (const name in routes) { + const subApp = app.basePath(`/${name}`) + routes[name]({ + get: (path, handler) => { + const wrapedHandler: Handler = async (ctx, ...args) => { + if (!ctx.get('data')) { + await handler(ctx, ...args) + } + } + subApp.get(path, wrapedHandler) + }, + }) +} + +app.get('/', (c) => { + return c.text('Hello Hono!') +}) + +const port = config.connect.port +console.log(`Server is running on port ${port}`) + +serve({ + fetch: app.fetch, + port +}) diff --git a/lib/middleware/access-control.js b/lib/middleware/access-control.ts similarity index 59% rename from lib/middleware/access-control.js rename to lib/middleware/access-control.ts index a7b4ab2ba5..4d1b41da0f 100644 --- a/lib/middleware/access-control.js +++ b/lib/middleware/access-control.ts @@ -1,9 +1,11 @@ -const config = require('@/config').value; -const md5 = require('@/utils/md5'); -const isLocalhost = require('is-localhost-ip'); +import type { MiddlewareHandler, Context } from 'hono'; +import { config } from '@/config'; +import md5 from '@/utils/md5'; +import isLocalhost from 'is-localhost-ip'; +import { getIp } from '@/utils/helpers'; -const reject = (ctx) => { - ctx.response.status = 403; +const reject = (ctx: Context) => { + ctx.status(403); throw new Error('Authentication failed. Access denied.'); }; @@ -11,7 +13,7 @@ const reject = (ctx) => { const ipv4Pattern = /^(\d{1,3}\.){3}\d{1,3}$/; const cidrPattern = /((?:\d{1,3}\.){3}\d{1,3})\/(\d{1,2})/; -const ipInCidr = (cidr, ip) => { +const ipInCidr = (cidr: string, ip: string) => { const cidrMatch = cidr.match(cidrPattern); const ipMatch = ip.match(ipv4Pattern); if (!cidrMatch || !ipMatch) { @@ -23,25 +25,25 @@ const ipInCidr = (cidr, ip) => { return cidrIpBits === ipBits; }; -const ipv4ToBitsring = (ip) => +const ipv4ToBitsring = (ip: string) => ip .split('.') .map((part) => ('00000000' + Number.parseInt(part).toString(2)).slice(-8)) .join(''); -module.exports = async (ctx, next) => { - const ip = ctx.ips[0] || ctx.ip; - const requestPath = ctx.request.path; - const requestUA = ctx.request.header['user-agent']; - const accessKey = ctx.query.key; - const accessCode = ctx.query.code; +const middleware: MiddlewareHandler = async (ctx, next) => { + const ip = getIp(ctx); + const requestPath = ctx.req.path; + const requestUA = ctx.req.header('user-agent'); + const accessKey = ctx.req.query('key'); + const accessCode = ctx.req.query('code'); const isControlled = config.accessKey || config.allowlist || config.denylist; - const allowLocalhost = config.allowLocalhost && (await isLocalhost(ip)); + const allowLocalhost = config.allowLocalhost && ip && (await isLocalhost(ip)); const grant = async () => { - if (ctx.response.status !== 403) { + if (ctx.res.status !== 403) { await next(); } }; @@ -57,14 +59,16 @@ module.exports = async (ctx, next) => { return grant(); } - if (config.allowlist && config.allowlist.some((item) => ip.includes(item) || ipInCidr(item, ip) || requestPath.includes(item) || requestUA.includes(item))) { + if (config.allowlist && config.allowlist.some((item) => ip?.includes(item) || (ip && ipInCidr(item, ip)) || requestPath.includes(item) || requestUA?.includes(item))) { return grant(); } - if (config.denylist && !config.denylist.some((item) => ip.includes(item) || ipInCidr(item, ip) || requestPath.includes(item) || requestUA.includes(item))) { + if (config.denylist && !config.denylist.some((item) => ip?.includes(item) || (ip && ipInCidr(item, ip)) || requestPath.includes(item) || requestUA?.includes(item))) { return grant(); } reject(ctx); } }; + +export default middleware; \ No newline at end of file diff --git a/lib/middleware/anti-hotlink.js b/lib/middleware/anti-hotlink.ts similarity index 68% rename from lib/middleware/anti-hotlink.js rename to lib/middleware/anti-hotlink.ts index 9a216126ab..5e5c8ce32f 100644 --- a/lib/middleware/anti-hotlink.js +++ b/lib/middleware/anti-hotlink.ts @@ -1,15 +1,16 @@ -const config = require('@/config').value; -const cheerio = require('cheerio'); -const logger = require('@/utils/logger'); -const path = require('path'); -const { art } = require('@/utils/render'); +import { config } from '@/config'; +import { load, type CheerioAPI } from 'cheerio'; +import logger from '@/utils/logger'; +import * as path from 'node:path'; +import render from '@/utils/render'; +import { type MiddlewareHandler } from 'hono'; const templateRegex = /\${([^{}]+)}/g; const allowedUrlProperties = new Set(['hash', 'host', 'hostname', 'href', 'origin', 'password', 'pathname', 'port', 'protocol', 'search', 'searchParams', 'username']); const IframeWrapperTemplate = path.join(__dirname, 'templates/iframe.art'); // match path or sub-path -const matchPath = (path, paths) => { +const matchPath = (path: string, paths: string[]) => { for (const p of paths) { if (path.startsWith(p) && (path.length === p.length || path[p.length] === '/')) { return true; @@ -19,13 +20,13 @@ const matchPath = (path, paths) => { }; // return ture if the path needs to be processed -const filterPath = (path) => { +const filterPath = (path: string) => { const include = config.hotlink.includePaths; const exclude = config.hotlink.excludePaths; return !(include && !matchPath(path, include)) && !(exclude && matchPath(path, exclude)); }; -const interpolate = (str, obj) => +const interpolate = (str: string, obj: Record) => str.replaceAll(templateRegex, (_, prop) => { let needEncode = false; if (prop.endsWith('_ue')) { @@ -35,7 +36,7 @@ const interpolate = (str, obj) => } return needEncode ? encodeURIComponent(obj[prop]) : obj[prop]; }); -const parseUrl = (str) => { +const parseUrl = (str: string) => { let url; try { url = new URL(str); @@ -45,7 +46,7 @@ const parseUrl = (str) => { return url; }; -const replaceUrls = ($, selector, template, attribute = 'src') => { +const replaceUrls = ($: CheerioAPI, selector: string, template: string, attribute = 'src') => { $(selector).each(function () { const old_src = $(this).attr(attribute); if (old_src) { @@ -58,15 +59,15 @@ const replaceUrls = ($, selector, template, attribute = 'src') => { }); }; -const wrapWithIframe = ($, selector) => { +const wrapWithIframe = ($: CheerioAPI, selector: string) => { $(selector).each((_, elem) => { - elem = $(elem); - elem.replaceWith(art(IframeWrapperTemplate, { content: elem.toString() })); + const $elem = $(elem); + $elem.replaceWith(render.art(IframeWrapperTemplate, { content: elem.toString() })); }); }; -const process = (html, image_hotlink_template, multimedia_hotlink_template, wrap_multimedia_in_iframe) => { - const $ = cheerio.load(html, undefined, false); +const process = (html: string, image_hotlink_template?: string, multimedia_hotlink_template?: string, wrap_multimedia_in_iframe?: boolean) => { + const $ = load(html, undefined, false); if (image_hotlink_template) { replaceUrls($, 'img, picture > source', image_hotlink_template); replaceUrls($, 'video[poster]', image_hotlink_template, 'poster'); @@ -83,7 +84,7 @@ const process = (html, image_hotlink_template, multimedia_hotlink_template, wrap return $.html(); }; -const validateTemplate = (template) => { +const validateTemplate = (template?: string) => { if (!template) { return; } @@ -95,12 +96,12 @@ const validateTemplate = (template) => { } }; -module.exports = async (ctx, next) => { +const middleware: MiddlewareHandler = async (ctx, next) => { await next(); let image_hotlink_template; let multimedia_hotlink_template; - const shouldWrapInIframe = ctx.query.wrap_multimedia_in_iframe === '1'; + const shouldWrapInIframe = ctx.req.query('wrap_multimedia_in_iframe') === '1'; // Read params if enabled if (config.feature.allow_user_hotlink_template) { @@ -109,13 +110,13 @@ module.exports = async (ctx, next) => { // A risk is that the media URLs will be replaced by user-supplied templates, // so a user could literally take the control of "where are the media from", // but only in their personal-use feed URL. - multimedia_hotlink_template = ctx.query.multimedia_hotlink_template; - image_hotlink_template = ctx.query.image_hotlink_template; + multimedia_hotlink_template = ctx.req.query('multimedia_hotlink_template'); + image_hotlink_template = ctx.req.query('image_hotlink_template'); } // Force config hotlink template on conflict if (config.hotlink.template) { - image_hotlink_template = filterPath(ctx.request.path) ? config.hotlink.template : undefined; + image_hotlink_template = filterPath(ctx.req.path) ? config.hotlink.template : undefined; } if (!image_hotlink_template && !multimedia_hotlink_template && !shouldWrapInIframe) { @@ -129,17 +130,22 @@ module.exports = async (ctx, next) => { // and here we will only check them in description. // Use Cheerio to load the description as html and filter all // image link - if (ctx.state.data) { - if (ctx.state.data.description) { - ctx.state.data.description = process(ctx.state.data.description, image_hotlink_template, multimedia_hotlink_template, shouldWrapInIframe); + const data = ctx.get('data'); + if (data) { + if (data.description) { + data.description = process(data.description, image_hotlink_template, multimedia_hotlink_template, shouldWrapInIframe); } - if (ctx.state.data.item) { - for (const item of ctx.state.data.item) { + if (data.item) { + for (const item of data.item) { if (item.description) { item.description = process(item.description, image_hotlink_template, multimedia_hotlink_template, shouldWrapInIframe); } } } + + ctx.set('data', data); } }; + +export default middleware; \ No newline at end of file diff --git a/lib/middleware/cache.ts b/lib/middleware/cache.ts new file mode 100644 index 0000000000..4140c5ec83 --- /dev/null +++ b/lib/middleware/cache.ts @@ -0,0 +1,68 @@ +import xxhash from 'xxhash-wasm'; +import type { MiddlewareHandler } from 'hono'; + +import { config } from '@/config'; +import { RequestInProgressError } from '@/errors'; +import cacheModule from '@/utils/cache/index' + +// only give cache string, as the `!` condition tricky +// md5 is used to shrink key size +// plz, write these tips in comments! +const middleware: MiddlewareHandler = async (ctx, next) => { + const { h64ToString } = await xxhash(); + const key = 'rsshub:koa-redis-cache:' + h64ToString(ctx.req.path); + const controlKey = 'rsshub:path-requested:' + h64ToString(ctx.req.path); + + if (!cacheModule.status.available) { + await next(); + return; + } + + const isRequesting = await cacheModule.globalCache.get(controlKey); + + if (isRequesting === '1') { + throw new RequestInProgressError('This path is currently fetching, please come back later!'); + } + + try { + const value = await cacheModule.globalCache.get(key); + + if (value) { + ctx.status(200) + if (config.cache.type === 'redis') { + ctx.header('X-Koa-Redis-Cache', 'true') + } else if (config.cache.type === 'memory') { + ctx.header('X-Koa-Memory-Cache', 'true') + } + ctx.set('data', JSON.parse(value)) + await next(); + return; + } + } catch (error) { + throw error; + } + + // Doesn't hit the cache? We need to let others know! + await cacheModule.globalCache.set(controlKey, '1', config.cache.requestTimeout); + + try { + await next(); + } catch (error) { + await cacheModule.globalCache.set(controlKey, '0', config.cache.requestTimeout); + throw error; + } + + const data = ctx.get('data') + if (ctx.res.headers.get('Cache-Control') !== 'no-cache' && data) { + data.lastBuildDate = new Date().toUTCString(); + ctx.set('data', data) + const body = JSON.stringify(data); + await cacheModule.globalCache.set(key, body, config.cache.routeExpire); + } + + // We need to let it go, even no cache set. + // Wait to set cache so the next request could be handled correctly + await cacheModule.globalCache.set(controlKey, '0', config.cache.requestTimeout); +}; + +export default middleware; diff --git a/lib/middleware/cache/index.js b/lib/middleware/cache/index.js deleted file mode 100644 index bde7cdc0aa..0000000000 --- a/lib/middleware/cache/index.js +++ /dev/null @@ -1,139 +0,0 @@ -const xxhash = require('xxhash-wasm'); -const config = require('@/config').value; -const logger = require('@/utils/logger'); -const { RequestInProgressError } = require('@/errors'); - -const globalCache = { - get: () => null, - set: () => null, -}; - -let cacheModule = { - get: () => null, - set: () => null, - status: { available: false }, - clients: {}, -}; - -if (config.cache.type === 'redis') { - cacheModule = require('./redis'); - const { redisClient } = cacheModule.clients; - globalCache.get = async (key) => { - if (key && cacheModule.status.available) { - const value = await redisClient.get(key); - return value; - } - }; - globalCache.set = cacheModule.set; -} else if (config.cache.type === 'memory') { - cacheModule = require('./memory'); - const { memoryCache } = cacheModule.clients; - globalCache.get = (key) => { - if (key && cacheModule.status.available) { - return memoryCache.get(key, { updateAgeOnGet: false }); - } - }; - globalCache.set = (key, value, maxAge) => { - if (!value || value === 'undefined') { - value = ''; - } - if (typeof value === 'object') { - value = JSON.stringify(value); - } - if (key) { - return memoryCache.set(key, value, { ttl: maxAge * 1000 }); - } - }; -} else { - logger.error('Cache not available, concurrent requests are not limited. This could lead to bad behavior.'); -} - -// only give cache string, as the `!` condition tricky -// md5 is used to shrink key size -// plz, write these tips in comments! -module.exports = function (app) { - const { get, set, status } = cacheModule; - app.context.cache = { - ...cacheModule, - tryGet: async (key, getValueFunc, maxAge = config.cache.contentExpire, refresh = true) => { - if (typeof key !== 'string') { - throw new TypeError('Cache key must be a string'); - } - let v = await get(key, refresh); - if (v) { - let parsed; - try { - parsed = JSON.parse(v); - } catch { - parsed = null; - } - if (parsed) { - v = parsed; - } - } else { - v = await getValueFunc(); - set(key, v, maxAge); - } - - return v; - }, - globalCache, - }; - - return async (ctx, next) => { - const { h64ToString } = await xxhash(); - const key = 'rsshub:koa-redis-cache:' + h64ToString(ctx.request.path); - const controlKey = 'rsshub:path-requested:' + h64ToString(ctx.request.path); - - if (!status.available) { - return next(); - } - - const isRequesting = await globalCache.get(controlKey); - - if (isRequesting === '1') { - throw new RequestInProgressError('This path is currently fetching, please come back later!'); - } - - try { - const value = await globalCache.get(key); - - if (value) { - ctx.response.status = 200; - if (config.cache.type === 'redis') { - ctx.response.set({ - 'X-Koa-Redis-Cache': 'true', - }); - } else if (config.cache.type === 'memory') { - ctx.response.set({ - 'X-Koa-Memory-Cache': 'true', - }); - } - ctx.state.data = JSON.parse(value); - return; - } - } catch { - // - } - - // Doesn't hit the cache? We need to let others know! - await globalCache.set(controlKey, '1', config.cache.requestTimeout); - - try { - await next(); - } catch (error) { - await globalCache.set(controlKey, '0', config.cache.requestTimeout); - throw error; - } - - if (ctx.response.get('Cache-Control') !== 'no-cache' && ctx.state && ctx.state.data) { - ctx.state.data.lastBuildDate = new Date().toUTCString(); - const body = JSON.stringify(ctx.state.data); - await globalCache.set(key, body, config.cache.routeExpire); - } - - // We need to let it go, even no cache set. - // Wait to set cache so the next request could be handled correctly - await globalCache.set(controlKey, '0', config.cache.requestTimeout); - }; -}; diff --git a/lib/middleware/debug.js b/lib/middleware/debug.js deleted file mode 100644 index 644795418a..0000000000 --- a/lib/middleware/debug.js +++ /dev/null @@ -1,25 +0,0 @@ -module.exports = async (ctx, next) => { - if (!ctx.debug.paths[ctx.request.path]) { - ctx.debug.paths[ctx.request.path] = 0; - } - ctx.debug.paths[ctx.request.path]++; - - ctx.debug.request++; - - await next(); - - if (!ctx.debug.routes[ctx._matchedRoute]) { - ctx._matchedRoute && (ctx.debug.routes[ctx._matchedRoute] = 0); - } - ctx._matchedRoute && ctx.debug.routes[ctx._matchedRoute]++; - - if (ctx.response.get('X-Koa-Redis-Cache') || ctx.response.get('X-Koa-Memory-Cache')) { - ctx.debug.hitCache++; - } - - ctx.state.debuged = true; - - if (ctx.status === 304) { - ctx.debug.etag++; - } -}; diff --git a/lib/middleware/debug.ts b/lib/middleware/debug.ts new file mode 100644 index 0000000000..e02e6ea739 --- /dev/null +++ b/lib/middleware/debug.ts @@ -0,0 +1,43 @@ +import { MiddlewareHandler } from "hono"; +import { getRouteNameFromPath } from '@/utils/helpers'; + +const middleware: MiddlewareHandler = async (ctx, next) => { + const debug = Object.assign({ + hitCache: 0, + request: 0, + etag: 0, + paths: [], + routes: [], + errorPaths: [], + errorRoutes: [], + }, ctx.get('debug')); + + if (!debug.paths[ctx.req.path]) { + debug.paths[ctx.req.path] = 0; + } + debug.paths[ctx.req.path]++; + + debug.request++; + + await next(); + + const routeName = getRouteNameFromPath(ctx.req.path); + if (routeName) { + if (!debug.routes[routeName]) { + debug.routes[routeName] = 0; + } + debug.routes[routeName]++; + } + + if (ctx.res.headers.get('X-Koa-Redis-Cache') || ctx.res.headers.get('X-Koa-Memory-Cache')) { + debug.hitCache++; + } + + ctx.set('debuged', true); + + if (ctx.res.status === 304) { + debug.etag++; + } +}; + +export default middleware; \ No newline at end of file diff --git a/lib/middleware/header.js b/lib/middleware/header.js deleted file mode 100644 index 762363f9d4..0000000000 --- a/lib/middleware/header.js +++ /dev/null @@ -1,45 +0,0 @@ -const etagCalculate = require('etag'); -const logger = require('@/utils/logger'); -const config = require('@/config').value; -const headers = { - 'Access-Control-Allow-Methods': 'GET', - 'Content-Type': 'application/xml; charset=utf-8', - 'Cache-Control': `public, max-age=${config.cache.routeExpire}`, - 'X-Content-Type-Options': 'nosniff', -}; -if (config.nodeName) { - headers['RSSHub-Node'] = config.nodeName; -} - -module.exports = async (ctx, next) => { - logger.info(`${ctx.url}, user IP: ${ctx.ips[0] || ctx.ip}`); - ctx.set(headers); - ctx.set({ - 'Access-Control-Allow-Origin': config.allowOrigin || ctx.host, - }); - - await next(); - - if (!ctx.body || typeof ctx.body !== 'string' || ctx.response.get('ETag')) { - return; - } - - const status = Math.trunc(ctx.status / 100); - if (2 !== status) { - return; - } - - ctx.set('ETag', etagCalculate(ctx.body.replace(/(.*)<\/lastBuildDate>/, '').replace(//, ''))); - - if (ctx.fresh) { - ctx.status = 304; - ctx.body = null; - } else { - const match = ctx.body.match(/(.*)<\/lastBuildDate>/); - if (match) { - ctx.set({ - 'Last-Modified': match[1], - }); - } - } -}; diff --git a/lib/middleware/header.ts b/lib/middleware/header.ts new file mode 100644 index 0000000000..5515085b20 --- /dev/null +++ b/lib/middleware/header.ts @@ -0,0 +1,59 @@ +import { getIp } from "@/utils/helpers"; +import { MiddlewareHandler } from "hono"; +import etagCalculate from "etag"; +import logger from "@/utils/logger"; +import { config } from "@/config"; + +const headers: Record = { + 'Access-Control-Allow-Methods': 'GET', + 'Content-Type': 'application/xml; charset=utf-8', + 'Cache-Control': `public, max-age=${config.cache.routeExpire}`, + 'X-Content-Type-Options': 'nosniff', +}; +if (config.nodeName) { + headers['RSSHub-Node'] = config.nodeName; +} + +function etagMatches(etag: string, ifNoneMatch: string | null) { + return ifNoneMatch != null && ifNoneMatch.split(/,\s*/).indexOf(etag) > -1 +} + +const middleware: MiddlewareHandler = async (ctx, next) => { + const ip = getIp(ctx); + logger.info(`${ctx.req.url}, user IP: ${ip}`); + + for (const key in headers) { + ctx.header(key, headers[key]) + } + ctx.header('Access-Control-Allow-Origin', config.allowOrigin || new URL(ctx.req.url).host); + + await next(); + + if (!ctx.res.body || ctx.res.headers.get('ETag')) { + return; + } + + const status = Math.trunc(ctx.res.status / 100); + if (2 !== status) { + return; + } + + const res = ctx.res as Response; + const body = await res.clone().text(); + const etag = etagCalculate(body.replace(/(.*)<\/lastBuildDate>/, '').replace(//, '')); + + ctx.set('ETag', etag); + + const ifNoneMatch = ctx.req.header('If-None-Match') ?? null; + if (etagMatches(etag, ifNoneMatch)) { + ctx.status(304); + ctx.body(null); + } else { + const match = body.match(/(.*)<\/lastBuildDate>/); + if (match) { + ctx.header('Last-Modified', match[1]); + } + } +}; + +export default middleware; \ No newline at end of file diff --git a/lib/middleware/onerror.js b/lib/middleware/onerror.js deleted file mode 100644 index a4272b0aae..0000000000 --- a/lib/middleware/onerror.js +++ /dev/null @@ -1,119 +0,0 @@ -const logger = require('@/utils/logger'); -const config = require('@/config').value; -const art = require('art-template'); -const path = require('path'); - -const { RequestInProgressError } = require('@/errors'); - -let Sentry; -let gitHash; - -if (config.sentry.dsn) { - Sentry = Sentry || require('@sentry/node'); - Sentry.init({ - dsn: config.sentry.dsn, - }); - Sentry.getCurrentScope().setTag('node_name', config.nodeName); - - logger.info('Sentry inited.'); -} - -try { - gitHash = require('git-rev-sync').short(); -} catch { - gitHash = (process.env.HEROKU_SLUG_COMMIT && process.env.HEROKU_SLUG_COMMIT.slice(0, 7)) || (process.env.VERCEL_GIT_COMMIT_SHA && process.env.VERCEL_GIT_COMMIT_SHA.slice(0, 7)) || 'unknown'; -} - -module.exports = async (ctx, next) => { - try { - const time = Date.now(); - await next(); - if (config.sentry.dsn && Date.now() - time >= config.sentry.routeTimeout) { - Sentry.withScope((scope) => { - scope.setTag('route', ctx._matchedRoute); - scope.setTag('name', ctx.request.path.split('/')[1]); - scope.addEventProcessor((event) => Sentry.Handlers.parseRequest(event, ctx.request)); - Sentry.captureException(new Error('Route Timeout')); - }); - } - } catch (error) { - if (error instanceof Error && !error.stack.split('\n')[1].includes('lib/middleware/parameter.js')) { - // Append v2 route path if a route throws an error - // since koa-mount will remove the mount path from ctx.request.path - // https://github.com/koajs/mount/issues/62 - ctx.request.path = (ctx.mountPath ?? '') + ctx.request.path; - ctx._matchedRoute = ctx._matchedRoute ? (ctx.mountPath ?? '') + ctx._matchedRoute : ctx._matchedRoute; - } - - let message = error; - if (error.name && (error.name === 'HTTPError' || error.name === 'RequestError')) { - message = `${error.message}: target website might be blocking our access, you can host your own RSSHub instance for a better usability.`; - } else if (error instanceof Error) { - message = process.env.NODE_ENV === 'production' ? error.message : error.stack; - } - - logger.error(`Error in ${ctx.request.path}: ${message}`); - - if (config.isPackage) { - ctx.body = { - error: { - message: error.message ?? error, - }, - }; - } else { - ctx.set({ - 'Content-Type': 'text/html; charset=UTF-8', - }); - - if (error instanceof RequestInProgressError) { - ctx.status = 503; - message = error.message; - ctx.set('Cache-Control', `public, max-age=${config.cache.requestTimeout}`); - } else if (ctx.status === 403) { - message = error.message; - } else { - ctx.status = 404; - } - - const requestPath = ctx.request.path; - - ctx.body = art(path.resolve(__dirname, '../views/error.art'), { - requestPath, - message, - errorPath: ctx.path, - nodeVersion: process.version, - gitHash, - }); - } - - if (!ctx.debug.errorPaths[ctx.request.path]) { - ctx.debug.errorPaths[ctx.request.path] = 0; - } - ctx.debug.errorPaths[ctx.request.path]++; - - if (!ctx.debug.errorRoutes[ctx._matchedRoute]) { - ctx._matchedRoute && (ctx.debug.errorRoutes[ctx._matchedRoute] = 0); - } - ctx._matchedRoute && ctx.debug.errorRoutes[ctx._matchedRoute]++; - - if (!ctx.state.debuged) { - if (!ctx.debug.routes[ctx._matchedRoute]) { - ctx._matchedRoute && (ctx.debug.routes[ctx._matchedRoute] = 0); - } - ctx._matchedRoute && ctx.debug.routes[ctx._matchedRoute]++; - - if (ctx.response.get('X-Koa-Redis-Cache') || ctx.response.get('X-Koa-Memory-Cache')) { - ctx.debug.hitCache++; - } - } - - if (config.sentry.dsn) { - Sentry.withScope((scope) => { - scope.setTag('route', ctx._matchedRoute); - scope.setTag('name', ctx.request.path.split('/')[1]); - scope.addEventProcessor((event) => Sentry.Handlers.parseRequest(event, ctx.request)); - Sentry.captureException(error); - }); - } - } -}; diff --git a/lib/middleware/onerror.ts b/lib/middleware/onerror.ts new file mode 100644 index 0000000000..a332f1ea3f --- /dev/null +++ b/lib/middleware/onerror.ts @@ -0,0 +1,92 @@ +import { MiddlewareHandler } from "hono"; +import logger from "@/utils/logger"; +import { config } from "@/config"; +import art from 'art-template'; +import * as path from 'node:path'; +import { RequestInProgressError } from '@/errors'; +import Sentry from '@sentry/node'; +import { getRouteNameFromPath } from "@/utils/helpers"; +import gitRevSync from 'git-rev-sync'; + +let gitHash; + +if (config.sentry.dsn) { + Sentry.init({ + dsn: config.sentry.dsn, + }); + Sentry.getCurrentScope().setTag('node_name', config.nodeName); + + logger.info('Sentry inited.'); +} + +try { + gitHash = gitRevSync.short(); +} catch { + gitHash = (process.env.HEROKU_SLUG_COMMIT && process.env.HEROKU_SLUG_COMMIT.slice(0, 7)) || (process.env.VERCEL_GIT_COMMIT_SHA && process.env.VERCEL_GIT_COMMIT_SHA.slice(0, 7)) || 'unknown'; +} + +const middleware: MiddlewareHandler = async (ctx, next) => { + try { + const time = Date.now(); + await next(); + if (config.sentry.dsn && Date.now() - time >= config.sentry.routeTimeout) { + Sentry.withScope((scope) => { + scope.setTag('name', getRouteNameFromPath(ctx.req.path)); + Sentry.captureException(new Error('Route Timeout')); + }); + } + } catch (error: any) { + let message = error; + if (error.name && (error.name === 'HTTPError' || error.name === 'RequestError')) { + message = `${error.message}: target website might be blocking our access, you can host your own RSSHub instance for a better usability.`; + } else if (error instanceof Error) { + message = process.env.NODE_ENV === 'production' ? error.message : error.stack; + } + + logger.error(`Error in ${ctx.req.path}: ${message}`); + + if (config.isPackage) { + ctx.json({ + error: { + message: error.message ?? error, + }, + }); + } else { + ctx.header('Content-Type', 'text/html; charset=UTF-8'); + + if (error instanceof RequestInProgressError) { + ctx.status(503); + message = error.message; + ctx.set('Cache-Control', `public, max-age=${config.cache.requestTimeout}`); + } else if (ctx.res.status === 403) { + message = error.message; + } else { + ctx.status(404); + } + + const requestPath = ctx.req.path; + + ctx.body = art(path.resolve(__dirname, '../views/error.art'), { + requestPath, + message, + errorPath: ctx.req.path, + nodeVersion: process.version, + gitHash, + }); + } + + const debug = ctx.get('debug'); + if (ctx.res.headers.get('X-Koa-Redis-Cache') || ctx.res.headers.get('X-Koa-Memory-Cache')) { + debug.hitCache++; + } + + if (config.sentry.dsn) { + Sentry.withScope((scope) => { + scope.setTag('name', ctx.req.path.split('/')[1]); + Sentry.captureException(error); + }); + } + } +}; + +export default middleware; \ No newline at end of file diff --git a/lib/middleware/parameter.js b/lib/middleware/parameter.js deleted file mode 100644 index 86f5fa85ee..0000000000 --- a/lib/middleware/parameter.js +++ /dev/null @@ -1,370 +0,0 @@ -const entities = require('entities'); -const cheerio = require('cheerio'); -const { simplecc } = require('simplecc-wasm'); -const got = require('@/utils/got'); -const config = require('@/config').value; -const { RE2JS } = require('re2js'); -const md = require('markdown-it')({ - html: true, -}); -const htmlToText = require('html-to-text'); - -let mercury_parser; - -const resolveRelativeLink = ($, elem, attr, baseUrl) => { - const $elem = $(elem); - - if (baseUrl) { - try { - const oldAttr = $elem.attr(attr); - if (oldAttr) { - // e.g. should leave