From 4591116733fc4fb65767b841ffef4680766a6b51 Mon Sep 17 00:00:00 2001 From: DIYgod Date: Sun, 21 Jan 2024 18:35:11 +0800 Subject: [PATCH] fix: middleware errors --- lib/errors/index.ts | 2 +- lib/index.ts | 2 +- lib/middleware/api-response-handler.js | 147 ------------------------- lib/middleware/api-template.js | 9 -- lib/middleware/cache.ts | 6 +- lib/middleware/debug.ts | 2 +- lib/middleware/header.ts | 26 ++--- lib/middleware/load-on-demand.js | 36 ------ lib/middleware/template.ts | 136 +++++++++++------------ lib/middleware/utf8.js | 6 - lib/types.ts | 62 +++++++---- lib/utils/common-utils.ts | 2 +- lib/utils/parse-date.ts | 6 +- lib/views/json.ts | 6 +- 14 files changed, 126 insertions(+), 322 deletions(-) delete mode 100644 lib/middleware/api-response-handler.js delete mode 100644 lib/middleware/api-template.js delete mode 100644 lib/middleware/load-on-demand.js delete mode 100644 lib/middleware/utf8.js diff --git a/lib/errors/index.ts b/lib/errors/index.ts index a400516a11..2c80ca006f 100644 --- a/lib/errors/index.ts +++ b/lib/errors/index.ts @@ -19,7 +19,7 @@ export const errorHandler: ErrorHandler = (error, ctx) => { } const debug = getDebugInfo(); - if (ctx.res.headers.get('X-Koa-Redis-Cache') || ctx.res.headers.get('X-Koa-Memory-Cache')) { + if (ctx.res.headers.get('RSSHub-Cache-Status')) { debug.hitCache++; setDebugInfo(debug); } diff --git a/lib/index.ts b/lib/index.ts index 3691ec74a0..a210498ec4 100644 --- a/lib/index.ts +++ b/lib/index.ts @@ -25,8 +25,8 @@ const app = new Hono() app.use('*', onerror); app.use('*', accessControl); app.use('*', debug); -app.use('*', header); app.use('*', template); +app.use('*', header); app.use('*', antiHotlink); app.use('*', parameter); app.use('*', cache); diff --git a/lib/middleware/api-response-handler.js b/lib/middleware/api-response-handler.js deleted file mode 100644 index e43954750c..0000000000 --- a/lib/middleware/api-response-handler.js +++ /dev/null @@ -1,147 +0,0 @@ -/** - * HTTP Status codes - */ -const statusCodes = { - CONTINUE: 100, - OK: 200, - CREATED: 201, - ACCEPTED: 202, - NO_CONTENT: 204, - BAD_REQUEST: 400, - UNAUTHORIZED: 401, - FORBIDDEN: 403, - NOT_FOUND: 404, - REQUEST_TIMEOUT: 408, - UNPROCESSABLE_ENTITY: 422, - INTERNAL_SERVER_ERROR: 500, - NOT_IMPLEMENTED: 501, - BAD_GATEWAY: 502, - SERVICE_UNAVAILABLE: 503, - GATEWAY_TIME_OUT: 504, -}; - -function responseHandler() { - return async (ctx, next) => { - ctx.res.statusCodes = statusCodes; - ctx.statusCodes = ctx.res.statusCodes; - - ctx.res.success = ({ statusCode, data = null, message = null }) => { - const status = 0; - - ctx.status = statusCode; - ctx.body = { status, data, message }; - }; - - // ctx.res.fail = ({ statusCode, code, data = null, message = null }) => { - // const status = -1; - - // if (!!statusCode && (statusCode >= 400 && statusCode < 500)) { - // ctx.status = statusCode; - // } else if (!(ctx.status >= 400 && ctx.status < 500)) { - // ctx.status = statusCodes.BAD_REQUEST; - // } - - // ctx.body = { status, code, data, message }; - // }; - - // ctx.res.error = ({ statusCode, code, data = null, message = null }) => { - // const status = -2; - - // if (!!statusCode && (statusCode >= 500 && statusCode < 600)) { - // ctx.status = statusCode; - // } else if (!(ctx.status >= 500 && ctx.status < 600)) { - // ctx.status = statusCodes.INTERNAL_SERVER_ERROR; - // } - - // ctx.body = { status, code, data, message }; - // }; - - ctx.res.ok = (params = {}) => { - ctx.res.success({ - ...params, - statusCode: statusCodes.OK, - }); - }; - - // ctx.res.noContent = (params = {}) => { - // ctx.res.success({ - // ...params, - // statusCode: statusCodes.NO_CONTENT, - // }); - // }; - - // ctx.res.badRequest = (params = {}) => { - // ctx.res.fail({ - // ...params, - // statusCode: statusCodes.BAD_REQUEST, - // }); - // }; - - // ctx.res.forbidden = (params = {}) => { - // ctx.res.fail({ - // ...params, - // statusCode: statusCodes.FORBIDDEN, - // }); - // }; - - // ctx.res.notFound = (params = {}) => { - // ctx.res.fail({ - // ...params, - // statusCode: statusCodes.NOT_FOUND, - // }); - // }; - - // ctx.res.requestTimeout = (params = {}) => { - // ctx.res.fail({ - // ...params, - // statusCode: statusCodes.REQUEST_TIMEOUT, - // }); - // }; - - // ctx.res.unprocessableEntity = (params = {}) => { - // ctx.res.fail({ - // ...params, - // statusCode: statusCodes.UNPROCESSABLE_ENTITY, - // }); - // }; - - // ctx.res.internalServerError = (params = {}) => { - // ctx.res.error({ - // ...params, - // statusCode: statusCodes.INTERNAL_SERVER_ERROR, - // }); - // }; - - // ctx.res.notImplemented = (params = {}) => { - // ctx.res.error({ - // ...params, - // statusCode: statusCodes.NOT_IMPLEMENTED, - // }); - // }; - - // ctx.res.badGateway = (params = {}) => { - // ctx.res.error({ - // ...params, - // statusCode: statusCodes.BAD_GATEWAY, - // }); - // }; - - // ctx.res.serviceUnavailable = (params = {}) => { - // ctx.res.error({ - // ...params, - // statusCode: statusCodes.SERVICE_UNAVAILABLE, - // }); - // }; - - // ctx.res.gatewayTimeOut = (params = {}) => { - // ctx.res.error({ - // ...params, - // statusCode: statusCodes.GATEWAY_TIME_OUT, - // }); - // }; - - await next(); - }; -} - -module.exports = responseHandler; diff --git a/lib/middleware/api-template.js b/lib/middleware/api-template.js deleted file mode 100644 index c3f10706db..0000000000 --- a/lib/middleware/api-template.js +++ /dev/null @@ -1,9 +0,0 @@ -module.exports = async (ctx, next) => { - await next(); - if (ctx.request.path.startsWith('/api/')) { - return ctx.res.ok({ - message: `request returned ${ctx.body.counter} ${ctx.body.counter > 1 ? 'routes' : 'route'}`, - data: ctx.body.result, - }); - } -}; diff --git a/lib/middleware/cache.ts b/lib/middleware/cache.ts index 4140c5ec83..47c1eaab3f 100644 --- a/lib/middleware/cache.ts +++ b/lib/middleware/cache.ts @@ -29,11 +29,7 @@ const middleware: MiddlewareHandler = async (ctx, next) => { if (value) { ctx.status(200) - if (config.cache.type === 'redis') { - ctx.header('X-Koa-Redis-Cache', 'true') - } else if (config.cache.type === 'memory') { - ctx.header('X-Koa-Memory-Cache', 'true') - } + ctx.header('RSSHub-Cache-Status', 'HIT') ctx.set('data', JSON.parse(value)) await next(); return; diff --git a/lib/middleware/debug.ts b/lib/middleware/debug.ts index ef6188e29e..d21a203f7e 100644 --- a/lib/middleware/debug.ts +++ b/lib/middleware/debug.ts @@ -12,7 +12,7 @@ const middleware: MiddlewareHandler = async (ctx, next) => { { const debug = getDebugInfo(); - if (ctx.res.headers.get('X-Koa-Redis-Cache') || ctx.res.headers.get('X-Koa-Memory-Cache')) { + if (ctx.res.headers.get('RSSHub-Cache-Status')) { debug.hitCache++; } diff --git a/lib/middleware/header.ts b/lib/middleware/header.ts index 5515085b20..c867e30f20 100644 --- a/lib/middleware/header.ts +++ b/lib/middleware/header.ts @@ -29,31 +29,25 @@ const middleware: MiddlewareHandler = async (ctx, next) => { await next(); - if (!ctx.res.body || ctx.res.headers.get('ETag')) { + const data = ctx.get('data'); + if (!data || ctx.res.headers.get('ETag')) { return; } - const status = Math.trunc(ctx.res.status / 100); - if (2 !== status) { - return; - } + const lastBuildDate = data.lastBuildDate; + delete data.lastBuildDate; + const etag = etagCalculate(JSON.stringify(data)); - const res = ctx.res as Response; - const body = await res.clone().text(); - const etag = etagCalculate(body.replace(/(.*)<\/lastBuildDate>/, '').replace(//, '')); - - ctx.set('ETag', etag); + ctx.header('ETag', etag); const ifNoneMatch = ctx.req.header('If-None-Match') ?? null; if (etagMatches(etag, ifNoneMatch)) { + console.log('in') ctx.status(304); - ctx.body(null); + return ctx.body(null); } else { - const match = body.match(/(.*)<\/lastBuildDate>/); - if (match) { - ctx.header('Last-Modified', match[1]); - } + ctx.header('Last-Modified', lastBuildDate); } }; -export default middleware; \ No newline at end of file +export default middleware; diff --git a/lib/middleware/load-on-demand.js b/lib/middleware/load-on-demand.js deleted file mode 100644 index 837a17cc77..0000000000 --- a/lib/middleware/load-on-demand.js +++ /dev/null @@ -1,36 +0,0 @@ -const mount = require('koa-mount'); -const Router = require('@koa/router'); -const routes = require('../v2router'); -const loadedRoutes = new Set(); - -module.exports = function (app) { - return async function (ctx, next) { - const p = ctx.request.path.split('/').filter(Boolean); - let modName = null; - let mounted = false; - - if (p.length > 0) { - modName = p[0]; - if (loadedRoutes.has(modName)) { - mounted = true; - } else { - const mod = routes[modName]; - // Mount module - if (mod) { - mounted = true; - loadedRoutes.add(modName); - const router = new Router(); - mod(router); - app.use(mount(`/${modName}`, router.routes())).use(router.allowedMethods()); - } - } - } - - await next(); - - // We should only add it when koa router matched - if (mounted && ctx._matchedRoute) { - ctx._matchedRoute = `/${modName}${ctx._matchedRoute}`; - } - }; -}; diff --git a/lib/middleware/template.ts b/lib/middleware/template.ts index b629efa9aa..0808b25bea 100644 --- a/lib/middleware/template.ts +++ b/lib/middleware/template.ts @@ -1,21 +1,15 @@ import render from '@/utils/render'; import * as path from 'node:path'; import { config } from '@/config'; -const typeRegex = /\.(atom|rss|ums|debug\.json|json|\d+\.debug\.html)$/; import utils from '@/utils/common-utils'; import type { MiddlewareHandler } from 'hono'; +import { Data } from '@/types'; const middleware: MiddlewareHandler = async (ctx, next) => { - if (ctx.req.header('user-agent')?.includes('Reeder')) { - ctx.req.path = ctx.req.path.replace(/.com$/, ''); - } - - ctx.set('type', ctx.req.path.match(typeRegex) || ['', '']); - ctx.req.path = ctx.req.path.replace(typeRegex, ''); - await next(); - const outputType = ctx.get('type')[1] || 'rss'; + const data: Data = ctx.get('data'); + const outputType = ctx.req.query('format') || 'rss'; // only enable when debugInfo=true if (config.debugInfo) { @@ -24,87 +18,83 @@ const middleware: MiddlewareHandler = async (ctx, next) => { return ctx.body(ctx.get('json') ? JSON.stringify(ctx.get('json'), null, 4) : JSON.stringify({ message: 'plugin does not set debug json' })); } - if (outputType.endsWith('.debug.html')) { + if (outputType === 'debug.html') { ctx.header('Content-Type', 'text/html; charset=UTF-8'); - const index = Number.parseInt(outputType.match(/(\d+)\.debug\.html$/)[1]); - return ctx.body(ctx.get('data')?.item?.[index]?.description || `ctx.get('data')?.item?.[${index}]?.description not found`); + const index = Number.parseInt(outputType.match(/(\d+)\.debug\.html$/)?.[1] || '0'); + return ctx.body(data?.item?.[index]?.description || `data?.item?.[${index}]?.description not found`); } } - const templateName = outputType === 'atom' ? 'atom.art' : 'rss.art'; - const template = path.resolve(__dirname, `../views/${templateName}`); + const templateName = outputType === 'atom' ? 'atom.art' : 'rss.art'; + const template = path.resolve(__dirname, `../views/${templateName}`); - if (ctx.get('data')) { - for (const prop of ['title', 'subtitle', 'author']) { - if (ctx.get('data')[prop]) { - ctx.get('data')[prop] = utils.collapseWhitespace(ctx.get('data')[prop]); + if (data) { + data.title = utils.collapseWhitespace(data.title); + data.description && (data.description = utils.collapseWhitespace(data.description)); + data.author && (data.author = utils.collapseWhitespace(data.author)); + + if (data.item) { + for (const item of data.item) { + if (item.title) { + item.title = utils.collapseWhitespace(item.title); + // trim title length + for (let length = 0, i = 0; i < item.title.length; i++) { + length += Buffer.from(item.title[i]).length === 1 ? 1 : 2; + if (length > config.titleLengthLimit) { + item.title = `${item.title.slice(0, i)}...`; + break; + } + } } - } - if (ctx.get('data').item) { - for (const item of ctx.get('data').item) { - if (item.title) { - item.title = utils.collapseWhitespace(item.title); - // trim title length - for (let length = 0, i = 0; i < item.title.length; i++) { - length += Buffer.from(item.title[i]).length === 1 ? 1 : 2; - if (length > config.titleLengthLimit) { - item.title = `${item.title.slice(0, i)}...`; - break; - } - } + if (typeof item.author === 'string') { + item.author = utils.collapseWhitespace(item.author); + } else if (typeof item.author === 'object' && item.author !== null) { + for (const a of item.author) { + a.name = utils.collapseWhitespace(a.name); } + if (outputType !== 'json') { + item.author = item.author.map((a: { name: string }) => a.name).join(', '); + } + } - if (typeof item.author === 'string') { - item.author = utils.collapseWhitespace(item.author); - } else if (typeof item.author === 'object' && item.author !== null) { - for (const a of item.author) { - a.name = utils.collapseWhitespace(a.name); - } - if (outputType !== 'json') { - item.author = item.author.map((a: { - name: string; - }) => a.name).join(', '); - } - } + if (item.itunes_duration && ((typeof item.itunes_duration === 'string' && !item.itunes_duration.includes(':')) || (typeof item.itunes_duration === 'number' && !isNaN(item.itunes_duration)))) { + item.itunes_duration = +item.itunes_duration; + item.itunes_duration = + Math.floor(item.itunes_duration / 3600) + ':' + (Math.floor((item.itunes_duration % 3600) / 60) / 100).toFixed(2).slice(-2) + ':' + (((item.itunes_duration % 3600) % 60) / 100).toFixed(2).slice(-2); + } - if (item.itunes_duration && ((typeof item.itunes_duration === 'string' && !item.itunes_duration.includes(':')) || (typeof item.itunes_duration === 'number' && !isNaN(item.itunes_duration)))) { - item.itunes_duration = +item.itunes_duration; - item.itunes_duration = - Math.floor(item.itunes_duration / 3600) + ':' + (Math.floor((item.itunes_duration % 3600) / 60) / 100).toFixed(2).slice(-2) + ':' + (((item.itunes_duration % 3600) % 60) / 100).toFixed(2).slice(-2); - } - - if (outputType !== 'rss') { - item.pubDate = utils.convertDateToISO8601(item.pubDate); - item.updated = utils.convertDateToISO8601(item.updated); - } + if (outputType !== 'rss') { + item.pubDate && (item.pubDate = utils.convertDateToISO8601(item.pubDate)); + item.updated && (item.updated = utils.convertDateToISO8601(item.updated)); } } } + } - const currentDate = new Date(); - const data = { - lastBuildDate: currentDate.toUTCString(), - updated: currentDate.toISOString(), - ttl: Math.trunc(config.cache.routeExpire / 60), - atomlink: ctx.req.url, - ...ctx.get('data'), - }; + const currentDate = new Date(); + const result = { + lastBuildDate: currentDate.toUTCString(), + updated: currentDate.toISOString(), + ttl: Math.trunc(config.cache.routeExpire / 60), + atomlink: ctx.req.url, + ...data, + }; - if (config.isPackage) { - return ctx.body(data); - } + if (config.isPackage) { + return ctx.json(result); + } - if (outputType === 'ums') { - ctx.header('Content-Type', 'application/json; charset=UTF-8'); - return ctx.body(render.rss3Ums(data)); - } else if (outputType === 'json') { - ctx.header('Content-Type', 'application/feed+json; charset=UTF-8'); - return ctx.body(render.json(data)); - } else { - return ctx.body(render.art(template, data)); - } + if (outputType === 'ums') { + ctx.header('Content-Type', 'application/json; charset=UTF-8'); + return ctx.body(render.rss3Ums(data)); + } else if (outputType === 'json') { + ctx.header('Content-Type', 'application/feed+json; charset=UTF-8'); + return ctx.body(render.json(data)); + } else { + return ctx.body(render.art(template, data)); + } }; export default middleware; diff --git a/lib/middleware/utf8.js b/lib/middleware/utf8.js deleted file mode 100644 index 57890897f4..0000000000 --- a/lib/middleware/utf8.js +++ /dev/null @@ -1,6 +0,0 @@ -// https://stackoverflow.com/questions/2507608/error-input-is-not-proper-utf-8-indicate-encoding-using-phps-simplexml-lo/40552083#40552083 -// https://stackoverflow.com/questions/1497885/remove-control-characters-from-php-string/1497928#1497928 -module.exports = async (ctx, next) => { - await next(); - ctx.body = typeof ctx.body === 'object' ? ctx.body : ctx.body.replaceAll(/[\u0000-\u0009\u000B\u000C\u000E-\u001F\u007F]/g, ''); -}; diff --git a/lib/types.ts b/lib/types.ts index f948fd9b7d..c983614112 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -1,24 +1,44 @@ export type DataItem = { - title: string; - description?: string; - pubDate: number | string; - link?: string; - category?: string[]; - author?: string; - doi?: string; + title: string; + description?: string; + pubDate?: number | string | Date; + link?: string; + category?: string[]; + author?: string | { name: string }[]; + doi?: string; + guid?: string; + id?: string; + content?: { + html: string; + text: string; + }; + image?: string; + banner?: string; + updated?: number | string | Date; + language?: string; + enclosure_url?: string; + enclosure_type?: string; + enclosure_title?: string; + enclosure_length?: number; + itunes_duration?: number | string; + + _extra?: Record & { + links?: { + url: string; + type: string; + content_html?: string; + }[]; + }; +}; - _extra?: Record & { - links?: { - url: string; - type: string; - content_html?: string; - }[]; - }; -} export type Data = { - title: string; - description: string; - link?: string; - item: DataItem[]; - allowEmpty?: boolean; -} \ No newline at end of file + title: string; + description?: string; + link?: string; + item?: DataItem[]; + allowEmpty?: boolean; + image?: string; + author?: string; + language?: string; + feedLink?: string; +}; diff --git a/lib/utils/common-utils.ts b/lib/utils/common-utils.ts index c5bb5c5e17..8ef37f1b90 100644 --- a/lib/utils/common-utils.ts +++ b/lib/utils/common-utils.ts @@ -15,7 +15,7 @@ const collapseWhitespace = (str: string) => { return str; }; -const convertDateToISO8601 = (date: string | Date) => { +const convertDateToISO8601 = (date: string | Date | number) => { if (!date) { return date; } diff --git a/lib/utils/parse-date.ts b/lib/utils/parse-date.ts index b0f7c286b2..7f43f0944c 100644 --- a/lib/utils/parse-date.ts +++ b/lib/utils/parse-date.ts @@ -126,7 +126,7 @@ const toDurations = (matches: string[]) => { }; export default { - parse: (date: string, ...options: any) => dayjs(date, ...options).toDate(), + parse: (date: string | number, ...options: any) => dayjs(date, ...options).toDate(), parseRelativeDate: (date: string) => { // 预处理日期字符串 date @@ -176,9 +176,9 @@ export default { const wordMatches = w.regExp.exec(firstMatch); if (wordMatches) { matches.unshift(wordMatches[1]); - + // 取特殊词对应日零时为起点，加上相应的时间长度 - + return w.startAt .set('hour', 0) .set('minute', 0) diff --git a/lib/views/json.ts b/lib/views/json.ts index 8e91d19637..cf9fb60746 100644 --- a/lib/views/json.ts +++ b/lib/views/json.ts @@ -1,9 +1,11 @@ +import { Data } from "@/types"; + /** * This function should be used by RSSHub middleware only. * @param {object} data ctx.state.data * @returns `JSON.stringify`-ed [JSON Feed](https://www.jsonfeed.org/) */ -const json = (data) => { +const json = (data: Data) => { const jsonFeed = { version: 'https://jsonfeed.org/version/1.1', title: data.title || 'RSSHub', @@ -13,7 +15,7 @@ const json = (data) => { icon: data.image, authors: typeof data.author === 'string' ? [{ name: data.author }] : data.author, language: data.language || 'zh-cn', - items: data.item.map((item) => ({ + items: data.item?.map((item) => ({ id: item.guid || item.id || item.link, url: item.link, title: item.title,