opensource/RSSHub
1
0
Fork 0
mirror of https://github.com/DIYgod/RSSHub.git synced 2025-12-17 03:58:49 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: middleware errors

Browse Source
This commit is contained in:
DIYgod
2024-01-21 18:35:11 +08:00
parent e7e3f689bd
commit 4591116733
14 changed files with 126 additions and 322 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
lib/errors/index.ts
View File

@@ -19,7 +19,7 @@ export const errorHandler: ErrorHandler = (error, ctx) => {
} }
const debug = getDebugInfo(); const debug = getDebugInfo();
if (ctx.res.headers.get('X-Koa-Redis-Cache') || ctx.res.headers.get('X-Koa-Memory-Cache')) { if (ctx.res.headers.get('RSSHub-Cache-Status')) {
debug.hitCache++; debug.hitCache++;
setDebugInfo(debug); setDebugInfo(debug);
} }

2
lib/index.ts
View File

@@ -25,8 +25,8 @@ const app = new Hono()
app.use('*', onerror); app.use('*', onerror);
app.use('*', accessControl); app.use('*', accessControl);
app.use('*', debug); app.use('*', debug);
app.use('*', header);
app.use('*', template); app.use('*', template);
app.use('*', header);
app.use('*', antiHotlink); app.use('*', antiHotlink);
app.use('*', parameter); app.use('*', parameter);
app.use('*', cache); app.use('*', cache);

147
lib/middleware/api-response-handler.js
View File

@@ -1,147 +0,0 @@
/**
* HTTP Status codes
*/
const statusCodes = {
CONTINUE: 100,
OK: 200,
CREATED: 201,
ACCEPTED: 202,
NO_CONTENT: 204,
BAD_REQUEST: 400,
UNAUTHORIZED: 401,
FORBIDDEN: 403,
NOT_FOUND: 404,
REQUEST_TIMEOUT: 408,
UNPROCESSABLE_ENTITY: 422,
INTERNAL_SERVER_ERROR: 500,
NOT_IMPLEMENTED: 501,
BAD_GATEWAY: 502,
SERVICE_UNAVAILABLE: 503,
GATEWAY_TIME_OUT: 504,
};
function responseHandler() {
return async (ctx, next) => {
ctx.res.statusCodes = statusCodes;
ctx.statusCodes = ctx.res.statusCodes;
ctx.res.success = ({ statusCode, data = null, message = null }) => {
const status = 0;
ctx.status = statusCode;
ctx.body = { status, data, message };
};
// ctx.res.fail = ({ statusCode, code, data = null, message = null }) => {
// const status = -1;
// if (!!statusCode && (statusCode >= 400 && statusCode < 500)) {
// ctx.status = statusCode;
// } else if (!(ctx.status >= 400 && ctx.status < 500)) {
// ctx.status = statusCodes.BAD_REQUEST;
// }
// ctx.body = { status, code, data, message };
// };
// ctx.res.error = ({ statusCode, code, data = null, message = null }) => {
// const status = -2;
// if (!!statusCode && (statusCode >= 500 && statusCode < 600)) {
// ctx.status = statusCode;
// } else if (!(ctx.status >= 500 && ctx.status < 600)) {
// ctx.status = statusCodes.INTERNAL_SERVER_ERROR;
// }
// ctx.body = { status, code, data, message };
// };
ctx.res.ok = (params = {}) => {
ctx.res.success({
...params,
statusCode: statusCodes.OK,
});
};
// ctx.res.noContent = (params = {}) => {
// ctx.res.success({
// ...params,
// statusCode: statusCodes.NO_CONTENT,
// });
// };
// ctx.res.badRequest = (params = {}) => {
// ctx.res.fail({
// ...params,
// statusCode: statusCodes.BAD_REQUEST,
// });
// };
// ctx.res.forbidden = (params = {}) => {
// ctx.res.fail({
// ...params,
// statusCode: statusCodes.FORBIDDEN,
// });
// };
// ctx.res.notFound = (params = {}) => {
// ctx.res.fail({
// ...params,
// statusCode: statusCodes.NOT_FOUND,
// });
// };
// ctx.res.requestTimeout = (params = {}) => {
// ctx.res.fail({
// ...params,
// statusCode: statusCodes.REQUEST_TIMEOUT,
// });
// };
// ctx.res.unprocessableEntity = (params = {}) => {
// ctx.res.fail({
// ...params,
// statusCode: statusCodes.UNPROCESSABLE_ENTITY,
// });
// };
// ctx.res.internalServerError = (params = {}) => {
// ctx.res.error({
// ...params,
// statusCode: statusCodes.INTERNAL_SERVER_ERROR,
// });
// };
// ctx.res.notImplemented = (params = {}) => {
// ctx.res.error({
// ...params,
// statusCode: statusCodes.NOT_IMPLEMENTED,
// });
// };
// ctx.res.badGateway = (params = {}) => {
// ctx.res.error({
// ...params,
// statusCode: statusCodes.BAD_GATEWAY,
// });
// };
// ctx.res.serviceUnavailable = (params = {}) => {
// ctx.res.error({
// ...params,
// statusCode: statusCodes.SERVICE_UNAVAILABLE,
// });
// };
// ctx.res.gatewayTimeOut = (params = {}) => {
// ctx.res.error({
// ...params,
// statusCode: statusCodes.GATEWAY_TIME_OUT,
// });
// };
await next();
};
}
module.exports = responseHandler;

9
lib/middleware/api-template.js
View File

@@ -1,9 +0,0 @@
module.exports = async (ctx, next) => {
await next();
if (ctx.request.path.startsWith('/api/')) {
return ctx.res.ok({
message: `request returned ${ctx.body.counter} ${ctx.body.counter > 1 ? 'routes' : 'route'}`,
data: ctx.body.result,
});
}
};

6
lib/middleware/cache.ts
View File

@@ -29,11 +29,7 @@ const middleware: MiddlewareHandler = async (ctx, next) => {
if (value) { if (value) {
ctx.status(200) ctx.status(200)
if (config.cache.type === 'redis') { ctx.header('RSSHub-Cache-Status', 'HIT')
ctx.header('X-Koa-Redis-Cache', 'true')
} else if (config.cache.type === 'memory') {
ctx.header('X-Koa-Memory-Cache', 'true')
}
ctx.set('data', JSON.parse(value)) ctx.set('data', JSON.parse(value))
await next(); await next();
return; return;

2
lib/middleware/debug.ts
View File

@@ -12,7 +12,7 @@ const middleware: MiddlewareHandler = async (ctx, next) => {
{ {
const debug = getDebugInfo(); const debug = getDebugInfo();
if (ctx.res.headers.get('X-Koa-Redis-Cache') || ctx.res.headers.get('X-Koa-Memory-Cache')) { if (ctx.res.headers.get('RSSHub-Cache-Status')) {
debug.hitCache++; debug.hitCache++;
} }

26
lib/middleware/header.ts
View File

@@ -29,31 +29,25 @@ const middleware: MiddlewareHandler = async (ctx, next) => {
await next(); await next();
if (!ctx.res.body || ctx.res.headers.get('ETag')) { const data = ctx.get('data');
if (!data || ctx.res.headers.get('ETag')) {
return; return;
} }
const status = Math.trunc(ctx.res.status / 100); const lastBuildDate = data.lastBuildDate;
if (2 !== status) { delete data.lastBuildDate;
return; const etag = etagCalculate(JSON.stringify(data));
}
const res = ctx.res as Response; ctx.header('ETag', etag);
const body = await res.clone().text();
const etag = etagCalculate(body.replace(/<lastBuildDate>(.*)<\/lastBuildDate>/, '').replace(/<atom:link(.*)\/>/, ''));
ctx.set('ETag', etag);
const ifNoneMatch = ctx.req.header('If-None-Match') ?? null; const ifNoneMatch = ctx.req.header('If-None-Match') ?? null;
if (etagMatches(etag, ifNoneMatch)) { if (etagMatches(etag, ifNoneMatch)) {
console.log('in')
ctx.status(304); ctx.status(304);
ctx.body(null); return ctx.body(null);
} else { } else {
const match = body.match(/<lastBuildDate>(.*)<\/lastBuildDate>/); ctx.header('Last-Modified', lastBuildDate);
if (match) {
ctx.header('Last-Modified', match[1]);
}
} }
}; };
export default middleware; export default middleware;

36
lib/middleware/load-on-demand.js
View File

@@ -1,36 +0,0 @@
const mount = require('koa-mount');
const Router = require('@koa/router');
const routes = require('../v2router');
const loadedRoutes = new Set();
module.exports = function (app) {
return async function (ctx, next) {
const p = ctx.request.path.split('/').filter(Boolean);
let modName = null;
let mounted = false;
if (p.length > 0) {
modName = p[0];
if (loadedRoutes.has(modName)) {
mounted = true;
} else {
const mod = routes[modName];
// Mount module
if (mod) {
mounted = true;
loadedRoutes.add(modName);
const router = new Router();
mod(router);
app.use(mount(`/${modName}`, router.routes())).use(router.allowedMethods());
}
}
}
await next();
// We should only add it when koa router matched
if (mounted && ctx._matchedRoute) {
ctx._matchedRoute = `/${modName}${ctx._matchedRoute}`;
}
};
};

136
lib/middleware/template.ts
View File

@@ -1,21 +1,15 @@
import render from '@/utils/render'; import render from '@/utils/render';
import * as path from 'node:path'; import * as path from 'node:path';
import { config } from '@/config'; import { config } from '@/config';
const typeRegex = /\.(atom|rss|ums|debug\.json|json|\d+\.debug\.html)$/;
import utils from '@/utils/common-utils'; import utils from '@/utils/common-utils';
import type { MiddlewareHandler } from 'hono'; import type { MiddlewareHandler } from 'hono';
import { Data } from '@/types';
const middleware: MiddlewareHandler = async (ctx, next) => { const middleware: MiddlewareHandler = async (ctx, next) => {
if (ctx.req.header('user-agent')?.includes('Reeder')) {
ctx.req.path = ctx.req.path.replace(/.com$/, '');
}
ctx.set('type', ctx.req.path.match(typeRegex) || ['', '']);
ctx.req.path = ctx.req.path.replace(typeRegex, '');
await next(); await next();
const outputType = ctx.get('type')[1] || 'rss'; const data: Data = ctx.get('data');
const outputType = ctx.req.query('format') || 'rss';
// only enable when debugInfo=true // only enable when debugInfo=true
if (config.debugInfo) { if (config.debugInfo) {
@@ -24,87 +18,83 @@ const middleware: MiddlewareHandler = async (ctx, next) => {
return ctx.body(ctx.get('json') ? JSON.stringify(ctx.get('json'), null, 4) : JSON.stringify({ message: 'plugin does not set debug json' })); return ctx.body(ctx.get('json') ? JSON.stringify(ctx.get('json'), null, 4) : JSON.stringify({ message: 'plugin does not set debug json' }));
} }
if (outputType.endsWith('.debug.html')) { if (outputType === 'debug.html') {
ctx.header('Content-Type', 'text/html; charset=UTF-8'); ctx.header('Content-Type', 'text/html; charset=UTF-8');
const index = Number.parseInt(outputType.match(/(\d+)\.debug\.html$/)[1]); const index = Number.parseInt(outputType.match(/(\d+)\.debug\.html$/)?.[1] || '0');
return ctx.body(ctx.get('data')?.item?.[index]?.description || `ctx.get('data')?.item?.[${index}]?.description not found`); return ctx.body(data?.item?.[index]?.description || `data?.item?.[${index}]?.description not found`);
} }
} }
const templateName = outputType === 'atom' ? 'atom.art' : 'rss.art'; const templateName = outputType === 'atom' ? 'atom.art' : 'rss.art';
const template = path.resolve(__dirname, `../views/${templateName}`); const template = path.resolve(__dirname, `../views/${templateName}`);
if (ctx.get('data')) { if (data) {
for (const prop of ['title', 'subtitle', 'author']) { data.title = utils.collapseWhitespace(data.title);
if (ctx.get('data')[prop]) { data.description && (data.description = utils.collapseWhitespace(data.description));
ctx.get('data')[prop] = utils.collapseWhitespace(ctx.get('data')[prop]); data.author && (data.author = utils.collapseWhitespace(data.author));
if (data.item) {
for (const item of data.item) {
if (item.title) {
item.title = utils.collapseWhitespace(item.title);
// trim title length
for (let length = 0, i = 0; i < item.title.length; i++) {
length += Buffer.from(item.title[i]).length === 1 ? 1 : 2;
if (length > config.titleLengthLimit) {
item.title = `${item.title.slice(0, i)}...`;
break;
}
}
} }
}
if (ctx.get('data').item) { if (typeof item.author === 'string') {
for (const item of ctx.get('data').item) { item.author = utils.collapseWhitespace(item.author);
if (item.title) { } else if (typeof item.author === 'object' && item.author !== null) {
item.title = utils.collapseWhitespace(item.title); for (const a of item.author) {
// trim title length a.name = utils.collapseWhitespace(a.name);
for (let length = 0, i = 0; i < item.title.length; i++) {
length += Buffer.from(item.title[i]).length === 1 ? 1 : 2;
if (length > config.titleLengthLimit) {
item.title = `${item.title.slice(0, i)}...`;
break;
}
}
} }
if (outputType !== 'json') {
item.author = item.author.map((a: { name: string }) => a.name).join(', ');
}
}
if (typeof item.author === 'string') { if (item.itunes_duration && ((typeof item.itunes_duration === 'string' && !item.itunes_duration.includes(':')) || (typeof item.itunes_duration === 'number' && !isNaN(item.itunes_duration)))) {
item.author = utils.collapseWhitespace(item.author); item.itunes_duration = +item.itunes_duration;
} else if (typeof item.author === 'object' && item.author !== null) { item.itunes_duration =
for (const a of item.author) { Math.floor(item.itunes_duration / 3600) + ':' + (Math.floor((item.itunes_duration % 3600) / 60) / 100).toFixed(2).slice(-2) + ':' + (((item.itunes_duration % 3600) % 60) / 100).toFixed(2).slice(-2);
a.name = utils.collapseWhitespace(a.name); }
}
if (outputType !== 'json') {
item.author = item.author.map((a: {
name: string;
}) => a.name).join(', ');
}
}
if (item.itunes_duration && ((typeof item.itunes_duration === 'string' && !item.itunes_duration.includes(':')) || (typeof item.itunes_duration === 'number' && !isNaN(item.itunes_duration)))) { if (outputType !== 'rss') {
item.itunes_duration = +item.itunes_duration; item.pubDate && (item.pubDate = utils.convertDateToISO8601(item.pubDate));
item.itunes_duration = item.updated && (item.updated = utils.convertDateToISO8601(item.updated));
Math.floor(item.itunes_duration / 3600) + ':' + (Math.floor((item.itunes_duration % 3600) / 60) / 100).toFixed(2).slice(-2) + ':' + (((item.itunes_duration % 3600) % 60) / 100).toFixed(2).slice(-2);
}
if (outputType !== 'rss') {
item.pubDate = utils.convertDateToISO8601(item.pubDate);
item.updated = utils.convertDateToISO8601(item.updated);
}
} }
} }
} }
}
const currentDate = new Date(); const currentDate = new Date();
const data = { const result = {
lastBuildDate: currentDate.toUTCString(), lastBuildDate: currentDate.toUTCString(),
updated: currentDate.toISOString(), updated: currentDate.toISOString(),
ttl: Math.trunc(config.cache.routeExpire / 60), ttl: Math.trunc(config.cache.routeExpire / 60),
atomlink: ctx.req.url, atomlink: ctx.req.url,
...ctx.get('data'), ...data,
}; };
if (config.isPackage) { if (config.isPackage) {
return ctx.body(data); return ctx.json(result);
} }
if (outputType === 'ums') { if (outputType === 'ums') {
ctx.header('Content-Type', 'application/json; charset=UTF-8'); ctx.header('Content-Type', 'application/json; charset=UTF-8');
return ctx.body(render.rss3Ums(data)); return ctx.body(render.rss3Ums(data));
} else if (outputType === 'json') { } else if (outputType === 'json') {
ctx.header('Content-Type', 'application/feed+json; charset=UTF-8'); ctx.header('Content-Type', 'application/feed+json; charset=UTF-8');
return ctx.body(render.json(data)); return ctx.body(render.json(data));
} else { } else {
return ctx.body(render.art(template, data)); return ctx.body(render.art(template, data));
} }
}; };
export default middleware; export default middleware;

6
lib/middleware/utf8.js
View File

@@ -1,6 +0,0 @@
// https://stackoverflow.com/questions/2507608/error-input-is-not-proper-utf-8-indicate-encoding-using-phps-simplexml-lo/40552083#40552083
// https://stackoverflow.com/questions/1497885/remove-control-characters-from-php-string/1497928#1497928
module.exports = async (ctx, next) => {
await next();
ctx.body = typeof ctx.body === 'object' ? ctx.body : ctx.body.replaceAll(/[\u0000-\u0009\u000B\u000C\u000E-\u001F\u007F]/g, '');
};

62
lib/types.ts
View File

@@ -1,24 +1,44 @@
export type DataItem = { export type DataItem = {
title: string; title: string;
description?: string; description?: string;
pubDate: number | string; pubDate?: number | string | Date;
link?: string; link?: string;
category?: string[]; category?: string[];
author?: string; author?: string | { name: string }[];
doi?: string; doi?: string;
guid?: string;
id?: string;
content?: {
html: string;
text: string;
};
image?: string;
banner?: string;
updated?: number | string | Date;
language?: string;
enclosure_url?: string;
enclosure_type?: string;
enclosure_title?: string;
enclosure_length?: number;
itunes_duration?: number | string;
_extra?: Record<string, any> & {
links?: {
url: string;
type: string;
content_html?: string;
}[];
};
};
_extra?: Record<string, any> & {
links?: {
url: string;
type: string;
content_html?: string;
}[];
};
}
export type Data = { export type Data = {
title: string; title: string;
description: string; description?: string;
link?: string; link?: string;
item: DataItem[]; item?: DataItem[];
allowEmpty?: boolean; allowEmpty?: boolean;
} image?: string;
author?: string;
language?: string;
feedLink?: string;
};

2
lib/utils/common-utils.ts
View File

@@ -15,7 +15,7 @@ const collapseWhitespace = (str: string) => {
return str; return str;
}; };
const convertDateToISO8601 = (date: string | Date) => { const convertDateToISO8601 = (date: string | Date | number) => {
if (!date) { if (!date) {
return date; return date;
} }

6
lib/utils/parse-date.ts
View File

@@ -126,7 +126,7 @@ const toDurations = (matches: string[]) => {
}; };
export default { export default {
parse: (date: string, ...options: any) => dayjs(date, ...options).toDate(), parse: (date: string | number, ...options: any) => dayjs(date, ...options).toDate(),
parseRelativeDate: (date: string) => { parseRelativeDate: (date: string) => {
// 预处理日期字符串 date // 预处理日期字符串 date
@@ -176,9 +176,9 @@ export default {
const wordMatches = w.regExp.exec(firstMatch); const wordMatches = w.regExp.exec(firstMatch);
if (wordMatches) { if (wordMatches) {
matches.unshift(wordMatches[1]); matches.unshift(wordMatches[1]);
// 取特殊词对应日零时为起点，加上相应的时间长度 // 取特殊词对应日零时为起点，加上相应的时间长度
return w.startAt return w.startAt
.set('hour', 0) .set('hour', 0)
.set('minute', 0) .set('minute', 0)

6
lib/views/json.ts
View File

@@ -1,9 +1,11 @@
import { Data } from "@/types";
/** /**
* This function should be used by RSSHub middleware only. * This function should be used by RSSHub middleware only.
* @param {object} data ctx.state.data * @param {object} data ctx.state.data
* @returns `JSON.stringify`-ed [JSON Feed](https://www.jsonfeed.org/) * @returns `JSON.stringify`-ed [JSON Feed](https://www.jsonfeed.org/)
*/ */
const json = (data) => { const json = (data: Data) => {
const jsonFeed = { const jsonFeed = {
version: 'https://jsonfeed.org/version/1.1', version: 'https://jsonfeed.org/version/1.1',
title: data.title || 'RSSHub', title: data.title || 'RSSHub',
@@ -13,7 +15,7 @@ const json = (data) => {
icon: data.image, icon: data.image,
authors: typeof data.author === 'string' ? [{ name: data.author }] : data.author, authors: typeof data.author === 'string' ? [{ name: data.author }] : data.author,
language: data.language || 'zh-cn', language: data.language || 'zh-cn',
items: data.item.map((item) => ({ items: data.item?.map((item) => ({
id: item.guid || item.id || item.link, id: item.guid || item.id || item.link,
url: item.link, url: item.link,
title: item.title, title: item.title,