opensource/RSSHub
1
0
Fork 0
mirror of https://github.com/DIYgod/RSSHub.git synced 2025-12-16 11:37:44 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor: optimize parameter (#3200)

Browse Source
This commit is contained in:
Cloud
2019-10-08 16:48:46 +08:00
committed by DIYgod
parent a2600c4c79
commit 90a87b1723
2 changed files with 150 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files

125
lib/middleware/parameter.js
View File

@@ -19,32 +19,17 @@ module.exports = async (ctx, next) => {
throw Error('this route is empty, please check the original site or <a href="https://github.com/DIYgod/RSSHub/issues/new/choose">create an issue</a>'); throw Error('this route is empty, please check the original site or <a href="https://github.com/DIYgod/RSSHub/issues/new/choose">create an issue</a>');
} }
if (ctx.query && ctx.query.mode && ctx.query.mode.toLowerCase() === 'fulltext') { // fix allowEmpty
const tasks = ctx.state.data.item.map(async (item) => { ctx.state.data.item = ctx.state.data.item || [];
const { link, author, description } = item;
const parsed_result = await ctx.cache.tryGet(`mercury-cache-${link}`, async () => {
// if parser failed, return default description and not report error
try {
const res = await got(link);
const $ = cheerio.load(res.data);
const result = await mercury_parser.parse(link, {
html: $.html(),
});
return result;
} catch (e) {
// no-empty
}
});
item.author = author || (parsed_result ? parsed_result.author : ''); // decode HTML entities
item.description = parsed_result ? parsed_result.content : description; ctx.state.data.title && (ctx.state.data.title = he.decode(ctx.state.data.title + ''));
}); ctx.state.data.description && (ctx.state.data.description = he.decode(ctx.state.data.description + ''));
await Promise.all(tasks);
}
// handle description // handle description
if (ctx.state.data.item && ctx.state.data.item.length) {
ctx.state.data.item.forEach((item) => { ctx.state.data.item.forEach((item) => {
item.title && (item.title = he.decode(item.title + ''));
if (item.description) { if (item.description) {
const $ = cheerio.load(item.description); const $ = cheerio.load(item.description);
let baseUrl = item.link || ctx.state.data.link; let baseUrl = item.link || ctx.state.data.link;
@@ -103,74 +88,96 @@ module.exports = async (ctx, next) => {
$ele.removeAttr(e); $ele.removeAttr(e);
}); });
}); });
item.description = $('body').html(); item.description = he.decode($('body').html() + '');
} }
}); });
}
// decode HTML entities if (ctx.query) {
ctx.state.data.title && (ctx.state.data.title = he.decode(ctx.state.data.title + '')); // limit
ctx.state.data.description && (ctx.state.data.description = he.decode(ctx.state.data.description + '')); if (ctx.query.limit) {
ctx.state.data.item && ctx.state.data.item = ctx.state.data.item.slice(0, parseInt(ctx.query.limit));
ctx.state.data.item.forEach((item) => { }
item.title && (item.title = he.decode(item.title + ''));
item.description && (item.description = he.decode(item.description + ''));
});
// filter // filter
if (ctx.query && (ctx.query.filter || ctx.query.filter_title || ctx.query.filter_description || ctx.query.filter_author)) { if (ctx.query.filter || ctx.query.filter_title || ctx.query.filter_description || ctx.query.filter_author) {
if (ctx.query.filter) {
ctx.query.filter_title = ctx.query.filter;
ctx.query.filter_description = ctx.query.filter;
}
ctx.state.data.item = ctx.state.data.item.filter((item) => { ctx.state.data.item = ctx.state.data.item.filter((item) => {
const title = item.title || ''; const title = item.title || '';
const description = item.description || title; const description = item.description || title;
const author = item.author || ''; const author = item.author || '';
return !( let isFilter = true;
(ctx.query.filter && !title.match(ctx.query.filter) && !description.match(ctx.query.filter)) || ctx.query.filter_title && (isFilter = isFilter && !title.match(ctx.query.filter_title));
(ctx.query.filter_title && !title.match(ctx.query.filter_title)) || ctx.query.filter_description && (isFilter = isFilter && !description.match(ctx.query.filter_description));
(ctx.query.filter_description && !description.match(ctx.query.filter_description)) || ctx.query.filter_author && (isFilter = isFilter && !author.match(ctx.query.filter_author));
(ctx.query.filter_author && !author.match(ctx.query.filter_author)) return !isFilter;
);
}); });
} }
if (ctx.query && (ctx.query.filterout || ctx.query.filterout_title || ctx.query.filterout_description || ctx.query.filterout_author)) {
if (ctx.query.filterout || ctx.query.filterout_title || ctx.query.filterout_description || ctx.query.filterout_author) {
if (ctx.query.filterout) {
ctx.query.filterout_title = ctx.query.filterout;
ctx.query.filterout_description = ctx.query.filterout;
}
ctx.state.data.item = ctx.state.data.item.filter((item) => { ctx.state.data.item = ctx.state.data.item.filter((item) => {
const title = item.title; const title = item.title;
const description = item.description || title; const description = item.description || title;
const author = item.author || ''; const author = item.author || '';
return ( let isFilter = true;
(ctx.query.filterout && !title.match(ctx.query.filterout) && !description.match(ctx.query.filterout)) || ctx.query.filterout_title && (isFilter = isFilter && !title.match(ctx.query.filterout_title));
(ctx.query.filterout_title && !title.match(ctx.query.filterout_title)) || ctx.query.filterout_description && (isFilter = isFilter && !description.match(ctx.query.filterout_description));
(ctx.query.filterout_description && !description.match(ctx.query.filterout_description)) || ctx.query.filterout_author && (isFilter = isFilter && !author.match(ctx.query.filterout_author));
(ctx.query.filterout_author && !author.match(ctx.query.filterout_author)) return isFilter;
);
}); });
} }
if (ctx.query && ctx.query.filter_time) {
if (ctx.query.filter_time) {
const now = Date.now(); const now = Date.now();
ctx.state.data.item = ctx.state.data.item.filter(({ pubDate }) => { ctx.state.data.item = ctx.state.data.item.filter(({ pubDate }) => {
if (!pubDate) { let isFilter = true;
return true;
}
try { try {
return now - new Date(pubDate).getTime() <= parseInt(ctx.query.filter_time) * 1000; isFilter = !pubDate || now - new Date(pubDate).getTime() <= parseInt(ctx.query.filter_time) * 1000;
} catch (err) { } catch (err) {
return true; // no-empty
} }
return isFilter;
}); });
} }
// limit
if (ctx.query && ctx.query.limit) {
ctx.state.data.item = ctx.state.data.item.slice(0, parseInt(ctx.query.limit));
}
// telegram instant view // telegram instant view
if (ctx.query && ctx.query.tgiv) { if (ctx.query.tgiv) {
ctx.state.data.item.map((item) => { ctx.state.data.item.map((item) => {
const encodedlink = encodeURIComponent(item.link); const encodedlink = encodeURIComponent(item.link);
item.link = `https://t.me/iv?url=${encodedlink}&rhash=${ctx.query.tgiv}`; item.link = `https://t.me/iv?url=${encodedlink}&rhash=${ctx.query.tgiv}`;
return item; return item;
}); });
} }
// fulltest
if (ctx.query.mode && ctx.query.mode.toLowerCase() === 'fulltext') {
const tasks = ctx.state.data.item.map(async (item) => {
const { link, author, description } = item;
const parsed_result = await ctx.cache.tryGet(`mercury-cache-${link}`, async () => {
// if parser failed, return default description and not report error
try {
const res = await got(link);
const $ = cheerio.load(res.data);
const result = await mercury_parser.parse(link, {
html: $.html(),
});
return result;
} catch (e) {
// no-empty
}
});
item.author = author || (parsed_result ? parsed_result.author : '');
item.description = parsed_result ? parsed_result.content : description;
});
await Promise.all(tasks);
}
}
} }
}; };

1
test/utils/got.js
View File

@@ -71,6 +71,7 @@ describe('got', () => {
params: { params: {
test: 1, test: 1,
}, },
responseType: 'buffer',
}); });
}); });
}); });