opensource/RSSHub
1
0
Fork 0
mirror of https://github.com/DIYgod/RSSHub.git synced 2025-12-05 04:11:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: anti-hotlink for images (#4481)

Browse Source
This commit is contained in:
NeverBehave
2020-04-27 22:27:47 -04:00
committed by GitHub
parent a36e6a0cb9
commit cde5d9af45
6 changed files with 149 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
docs/en/install/README.md
View File

@@ -349,6 +349,8 @@ Access control includes a whitelist and a blacklist, support IP and route, use `
`DISALLOW_ROBOT`: prevent indexing by search engine
`HOTLINK_TEMPLATE`: Replace image link in description to avoid anti-hotlink protection, leave blank to disable this function. Usage reference [#2769](https://github.com/DIYgod/RSSHub/issues/2769). You may use any properity listed in [URL](https://developer.mozilla.org/en-US/docs/Web/API/URL#Properties), format of JS template literal. e.g. `${protocol}//${host}${pathname}`, `https://i3.wp.com/${host}${pathname}`
### Route-specific Configurations
- pixiv: [Registration](https://accounts.pixiv.net/signup)

2
docs/install/README.md
View File

@@ -353,6 +353,8 @@ RSSHub 支持 `memory` 和 `redis` 两种缓存方式
`DISALLOW_ROBOT`: 防止被搜索引擎收录
`HOTLINK_TEMPLATE`: 用于处理描述中图片的链接，绕过防盗链等限制，留空不生效。用法参考[#2769](https://github.com/DIYgod/RSSHub/issues/2769)。可以使用[URL](https://developer.mozilla.org/en-US/docs/Web/API/URL#Properties)的所有属性，格式为 JS 变量模板。例子：`${protocol}//${host}${pathname}`, `https://i3.wp.com/${host}${pathname}`
### 部分 RSS 模块配置
- pixiv 全部路由: [注册地址](https://accounts.pixiv.net/signup)

4
lib/app.js
View File

@@ -15,6 +15,7 @@ const template = require('./middleware/template');
const favicon = require('koa-favicon');
const debug = require('./middleware/debug');
const accessControl = require('./middleware/access-control');
const antiHotlink = require('./middleware/anti-hotlink');
const router = require('./router');
const protected_router = require('./protected_router');
@@ -63,6 +64,9 @@ app.use(apiResponseHandler());
// 4 generate body
app.use(template);
// anti-hotlink
app.use(antiHotlink);
// 3 filter content
app.use(parameter);

3
lib/config.js
View File

@@ -128,6 +128,9 @@ const calculateValue = () => {
scihub: {
host: envs.SCIHUB_HOST || 'https://sci-hub.tw/',
},
hotlink: {
template: envs.HOTLINK_TEMPLATE,
},
};
};
calculateValue();

54
lib/middleware/anti-hotlink.js Normal file
View File

@@ -0,0 +1,54 @@
const config = require('@/config').value;
const cheerio = require('cheerio');
const logger = require('@/utils/logger');
const interpolate = (str, obj) => str.replace(/\${([^}]+)}/g, (_, prop) => obj[prop]);
// I don't want to keep another regex and
// URL will be the standard way to parse URL
const parseUrl = (str) => {
let url;
try {
url = new URL(str);
} catch (e) {
logger.error(`Failed to parse ${str}`);
}
return url;
};
const replaceUrls = (body, template) => {
const $ = cheerio.load(body, { decodeEntities: false, xmlMode: true });
$('img').each(function () {
const old_src = $(this).attr('src');
const url = parseUrl(old_src);
if (url) {
const new_src = interpolate(template, url);
$(this).attr('src', new_src);
}
});
return $.root().html();
};
module.exports = async (ctx, next) => {
await next();
const template = config.hotlink.template;
// Assume that only description include image link
// and here we will only check them in description.
// Use Cherrio to load the description as html and filter all
// image link
if (template) {
if (ctx.state.data) {
if (ctx.state.data.description) {
ctx.state.data.description = replaceUrls(ctx.state.data.description, template);
}
ctx.state.data.item &&
ctx.state.data.item.forEach((item) => {
if (item.description) {
item.description = replaceUrls(item.description, template);
}
});
}
}
};

84
test/middleware/anti-hotlink.js Normal file
View File

@@ -0,0 +1,84 @@
const supertest = require('supertest');
jest.mock('request-promise-native');
const Parser = require('rss-parser');
const parser = new Parser();
let server;
afterAll(() => {
delete process.env.HOTLINK_TEMPLATE;
});
afterEach(() => {
delete process.env.HOTLINK_TEMPLATE;
jest.resetModules();
server.close();
});
describe('anti-hotlink', () => {
it('template', async () => {
process.env.HOTLINK_TEMPLATE = 'https://i3.wp.com/${host}${pathname}';
server = require('../../lib/index');
const request = supertest(server);
const response = await request.get('/test/complicated');
const parsed = await parser.parseString(response.text);
expect(parsed.items[0].content).toBe(
`<a href="https://mock.com/DIYgod/RSSHub"/>
<img src="https://i3.wp.com/mock.com/DIYgod/RSSHub.jpg" referrerpolicy="no-referrer">
<a href="http://mock.com/DIYgod/RSSHub"/>
<img src="https://i3.wp.com/mock.com/DIYgod/RSSHub.jpg" data-src="/DIYgod/RSSHub0.jpg" referrerpolicy="no-referrer">
<img data-src="/DIYgod/RSSHub.jpg" src="https://i3.wp.com/mock.com/DIYgod/RSSHub.jpg" referrerpolicy="no-referrer">
<img data-mock="/DIYgod/RSSHub.png" src="https://i3.wp.com/mock.com/DIYgod/RSSHub.png" referrerpolicy="no-referrer">
<img mock="/DIYgod/RSSHub.gif" src="https://i3.wp.com/mock.com/DIYgod/RSSHub.gif" referrerpolicy="no-referrer">
<img src="https://i3.wp.com/mock.com/DIYgod/DIYgod/RSSHub" referrerpolicy="no-referrer">
<img src="https://i3.wp.com/mock.com/DIYgod/RSSHub.jpg" referrerpolicy="no-referrer"/></img></img></img></img></img></img>`
);
expect(parsed.items[1].content).toBe(`<a href="https://mock.com/DIYgod/RSSHub"/>
<img src="https://i3.wp.com/mock.com/DIYgod/RSSHub.jpg" referrerpolicy="no-referrer"/>`);
});
it('url', async () => {
process.env.HOTLINK_TEMPLATE = '${protocol}//${host}${pathname}';
server = require('../../lib/index');
const request = supertest(server);
const response = await request.get('/test/complicated');
const parsed = await parser.parseString(response.text);
expect(parsed.items[0].content).toBe(
`<a href="https://mock.com/DIYgod/RSSHub"/>
<img src="https://mock.com/DIYgod/RSSHub.jpg" referrerpolicy="no-referrer">
<a href="http://mock.com/DIYgod/RSSHub"/>
<img src="https://mock.com/DIYgod/RSSHub.jpg" data-src="/DIYgod/RSSHub0.jpg" referrerpolicy="no-referrer">
<img data-src="/DIYgod/RSSHub.jpg" src="https://mock.com/DIYgod/RSSHub.jpg" referrerpolicy="no-referrer">
<img data-mock="/DIYgod/RSSHub.png" src="https://mock.com/DIYgod/RSSHub.png" referrerpolicy="no-referrer">
<img mock="/DIYgod/RSSHub.gif" src="https://mock.com/DIYgod/RSSHub.gif" referrerpolicy="no-referrer">
<img src="http://mock.com/DIYgod/DIYgod/RSSHub" referrerpolicy="no-referrer">
<img src="https://mock.com/DIYgod/RSSHub.jpg" referrerpolicy="no-referrer"/></img></img></img></img></img></img>`
);
expect(parsed.items[1].content).toBe(`<a href="https://mock.com/DIYgod/RSSHub"/>
<img src="https://mock.com/DIYgod/RSSHub.jpg" referrerpolicy="no-referrer"/>`);
});
it('no-template', async () => {
process.env.HOTLINK_TEMPLATE = '';
server = require('../../lib/index');
const request = supertest(server);
const response = await request.get('/test/complicated');
const parsed = await parser.parseString(response.text);
expect(parsed.items[0].content).toBe(
`<a href="https://mock.com/DIYgod/RSSHub"></a>
<img src="https://mock.com/DIYgod/RSSHub.jpg" referrerpolicy="no-referrer">
<a href="http://mock.com/DIYgod/RSSHub"></a>
<img src="https://mock.com/DIYgod/RSSHub.jpg" data-src="/DIYgod/RSSHub0.jpg" referrerpolicy="no-referrer">
<img data-src="/DIYgod/RSSHub.jpg" src="https://mock.com/DIYgod/RSSHub.jpg" referrerpolicy="no-referrer">
<img data-mock="/DIYgod/RSSHub.png" src="https://mock.com/DIYgod/RSSHub.png" referrerpolicy="no-referrer">
<img mock="/DIYgod/RSSHub.gif" src="https://mock.com/DIYgod/RSSHub.gif" referrerpolicy="no-referrer">
<img src="http://mock.com/DIYgod/DIYgod/RSSHub" referrerpolicy="no-referrer">
<img src="https://mock.com/DIYgod/RSSHub.jpg" referrerpolicy="no-referrer">`
);
expect(parsed.items[1].content).toBe(`<a href="https://mock.com/DIYgod/RSSHub"></a>
<img src="https://mock.com/DIYgod/RSSHub.jpg" referrerpolicy="no-referrer">`);
});
});