feat: robots not index config (#4503)

lib/router.js

@@ -1,4 +1,5 @@
 const Router = require('@koa/router');
+const config = require('@/config').value;
 const router = new Router();
 
 // 遍历整个 routes 文件夹，导入模块路由 router.js 和 router-custom.js 文件
@@ -24,6 +25,16 @@ router.get('/', require('./routes/index'));
 // test
 router.get('/test/:id', require('./routes/test'));
 
+router.get('/robots.txt', async (ctx, next) => {
+    if (config.disallowRobot) {
+        ctx.set('Content-Type', 'text/plain');
+        ctx.body = 'User-agent: *\nDisallow: /';
+    } else {
+        ctx.throw(404, 'Not Found');
+    }
+    await next();
+});
+
 // RSSHub
 router.get('/rsshub/rss', require('./routes/rsshub/routes')); // 弃用
 router.get('/rsshub/routes', require('./routes/rsshub/routes'));