app: fix redis cache

config.js

@@ -3,6 +3,12 @@ module.exports = {
     cacheType: process.env.CACHE_TYPE || 'memory', // support memory and redis, set empty to disable cache
     cacheExpire: process.env.CACHE_EXPIRE || 5 * 60,
     ua: process.env.UA || 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36',
+    redis: {
+        url: process.env.REDIS_URL || 'redis://localhost:6379/',
+        options: {                                 // 支持这些参数 https://github.com/NodeRedis/node_redis#options-object-properties
+            password: process.env.REDIS_PASSWORD || null,
+        }
+    },
     pixiv: {
         client_id: 'MOBrBDS8blbauoSck0ZfDbtuzpyT',
         client_secret: 'lsACyCD94FhDUtGTXi3QzcFE2uU1hqtDaKeqrdwj',

index.js

@@ -6,8 +6,8 @@ const config = require('./config');
 const onerror = require('./middleware/onerror');
 const header = require('./middleware/header.js');
 const utf8 = require('./middleware/utf8');
-const memoryCache = require('./middleware/cache.js');
-const redisCache = require('koa-redis-cache');
+const memoryCache = require('./middleware/lru-cache.js');
+const redisCache = require('./middleware/redis-cache.js');
 const filter = require('./middleware/filter.js');
 const template = require('./middleware/template.js');
 const favicon = require('koa-favicon');
@@ -52,8 +52,12 @@ if (config.cacheType === 'memory') {
     app.use(
         redisCache({
             expire: config.cacheExpire,
+            ignoreQuery: true,
             onerror: (e) => {
-                logger.error('cache error', e);
+                logger.error('Redis error: ', e);
+            },
+            onconnect: () => {
+                logger.info('Redis connect.');
             }
         })
     );

middleware/lru-cache.js

@@ -1,4 +1,4 @@
-// transform from https://github.com/coderhaoxin/koa-redis-cache
+// baed on https://github.com/coderhaoxin/koa-redis-cache
 
 const pathToRegExp = require('path-to-regexp');
 const readall = require('readall');

middleware/redis-cache.js

@@ -0,0 +1,177 @@
+// baed on https://github.com/coderhaoxin/koa-redis-cache
+
+const pathToRegExp = require('path-to-regexp');
+const wrapper = require('co-redis');
+const readall = require('readall');
+const crypto = require('crypto');
+const Redis = require('redis');
+
+module.exports = function (options = {}) {
+    let redisAvailable = false;
+
+    const {
+        prefix = 'koa-redis-cache:',
+        expire = 30 * 60, // 30 min
+        routes = ['(.*)'],
+        exclude = [],
+        passParam = '',
+        maxLength = Infinity,
+        ignoreQuery = false,
+        onerror = function () { },
+        onconnect = function () {},
+    } = options;
+
+    const {
+        host:redisHost = 'localhost',
+        port:redisPort = 6379,
+        url:redisUrl = `redis://${redisHost}:${redisPort}/`,
+        options:redisOptions = {}
+    } = options.redis || {};
+
+    /**
+   * redisClient
+   */
+    const redisClient = wrapper(Redis.createClient(redisUrl, redisOptions));
+    redisClient.on('error', (error) => {
+        redisAvailable = false;
+        onerror(error);
+    });
+    redisClient.on('end', () => {
+        redisAvailable = false;
+    });
+    redisClient.on('connect', () => {
+        redisAvailable = true;
+        onconnect();
+    });
+
+    return async function cache (ctx, next) {
+        const { url, path } = ctx.request;
+        const resolvedPrefix = typeof prefix === 'function' ? prefix.call(ctx, ctx) : prefix;
+        const key = resolvedPrefix + md5(ignoreQuery ? path : url);
+        const tkey = key + ':type';
+        let match = false;
+        let routeExpire = false;
+
+        for (let i = 0; i < routes.length; i++) {
+            let route = routes[i];
+
+            if (typeof routes[i] === 'object') {
+                route = routes[i].path;
+                routeExpire = routes[i].expire;
+            }
+
+            if (paired(route, path)) {
+                match = true;
+                break;
+            }
+        }
+
+        for (let j = 0; j < exclude.length; j++) {
+            if (paired(exclude[j], path)) {
+                match = false;
+                break;
+            }
+        }
+
+        if (!redisAvailable || !match || passParam && ctx.request.query[passParam]) {
+            return await next();
+        }
+
+        let ok = false;
+        try {
+            ok = await getCache(ctx, key, tkey);
+        } catch (e) {
+            ok = false;
+        }
+        if (ok) {
+            return;
+        }
+
+        await next();
+
+        try {
+            const trueExpire = routeExpire || expire;
+            await setCache(ctx, key, tkey, trueExpire);
+        } catch (e) { }
+        routeExpire = false;
+    };
+
+    /**
+   * getCache
+   */
+    async function getCache (ctx, key, tkey) {
+        const value = await redisClient.get(key);
+        let type;
+        let ok = false;
+
+        if (value) {
+            ctx.response.status = 200;
+            type = await redisClient.get(tkey) || 'text/html';
+            // can happen if user specified return_buffers: true in redis options
+            if (Buffer.isBuffer(type)) {type = type.toString();}
+            ctx.response.set('X-Koa-Redis-Cache', 'true');
+            ctx.response.type = type;
+            try {
+                ctx.state.data = JSON.parse(value);
+            } catch (e) {
+                ctx.state.data = {};
+            }
+            ok = true;
+        }
+
+        return ok;
+    }
+
+    /**
+   * setCache
+   */
+    async function setCache (ctx, key, tkey, expire) {
+        ctx.state.data.lastBuildDate = new Date().toUTCString();
+        const body = JSON.stringify(ctx.state.data);
+
+        if (ctx.request.method !== 'GET' || !body) {
+            return;
+        }
+        if (Buffer.byteLength(body) > maxLength) {
+            return;
+        }
+        await redisClient.setex(key, expire, body);
+
+        await cacheType(ctx, tkey, expire);
+    }
+
+    /**
+   * cacheType
+   */
+    async function cacheType (ctx, tkey, expire) {
+        const type = ctx.response.type;
+        if (type) {
+            await redisClient.setex(tkey, expire, type);
+        }
+    }
+};
+
+function paired (route, path) {
+    const options = {
+        sensitive: true,
+        strict: true,
+    };
+
+    return pathToRegExp(route, [], options).exec(path);
+}
+
+function read (stream) {
+    return new Promise((resolve, reject) => {
+        readall(stream, (err, data) => {
+            if (err) {
+                reject(err);
+            } else {
+                resolve(data);
+            }
+        });
+    });
+}
+
+function md5 (str) {
+    return crypto.createHash('md5').update(str).digest('hex');
+}

package.json

@@ -28,6 +28,7 @@
     "art-template": "4.12.2",
     "axios": "0.18.0",
     "cheerio": "1.0.0-rc.2",
+    "co-redis": "2.1.1",
     "crypto": "1.0.1",
     "eslint": "4.19.1",
     "form-data": "^2.3.2",
@@ -35,11 +36,11 @@
     "json-bigint": "0.2.3",
     "koa": "2.5.1",
     "koa-favicon": "2.0.1",
-    "koa-redis-cache": "3.0.0",
     "koa-router": "7.4.0",
     "lru-cache": "4.1.3",
     "path-to-regexp": "2.2.1",
     "readall": "1.0.0",
+    "redis": "2.8.0",
     "twit": "2.2.9",
     "winston": "3.0.0-rc3"
   },

yarn.lock

@@ -807,7 +807,7 @@ clone@^1.0.2:
   version "1.0.4"
   resolved "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz#da309cc263df15994c688ca902179ca3c7cd7c7e"
 
-co-redis@2:
+co-redis@2.1.1:
   version "2.1.1"
   resolved "https://registry.npmjs.org/co-redis/-/co-redis-2.1.1.tgz#8bc3717977c9ee5bfb6b28f39ae01bcf724d2dde"
   dependencies:
@@ -2980,15 +2980,6 @@ koa-mount@^3.0.0:
     debug "^2.6.1"
     koa-compose "^3.2.1"
 
-koa-redis-cache@3.0.0:
-  version "3.0.0"
-  resolved "https://registry.npmjs.org/koa-redis-cache/-/koa-redis-cache-3.0.0.tgz#e4c33ebc9d9c08fd50a669bca4573fabb4b2c584"
-  dependencies:
-    co-redis "2"
-    path-to-regexp "1"
-    readall "1"
-    redis "2"
-
 koa-router@7.4.0:
   version "7.4.0"
   resolved "https://registry.npmjs.org/koa-router/-/koa-router-7.4.0.tgz#aee1f7adc02d5cb31d7d67465c9eacc825e8c5e0"
@@ -3960,16 +3951,16 @@ path-parse@^1.0.5:
   version "1.0.5"
   resolved "https://registry.npmjs.org/path-parse/-/path-parse-1.0.5.tgz#3c1adf871ea9cd6c9431b6ea2bd74a0ff055c4c1"
 
-path-to-regexp@1, path-to-regexp@^1.1.1:
+path-to-regexp@2.2.1:
+  version "2.2.1"
+  resolved "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-2.2.1.tgz#90b617025a16381a879bc82a38d4e8bdeb2bcf45"
+
+path-to-regexp@^1.1.1:
   version "1.7.0"
   resolved "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-1.7.0.tgz#59fde0f435badacba103a84e9d3bc64e96b9937d"
   dependencies:
     isarray "0.0.1"
 
-path-to-regexp@2.2.1:
-  version "2.2.1"
-  resolved "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-2.2.1.tgz#90b617025a16381a879bc82a38d4e8bdeb2bcf45"
-
 path-type@^3.0.0:
   version "3.0.0"
   resolved "https://registry.npmjs.org/path-type/-/path-type-3.0.0.tgz#cef31dc8e0a1a3bb0d105c0cd97cf3bf47f4e36f"
@@ -4506,7 +4497,7 @@ readable-stream@1.0:
     isarray "0.0.1"
     string_decoder "~0.10.x"
 
-readall@1, readall@1.0.0:
+readall@1.0.0:
   version "1.0.0"
   resolved "https://registry.npmjs.org/readall/-/readall-1.0.0.tgz#8b4bfc412e687dbfbb00e0a6ede7e3a0f001b7c7"
   dependencies:
@@ -4536,7 +4527,7 @@ redis-parser@^2.6.0:
   version "2.6.0"
   resolved "https://registry.npmjs.org/redis-parser/-/redis-parser-2.6.0.tgz#52ed09dacac108f1a631c07e9b69941e7a19504b"
 
-redis@2:
+redis@2.8.0:
   version "2.8.0"
   resolved "https://registry.npmjs.org/redis/-/redis-2.8.0.tgz#202288e3f58c49f6079d97af7a10e1303ae14b02"
   dependencies: