diff --git a/config.js b/config.js index 2239e88f0d..3cec13fa72 100644 --- a/config.js +++ b/config.js @@ -3,6 +3,12 @@ module.exports = { cacheType: process.env.CACHE_TYPE || 'memory', // support memory and redis, set empty to disable cache cacheExpire: process.env.CACHE_EXPIRE || 5 * 60, ua: process.env.UA || 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36', + redis: { + url: process.env.REDIS_URL || 'redis://localhost:6379/', + options: { // 支持这些参数 https://github.com/NodeRedis/node_redis#options-object-properties + password: process.env.REDIS_PASSWORD || null, + } + }, pixiv: { client_id: 'MOBrBDS8blbauoSck0ZfDbtuzpyT', client_secret: 'lsACyCD94FhDUtGTXi3QzcFE2uU1hqtDaKeqrdwj', diff --git a/index.js b/index.js index 56293d0ac7..2997529777 100644 --- a/index.js +++ b/index.js @@ -6,8 +6,8 @@ const config = require('./config'); const onerror = require('./middleware/onerror'); const header = require('./middleware/header.js'); const utf8 = require('./middleware/utf8'); -const memoryCache = require('./middleware/cache.js'); -const redisCache = require('koa-redis-cache'); +const memoryCache = require('./middleware/lru-cache.js'); +const redisCache = require('./middleware/redis-cache.js'); const filter = require('./middleware/filter.js'); const template = require('./middleware/template.js'); const favicon = require('koa-favicon'); @@ -52,8 +52,12 @@ if (config.cacheType === 'memory') { app.use( redisCache({ expire: config.cacheExpire, + ignoreQuery: true, onerror: (e) => { - logger.error('cache error', e); + logger.error('Redis error: ', e); + }, + onconnect: () => { + logger.info('Redis connect.'); } }) ); diff --git a/middleware/cache.js b/middleware/lru-cache.js similarity index 98% rename from middleware/cache.js rename to middleware/lru-cache.js index 9b12420599..b2c9aad3df 100644 --- a/middleware/cache.js +++ b/middleware/lru-cache.js @@ -1,4 +1,4 @@ -// transform from https://github.com/coderhaoxin/koa-redis-cache +// baed on https://github.com/coderhaoxin/koa-redis-cache const pathToRegExp = require('path-to-regexp'); const readall = require('readall'); diff --git a/middleware/redis-cache.js b/middleware/redis-cache.js new file mode 100644 index 0000000000..74e2ff9ba9 --- /dev/null +++ b/middleware/redis-cache.js @@ -0,0 +1,177 @@ +// baed on https://github.com/coderhaoxin/koa-redis-cache + +const pathToRegExp = require('path-to-regexp'); +const wrapper = require('co-redis'); +const readall = require('readall'); +const crypto = require('crypto'); +const Redis = require('redis'); + +module.exports = function (options = {}) { + let redisAvailable = false; + + const { + prefix = 'koa-redis-cache:', + expire = 30 * 60, // 30 min + routes = ['(.*)'], + exclude = [], + passParam = '', + maxLength = Infinity, + ignoreQuery = false, + onerror = function () { }, + onconnect = function () {}, + } = options; + + const { + host:redisHost = 'localhost', + port:redisPort = 6379, + url:redisUrl = `redis://${redisHost}:${redisPort}/`, + options:redisOptions = {} + } = options.redis || {}; + + /** + * redisClient + */ + const redisClient = wrapper(Redis.createClient(redisUrl, redisOptions)); + redisClient.on('error', (error) => { + redisAvailable = false; + onerror(error); + }); + redisClient.on('end', () => { + redisAvailable = false; + }); + redisClient.on('connect', () => { + redisAvailable = true; + onconnect(); + }); + + return async function cache (ctx, next) { + const { url, path } = ctx.request; + const resolvedPrefix = typeof prefix === 'function' ? prefix.call(ctx, ctx) : prefix; + const key = resolvedPrefix + md5(ignoreQuery ? path : url); + const tkey = key + ':type'; + let match = false; + let routeExpire = false; + + for (let i = 0; i < routes.length; i++) { + let route = routes[i]; + + if (typeof routes[i] === 'object') { + route = routes[i].path; + routeExpire = routes[i].expire; + } + + if (paired(route, path)) { + match = true; + break; + } + } + + for (let j = 0; j < exclude.length; j++) { + if (paired(exclude[j], path)) { + match = false; + break; + } + } + + if (!redisAvailable || !match || passParam && ctx.request.query[passParam]) { + return await next(); + } + + let ok = false; + try { + ok = await getCache(ctx, key, tkey); + } catch (e) { + ok = false; + } + if (ok) { + return; + } + + await next(); + + try { + const trueExpire = routeExpire || expire; + await setCache(ctx, key, tkey, trueExpire); + } catch (e) { } + routeExpire = false; + }; + + /** + * getCache + */ + async function getCache (ctx, key, tkey) { + const value = await redisClient.get(key); + let type; + let ok = false; + + if (value) { + ctx.response.status = 200; + type = await redisClient.get(tkey) || 'text/html'; + // can happen if user specified return_buffers: true in redis options + if (Buffer.isBuffer(type)) {type = type.toString();} + ctx.response.set('X-Koa-Redis-Cache', 'true'); + ctx.response.type = type; + try { + ctx.state.data = JSON.parse(value); + } catch (e) { + ctx.state.data = {}; + } + ok = true; + } + + return ok; + } + + /** + * setCache + */ + async function setCache (ctx, key, tkey, expire) { + ctx.state.data.lastBuildDate = new Date().toUTCString(); + const body = JSON.stringify(ctx.state.data); + + if (ctx.request.method !== 'GET' || !body) { + return; + } + if (Buffer.byteLength(body) > maxLength) { + return; + } + await redisClient.setex(key, expire, body); + + await cacheType(ctx, tkey, expire); + } + + /** + * cacheType + */ + async function cacheType (ctx, tkey, expire) { + const type = ctx.response.type; + if (type) { + await redisClient.setex(tkey, expire, type); + } + } +}; + +function paired (route, path) { + const options = { + sensitive: true, + strict: true, + }; + + return pathToRegExp(route, [], options).exec(path); +} + +function read (stream) { + return new Promise((resolve, reject) => { + readall(stream, (err, data) => { + if (err) { + reject(err); + } else { + resolve(data); + } + }); + }); +} + +function md5 (str) { + return crypto.createHash('md5').update(str).digest('hex'); +} diff --git a/package.json b/package.json index a09a6969ea..cafa308f4b 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "art-template": "4.12.2", "axios": "0.18.0", "cheerio": "1.0.0-rc.2", + "co-redis": "2.1.1", "crypto": "1.0.1", "eslint": "4.19.1", "form-data": "^2.3.2", @@ -35,11 +36,11 @@ "json-bigint": "0.2.3", "koa": "2.5.1", "koa-favicon": "2.0.1", - "koa-redis-cache": "3.0.0", "koa-router": "7.4.0", "lru-cache": "4.1.3", "path-to-regexp": "2.2.1", "readall": "1.0.0", + "redis": "2.8.0", "twit": "2.2.9", "winston": "3.0.0-rc3" }, diff --git a/yarn.lock b/yarn.lock index 04d48ec962..190109fcae 100644 --- a/yarn.lock +++ b/yarn.lock @@ -807,7 +807,7 @@ clone@^1.0.2: version "1.0.4" resolved "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz#da309cc263df15994c688ca902179ca3c7cd7c7e" -co-redis@2: +co-redis@2.1.1: version "2.1.1" resolved "https://registry.npmjs.org/co-redis/-/co-redis-2.1.1.tgz#8bc3717977c9ee5bfb6b28f39ae01bcf724d2dde" dependencies: @@ -2980,15 +2980,6 @@ koa-mount@^3.0.0: debug "^2.6.1" koa-compose "^3.2.1" -koa-redis-cache@3.0.0: - version "3.0.0" - resolved "https://registry.npmjs.org/koa-redis-cache/-/koa-redis-cache-3.0.0.tgz#e4c33ebc9d9c08fd50a669bca4573fabb4b2c584" - dependencies: - co-redis "2" - path-to-regexp "1" - readall "1" - redis "2" - koa-router@7.4.0: version "7.4.0" resolved "https://registry.npmjs.org/koa-router/-/koa-router-7.4.0.tgz#aee1f7adc02d5cb31d7d67465c9eacc825e8c5e0" @@ -3960,16 +3951,16 @@ path-parse@^1.0.5: version "1.0.5" resolved "https://registry.npmjs.org/path-parse/-/path-parse-1.0.5.tgz#3c1adf871ea9cd6c9431b6ea2bd74a0ff055c4c1" -path-to-regexp@1, path-to-regexp@^1.1.1: +path-to-regexp@2.2.1: + version "2.2.1" + resolved "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-2.2.1.tgz#90b617025a16381a879bc82a38d4e8bdeb2bcf45" + +path-to-regexp@^1.1.1: version "1.7.0" resolved "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-1.7.0.tgz#59fde0f435badacba103a84e9d3bc64e96b9937d" dependencies: isarray "0.0.1" -path-to-regexp@2.2.1: - version "2.2.1" - resolved "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-2.2.1.tgz#90b617025a16381a879bc82a38d4e8bdeb2bcf45" - path-type@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/path-type/-/path-type-3.0.0.tgz#cef31dc8e0a1a3bb0d105c0cd97cf3bf47f4e36f" @@ -4506,7 +4497,7 @@ readable-stream@1.0: isarray "0.0.1" string_decoder "~0.10.x" -readall@1, readall@1.0.0: +readall@1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/readall/-/readall-1.0.0.tgz#8b4bfc412e687dbfbb00e0a6ede7e3a0f001b7c7" dependencies: @@ -4536,7 +4527,7 @@ redis-parser@^2.6.0: version "2.6.0" resolved "https://registry.npmjs.org/redis-parser/-/redis-parser-2.6.0.tgz#52ed09dacac108f1a631c07e9b69941e7a19504b" -redis@2: +redis@2.8.0: version "2.8.0" resolved "https://registry.npmjs.org/redis/-/redis-2.8.0.tgz#202288e3f58c49f6079d97af7a10e1303ae14b02" dependencies: