diff --git a/lib/index.js b/lib/index.js index f97315be9e..106db3ded5 100644 --- a/lib/index.js +++ b/lib/index.js @@ -6,8 +6,7 @@ const logger = require('./utils/logger'); const onerror = require('./middleware/onerror'); const header = require('./middleware/header'); const utf8 = require('./middleware/utf8'); -const memoryCache = require('./middleware/lru-cache'); -const redisCache = require('./middleware/redis-cache'); +const cache = require('./middleware/cache'); const parameter = require('./middleware/parameter'); const template = require('./middleware/template'); const favicon = require('koa-favicon'); @@ -65,57 +64,9 @@ app.use(template); app.use(parameter); // 2 cache -if (config.cacheType === 'memory') { - app.use( - memoryCache({ - app: app, - expire: config.cacheExpire, - ignoreQuery: true, - }) - ); -} else if (config.cacheType === 'redis') { - app.use( - redisCache({ - app: app, - expire: config.cacheExpire, - ignoreQuery: true, - redis: config.redis, - onerror: (e) => { - logger.error('Redis error: ', e); - }, - onconnect: () => { - logger.info('Redis connected.'); - }, - }) - ); -} else { - app.context.cache = { - get: () => null, - set: () => null, - }; -} -app.context.cache.tryGet = async function(key, getValueFunc, maxAge = 24 * 60 * 60) { - let v = await this.get(key); - if (!v) { - v = await getValueFunc(); - this.set(key, v, maxAge); - } else { - let parsed; - try { - parsed = JSON.parse(v); - } catch (e) { - parsed = null; - } - if (parsed) { - v = parsed; - } - } - - return v; -}; +app.use(cache(app)); // router - app.use(mount('/', router.routes())).use(router.allowedMethods()); // routes the require authentication diff --git a/lib/middleware/cache.js b/lib/middleware/cache.js new file mode 100644 index 0000000000..0ae37cdb14 --- /dev/null +++ b/lib/middleware/cache.js @@ -0,0 +1,254 @@ +const wrapper = require('co-redis'); +const Redis = require('redis'); +const Lru = require('lru-cache'); +const md5 = require('../utils/md5'); +const config = require('../config'); +const logger = require('../utils/logger'); +const pathToRegExp = require('path-to-regexp'); + +module.exports = function(app, options = {}) { + let available = false; + + const { prefix = 'koa-redis-cache:', expire = config.cacheExpire, routes = ['(.*)'], exclude = ['/'], passParam = '', maxLength = Infinity, ignoreQuery = true } = options; + + const globalCache = { + get: null, + set: null, + }; + if (config.cacheType === 'redis') { + const { host: redisHost = 'localhost', port: redisPort = 6379, url: redisUrl = `redis://${redisHost}:${redisPort}/`, options: redisOptions = {} } = config.redis || {}; + if (!redisOptions.password) { + delete redisOptions.password; + } + const redisClient = wrapper(Redis.createClient(redisUrl, redisOptions)); + redisClient.on('error', (error) => { + available = false; + logger.error('Redis error: ', error); + }); + redisClient.on('end', () => { + available = false; + }); + redisClient.on('connect', () => { + available = true; + logger.info('Redis connected.'); + }); + + app.context.cache = { + get: async (key) => { + if (key) { + const value = await redisClient.get(key); + if (value) { + await redisClient.expire(key, 24 * 60 * 60); + } + return value; + } + }, + set: async (key, value, maxAge) => { + if (!value || value === 'undefined') { + value = ''; + } + if (typeof value === 'object') { + value = JSON.stringify(value); + } + if (key) { + await redisClient.setex(key, maxAge, value); + } + }, + }; + globalCache.get = async (key) => { + if (key) { + const value = await redisClient.get(key); + return value; + } + }; + globalCache.set = app.context.cache.set; + } else if (config.cacheType === 'memory') { + const pageCache = new Lru({ + maxAge: expire * 1000, + max: maxLength, + }); + + const routeCache = new Lru({ + maxAge: expire * 1000, + max: maxLength, + updateAgeOnGet: true, + }); + + app.context.cache = { + get: (key) => { + if (key) { + return routeCache.get(key); + } + }, + set: (key, value, maxAge) => { + if (!value || value === 'undefined') { + value = ''; + } + if (typeof value === 'object') { + value = JSON.stringify(value); + } + if (key) { + return routeCache.set(key, value, maxAge * 1000); + } + }, + }; + globalCache.get = (key) => { + if (key) { + return pageCache.get(key); + } + }; + globalCache.set = (key, value, maxAge) => { + if (!value || value === 'undefined') { + value = ''; + } + if (typeof value === 'object') { + value = JSON.stringify(value); + } + if (key) { + return pageCache.set(key, value, maxAge * 1000); + } + }; + available = true; + } + + app.context.cache.tryGet = async function(key, getValueFunc, maxAge = 24 * 60 * 60) { + let v = await this.get(key); + if (!v) { + v = await getValueFunc(); + this.set(key, v, maxAge); + } else { + let parsed; + try { + parsed = JSON.parse(v); + } catch (e) { + parsed = null; + } + if (parsed) { + v = parsed; + } + } + + return v; + }; + + async function getCache(ctx, key, tkey) { + const value = await globalCache.get(key); + let type; + let ok = false; + + if (value) { + ctx.response.status = 200; + type = (await globalCache.get(tkey)) || 'text/html'; + // can happen if user specified return_buffers: true in redis options + if (Buffer.isBuffer(type)) { + type = type.toString(); + } + if (config.cacheType === 'redis') { + ctx.response.set({ + 'X-Koa-Redis-Cache': 'true', + 'Content-Type': type, + }); + } else if (config.cacheType === 'memory') { + ctx.response.set({ + 'X-Koa-Memory-Cache': 'true', + 'Content-Type': type, + }); + } + try { + ctx.state.data = JSON.parse(value); + } catch (e) { + ctx.state.data = {}; + } + ok = true; + } + + return ok; + } + + async function setCache(ctx, key, tkey, expire) { + ctx.state.data.lastBuildDate = new Date().toUTCString(); + const body = JSON.stringify(ctx.state.data); + + if (ctx.request.method !== 'GET' || !body) { + return; + } + if (Buffer.byteLength(body) > maxLength) { + return; + } + await globalCache.set(key, body, expire); + + const type = ctx.response.headers['content-type']; + if (type) { + await globalCache.set(tkey, type, expire); + } + } + + return async function cache(ctx, next) { + const { url, path } = ctx.request; + const resolvedPrefix = typeof prefix === 'function' ? prefix.call(ctx, ctx) : prefix; + const key = resolvedPrefix + md5(ignoreQuery ? path : url); + const tkey = key + ':type'; + + const validityCheck = (routes, exclude, path) => { + let match = false; + let routeExpire = false; + + const paired = (route, path) => { + const options = { + sensitive: true, + strict: true, + }; + return pathToRegExp(route, [], options).exec(path); + }; + + for (let i = 0; i < routes.length; i++) { + let route = routes[i]; + if (typeof routes[i] === 'object') { + route = routes[i].path; + routeExpire = routes[i].expire; + } + if (paired(route, path)) { + match = true; + break; + } + } + + for (let j = 0; j < exclude.length; j++) { + if (paired(exclude[j], path)) { + match = false; + break; + } + } + + return { match, routeExpire }; + }; + + const validity = validityCheck(routes, exclude, path); + const match = validity.match; + let routeExpire = validity.routeExpire; + + if (!available || !match || (passParam && ctx.request.query[passParam])) { + return await next(); + } + + let ok = false; + try { + ok = await getCache(ctx, key, tkey); + } catch (e) { + ok = false; + } + if (ok) { + return; + } + + await next(); + + try { + const trueExpire = routeExpire || expire; + await setCache(ctx, key, tkey, trueExpire); + } catch (e) { + // + } + routeExpire = false; + }; +}; diff --git a/lib/middleware/lru-cache.js b/lib/middleware/lru-cache.js deleted file mode 100644 index 90622038f9..0000000000 --- a/lib/middleware/lru-cache.js +++ /dev/null @@ -1,139 +0,0 @@ -// based on https://github.com/coderhaoxin/koa-redis-cache - -const lru = require('lru-cache'); -const common = require('./cache-common'); -const md5 = require('../utils/md5'); - -module.exports = function(options = {}) { - const { - prefix = 'koa-cache:', - expire = 30 * 60, // 30 min - routes = ['(.*)'], - exclude = ['/'], - passParam = '', - maxLength = Infinity, - ignoreQuery = false, - } = options; - - const pageCache = new lru({ - maxAge: expire * 1000, - max: maxLength, - }); - - const routeCache = new lru({ - maxAge: expire * 1000, - max: maxLength, - updateAgeOnGet: true, - }); - - options.app.context.cache = { - get: (key) => { - if (key) { - return routeCache.get(key); - } - }, - set: (key, value, maxAge) => { - if (!value || value === 'undefined') { - value = ''; - } - if (typeof value === 'object') { - value = JSON.stringify(value); - } - if (key) { - return routeCache.set(key, value, maxAge * 1000); - } - }, - }; - - return async function cache(ctx, next) { - const { url, path } = ctx.request; - const resolvedPrefix = typeof prefix === 'function' ? prefix.call(ctx, ctx) : prefix; - const key = resolvedPrefix + md5(ignoreQuery ? path : url); - const tkey = key + ':type'; - - const validityCheck = common.validityCheck(routes, exclude, path); - const match = validityCheck.match; - let routeExpire = validityCheck.routeExpire; - - if (!match || (passParam && ctx.request.query[passParam])) { - return await next(); - } - - let ok = false; - try { - ok = await getCache(ctx, key, tkey); - } catch (e) { - ok = false; - } - if (ok) { - return; - } - - await next(); - - try { - const trueExpire = routeExpire || expire; - await setCache(ctx, key, tkey, trueExpire); - } catch (e) {} // eslint-disable-line no-empty - routeExpire = false; - }; - - /** - * getCache - */ - async function getCache(ctx, key, tkey) { - let type; - const value = pageCache.get(key); - - let ok = false; - - if (value) { - ctx.response.status = 200; - type = pageCache.get(tkey) || 'text/html'; - // can happen if user specified return_buffers: true in redis options - if (Buffer.isBuffer(type)) { - type = type.toString(); - } - ctx.response.set({ - 'X-Koa-Memory-Cache': 'true', - 'Content-Type': type, - }); - try { - ctx.state.data = JSON.parse(value); - } catch (e) { - ctx.state.data = {}; - } - ok = true; - } - - return ok; - } - - /** - * setCache - */ - async function setCache(ctx, key, tkey) { - ctx.state.data.lastBuildDate = new Date().toUTCString(); - const body = JSON.stringify(ctx.state.data); - - if (ctx.request.method !== 'GET' || !body) { - return; - } - if (Buffer.byteLength(body) > maxLength) { - return; - } - pageCache.set(key, body); - - await cacheType(ctx, tkey); - } - - /** - * cacheType - */ - async function cacheType(ctx, tkey) { - const type = ctx.response.headers['content-type']; - if (type) { - pageCache.set(tkey, type); - } - } -}; diff --git a/lib/middleware/redis-cache.js b/lib/middleware/redis-cache.js deleted file mode 100644 index 6ecbeaca4c..0000000000 --- a/lib/middleware/redis-cache.js +++ /dev/null @@ -1,159 +0,0 @@ -// based on https://github.com/coderhaoxin/koa-redis-cache - -const wrapper = require('co-redis'); -const Redis = require('redis'); -const common = require('./cache-common'); -const md5 = require('../utils/md5'); - -module.exports = function(options = {}) { - let redisAvailable = false; - - const { - prefix = 'koa-redis-cache:', - expire = 30 * 60, // 30 min - routes = ['(.*)'], - exclude = ['/'], - passParam = '', - maxLength = Infinity, - ignoreQuery = false, - onerror = function() {}, - onconnect = function() {}, - } = options; - - const { host: redisHost = 'localhost', port: redisPort = 6379, url: redisUrl = `redis://${redisHost}:${redisPort}/`, options: redisOptions = {} } = options.redis || {}; - - /** - * redisClient - */ - if (!redisOptions.password) { - delete redisOptions.password; - } - const redisClient = wrapper(Redis.createClient(redisUrl, redisOptions)); - redisClient.on('error', (error) => { - redisAvailable = false; - onerror(error); - }); - redisClient.on('end', () => { - redisAvailable = false; - }); - redisClient.on('connect', () => { - redisAvailable = true; - onconnect(); - }); - - options.app.context.cache = { - get: async (key) => { - if (key) { - const value = await redisClient.get(key); - if (value) { - let ttl = await redisClient.ttl(key); - ttl = ttl > expire ? ttl : expire * 2; - await redisClient.setex(key, ttl, value); - } - return value; - } - }, - set: async (key, value, maxAge) => { - if (!value || value === 'undefined') { - value = ''; - } - if (typeof value === 'object') { - value = JSON.stringify(value); - } - if (key) { - await redisClient.setex(key, maxAge, value); - } - }, - }; - - return async function cache(ctx, next) { - const { url, path } = ctx.request; - const resolvedPrefix = typeof prefix === 'function' ? prefix.call(ctx, ctx) : prefix; - const key = resolvedPrefix + md5(ignoreQuery ? path : url); - const tkey = key + ':type'; - - const validityCheck = common.validityCheck(routes, exclude, path); - const match = validityCheck.match; - let routeExpire = validityCheck.routeExpire; - - if (!redisAvailable || !match || (passParam && ctx.request.query[passParam])) { - return await next(); - } - - let ok = false; - try { - ok = await getCache(ctx, key, tkey); - } catch (e) { - ok = false; - } - if (ok) { - return; - } - - await next(); - - try { - const trueExpire = routeExpire || expire; - await setCache(ctx, key, tkey, trueExpire); - } catch (e) {} // eslint-disable-line no-empty - routeExpire = false; - }; - - /** - * getCache - */ - async function getCache(ctx, key, tkey) { - const value = await redisClient.get(key); - let type; - let ok = false; - - if (value) { - ctx.response.status = 200; - type = (await redisClient.get(tkey)) || 'text/html'; - // can happen if user specified return_buffers: true in redis options - if (Buffer.isBuffer(type)) { - type = type.toString(); - } - ctx.response.set({ - 'X-Koa-Redis-Cache': 'true', - 'Content-Type': type, - }); - try { - ctx.state.data = JSON.parse(value); - } catch (e) { - ctx.state.data = {}; - } - ok = true; - } - - return ok; - } - - /** - * setCache - */ - async function setCache(ctx, key, tkey, expire) { - ctx.state.data.lastBuildDate = new Date().toUTCString(); - const body = JSON.stringify(ctx.state.data); - - if (ctx.request.method !== 'GET' || !body) { - return; - } - if (Buffer.byteLength(body) > maxLength) { - return; - } - await redisClient.setex(key, expire, body); - - await cacheType(ctx, tkey, expire); - } - - /** - * cacheType - */ - async function cacheType(ctx, tkey, expire) { - const type = ctx.response.headers['content-type']; - if (type) { - await redisClient.setex(tkey, expire, type); - } - } -}; diff --git a/lib/routes/zhihu/daily.js b/lib/routes/zhihu/daily.js index 92f6e0a48e..1dd693e08d 100644 --- a/lib/routes/zhihu/daily.js +++ b/lib/routes/zhihu/daily.js @@ -24,7 +24,7 @@ module.exports = async (ctx) => { link: 'https://news-at.zhihu.com/story/' + story.id, }; const key = 'daily' + story.id; - const value = await ctx.cache.get(key, true); + const value = await ctx.cache.get(key); if (value) { item.description = value; @@ -37,7 +37,7 @@ module.exports = async (ctx) => { }, }); item.description = utils.ProcessImage(storyDetail.data.body.replace(/
([\s\S]*?)<\/div>/g, '$1').replace(//g, '')); - ctx.cache.set(key, item.description, 24 * 60 * 60, true); + ctx.cache.set(key, item.description, 24 * 60 * 60); } return Promise.resolve(item);