diff --git a/lib/v2/gov/miit/wjgs.js b/lib/v2/gov/miit/wjgs.js index 6a506d3514..37773fad52 100644 --- a/lib/v2/gov/miit/wjgs.js +++ b/lib/v2/gov/miit/wjgs.js @@ -1,53 +1,62 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); +const { parseDate } = require('@/utils/parse-date'); +const timezone = require('@/utils/timezone'); +const baseUrl = 'https://www.miit.gov.cn'; +const siteUrl = `${baseUrl}/zwgk/wjgs/index.html`; module.exports = async (ctx) => { - const base_url = 'http://www.miit.gov.cn/n1146295/n7281310/'; - const response = await got.get(base_url); + const response = await got(siteUrl); const $ = cheerio.load(response.data); - const list = $('.clist_con li').get(); + const buildStatic = $('script[parseType=buildstatic]'); + const requestUrl = buildStatic.attr('url'); + const queryData = JSON.parse(buildStatic.attr('querydata').replace(/'/g, '"')); - const ProcessFeed = (data) => { - const $ = cheerio.load(data); + const { data } = await got(`${baseUrl}${requestUrl}`, { + headers: { + referer: siteUrl, + }, + searchParams: queryData, + }); + const list = cheerio.load(data.data.html, null, false); - const content = $('p'); - return content.html(); - }; - - const items = await Promise.all( - list.map(async (item) => { - const $ = cheerio.load(item); - const $a = $('a'); - let link = $a.attr('href'); - if (link.startsWith('..')) { - link = base_url + link; - } - - const cache = await ctx.cache.get(link); - if (cache) { - return Promise.resolve(JSON.parse(cache)); - } - - const response = await got({ - method: 'get', - url: link, - }); - - const single = { - title: $a.text(), - description: ProcessFeed(response.data), - link, + let items = list('.page-content ul li') + .toArray() + .map((item) => { + item = list(item); + return { + title: item.find('a').attr('title'), + link: new URL(item.find('a').attr('href'), baseUrl).href, + pubDate: parseDate(item.find('.fr').text(), 'YYYY-MM-DD'), }; + }); - ctx.cache.set(link, JSON.stringify(single)); - return Promise.resolve(single); - }) + items = await Promise.all( + items.map((item) => + ctx.cache.tryGet(item.link, async () => { + const { data } = await got(item.link); + const $ = cheerio.load(data); + + $('iframe').each((_, e) => { + e = $(e); + if (e.attr('src').startsWith('/')) { + e.attr('src', new URL(e.attr('src'), baseUrl).href); + } + }); + item.author = $('.cinfo') + .text() + .match(/来源:(.*)/)[1]; + item.pubDate = timezone(parseDate($('#con_time').text(), 'YYYY-MM-DD HH:mm'), +8); + item.description = $('.ccontent').html(); + + return item; + }) + ) ); ctx.state.data = { - title: '中国工业化和信息部', - link: 'http://www.miit.gov.cn', - description: '文件公示', + title: `${$('head title').text()} - ${$('meta[name=SiteName]').attr('content')}`, + link: siteUrl, item: items, }; }; diff --git a/lib/v2/gov/miit/zcjd.js b/lib/v2/gov/miit/zcjd.js index c5e3c46690..7516268c9d 100644 --- a/lib/v2/gov/miit/zcjd.js +++ b/lib/v2/gov/miit/zcjd.js @@ -1,53 +1,62 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); +const { parseDate } = require('@/utils/parse-date'); +const timezone = require('@/utils/timezone'); +const baseUrl = 'https://www.miit.gov.cn'; +const siteUrl = `${baseUrl}/zwgk/zcjd/index.html`; module.exports = async (ctx) => { - const base_url = 'http://www.miit.gov.cn/n1146295/n7281315/'; - const response = await got.get(base_url); + const response = await got(siteUrl); const $ = cheerio.load(response.data); - const list = $('.clist_con li').get(); + const buildStatic = $('script[parseType=buildstatic]'); + const requestUrl = buildStatic.attr('url'); + const queryData = JSON.parse(buildStatic.attr('querydata').replace(/'/g, '"')); - const ProcessFeed = (data) => { - const $ = cheerio.load(data); + const { data } = await got(`${baseUrl}${requestUrl}`, { + headers: { + referer: siteUrl, + }, + searchParams: queryData, + }); + const list = cheerio.load(data.data.html, null, false); - const content = $('p'); - return content.html(); - }; - - const items = await Promise.all( - list.map(async (item) => { - const $ = cheerio.load(item); - const $a = $('a'); - let link = $a.attr('href'); - if (link.startsWith('..')) { - link = base_url + link; - } - - const cache = await ctx.cache.get(link); - if (cache) { - return Promise.resolve(JSON.parse(cache)); - } - - const response = await got({ - method: 'get', - url: link, - }); - - const single = { - title: $a.text(), - description: ProcessFeed(response.data), - link, + let items = list('.page-content ul li') + .toArray() + .map((item) => { + item = list(item); + return { + title: item.find('a').attr('title'), + link: new URL(item.find('a').attr('href'), baseUrl).href, + pubDate: parseDate(item.find('.fr').text(), 'YYYY-MM-DD'), }; + }); - ctx.cache.set(link, JSON.stringify(single)); - return Promise.resolve(single); - }) + items = await Promise.all( + items.map((item) => + ctx.cache.tryGet(item.link, async () => { + const { data } = await got(item.link); + const $ = cheerio.load(data); + + $('iframe').each((_, e) => { + e = $(e); + if (e.attr('src').startsWith('/')) { + e.attr('src', new URL(e.attr('src'), baseUrl).href); + } + }); + item.author = $('.cinfo') + .text() + .match(/来源:(.*)/)[1]; + item.pubDate = timezone(parseDate($('#con_time').text(), 'YYYY-MM-DD HH:mm'), +8); + item.description = $('.ccontent').html(); + + return item; + }) + ) ); ctx.state.data = { - title: '中国工业化和信息部', - link: 'http://www.miit.gov.cn', - description: '政策解读', + title: `${$('head title').text()} - ${$('meta[name=SiteName]').attr('content')}`, + link: siteUrl, item: items, }; };