diff --git a/lib/routes/galgame/hhgal.js b/lib/routes/galgame/hhgal.js index c27ab0cb0c..73d2e8948d 100644 --- a/lib/routes/galgame/hhgal.js +++ b/lib/routes/galgame/hhgal.js @@ -1,5 +1,6 @@ const cheerio = require('cheerio'); -const cloudscraper = require('cloudscraper'); +const got = require('hooman'); +const { CookieJar } = require('tough-cookie'); module.exports = async (ctx) => { const baseUrl = 'https://www.hhgal.com/'; @@ -15,25 +16,29 @@ module.exports = async (ctx) => { return val; }; - const jar = cloudscraper.jar(); - const response = await cloudscraper({ - uri: baseUrl, - jar: jar, - }); + const cookieJar = new CookieJar(); + const response = ( + await got({ + url: baseUrl, + cookieJar, + }) + ).body; let html = null; if (response.indexOf('YunSuoAutoJump') > -1) { const screenData = stringToHex('1280,720'); const urlData = stringToHex(baseUrl); - jar.setCookie('srcurl=' + urlData, baseUrl); - await cloudscraper({ - uri: baseUrl + '?security_verify_data=' + screenData, - jar: jar, - }); - html = await cloudscraper({ - uri: baseUrl, - jar: jar, + cookieJar.setCookie('srcurl=' + urlData, baseUrl); + await got({ + url: baseUrl + '?security_verify_data=' + screenData, + cookieJar, }); + html = ( + await got({ + url: baseUrl, + cookieJar, + }) + ).body; } const $ = html === null ? cheerio.load(response) : cheerio.load(html); diff --git a/lib/routes/javlibrary/utils.js b/lib/routes/javlibrary/utils.js index dbef298c36..d3671c6795 100644 --- a/lib/routes/javlibrary/utils.js +++ b/lib/routes/javlibrary/utils.js @@ -1,13 +1,13 @@ const url = require('url'); const cheerio = require('cheerio'); -const cloudscraper = require('cloudscraper'); +const got = require('hooman'); const dateUtil = require('@/utils/date'); // 通过传入不同的ctx.state.data.link 返回javlibrary 不同link的rss exports.template = async function template(ctx) { const link = ctx.state.data.link; - const response = await cloudscraper.get(link); - const $ = cheerio.load(response); + const response = await got.get(link); + const $ = cheerio.load(response.body); const list = $('.videothumblist .video').get(); const items = await Promise.all( @@ -20,8 +20,8 @@ exports.template = async function template(ctx) { }; const details = await ctx.cache.tryGet(itemLink, async () => { - const response = await cloudscraper.get(itemLink); - const $ = cheerio.load(response); + const response = await got.get(itemLink); + const $ = cheerio.load(response.body); $('#video_info #video_review .icon').remove(); return { author: $('.star').text(), @@ -40,8 +40,8 @@ exports.template = async function template(ctx) { }; exports.getVideoComments = async function getVideoComments(link) { - const response = await cloudscraper.get(link); - const $ = cheerio.load(response); + const response = await got.get(link); + const $ = cheerio.load(response.body); const list = $('#video_comments > table'); return ( diff --git a/lib/routes/steam/steamdb/free.js b/lib/routes/steam/steamdb/free.js index e78a9bd874..445f2d5874 100644 --- a/lib/routes/steam/steamdb/free.js +++ b/lib/routes/steam/steamdb/free.js @@ -1,10 +1,10 @@ -const cloudscraper = require('cloudscraper'); +const got = require('hooman'); const cheerio = require('cheerio'); module.exports = async (ctx) => { const { type } = ctx.params; const url = 'https://steamdb.info/upcoming/free'; - const html = await cloudscraper.get(url); + const html = (await got.get(url)).body; const $ = cheerio.load(html); const $trs = $('.container table.table-products:first-of-type tbody tr'); @@ -42,7 +42,7 @@ module.exports = async (ctx) => { const hoverLink = `https://steamdb.info/api/RenderAppHover/?appid=${appid}`; const description = await ctx.cache.tryGet(hoverLink, async () => { - const description = await cloudscraper.get(hoverLink); + const description = (await got.get(hoverLink)).body; return description; }); diff --git a/package.json b/package.json index 4c8293b737..e11cc86613 100644 --- a/package.json +++ b/package.json @@ -66,7 +66,6 @@ "art-template": "4.13.2", "cheerio": "1.0.0-rc.3", "chrono-node": "1.4.6", - "cloudscraper": "4.6.0", "co-redis": "2.1.1", "crypto-js": "4.0.0", "currency-symbol-map": "4.0.4", @@ -79,6 +78,7 @@ "googleapis": "50.0.0", "got": "11.1.1", "he": "1.2.0", + "hooman": "1.0.0", "iconv-lite": "0.5.1", "jsdom": "16.2.2", "json-bigint": "0.3.0", @@ -101,6 +101,7 @@ "require-all": "3.0.0", "rss-parser": "3.8.0", "socks-proxy-agent": "5.0.0", + "tough-cookie": "4.0.0", "tunnel": "0.0.6", "twit": "2.2.11", "winston": "3.2.1" diff --git a/yarn.lock b/yarn.lock index cfca01d2f2..2d33f59486 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3409,13 +3409,6 @@ clone-response@^1.0.2: dependencies: mimic-response "^1.0.0" -cloudscraper@4.6.0: - version "4.6.0" - resolved "https://registry.yarnpkg.com/cloudscraper/-/cloudscraper-4.6.0.tgz#bb83157cd18c2e24a0a2de748e2cbde23f6ff684" - integrity sha512-42g6atOAQwhoMlzCYsB1238RYEQa3ibcxhjVeYuZQDLGSZjBNAKOlF/2kcPwZUhlRKA9LDwuYQ7/0LCoMui2ww== - dependencies: - request-promise "^4.2.4" - co-redis@2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/co-redis/-/co-redis-2.1.1.tgz#8bc3717977c9ee5bfb6b28f39ae01bcf724d2dde" @@ -4314,6 +4307,11 @@ destroy@^1.0.4, destroy@~1.0.4: resolved "https://registry.yarnpkg.com/destroy/-/destroy-1.0.4.tgz#978857442c44749e4206613e37946205826abd80" integrity sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA= +detect-indent@~6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/detect-indent/-/detect-indent-6.0.0.tgz#0abd0f549f69fc6659a254fe96786186b6f528fd" + integrity sha512-oSyFlqaTHCItVRGK5RmrmjB+CmaMOW7IaNA/kdxqhoa6d17j/5ce9O9eWXmV/KEdRwqpQA+Vqe8a8Bsybu4YnA== + detect-libc@^1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-1.0.3.tgz#fa137c4bd698edf55cd5cd02ac559f91a4c4ba9b" @@ -4390,6 +4388,11 @@ dns-txt@^2.0.2: dependencies: buffer-indexof "^1.0.0" +docopt@~0.6.2: + version "0.6.2" + resolved "https://registry.yarnpkg.com/docopt/-/docopt-0.6.2.tgz#b28e9e2220da5ec49f7ea5bb24a47787405eeb11" + integrity sha1-so6eIiDaXsSffqW7JKR3h0Be6xE= + docsearch.js@^2.5.2: version "2.6.3" resolved "https://registry.yarnpkg.com/docsearch.js/-/docsearch.js-2.6.3.tgz#57cb4600d3b6553c677e7cbbe6a734593e38625d" @@ -4477,6 +4480,15 @@ domutils@^1.5.1, domutils@^1.7.0: dom-serializer "0" domelementtype "1" +dot-json@^1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/dot-json/-/dot-json-1.2.0.tgz#16f62bbb15b0b834f5c8de299cfb9e4d494f2ac1" + integrity sha512-4bEM7KHFl/U9gAI5nIvU0/fwVzNnE713K339vcxAMtxd2D9mZP6o65UwlcXigJL4rfk90UM0J+D7IPIFYZMQ8Q== + dependencies: + detect-indent "~6.0.0" + docopt "~0.6.2" + underscore-keypath "~0.0.22" + dot-prop@^4.1.1: version "4.2.0" resolved "https://registry.yarnpkg.com/dot-prop/-/dot-prop-4.2.0.tgz#1f19e0c2e1aa0e32797c49799f2837ac6af69c57" @@ -5762,7 +5774,7 @@ googleapis@50.0.0: google-auth-library "^6.0.0" googleapis-common "^4.1.0" -got@11.1.1: +got@11.1.1, got@^11.1.0: version "11.1.1" resolved "https://registry.yarnpkg.com/got/-/got-11.1.1.tgz#11f83049df8155b384413547fe163dd4b35b4240" integrity sha512-7WxfuTyT02hMZZdDvaAprEoxsU13boxI8rWMpqk/5Mq6t+YVbExVB2L6yRLh2Nw3TeJyFpanId41+ZyXGesmbw== @@ -5978,6 +5990,16 @@ hogan.js@^3.0.2: mkdirp "0.3.0" nopt "1.0.10" +hooman@1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/hooman/-/hooman-1.0.0.tgz#0769b81d6e3ce94db3c1b1a35d766531403a53a1" + integrity sha512-XkbN3bTqqkkqU4KfdsKEAmjtxJMJkVD4nlFOcFQEuYXUpnPnZbN3UOjvoR+oWtEsX0Vtjz5ffrJUtdjtzz89mw== + dependencies: + got "^11.1.0" + jsdom "^16.2.2" + tough-cookie "^4.0.0" + user-agents "^1.0.559" + hosted-git-info@^2.1.4: version "2.8.8" resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.8.8.tgz#7539bd4bc1e0e0a895815a2e0262420b12858488" @@ -9762,6 +9784,11 @@ psl@^1.1.28: resolved "https://registry.yarnpkg.com/psl/-/psl-1.1.32.tgz#3f132717cf2f9c169724b2b6caf373cf694198db" integrity sha512-MHACAkHpihU/REGGPLj4sEfc/XKW2bheigvHO1dUqjaKigMp1C8+WLQYRGgeKFMsw5PMfegZcaN8IDXK/cD0+g== +psl@^1.1.33: + version "1.8.0" + resolved "https://registry.yarnpkg.com/psl/-/psl-1.8.0.tgz#9326f8bcfb013adcc005fdff056acce020e51c24" + integrity sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ== + pstree.remy@^1.1.7: version "1.1.7" resolved "https://registry.yarnpkg.com/pstree.remy/-/pstree.remy-1.1.7.tgz#c76963a28047ed61542dc361aa26ee55a7fa15f3" @@ -10194,7 +10221,7 @@ request-promise-native@^1.0.5, request-promise-native@^1.0.8: stealthy-require "^1.1.1" tough-cookie "^2.3.3" -request-promise@^4.2.2, request-promise@^4.2.4: +request-promise@^4.2.2: version "4.2.5" resolved "https://registry.npm.taobao.org/request-promise/download/request-promise-4.2.5.tgz?cache=0&sync_timestamp=1572828627911&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Frequest-promise%2Fdownload%2Frequest-promise-4.2.5.tgz#186222c59ae512f3497dfe4d75a9c8461bd0053c" integrity sha1-GGIixZrlEvNJff5NdanIRhvQBTw= @@ -11578,6 +11605,15 @@ touch@^3.1.0: dependencies: nopt "~1.0.10" +tough-cookie@4.0.0, tough-cookie@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.0.0.tgz#d822234eeca882f991f0f908824ad2622ddbece4" + integrity sha512-tHdtEpQCMrc1YLrMaqXXcj6AxhYi/xgit6mZu1+EDWUn+qhUf8wMQoFIy9NXuq23zAwtcB0t/MjACGR18pcRbg== + dependencies: + psl "^1.1.33" + punycode "^2.1.1" + universalify "^0.1.2" + tough-cookie@^2.3.3, tough-cookie@^2.3.4, tough-cookie@~2.5.0: version "2.5.0" resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.5.0.tgz#cd9fb2a0aa1d5a12b473bd9fb96fa3dcff65ade2" @@ -11747,6 +11783,18 @@ undefsafe@^2.0.2: dependencies: debug "^2.2.0" +underscore-keypath@~0.0.22: + version "0.0.22" + resolved "https://registry.yarnpkg.com/underscore-keypath/-/underscore-keypath-0.0.22.tgz#48a528392bb6efc424be1caa56da4b5faccf264d" + integrity sha1-SKUoOSu278QkvhyqVtpLX6zPJk0= + dependencies: + underscore "*" + +underscore@*: + version "1.10.2" + resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.10.2.tgz#73d6aa3668f3188e4adb0f1943bd12cfd7efaaaf" + integrity sha512-N4P+Q/BuyuEKFJ43B9gYuOj4TQUHXX+j2FqguVOpjkssLUUrnJofCcBccJSCoeturDoZU6GorDTHSvUDlSQbTg== + unicode-canonical-property-names-ecmascript@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-1.0.4.tgz#2619800c4c825800efdd8343af7dd9933cbe2818" @@ -11811,7 +11859,7 @@ unique-string@^2.0.0: dependencies: crypto-random-string "^2.0.0" -universalify@^0.1.0: +universalify@^0.1.0, universalify@^0.1.2: version "0.1.2" resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.1.2.tgz#b646f69be3942dabcecc9d6639c80dc105efaa66" integrity sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg== @@ -11921,6 +11969,14 @@ use@^3.1.0: resolved "https://registry.yarnpkg.com/use/-/use-3.1.1.tgz#d50c8cac79a19fbc20f2911f56eb973f4e10070f" integrity sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ== +user-agents@^1.0.559: + version "1.0.559" + resolved "https://registry.yarnpkg.com/user-agents/-/user-agents-1.0.559.tgz#389817f79dff667d9e5c8803631c41043e8eccd8" + integrity sha512-HdAlNS3vDxOGMRwmv8or05xL96MV3CEwQhUSFTCRoOvTOEnWhTEBPAHRry/xZpVTTOtx77UHMal8YKcx6fs7Lg== + dependencies: + dot-json "^1.2.0" + lodash.clonedeep "^4.5.0" + util-deprecate@^1.0.1, util-deprecate@~1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"