feat: deps cloudscraper -> hooman, close #4538

lib/routes/galgame/hhgal.js

@@ -1,5 +1,6 @@
 const cheerio = require('cheerio');
-const cloudscraper = require('cloudscraper');
+const got = require('hooman');
+const { CookieJar } = require('tough-cookie');
 
 module.exports = async (ctx) => {
     const baseUrl = 'https://www.hhgal.com/';
@@ -15,25 +16,29 @@ module.exports = async (ctx) => {
         return val;
     };
 
-    const jar = cloudscraper.jar();
+    const cookieJar = new CookieJar();
-    const response = await cloudscraper({
+    const response = (
-        uri: baseUrl,
+        await got({
-        jar: jar,
+            url: baseUrl,
-    });
+            cookieJar,
+        })
+    ).body;
     let html = null;
 
     if (response.indexOf('YunSuoAutoJump') > -1) {
         const screenData = stringToHex('1280,720');
         const urlData = stringToHex(baseUrl);
-        jar.setCookie('srcurl=' + urlData, baseUrl);
+        cookieJar.setCookie('srcurl=' + urlData, baseUrl);
-        await cloudscraper({
+        await got({
-            uri: baseUrl + '?security_verify_data=' + screenData,
+            url: baseUrl + '?security_verify_data=' + screenData,
-            jar: jar,
+            cookieJar,
-        });
-        html = await cloudscraper({
-            uri: baseUrl,
-            jar: jar,
         });
+        html = (
+            await got({
+                url: baseUrl,
+                cookieJar,
+            })
+        ).body;
     }
 
     const $ = html === null ? cheerio.load(response) : cheerio.load(html);

lib/routes/javlibrary/utils.js

@@ -1,13 +1,13 @@
 const url = require('url');
 const cheerio = require('cheerio');
-const cloudscraper = require('cloudscraper');
+const got = require('hooman');
 const dateUtil = require('@/utils/date');
 
 // 通过传入不同的ctx.state.data.link 返回javlibrary 不同link的rss
 exports.template = async function template(ctx) {
     const link = ctx.state.data.link;
-    const response = await cloudscraper.get(link);
+    const response = await got.get(link);
-    const $ = cheerio.load(response);
+    const $ = cheerio.load(response.body);
     const list = $('.videothumblist .video').get();
 
     const items = await Promise.all(
@@ -20,8 +20,8 @@ exports.template = async function template(ctx) {
             };
 
             const details = await ctx.cache.tryGet(itemLink, async () => {
-                const response = await cloudscraper.get(itemLink);
+                const response = await got.get(itemLink);
-                const $ = cheerio.load(response);
+                const $ = cheerio.load(response.body);
                 $('#video_info #video_review .icon').remove();
                 return {
                     author: $('.star').text(),
@@ -40,8 +40,8 @@ exports.template = async function template(ctx) {
 };
 
 exports.getVideoComments = async function getVideoComments(link) {
-    const response = await cloudscraper.get(link);
+    const response = await got.get(link);
-    const $ = cheerio.load(response);
+    const $ = cheerio.load(response.body);
     const list = $('#video_comments > table');
 
     return (

lib/routes/steam/steamdb/free.js

@@ -1,10 +1,10 @@
-const cloudscraper = require('cloudscraper');
+const got = require('hooman');
 const cheerio = require('cheerio');
 
 module.exports = async (ctx) => {
     const { type } = ctx.params;
     const url = 'https://steamdb.info/upcoming/free';
-    const html = await cloudscraper.get(url);
+    const html = (await got.get(url)).body;
     const $ = cheerio.load(html);
 
     const $trs = $('.container table.table-products:first-of-type tbody tr');
@@ -42,7 +42,7 @@ module.exports = async (ctx) => {
 
             const hoverLink = `https://steamdb.info/api/RenderAppHover/?appid=${appid}`;
             const description = await ctx.cache.tryGet(hoverLink, async () => {
-                const description = await cloudscraper.get(hoverLink);
+                const description = (await got.get(hoverLink)).body;
                 return description;
             });
 

package.json

@@ -66,7 +66,6 @@
     "art-template": "4.13.2",
     "cheerio": "1.0.0-rc.3",
     "chrono-node": "1.4.6",
-    "cloudscraper": "4.6.0",
     "co-redis": "2.1.1",
     "crypto-js": "4.0.0",
     "currency-symbol-map": "4.0.4",
@@ -79,6 +78,7 @@
     "googleapis": "50.0.0",
     "got": "11.1.1",
     "he": "1.2.0",
+    "hooman": "1.0.0",
     "iconv-lite": "0.5.1",
     "jsdom": "16.2.2",
     "json-bigint": "0.3.0",
@@ -101,6 +101,7 @@
     "require-all": "3.0.0",
     "rss-parser": "3.8.0",
     "socks-proxy-agent": "5.0.0",
+    "tough-cookie": "4.0.0",
     "tunnel": "0.0.6",
     "twit": "2.2.11",
     "winston": "3.2.1"

yarn.lock

@@ -3409,13 +3409,6 @@ clone-response@^1.0.2:
   dependencies:
     mimic-response "^1.0.0"
 
-cloudscraper@4.6.0:
-  version "4.6.0"
-  resolved "https://registry.yarnpkg.com/cloudscraper/-/cloudscraper-4.6.0.tgz#bb83157cd18c2e24a0a2de748e2cbde23f6ff684"
-  integrity sha512-42g6atOAQwhoMlzCYsB1238RYEQa3ibcxhjVeYuZQDLGSZjBNAKOlF/2kcPwZUhlRKA9LDwuYQ7/0LCoMui2ww==
-  dependencies:
-    request-promise "^4.2.4"
-
 co-redis@2.1.1:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/co-redis/-/co-redis-2.1.1.tgz#8bc3717977c9ee5bfb6b28f39ae01bcf724d2dde"
@@ -4314,6 +4307,11 @@ destroy@^1.0.4, destroy@~1.0.4:
   resolved "https://registry.yarnpkg.com/destroy/-/destroy-1.0.4.tgz#978857442c44749e4206613e37946205826abd80"
   integrity sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=
 
+detect-indent@~6.0.0:
+  version "6.0.0"
+  resolved "https://registry.yarnpkg.com/detect-indent/-/detect-indent-6.0.0.tgz#0abd0f549f69fc6659a254fe96786186b6f528fd"
+  integrity sha512-oSyFlqaTHCItVRGK5RmrmjB+CmaMOW7IaNA/kdxqhoa6d17j/5ce9O9eWXmV/KEdRwqpQA+Vqe8a8Bsybu4YnA==
+
 detect-libc@^1.0.2:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-1.0.3.tgz#fa137c4bd698edf55cd5cd02ac559f91a4c4ba9b"
@@ -4390,6 +4388,11 @@ dns-txt@^2.0.2:
   dependencies:
     buffer-indexof "^1.0.0"
 
+docopt@~0.6.2:
+  version "0.6.2"
+  resolved "https://registry.yarnpkg.com/docopt/-/docopt-0.6.2.tgz#b28e9e2220da5ec49f7ea5bb24a47787405eeb11"
+  integrity sha1-so6eIiDaXsSffqW7JKR3h0Be6xE=
+
 docsearch.js@^2.5.2:
   version "2.6.3"
   resolved "https://registry.yarnpkg.com/docsearch.js/-/docsearch.js-2.6.3.tgz#57cb4600d3b6553c677e7cbbe6a734593e38625d"
@@ -4477,6 +4480,15 @@ domutils@^1.5.1, domutils@^1.7.0:
     dom-serializer "0"
     domelementtype "1"
 
+dot-json@^1.2.0:
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/dot-json/-/dot-json-1.2.0.tgz#16f62bbb15b0b834f5c8de299cfb9e4d494f2ac1"
+  integrity sha512-4bEM7KHFl/U9gAI5nIvU0/fwVzNnE713K339vcxAMtxd2D9mZP6o65UwlcXigJL4rfk90UM0J+D7IPIFYZMQ8Q==
+  dependencies:
+    detect-indent "~6.0.0"
+    docopt "~0.6.2"
+    underscore-keypath "~0.0.22"
+
 dot-prop@^4.1.1:
   version "4.2.0"
   resolved "https://registry.yarnpkg.com/dot-prop/-/dot-prop-4.2.0.tgz#1f19e0c2e1aa0e32797c49799f2837ac6af69c57"
@@ -5762,7 +5774,7 @@ googleapis@50.0.0:
     google-auth-library "^6.0.0"
     googleapis-common "^4.1.0"
 
-got@11.1.1:
+got@11.1.1, got@^11.1.0:
   version "11.1.1"
   resolved "https://registry.yarnpkg.com/got/-/got-11.1.1.tgz#11f83049df8155b384413547fe163dd4b35b4240"
   integrity sha512-7WxfuTyT02hMZZdDvaAprEoxsU13boxI8rWMpqk/5Mq6t+YVbExVB2L6yRLh2Nw3TeJyFpanId41+ZyXGesmbw==
@@ -5978,6 +5990,16 @@ hogan.js@^3.0.2:
     mkdirp "0.3.0"
     nopt "1.0.10"
 
+hooman@1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/hooman/-/hooman-1.0.0.tgz#0769b81d6e3ce94db3c1b1a35d766531403a53a1"
+  integrity sha512-XkbN3bTqqkkqU4KfdsKEAmjtxJMJkVD4nlFOcFQEuYXUpnPnZbN3UOjvoR+oWtEsX0Vtjz5ffrJUtdjtzz89mw==
+  dependencies:
+    got "^11.1.0"
+    jsdom "^16.2.2"
+    tough-cookie "^4.0.0"
+    user-agents "^1.0.559"
+
 hosted-git-info@^2.1.4:
   version "2.8.8"
   resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.8.8.tgz#7539bd4bc1e0e0a895815a2e0262420b12858488"
@@ -9762,6 +9784,11 @@ psl@^1.1.28:
   resolved "https://registry.yarnpkg.com/psl/-/psl-1.1.32.tgz#3f132717cf2f9c169724b2b6caf373cf694198db"
   integrity sha512-MHACAkHpihU/REGGPLj4sEfc/XKW2bheigvHO1dUqjaKigMp1C8+WLQYRGgeKFMsw5PMfegZcaN8IDXK/cD0+g==
 
+psl@^1.1.33:
+  version "1.8.0"
+  resolved "https://registry.yarnpkg.com/psl/-/psl-1.8.0.tgz#9326f8bcfb013adcc005fdff056acce020e51c24"
+  integrity sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ==
+
 pstree.remy@^1.1.7:
   version "1.1.7"
   resolved "https://registry.yarnpkg.com/pstree.remy/-/pstree.remy-1.1.7.tgz#c76963a28047ed61542dc361aa26ee55a7fa15f3"
@@ -10194,7 +10221,7 @@ request-promise-native@^1.0.5, request-promise-native@^1.0.8:
     stealthy-require "^1.1.1"
     tough-cookie "^2.3.3"
 
-request-promise@^4.2.2, request-promise@^4.2.4:
+request-promise@^4.2.2:
   version "4.2.5"
   resolved "https://registry.npm.taobao.org/request-promise/download/request-promise-4.2.5.tgz?cache=0&sync_timestamp=1572828627911&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Frequest-promise%2Fdownload%2Frequest-promise-4.2.5.tgz#186222c59ae512f3497dfe4d75a9c8461bd0053c"
   integrity sha1-GGIixZrlEvNJff5NdanIRhvQBTw=
@@ -11578,6 +11605,15 @@ touch@^3.1.0:
   dependencies:
     nopt "~1.0.10"
 
+tough-cookie@4.0.0, tough-cookie@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.0.0.tgz#d822234eeca882f991f0f908824ad2622ddbece4"
+  integrity sha512-tHdtEpQCMrc1YLrMaqXXcj6AxhYi/xgit6mZu1+EDWUn+qhUf8wMQoFIy9NXuq23zAwtcB0t/MjACGR18pcRbg==
+  dependencies:
+    psl "^1.1.33"
+    punycode "^2.1.1"
+    universalify "^0.1.2"
+
 tough-cookie@^2.3.3, tough-cookie@^2.3.4, tough-cookie@~2.5.0:
   version "2.5.0"
   resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.5.0.tgz#cd9fb2a0aa1d5a12b473bd9fb96fa3dcff65ade2"
@@ -11747,6 +11783,18 @@ undefsafe@^2.0.2:
   dependencies:
     debug "^2.2.0"
 
+underscore-keypath@~0.0.22:
+  version "0.0.22"
+  resolved "https://registry.yarnpkg.com/underscore-keypath/-/underscore-keypath-0.0.22.tgz#48a528392bb6efc424be1caa56da4b5faccf264d"
+  integrity sha1-SKUoOSu278QkvhyqVtpLX6zPJk0=
+  dependencies:
+    underscore "*"
+
+underscore@*:
+  version "1.10.2"
+  resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.10.2.tgz#73d6aa3668f3188e4adb0f1943bd12cfd7efaaaf"
+  integrity sha512-N4P+Q/BuyuEKFJ43B9gYuOj4TQUHXX+j2FqguVOpjkssLUUrnJofCcBccJSCoeturDoZU6GorDTHSvUDlSQbTg==
+
 unicode-canonical-property-names-ecmascript@^1.0.4:
   version "1.0.4"
   resolved "https://registry.yarnpkg.com/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-1.0.4.tgz#2619800c4c825800efdd8343af7dd9933cbe2818"
@@ -11811,7 +11859,7 @@ unique-string@^2.0.0:
   dependencies:
     crypto-random-string "^2.0.0"
 
-universalify@^0.1.0:
+universalify@^0.1.0, universalify@^0.1.2:
   version "0.1.2"
   resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.1.2.tgz#b646f69be3942dabcecc9d6639c80dc105efaa66"
   integrity sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==
@@ -11921,6 +11969,14 @@ use@^3.1.0:
   resolved "https://registry.yarnpkg.com/use/-/use-3.1.1.tgz#d50c8cac79a19fbc20f2911f56eb973f4e10070f"
   integrity sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ==
 
+user-agents@^1.0.559:
+  version "1.0.559"
+  resolved "https://registry.yarnpkg.com/user-agents/-/user-agents-1.0.559.tgz#389817f79dff667d9e5c8803631c41043e8eccd8"
+  integrity sha512-HdAlNS3vDxOGMRwmv8or05xL96MV3CEwQhUSFTCRoOvTOEnWhTEBPAHRry/xZpVTTOtx77UHMal8YKcx6fs7Lg==
+  dependencies:
+    dot-json "^1.2.0"
+    lodash.clonedeep "^4.5.0"
+
 util-deprecate@^1.0.1, util-deprecate@~1.0.1:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"