From 7e45fe12e959b13458f48c7893f0bb92440a1fe0 Mon Sep 17 00:00:00 2001 From: Rongrong Date: Wed, 5 Jul 2023 01:24:09 +0800 Subject: [PATCH] fix(route/twitter): Web API authentication (#12754) --- docs/en/install/README.md | 2 +- docs/en/social-media.md | 2 +- docs/install/README.md | 2 +- docs/social-media.md | 2 +- lib/config.js | 3 +- lib/v2/twitter/maintainer.js | 2 +- lib/v2/twitter/web-api/constants.js | 63 +++++++++++++++++++++++++++ lib/v2/twitter/web-api/twitter-api.js | 19 ++++---- lib/v2/twitter/web-api/twitter-got.js | 30 ++++++++++--- 9 files changed, 104 insertions(+), 21 deletions(-) create mode 100644 lib/v2/twitter/web-api/constants.js diff --git a/docs/en/install/README.md b/docs/en/install/README.md index 48b1d38823..0c85c3c248 100644 --- a/docs/en/install/README.md +++ b/docs/en/install/README.md @@ -823,7 +823,7 @@ See docs of the specified route and `lib/config.js` for detailed information. - `TWITTER_CONSUMER_KEY`: Twitter Developer API key, support multiple keys, split them with `,` - `TWITTER_CONSUMER_SECRET`: Twitter Developer API key secret, support multiple keys, split them with `,` - - `TWITTER_WEBAPI_AUTHORIZAION`: Twitter Web API authorization. If either of the above environment variables is not set, the Twitter Web API will be used. However, no need to set this environment var since every single user and guest share the same authorization token which has already been built into RSSHub. + - `TWITTER_WEBAPI_AUTHORIZAION`: Twitter Web API authorization, in format `key:secret`, support multiple ones, split them with `,`. If either of the above environment variables is not set, the Twitter Web API will be used. However, no need to set this environment var since currently known tokens have already been built into RSSHub. - `TWITTER_TOKEN_{handler}`: The token generated by the corresponding Twitter handler, replace `{handler}` with the Twitter handler, the value is a combination of `Twitter API key, Twitter API key secret, Access token, Access token secret` connected by a comma `,`. Eg. `TWITTER_TOKEN_RSSHub=bX1zry5nG4d1RbESQbnADpVIo,2YrD8qo9sXbB8VlYfVmo1Qtw0xsexnOliU5oZofq7aPIGou0Xx,123456789-hlkUHFYmeXrRcf6SEQciP8rP4lzmRgMgwdqIN9aK,pHcPnfa28rCIKhSICUCiaw9ppuSSl7T2f3dnGYpSM0bod`. - Wordpress: diff --git a/docs/en/social-media.md b/docs/en/social-media.md index f06f6bd498..6163d82e42 100644 --- a/docs/en/social-media.md +++ b/docs/en/social-media.md @@ -532,7 +532,7 @@ This route requires Twitter token's corresponding id, therefore it's only availa ### Tweet Details - + ## Vimeo diff --git a/docs/install/README.md b/docs/install/README.md index 6382f2e767..ca9cadcdeb 100644 --- a/docs/install/README.md +++ b/docs/install/README.md @@ -867,7 +867,7 @@ RSSHub 支持使用访问密钥 / 码，白名单和黑名单三种方式进行 - `TWITTER_CONSUMER_KEY`: Twitter Developer API key，支持多个 key，用英文逗号 `,` 隔开 - `TWITTER_CONSUMER_SECRET`: Twitter Developer API key secret，支持多个 key，用英文逗号 `,` 隔开，顺序与 key 对应 - - `TWITTER_WEBAPI_AUTHORIZAION`: Twitter Web API authorization。如果上述两个环境变量中的任意一个未设置，就会使用 Twitter Web API。然而，没有必要设置这个环境变量，因为所有用户和访客共享同一个 authorization token 且已经内置于 RSSHub 之中。 + - `TWITTER_WEBAPI_AUTHORIZAION`: Twitter Web API authorization，格式为 `key:secret`，支持多个，用英文逗号 `,` 隔开。如果上述两个环境变量中的任意一个未设置，就会使用 Twitter Web API。然而，没有必要设置这个环境变量，因为 RSSHub 已经内置了目前已知可用的 token。 - `TWITTER_TOKEN_{handler}`: 对应 Twitter 用户名生成的 token，`{handler}` 替换为用于生成该 token 的 Twitter 用户名，值为 `Twitter API key, Twitter API key secret, Access token, Access token secret` 用逗号隔开，例如：`TWITTER_TOKEN_RSSHub=bX1zry5nG4d1RbESQbnADpVIo,2YrD8qo9sXbB8VlYfVmo1Qtw0xsexnOliU5oZofq7aPIGou0Xx,123456789-hlkUHFYmeXrRcf6SEQciP8rP4lzmRgMgwdqIN9aK,pHcPnfa28rCIKhSICUCiaw9ppuSSl7T2f3dnGYpSM0bod` - Wordpress diff --git a/docs/social-media.md b/docs/social-media.md index 67569f3467..537e1d27ec 100644 --- a/docs/social-media.md +++ b/docs/social-media.md @@ -870,7 +870,7 @@ Instagram Stories 没有可靠的 guid，你的 RSS 阅读器可能将同一条 ### 推文详情 - + ## Vimeo diff --git a/lib/config.js b/lib/config.js index 140eef2583..55ea214517 100644 --- a/lib/config.js +++ b/lib/config.js @@ -277,8 +277,7 @@ const calculateValue = () => { consumer_key: envs.TWITTER_CONSUMER_KEY, consumer_secret: envs.TWITTER_CONSUMER_SECRET, tokens: twitter_tokens, - authorization: envs.TWITTER_WEBAPI_AUTHORIZAION || 'Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw', - // reference: https://github.com/dangeredwolf/FixTweet/blob/f3082bbb0d69798687481a605f6760b2eb7558e0/src/constants.ts#L23-L25 + authorization: envs.TWITTER_WEBAPI_AUTHORIZAION && envs.TWITTER_WEBAPI_AUTHORIZAION.split(','), }, weibo: { app_key: envs.WEIBO_APP_KEY, diff --git a/lib/v2/twitter/maintainer.js b/lib/v2/twitter/maintainer.js index efad7dd809..f0c31905b4 100644 --- a/lib/v2/twitter/maintainer.js +++ b/lib/v2/twitter/maintainer.js @@ -6,6 +6,6 @@ module.exports = { '/list/:id/:name/:routeParams?': ['xyqfer'], '/media/:id/:routeParams?': ['yindaheng98', 'Rongronggg9'], '/trends/:woeid?': ['sakamossan'], - '/tweet/:id/status/:status/:original?': ['LarchLiu'], + '/tweet/:id/status/:status/:original?': ['LarchLiu', 'Rongronggg9'], '/user/:id/:routeParams?': ['DIYgod', 'yindaheng98', 'Rongronggg9'], }; diff --git a/lib/v2/twitter/web-api/constants.js b/lib/v2/twitter/web-api/constants.js new file mode 100644 index 0000000000..8f9aa30205 --- /dev/null +++ b/lib/v2/twitter/web-api/constants.js @@ -0,0 +1,63 @@ +// https://github.com/zedeus/nitter/issues/919#issuecomment-1619067142 +// https://git.sr.ht/~cloutier/bird.makeup/tree/087a8e3e98b642841dde84465c19121fc3b4c6ee/item/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs#L36 +const tokens = [ + 'CjulERsDeqhhjSme66ECg:IQWdVyqFxghAtURHGeGiWAsmCAGmdW3WmbEx6Hck', // iPad + // valid, but endpoints differ + // 'IQKbtAYlXLripLGPWd0HUA:GgDYlkSvaPxGxC4X8liwpUoqKwwr3lCADbz8A7ADU', // iPhone + // '3nVuSoBZnx6U4vzUxf5w:Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys', // Android + // '3rJOl1ODzm9yZy63FACdg:5jPoQ5kQvMJFDYRNE8bQ4rHuds4xJqhvgNJM4awaE8', // Mac +]; + +const graphQLEndpointsPlain = [ + '/graphql/oUZZZ8Oddwxs8Cd3iW3UEA/UserByScreenName', + '/graphql/3XDB26fBve-MmjHaWTUZxA/TweetDetail', + '/graphql/QqZBEqganhHwmU9QscmIug/UserTweets', + '/graphql/wxoVeDnl0mP7VLhe6mTOdg/UserTweetsAndReplies', + '/graphql/Az0-KW6F-FyYTc2OJmvUhg/UserMedia', + '/graphql/kgZtsNyE46T3JaEf2nF9vw/Likes', + // these endpoints are not available if authenticated as other clients + // FYI, endpoints for Android: https://gist.github.com/ScamCast/2e40befbd1b61c4a80cda2745d4df1f4 +]; + +const graphQLMap = Object.fromEntries(graphQLEndpointsPlain.map((endpoint) => [endpoint.split('/')[3], endpoint])); + +const featuresMap = { + UserByScreenName: JSON.stringify({ + hidden_profile_likes_enabled: false, + responsive_web_graphql_exclude_directive_enabled: true, + verified_phone_label_enabled: false, + subscriptions_verification_info_verified_since_enabled: true, + highlights_tweets_tab_ui_enabled: true, + creator_subscriptions_tweet_preview_api_enabled: true, + responsive_web_graphql_skip_user_profile_image_extensions_enabled: false, + responsive_web_graphql_timeline_navigation_enabled: true, + }), + UserTweets: JSON.stringify({ + rweb_lists_timeline_redesign_enabled: true, + responsive_web_graphql_exclude_directive_enabled: true, + verified_phone_label_enabled: false, + creator_subscriptions_tweet_preview_api_enabled: true, + responsive_web_graphql_timeline_navigation_enabled: true, + responsive_web_graphql_skip_user_profile_image_extensions_enabled: false, + tweetypie_unmention_optimization_enabled: true, + responsive_web_edit_tweet_api_enabled: true, + graphql_is_translatable_rweb_tweet_is_translatable_enabled: true, + view_counts_everywhere_api_enabled: true, + longform_notetweets_consumption_enabled: true, + responsive_web_twitter_article_tweet_consumption_enabled: false, + tweet_awards_web_tipping_enabled: false, + freedom_of_speech_not_reach_fetch_enabled: true, + standardized_nudges_misinfo: true, + tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true, + longform_notetweets_rich_text_read_enabled: true, + longform_notetweets_inline_media_enabled: true, + responsive_web_media_download_video_enabled: false, + responsive_web_enhance_cards_enabled: false, + }), +}; + +module.exports = { + tokens, + graphQLMap, + featuresMap, +}; diff --git a/lib/v2/twitter/web-api/twitter-api.js b/lib/v2/twitter/web-api/twitter-api.js index ac7bbb5164..64408334bf 100644 --- a/lib/v2/twitter/web-api/twitter-api.js +++ b/lib/v2/twitter/web-api/twitter-api.js @@ -1,4 +1,5 @@ const twitterGot = require('./twitter-got'); +const { graphQLMap, featuresMap } = require('./constants'); const config = require('@/config').value; // https://github.com/mikf/gallery-dl/blob/a53cfc845e12d9e98fefd07e43ebffaec488c18f/gallery_dl/extractor/twitter.py#L727-L755 @@ -65,6 +66,7 @@ const paginationTweets = async (endpoint, userId, variables, path) => { ...variables, userId, }), + features: featuresMap.UserTweets, }); let instructions; @@ -81,24 +83,24 @@ const paginationTweets = async (endpoint, userId, variables, path) => { // https://github.com/mikf/gallery-dl/blob/a53cfc845e12d9e98fefd07e43ebffaec488c18f/gallery_dl/extractor/twitter.py#L807-L814 const timelineTweets = (userId, params = {}) => - paginationTweets('/graphql/WZT7sCTrLvSOaWOXLDsWbQ/UserTweets', userId, { + paginationTweets(graphQLMap.UserTweets, userId, { ...params, withQuickPromoteEligibilityTweetFields: true, }); // https://github.com/mikf/gallery-dl/blob/a53cfc845e12d9e98fefd07e43ebffaec488c18f/gallery_dl/extractor/twitter.py#L816-L823 const timelineTweetsAndReplies = (userId, params = {}) => - paginationTweets('/graphql/t4wEKVulW4Mbv1P0kgxTEw/UserTweetsAndReplies', userId, { + paginationTweets(graphQLMap.UserTweetsAndReplies, userId, { ...params, withCommunity: true, }); // https://github.com/mikf/gallery-dl/blob/a53cfc845e12d9e98fefd07e43ebffaec488c18f/gallery_dl/extractor/twitter.py#L825-L831 -const timelineMedia = (userId, params = {}) => paginationTweets('/graphql/nRybED9kRbN-TOWioHq1ng/UserMedia', userId, params); +const timelineMedia = (userId, params = {}) => paginationTweets(graphQLMap.UserMedia, userId, params); // this query requires login // https://github.com/mikf/gallery-dl/blob/a53cfc845e12d9e98fefd07e43ebffaec488c18f/gallery_dl/extractor/twitter.py#L833-L839 -const timelineLikes = (userId, params = {}) => paginationTweets(`/graphql/9MSTt44HoGjVFSg_u3rHDw/Likes`, userId, params); +const timelineLikes = (userId, params = {}) => paginationTweets(graphQLMap.Likes, userId, params); // https://github.com/mikf/gallery-dl/blob/a53cfc845e12d9e98fefd07e43ebffaec488c18f/gallery_dl/extractor/twitter.py#L858-L866 const timelineKeywords = (keywords, params = {}) => @@ -114,7 +116,7 @@ const timelineKeywords = (keywords, params = {}) => // https://github.com/mikf/gallery-dl/blob/a53cfc845e12d9e98fefd07e43ebffaec488c18f/gallery_dl/extractor/twitter.py#L795-L805 const tweetDetail = (userId, params) => paginationTweets( - '/graphql/ItejhtHVxU7ksltgMmyaLA/TweetDetail', + graphQLMap.TweetDetail, userId, { ...params, @@ -225,12 +227,13 @@ const excludeRetweet = function (tweets) { }; const userByScreenName = (screenName) => - twitterGot('https://twitter.com/i/api/graphql/hc-pka9A7gyS3xODIafnrQ/UserByScreenName', { + twitterGot(`https://twitter.com/i/api${graphQLMap.UserByScreenName}`, { variables: `{"screen_name":"${screenName}","withHighlightedLabel":true}`, + features: featuresMap.UserByScreenName, }); const getUserData = (cache, screenName) => cache.tryGet(`twitter-userdata-${screenName}`, () => userByScreenName(screenName)); -const getUserID = async (cache, screenName) => (await getUserData(cache, screenName)).data.user.rest_id; -const getUser = async (cache, screenName) => (await getUserData(cache, screenName)).data.user.legacy; +const getUserID = async (cache, screenName) => (await getUserData(cache, screenName)).data.user.result.rest_id; +const getUser = async (cache, screenName) => (await getUserData(cache, screenName)).data.user.result.legacy; const cacheTryGet = async (cache, screenName, params, func) => { const id = await getUserID(cache, screenName); diff --git a/lib/v2/twitter/web-api/twitter-got.js b/lib/v2/twitter/web-api/twitter-got.js index 221d1b1149..96720ab28b 100644 --- a/lib/v2/twitter/web-api/twitter-got.js +++ b/lib/v2/twitter/web-api/twitter-got.js @@ -3,11 +3,13 @@ const { promisify } = require('util'); const queryString = require('query-string'); const got = require('@/utils/got'); const config = require('@/config').value; +const constants = require('./constants'); + +const tokens = config.twitter.authorization && config.twitter.authorization.length ? config.twitter.authorization : constants.tokens; + // https://github.com/mikf/gallery-dl/blob/a53cfc845e12d9e98fefd07e43ebffaec488c18f/gallery_dl/extractor/twitter.py#L716-L726 const headers = { - authorization: config.twitter.authorization, - // Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw - // reference: https://github.com/dangeredwolf/FixTweet/blob/f3082bbb0d69798687481a605f6760b2eb7558e0/src/constants.ts#L23-L25 + authorization: undefined, 'x-guest-token': undefined, 'x-twitter-auth-type': undefined, 'x-twitter-client-language': 'en', @@ -15,7 +17,11 @@ const headers = { 'x-csrf-token': undefined, Referer: 'https://twitter.com/', }; -let cookieJar, setCookie, getCookies; + +let cookieJar, + setCookie, + getCookies, + tries = 0; const cookiedomain = 'twitter.com'; const cookieurl = 'https://twitter.com'; @@ -39,13 +45,22 @@ async function resetSession() { cookieJar = new CookieJar(); getCookies = promisify(cookieJar.getCookies.bind(cookieJar)); setCookie = promisify(cookieJar.setCookie.bind(cookieJar)); + let response; + // auth + headers.authorization = `Basic ${Buffer.from(tokens[tries++ % tokens.length]).toString('base64')}`; + response = await twitterGot({ + url: 'https://api.twitter.com/oauth2/token', + method: 'POST', + searchParams: queryString.stringify({ grant_type: 'client_credentials' }), + }); + headers.authorization = `Bearer ${response.data.access_token}`; // 生成csrf-token const csrfToken = [...Array(16 * 2)].map(() => Math.floor(Math.random() * 16).toString(16)).join(''); await setCookie(new Cookie({ key: 'ct0', value: csrfToken, domain: cookiedomain, secure: false }), cookieurl); headers['x-csrf-token'] = csrfToken; headers['x-guest-token'] = undefined; // 发起初始化请求 - const response = await twitterGot({ + response = await twitterGot({ url: 'https://api.twitter.com/1.1/guest/activate.json', method: 'POST', }); @@ -78,13 +93,16 @@ async function twitterRequest(url, params, method) { try { response = await request(); } catch (e) { - if (e.response.status === 403) { + if (e.response.status === 403 || (e.response.status === 404 && !e.response.data)) { await resetSession(); response = await request(); } else { throw e; } } + if (response.data.errors) { + throw Error('API reports error:\n' + response.data.errors.map((e) => `${e.code}: ${e.message}`).join('\n')); + } return response.data; }