fix(route/twitter): Web API authentication (#12754)

docs/en/install/README.md

@@ -823,7 +823,7 @@ See docs of the specified route and `lib/config.js` for detailed information.
 
     -   `TWITTER_CONSUMER_KEY`: Twitter Developer API key, support multiple keys, split them with `,`
     -   `TWITTER_CONSUMER_SECRET`: Twitter Developer API key secret, support multiple keys, split them with `,`
-    -   `TWITTER_WEBAPI_AUTHORIZAION`: Twitter Web API authorization. If either of the above environment variables is not set, the Twitter Web API will be used. However, no need to set this environment var since every single user and guest share the same authorization token which has already been built into RSSHub.
+    -   `TWITTER_WEBAPI_AUTHORIZAION`: Twitter Web API authorization, in format `key:secret`, support multiple ones, split them with `,`. If either of the above environment variables is not set, the Twitter Web API will be used. However, no need to set this environment var since currently known tokens have already been built into RSSHub.
     -   `TWITTER_TOKEN_{handler}`: The token generated by the corresponding Twitter handler, replace `{handler}` with the Twitter handler, the value is a combination of `Twitter API key, Twitter API key secret, Access token, Access token secret` connected by a comma `,`. Eg. `TWITTER_TOKEN_RSSHub=bX1zry5nG4d1RbESQbnADpVIo,2YrD8qo9sXbB8VlYfVmo1Qtw0xsexnOliU5oZofq7aPIGou0Xx,123456789-hlkUHFYmeXrRcf6SEQciP8rP4lzmRgMgwdqIN9aK,pHcPnfa28rCIKhSICUCiaw9ppuSSl7T2f3dnGYpSM0bod`.
 
 -   Wordpress:

docs/en/social-media.md

@@ -532,7 +532,7 @@ This route requires Twitter token's corresponding id, therefore it's only availa
 
 ### Tweet Details
 
-<Route author="LarchLiu" example="/twitter/tweet/DIYgod/status/1650844643997646852" path="/twitter/tweet/:id/status/:status/:original?" :paramsDesc="['User name', 'Tweet ID', 'extra parameters, data type of return, if the value is not `0`/`false` and `config.isPackage` is `true`, return the original data of twitter']" radar="1" rssbud="1"/>
+<Route author="LarchLiu Rongronggg9" example="/twitter/tweet/DIYgod/status/1650844643997646852" path="/twitter/tweet/:id/status/:status/:original?" :paramsDesc="['User name', 'Tweet ID', 'extra parameters, data type of return, if the value is not `0`/`false` and `config.isPackage` is `true`, return the original data of twitter']" radar="1" rssbud="1"/>
 
 ## Vimeo
 

docs/install/README.md

@@ -867,7 +867,7 @@ RSSHub 支持使用访问密钥 / 码，白名单和黑名单三种方式进行
 
     -   `TWITTER_CONSUMER_KEY`: Twitter Developer API key，支持多个 key，用英文逗号 `,` 隔开
     -   `TWITTER_CONSUMER_SECRET`: Twitter Developer API key secret，支持多个 key，用英文逗号 `,` 隔开，顺序与 key 对应
-    -   `TWITTER_WEBAPI_AUTHORIZAION`: Twitter Web API authorization。如果上述两个环境变量中的任意一个未设置，就会使用 Twitter Web API。然而，没有必要设置这个环境变量，因为所有用户和访客共享同一个 authorization token 且已经内置于 RSSHub 之中。
+    -   `TWITTER_WEBAPI_AUTHORIZAION`: Twitter Web API authorization，格式为 `key:secret`，支持多个，用英文逗号 `,` 隔开。如果上述两个环境变量中的任意一个未设置，就会使用 Twitter Web API。然而，没有必要设置这个环境变量，因为 RSSHub 已经内置了目前已知可用的 token。
     -   `TWITTER_TOKEN_{handler}`: 对应 Twitter 用户名生成的 token，`{handler}` 替换为用于生成该 token 的 Twitter 用户名，值为 `Twitter API key, Twitter API key secret, Access token, Access token secret` 用逗号隔开，例如：`TWITTER_TOKEN_RSSHub=bX1zry5nG4d1RbESQbnADpVIo,2YrD8qo9sXbB8VlYfVmo1Qtw0xsexnOliU5oZofq7aPIGou0Xx,123456789-hlkUHFYmeXrRcf6SEQciP8rP4lzmRgMgwdqIN9aK,pHcPnfa28rCIKhSICUCiaw9ppuSSl7T2f3dnGYpSM0bod`
 
 -   Wordpress

docs/social-media.md

@@ -870,7 +870,7 @@ Instagram Stories 没有可靠的 guid，你的 RSS 阅读器可能将同一条
 
 ### 推文详情
 
-<Route author="LarchLiu" example="/twitter/tweet/DIYgod/status/1650844643997646852" path="/twitter/tweet/:id/status/:status/:original?" :paramsDesc="['用户名', '推文 ID', '额外参数；返回数据类型，当非 `0`/`false` 且 `config.isPackage` 为 `true`时，返回 twitter 原始数据']" radar="1" rssbud="1"/>
+<Route author="LarchLiu Rongronggg9" example="/twitter/tweet/DIYgod/status/1650844643997646852" path="/twitter/tweet/:id/status/:status/:original?" :paramsDesc="['用户名', '推文 ID', '额外参数；返回数据类型，当非 `0`/`false` 且 `config.isPackage` 为 `true`时，返回 twitter 原始数据']" radar="1" rssbud="1"/>
 
 ## Vimeo
 

lib/config.js

@@ -277,8 +277,7 @@ const calculateValue = () => {
             consumer_key: envs.TWITTER_CONSUMER_KEY,
             consumer_secret: envs.TWITTER_CONSUMER_SECRET,
             tokens: twitter_tokens,
-            authorization: envs.TWITTER_WEBAPI_AUTHORIZAION || 'Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw',
-            // reference: https://github.com/dangeredwolf/FixTweet/blob/f3082bbb0d69798687481a605f6760b2eb7558e0/src/constants.ts#L23-L25
+            authorization: envs.TWITTER_WEBAPI_AUTHORIZAION && envs.TWITTER_WEBAPI_AUTHORIZAION.split(','),
         },
         weibo: {
             app_key: envs.WEIBO_APP_KEY,

lib/v2/twitter/maintainer.js

@@ -6,6 +6,6 @@ module.exports = {
     '/list/:id/:name/:routeParams?': ['xyqfer'],
     '/media/:id/:routeParams?': ['yindaheng98', 'Rongronggg9'],
     '/trends/:woeid?': ['sakamossan'],
-    '/tweet/:id/status/:status/:original?': ['LarchLiu'],
+    '/tweet/:id/status/:status/:original?': ['LarchLiu', 'Rongronggg9'],
     '/user/:id/:routeParams?': ['DIYgod', 'yindaheng98', 'Rongronggg9'],
 };

lib/v2/twitter/web-api/constants.js

@@ -0,0 +1,63 @@
+// https://github.com/zedeus/nitter/issues/919#issuecomment-1619067142
+// https://git.sr.ht/~cloutier/bird.makeup/tree/087a8e3e98b642841dde84465c19121fc3b4c6ee/item/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs#L36
+const tokens = [
+    'CjulERsDeqhhjSme66ECg:IQWdVyqFxghAtURHGeGiWAsmCAGmdW3WmbEx6Hck', // iPad
+    // valid, but endpoints differ
+    // 'IQKbtAYlXLripLGPWd0HUA:GgDYlkSvaPxGxC4X8liwpUoqKwwr3lCADbz8A7ADU', // iPhone
+    // '3nVuSoBZnx6U4vzUxf5w:Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys', // Android
+    // '3rJOl1ODzm9yZy63FACdg:5jPoQ5kQvMJFDYRNE8bQ4rHuds4xJqhvgNJM4awaE8', // Mac
+];
+
+const graphQLEndpointsPlain = [
+    '/graphql/oUZZZ8Oddwxs8Cd3iW3UEA/UserByScreenName',
+    '/graphql/3XDB26fBve-MmjHaWTUZxA/TweetDetail',
+    '/graphql/QqZBEqganhHwmU9QscmIug/UserTweets',
+    '/graphql/wxoVeDnl0mP7VLhe6mTOdg/UserTweetsAndReplies',
+    '/graphql/Az0-KW6F-FyYTc2OJmvUhg/UserMedia',
+    '/graphql/kgZtsNyE46T3JaEf2nF9vw/Likes',
+    // these endpoints are not available if authenticated as other clients
+    // FYI, endpoints for Android: https://gist.github.com/ScamCast/2e40befbd1b61c4a80cda2745d4df1f4
+];
+
+const graphQLMap = Object.fromEntries(graphQLEndpointsPlain.map((endpoint) => [endpoint.split('/')[3], endpoint]));
+
+const featuresMap = {
+    UserByScreenName: JSON.stringify({
+        hidden_profile_likes_enabled: false,
+        responsive_web_graphql_exclude_directive_enabled: true,
+        verified_phone_label_enabled: false,
+        subscriptions_verification_info_verified_since_enabled: true,
+        highlights_tweets_tab_ui_enabled: true,
+        creator_subscriptions_tweet_preview_api_enabled: true,
+        responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
+        responsive_web_graphql_timeline_navigation_enabled: true,
+    }),
+    UserTweets: JSON.stringify({
+        rweb_lists_timeline_redesign_enabled: true,
+        responsive_web_graphql_exclude_directive_enabled: true,
+        verified_phone_label_enabled: false,
+        creator_subscriptions_tweet_preview_api_enabled: true,
+        responsive_web_graphql_timeline_navigation_enabled: true,
+        responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
+        tweetypie_unmention_optimization_enabled: true,
+        responsive_web_edit_tweet_api_enabled: true,
+        graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
+        view_counts_everywhere_api_enabled: true,
+        longform_notetweets_consumption_enabled: true,
+        responsive_web_twitter_article_tweet_consumption_enabled: false,
+        tweet_awards_web_tipping_enabled: false,
+        freedom_of_speech_not_reach_fetch_enabled: true,
+        standardized_nudges_misinfo: true,
+        tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
+        longform_notetweets_rich_text_read_enabled: true,
+        longform_notetweets_inline_media_enabled: true,
+        responsive_web_media_download_video_enabled: false,
+        responsive_web_enhance_cards_enabled: false,
+    }),
+};
+
+module.exports = {
+    tokens,
+    graphQLMap,
+    featuresMap,
+};

lib/v2/twitter/web-api/twitter-api.js

@@ -1,4 +1,5 @@
 const twitterGot = require('./twitter-got');
+const { graphQLMap, featuresMap } = require('./constants');
 const config = require('@/config').value;
 
 // https://github.com/mikf/gallery-dl/blob/a53cfc845e12d9e98fefd07e43ebffaec488c18f/gallery_dl/extractor/twitter.py#L727-L755
@@ -65,6 +66,7 @@ const paginationTweets = async (endpoint, userId, variables, path) => {
             ...variables,
             userId,
         }),
+        features: featuresMap.UserTweets,
     });
 
     let instructions;
@@ -81,24 +83,24 @@ const paginationTweets = async (endpoint, userId, variables, path) => {
 
 // https://github.com/mikf/gallery-dl/blob/a53cfc845e12d9e98fefd07e43ebffaec488c18f/gallery_dl/extractor/twitter.py#L807-L814
 const timelineTweets = (userId, params = {}) =>
-    paginationTweets('/graphql/WZT7sCTrLvSOaWOXLDsWbQ/UserTweets', userId, {
+    paginationTweets(graphQLMap.UserTweets, userId, {
         ...params,
         withQuickPromoteEligibilityTweetFields: true,
     });
 
 // https://github.com/mikf/gallery-dl/blob/a53cfc845e12d9e98fefd07e43ebffaec488c18f/gallery_dl/extractor/twitter.py#L816-L823
 const timelineTweetsAndReplies = (userId, params = {}) =>
-    paginationTweets('/graphql/t4wEKVulW4Mbv1P0kgxTEw/UserTweetsAndReplies', userId, {
+    paginationTweets(graphQLMap.UserTweetsAndReplies, userId, {
         ...params,
         withCommunity: true,
     });
 
 // https://github.com/mikf/gallery-dl/blob/a53cfc845e12d9e98fefd07e43ebffaec488c18f/gallery_dl/extractor/twitter.py#L825-L831
-const timelineMedia = (userId, params = {}) => paginationTweets('/graphql/nRybED9kRbN-TOWioHq1ng/UserMedia', userId, params);
+const timelineMedia = (userId, params = {}) => paginationTweets(graphQLMap.UserMedia, userId, params);
 
 // this query requires login
 // https://github.com/mikf/gallery-dl/blob/a53cfc845e12d9e98fefd07e43ebffaec488c18f/gallery_dl/extractor/twitter.py#L833-L839
-const timelineLikes = (userId, params = {}) => paginationTweets(`/graphql/9MSTt44HoGjVFSg_u3rHDw/Likes`, userId, params);
+const timelineLikes = (userId, params = {}) => paginationTweets(graphQLMap.Likes, userId, params);
 
 // https://github.com/mikf/gallery-dl/blob/a53cfc845e12d9e98fefd07e43ebffaec488c18f/gallery_dl/extractor/twitter.py#L858-L866
 const timelineKeywords = (keywords, params = {}) =>
@@ -114,7 +116,7 @@ const timelineKeywords = (keywords, params = {}) =>
 // https://github.com/mikf/gallery-dl/blob/a53cfc845e12d9e98fefd07e43ebffaec488c18f/gallery_dl/extractor/twitter.py#L795-L805
 const tweetDetail = (userId, params) =>
     paginationTweets(
-        '/graphql/ItejhtHVxU7ksltgMmyaLA/TweetDetail',
+        graphQLMap.TweetDetail,
         userId,
         {
             ...params,
@@ -225,12 +227,13 @@ const excludeRetweet = function (tweets) {
 };
 
 const userByScreenName = (screenName) =>
-    twitterGot('https://twitter.com/i/api/graphql/hc-pka9A7gyS3xODIafnrQ/UserByScreenName', {
+    twitterGot(`https://twitter.com/i/api${graphQLMap.UserByScreenName}`, {
         variables: `{"screen_name":"${screenName}","withHighlightedLabel":true}`,
+        features: featuresMap.UserByScreenName,
     });
 const getUserData = (cache, screenName) => cache.tryGet(`twitter-userdata-${screenName}`, () => userByScreenName(screenName));
-const getUserID = async (cache, screenName) => (await getUserData(cache, screenName)).data.user.rest_id;
-const getUser = async (cache, screenName) => (await getUserData(cache, screenName)).data.user.legacy;
+const getUserID = async (cache, screenName) => (await getUserData(cache, screenName)).data.user.result.rest_id;
+const getUser = async (cache, screenName) => (await getUserData(cache, screenName)).data.user.result.legacy;
 
 const cacheTryGet = async (cache, screenName, params, func) => {
     const id = await getUserID(cache, screenName);

lib/v2/twitter/web-api/twitter-got.js

@@ -3,11 +3,13 @@ const { promisify } = require('util');
 const queryString = require('query-string');
 const got = require('@/utils/got');
 const config = require('@/config').value;
+const constants = require('./constants');
+
+const tokens = config.twitter.authorization && config.twitter.authorization.length ? config.twitter.authorization : constants.tokens;
+
 // https://github.com/mikf/gallery-dl/blob/a53cfc845e12d9e98fefd07e43ebffaec488c18f/gallery_dl/extractor/twitter.py#L716-L726
 const headers = {
-    authorization: config.twitter.authorization,
-    // Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw
-    // reference: https://github.com/dangeredwolf/FixTweet/blob/f3082bbb0d69798687481a605f6760b2eb7558e0/src/constants.ts#L23-L25
+    authorization: undefined,
     'x-guest-token': undefined,
     'x-twitter-auth-type': undefined,
     'x-twitter-client-language': 'en',
@@ -15,7 +17,11 @@ const headers = {
     'x-csrf-token': undefined,
     Referer: 'https://twitter.com/',
 };
-let cookieJar, setCookie, getCookies;
+
+let cookieJar,
+    setCookie,
+    getCookies,
+    tries = 0;
 
 const cookiedomain = 'twitter.com';
 const cookieurl = 'https://twitter.com';
@@ -39,13 +45,22 @@ async function resetSession() {
     cookieJar = new CookieJar();
     getCookies = promisify(cookieJar.getCookies.bind(cookieJar));
     setCookie = promisify(cookieJar.setCookie.bind(cookieJar));
+    let response;
+    // auth
+    headers.authorization = `Basic ${Buffer.from(tokens[tries++ % tokens.length]).toString('base64')}`;
+    response = await twitterGot({
+        url: 'https://api.twitter.com/oauth2/token',
+        method: 'POST',
+        searchParams: queryString.stringify({ grant_type: 'client_credentials' }),
+    });
+    headers.authorization = `Bearer ${response.data.access_token}`;
     // 生成csrf-token
     const csrfToken = [...Array(16 * 2)].map(() => Math.floor(Math.random() * 16).toString(16)).join('');
     await setCookie(new Cookie({ key: 'ct0', value: csrfToken, domain: cookiedomain, secure: false }), cookieurl);
     headers['x-csrf-token'] = csrfToken;
     headers['x-guest-token'] = undefined;
     // 发起初始化请求
-    const response = await twitterGot({
+    response = await twitterGot({
         url: 'https://api.twitter.com/1.1/guest/activate.json',
         method: 'POST',
     });
@@ -78,13 +93,16 @@ async function twitterRequest(url, params, method) {
     try {
         response = await request();
     } catch (e) {
-        if (e.response.status === 403) {
+        if (e.response.status === 403 || (e.response.status === 404 && !e.response.data)) {
             await resetSession();
             response = await request();
         } else {
             throw e;
         }
     }
+    if (response.data.errors) {
+        throw Error('API reports error:\n' + response.data.errors.map((e) => `${e.code}: ${e.message}`).join('\n'));
+    }
     return response.data;
 }