From 7e5bb7f46286dbde4d14b538429fabce415c917c Mon Sep 17 00:00:00 2001
From: Nishant Singh <57475999+Rjnishant530@users.noreply.github.com>
Date: Mon, 21 Aug 2023 21:43:16 +0530
Subject: [PATCH] feat(route): add dlnews (#12997)

* feat(route):add DLNEWS

* clean up

* use embedded JSON

* fix: docusaurus style md

* add description.art and rate limit

* increase the concurency

* fix: fix radar docs link

* docs: move to finance

---------
---
 lib/v2/dlnews/category.js               | 81 +++++++++++++++++++++++++
 lib/v2/dlnews/maintainer.js             |  3 +
 lib/v2/dlnews/radar.js                  | 19 ++++++
 lib/v2/dlnews/router.js                 |  3 +
 lib/v2/dlnews/templates/description.art | 24 ++++++++
 lib/v2/dlnews/utils.js                  | 22 +++++++
 website/docs/routes/finance.md          | 24 ++++++++
 website/docs/routes/new-media.md        |  1 -
 8 files changed, 176 insertions(+), 1 deletion(-)
 create mode 100644 lib/v2/dlnews/category.js
 create mode 100644 lib/v2/dlnews/maintainer.js
 create mode 100644 lib/v2/dlnews/radar.js
 create mode 100644 lib/v2/dlnews/router.js
 create mode 100644 lib/v2/dlnews/templates/description.art
 create mode 100644 lib/v2/dlnews/utils.js

diff --git a/lib/v2/dlnews/category.js b/lib/v2/dlnews/category.js
new file mode 100644
index 0000000000..3b7f5d6744
--- /dev/null
+++ b/lib/v2/dlnews/category.js
@@ -0,0 +1,81 @@
+const cheerio = require('cheerio');
+const got = require('@/utils/got');
+const { getData, getList } = require('./utils');
+const { art } = require('@/utils/render');
+const path = require('path');
+const asyncPool = require('tiny-async-pool');
+
+const _website = 'dlnews';
+const topics = {
+    defi: 'DeFi',
+    fintech: 'Fintech/VC/Deals',
+    'llama-u': 'Llama U',
+    markets: 'Markets',
+    'people-culture': 'People & Culture',
+    regulation: 'Regulation',
+    snapshot: 'Snapshot',
+    web3: 'Web3',
+};
+const extractArticle = (ctx, item) =>
+    ctx.cache.tryGet(item.link, async () => {
+        const { data: response } = await got(item.link);
+        const $ = cheerio.load(response);
+        const scriptTagContent = $('script#fusion-metadata').text();
+        const jsonData = JSON.parse(scriptTagContent.match(/Fusion\.globalContent=({.*?});Fusion\.globalContentConfig/)[1]).content_elements;
+        const filteredData = [];
+        for (const v of jsonData) {
+            if (v.type === 'header' && v.content.includes('What we’re reading')) {
+                break;
+            } else if (v.type === 'custom_embed' && Boolean(v.embed.config.text)) {
+                filteredData.push({ type: v.type, data: v.embed.config.text });
+            } else if (v.type === 'text' && !v.content.includes('<b>NOW READ: </b>')) {
+                filteredData.push({ type: v.type, data: v.content });
+            } else if (v.type === 'header') {
+                filteredData.push({ type: v.type, data: v.content });
+            } else if (v.type === 'list') {
+                filteredData.push({ type: v.type, list_type: v.list_type, items: v.items });
+            } else if (v.type === 'image') {
+                filteredData.push({ type: v.type, src: v.url, alt: v.alt_text, caption: v.subtitle });
+            }
+        }
+        item.description = art(path.resolve(__dirname, 'templates/description.art'), filteredData);
+        return item;
+    });
+
+module.exports = async (ctx) => {
+    const { category } = ctx.params;
+    const baseUrl = 'https://www.dlnews.com';
+    const apiPath = '/pf/api/v3/content/fetch/articles-api';
+    let vertical;
+    if (category) {
+        vertical = category;
+    } else {
+        vertical = '';
+    }
+
+    const query = {
+        author: '',
+        date: 'now-1y/d',
+        offset: 0,
+        query: '',
+        size: 15,
+        sort: 'display_date:desc',
+        vertical,
+    };
+    const data = await getData(`${baseUrl}${apiPath}?query=${encodeURIComponent(JSON.stringify(query))}&_website=${_website}`);
+    const list = getList(data);
+    const items = [];
+    for await (const data of asyncPool(3, list, (item) => extractArticle(ctx, item))) {
+        items.push(data);
+    }
+
+    ctx.state.data = {
+        title: topics.hasOwnProperty(category) ? `${topics[category]} : DL News` : 'DL News',
+        link: baseUrl,
+        item: items,
+        description: topics.hasOwnProperty(category) ? `${topics[category]} : News on dlnews.com` : 'Latest News on dlnews.com',
+        logo: 'https://www.dlnews.com/pf/resources/favicon.ico?d=284',
+        icon: 'https://www.dlnews.com/pf/resources/favicon.ico?d=284',
+        language: 'en-us',
+    };
+};
diff --git a/lib/v2/dlnews/maintainer.js b/lib/v2/dlnews/maintainer.js
new file mode 100644
index 0000000000..7b13511df5
--- /dev/null
+++ b/lib/v2/dlnews/maintainer.js
@@ -0,0 +1,3 @@
+module.exports = {
+    '/:category?': ['Rjnishant530'],
+};
diff --git a/lib/v2/dlnews/radar.js b/lib/v2/dlnews/radar.js
new file mode 100644
index 0000000000..c5946409cd
--- /dev/null
+++ b/lib/v2/dlnews/radar.js
@@ -0,0 +1,19 @@
+module.exports = {
+    'dlnews.com': {
+        _name: 'DL NEWS',
+        '.': [
+            {
+                title: 'All Articles',
+                docs: 'https://docs.rsshub.app/routes/finance#dl-news',
+                source: ['/articles/'],
+                target: '/dlnews/',
+            },
+            {
+                title: 'Topic',
+                docs: 'https://docs.rsshub.app/routes/finance#dl-news',
+                source: ['/articles/:category'],
+                target: '/dlnews/:category',
+            },
+        ],
+    },
+};
diff --git a/lib/v2/dlnews/router.js b/lib/v2/dlnews/router.js
new file mode 100644
index 0000000000..7eece4711e
--- /dev/null
+++ b/lib/v2/dlnews/router.js
@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/:category?', require('./category'));
+};
diff --git a/lib/v2/dlnews/templates/description.art b/lib/v2/dlnews/templates/description.art
new file mode 100644
index 0000000000..bcfe4d7245
--- /dev/null
+++ b/lib/v2/dlnews/templates/description.art
@@ -0,0 +1,24 @@
+{{ each $data d }}
+    {{ if d.type == 'custom_embed' }}
+        <ul>
+            {{ each d.data.split('\n') line }}
+                <li>{{@  line }}</li>
+            {{ /each }}
+        </ul>
+    {{ else if d.type == 'header' }}
+        <h2>{{@ d.data }}</h2>
+    {{ else if d.type == 'list' }}
+        {{ if d.list_type == 'unordered' }}<ul>{{ else }}<ol>{{ /if }}
+            {{ each d.items item }}
+                <li>{{@ item.content }}</li>
+            {{ /each }}
+        {{ if d.list_type == 'unordered' }}</ul>{{ else }}</ol>{{ /if }}
+    {{ else if d.type == 'image' }}
+        <figure>
+            <img src="{{ d.src }}" alt="{{ d.alt }}">
+            <figcaption>{{@  d.caption }}</figcaption>
+        </figure>
+    {{ else if d.type == 'text' }}
+        <p>{{@ d.data }}</p>
+    {{ /if }}
+{{ /each }}
\ No newline at end of file
diff --git a/lib/v2/dlnews/utils.js b/lib/v2/dlnews/utils.js
new file mode 100644
index 0000000000..4a820ab2ad
--- /dev/null
+++ b/lib/v2/dlnews/utils.js
@@ -0,0 +1,22 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+
+const baseUrl = 'https://www.dlnews.com';
+const getData = async (url) => (await got.get(url).json()).content_elements;
+
+const getList = (data) =>
+    data.map((value) => {
+        const { _id, headlines, description, publish_date, website_url, taxonomy, credits, promo_items } = value;
+        return {
+            id: _id,
+            title: headlines.basic,
+            link: `${baseUrl}${website_url}`,
+            description: description.basic,
+            author: credits.by.map((v) => v.name).join(', '),
+            itunes_item_image: promo_items.basic.url,
+            pubDate: parseDate(publish_date),
+            category: taxonomy.sections.map((v) => v.name).join(', '),
+        };
+    });
+
+module.exports = { getData, getList };
diff --git a/website/docs/routes/finance.md b/website/docs/routes/finance.md
index 0a8cc00138..7f5a6e6d52 100644
--- a/website/docs/routes/finance.md
+++ b/website/docs/routes/finance.md
@@ -77,6 +77,29 @@
 
 <Route author="HenryQW" example="/cfd/div_gbp" path="/cfd/div_gbp" />
 
+## DL NEWS {#dl-news}
+
+### All Articles {#dl-news-all-articles}
+
+<Route author="Rjnishant530" example="/dlnews" path="/dlnews" radar="1"/>
+
+### Topic {#dl-news-topic}
+
+<Route author="Rjnishant530" example="/dlnews/fintech" path="/dlnews/:category" paramsDesc={['Find in Table. Defaults to All articles']} radar="1">
+
+| Topic              | Link                |
+|--------------------|---------------------|
+| DeFi               | defi    |
+| Fintech/VC/Deals   | fintech  |
+| Llama U            | llama-u  |
+| Markets            | markets  |
+| People & Culture   | people-culture |
+| Regulation         | regulation |
+| Snapshot           | snapshot |
+| Web3               | web3     |
+
+</Route>
+
 ## DT 财经 {#dt-cai-jing}
 
 ### 数据洞察 {#dt-cai-jing-shu-ju-dong-cha}
@@ -99,6 +122,7 @@
 
 </Route>
 
+
 ## Finology Insider {#finology-insider}
 
 ### Bullets {#finology-insider-bullets}
diff --git a/website/docs/routes/new-media.md b/website/docs/routes/new-media.md
index b19aebee11..9cb6d62bd3 100644
--- a/website/docs/routes/new-media.md
+++ b/website/docs/routes/new-media.md
@@ -5220,4 +5220,3 @@ QueryString:
 ### 公众号 {#zi-you-wei-xin-gong-zhong-hao}
 
 <Route author="TonyRL" example="/freewechat/profile/MzI5NTUxNzk3OA==" path="/freewechat/profile/:id" paramsDesc={['公众号 ID，可在URL中找到']} radar="1" rssbud="1" anticrawler="1"/>
-