From dbb692462da0173f1dd7d6f5de38d3d779252d41 Mon Sep 17 00:00:00 2001 From: Dzming Li Date: Tue, 9 Dec 2025 03:43:06 -0800 Subject: [PATCH] chore: add Nix flake and NixOS module support (#20597) * feat(nix): add flake module and offline-friendly build * chore(nix): upgrade to Node.js 24 and pnpm 10 Update devenv.nix to match project dependencies: - Upgrade Node.js from 22 to 24 (aligns with Dockerfile node:24-bookworm) - Upgrade pnpm from 9 to 10 (aligns with package.json pnpm@10.22.0) * test: fix buffer-get test timeout Replace external URL with mock server endpoint to prevent test timeout. The test was trying to fetch from http://example.com which is not mocked, causing it to timeout. Now uses http://rsshub.test/headers which is properly mocked in the test setup. --- .envrc | 1 + .gitignore | 2 +- devenv.nix | 108 ++++++++++ flake.lock | 259 ++++++++++++++++++++++++ flake.nix | 308 +++++++++++++++++++++++++++++ lib/routes/youtube/api/youtubei.ts | 30 +-- lib/utils/got.test.ts | 2 +- scripts/workflow/build-routes.ts | 33 +++- 8 files changed, 721 insertions(+), 22 deletions(-) create mode 100644 .envrc create mode 100644 devenv.nix create mode 100644 flake.lock create mode 100644 flake.nix diff --git a/.envrc b/.envrc new file mode 100644 index 0000000000..a5dbbcba7b --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +use flake . diff --git a/.gitignore b/.gitignore index b529e4c644..ea5c4e4faa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ .DS_Store .cursorrules -.env* +.env .eslintcache .idea .log diff --git a/devenv.nix b/devenv.nix new file mode 100644 index 0000000000..5df86485fe --- /dev/null +++ b/devenv.nix @@ -0,0 +1,108 @@ +{ pkgs, lib, config, ... }: + +{ + # https://devenv.sh/basics/ + env = { + NODE_ENV = "dev"; + NODE_OPTIONS = "--max-http-header-size=32768"; + }; + + # https://devenv.sh/packages/ + packages = with pkgs; [ + git + + # Optional: Uncomment if you need browser automation + # chromium + ]; + + # https://devenv.sh/languages/ + languages.javascript = { + enable = true; + package = pkgs.nodejs_24; + pnpm = { + enable = true; + package = pkgs.pnpm_10; + }; + }; + + # https://devenv.sh/services/ + services.redis = { + enable = lib.mkDefault false; # Disabled by default, users can enable in devenv.local.nix + port = 6379; + }; + + # https://devenv.sh/scripts/ + scripts.rsshub-dev.exec = '' + pnpm run dev + ''; + + scripts.rsshub-build.exec = '' + pnpm run build + ''; + + scripts.rsshub-start.exec = '' + pnpm start + ''; + + scripts.rsshub-test.exec = '' + pnpm test + ''; + + # https://devenv.sh/processes/ + processes = { + # Uncomment to auto-start RSSHub in dev mode when entering the shell + # rsshub.exec = "pnpm run dev"; + + # Example: Auto-start with Redis + # rsshub.exec = "pnpm run dev"; + }; + + # https://devenv.sh/pre-commit-hooks/ + pre-commit.hooks = { + # Lint staged files + eslint = { + enable = true; + entry = lib.mkForce "pnpm run format:staged"; + }; + }; + + enterShell = '' + echo "" + echo "๐Ÿš€ RSSHub Development Environment" + echo "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”" + echo "" + echo "Node.js: $(node --version)" + echo "pnpm: $(pnpm --version)" + ${lib.optionalString config.services.redis.enable '' + echo "Redis: Running on port ${toString config.services.redis.port}" + ''} + echo "" + echo "Available commands:" + echo " rsshub-dev - Start development server (pnpm run dev)" + echo " rsshub-build - Build the project (pnpm run build)" + echo " rsshub-start - Start production server (pnpm start)" + echo " rsshub-test - Run tests (pnpm test)" + ${lib.optionalString (!config.services.redis.enable) '' + echo "" + echo "๐Ÿ’ก Tip: Enable Redis by creating devenv.local.nix:" + echo " { services.redis.enable = true; }" + ''} + echo "" + echo "Documentation: https://docs.rsshub.app" + echo "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”" + echo "" + + # Install dependencies if node_modules doesn't exist + if [ ! -d "node_modules" ]; then + echo "๐Ÿ“ฆ Installing dependencies..." + pnpm install + fi + ''; + + # https://devenv.sh/integrations/dotenv/ + dotenv.enable = true; # Automatically load .env file + + # Load local overrides if they exist + # Users can create devenv.local.nix for personal customizations + imports = lib.optional (builtins.pathExists ./devenv.local.nix) ./devenv.local.nix; +} diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000000..cdfc46ed3b --- /dev/null +++ b/flake.lock @@ -0,0 +1,259 @@ +{ + "nodes": { + "cachix": { + "inputs": { + "devenv": [ + "devenv" + ], + "flake-compat": [ + "devenv", + "flake-compat" + ], + "git-hooks": [ + "devenv", + "git-hooks" + ], + "nixpkgs": [ + "devenv", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1760971495, + "narHash": "sha256-IwnNtbNVrlZIHh7h4Wz6VP0Furxg9Hh0ycighvL5cZc=", + "owner": "cachix", + "repo": "cachix", + "rev": "c5bfd933d1033672f51a863c47303fc0e093c2d2", + "type": "github" + }, + "original": { + "owner": "cachix", + "ref": "latest", + "repo": "cachix", + "type": "github" + } + }, + "devenv": { + "inputs": { + "cachix": "cachix", + "flake-compat": "flake-compat", + "flake-parts": "flake-parts", + "git-hooks": "git-hooks", + "nix": "nix", + "nixpkgs": "nixpkgs" + }, + "locked": { + "lastModified": 1764368166, + "narHash": "sha256-FktN7dtYlC/sgLGBCGFXzNOvwgB7MSujp6cooJE48Ac=", + "owner": "cachix", + "repo": "devenv", + "rev": "47a243b97499bfe5d5783d1fc86d9fe776b2497f", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "devenv", + "type": "github" + } + }, + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1761588595, + "narHash": "sha256-XKUZz9zewJNUj46b4AJdiRZJAvSZ0Dqj2BNfXvFlJC4=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "f387cd2afec9419c8ee37694406ca490c3f34ee5", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-parts": { + "inputs": { + "nixpkgs-lib": [ + "devenv", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1760948891, + "narHash": "sha256-TmWcdiUUaWk8J4lpjzu4gCGxWY6/Ok7mOK4fIFfBuU4=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "864599284fc7c0ba6357ed89ed5e2cd5040f0c04", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "git-hooks": { + "inputs": { + "flake-compat": [ + "devenv", + "flake-compat" + ], + "gitignore": "gitignore", + "nixpkgs": [ + "devenv", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1760663237, + "narHash": "sha256-BflA6U4AM1bzuRMR8QqzPXqh8sWVCNDzOdsxXEguJIc=", + "owner": "cachix", + "repo": "git-hooks.nix", + "rev": "ca5b894d3e3e151ffc1db040b6ce4dcc75d31c37", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "git-hooks.nix", + "type": "github" + } + }, + "gitignore": { + "inputs": { + "nixpkgs": [ + "devenv", + "git-hooks", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1709087332, + "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=", + "owner": "hercules-ci", + "repo": "gitignore.nix", + "rev": "637db329424fd7e46cf4185293b9cc8c88c95394", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "gitignore.nix", + "type": "github" + } + }, + "nix": { + "inputs": { + "flake-compat": [ + "devenv", + "flake-compat" + ], + "flake-parts": [ + "devenv", + "flake-parts" + ], + "git-hooks-nix": [ + "devenv", + "git-hooks" + ], + "nixpkgs": [ + "devenv", + "nixpkgs" + ], + "nixpkgs-23-11": [ + "devenv" + ], + "nixpkgs-regression": [ + "devenv" + ] + }, + "locked": { + "lastModified": 1761648602, + "narHash": "sha256-H97KSB/luq/aGobKRuHahOvT1r7C03BgB6D5HBZsbN8=", + "owner": "cachix", + "repo": "nix", + "rev": "3e5644da6830ef65f0a2f7ec22830c46285bfff6", + "type": "github" + }, + "original": { + "owner": "cachix", + "ref": "devenv-2.30.6", + "repo": "nix", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1761313199, + "narHash": "sha256-wCIACXbNtXAlwvQUo1Ed++loFALPjYUA3dpcUJiXO44=", + "owner": "cachix", + "repo": "devenv-nixpkgs", + "rev": "d1c30452ebecfc55185ae6d1c983c09da0c274ff", + "type": "github" + }, + "original": { + "owner": "cachix", + "ref": "rolling", + "repo": "devenv-nixpkgs", + "type": "github" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1764242076, + "narHash": "sha256-sKoIWfnijJ0+9e4wRvIgm/HgE27bzwQxcEmo2J/gNpI=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "2fad6eac6077f03fe109c4d4eb171cf96791faa4", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "devenv": "devenv", + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs_2" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000000..03bab86f56 --- /dev/null +++ b/flake.nix @@ -0,0 +1,308 @@ +{ + description = "RSSHub - Make RSS Great Again!"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + flake-utils.url = "github:numtide/flake-utils"; + devenv.url = "github:cachix/devenv"; + }; + + outputs = inputs@{ self, nixpkgs, flake-utils, devenv }: + let + # Helper to define the RSSHub package + makeRSSHub = pkgs: + let + pnpm = pkgs.pnpm_9; + deps = pnpm.fetchDeps { + pname = "rsshub"; + src = ./.; + hash = "sha256-ErMPvlOIDqn03s2P+tzbQbYPZFEax5P61O1DJputvo4="; + fetcherVersion = 2; + }; + in + pkgs.stdenv.mkDerivation rec { + pname = "rsshub"; + version = "1.0.0"; + + src = ./.; + + nativeBuildInputs = with pkgs; [ + nodejs_22 + pnpm.configHook + git + ]; + + buildInputs = with pkgs; [ + # Optional: Add chromium for routes that need browser automation + # chromium + ]; + + pnpmDeps = deps; + + # ไฟฎ่กฅๆž„ๅปบ่„šๆœฌไปฅๆ”ฏๆŒ็ฆป็บฟๆž„ๅปบ๏ผˆNix ๆž„ๅปบ็Žฏๅขƒๆ— ็ฝ‘็ปœ่ฎฟ้—ฎ๏ผ‰ + postPatch = '' + # ๅœจ registry.ts ไธญๆทปๅŠ  BUILD_ROUTES ๆจกๅผ๏ผŒไฝฟ็”จ directoryImport ไฝ†ไธๅฎž้™…ๅฏผๅ…ฅๆจกๅ— + substituteInPlace lib/registry.ts \ + --replace-fail 'if (config.isPackage)' \ + 'if (process.env.BUILD_ROUTES_MODE) { + modules = directoryImport({ + targetDirectoryPath: path.join(__dirname, "./routes"), + importPattern: /\.ts$/, + }) as typeof modules; +} else if (config.isPackage)' + ''; + + # The build phase + buildPhase = '' + runHook preBuild + + # ๅ…ˆๆž„ๅปบ่ทฏ็”ฑๅ…ƒๆ•ฐๆฎ๏ผˆไฝฟ็”จ directoryImport ไฝ†้ฟๅ…ๆ‰ง่กŒๆจกๅ—้กถๅฑ‚ไปฃ็ ๏ผ‰ + export BUILD_ROUTES_MODE=1 + pnpm run build:routes + unset BUILD_ROUTES_MODE + + # ็„ถๅŽๆž„ๅปบๅบ”็”จ + export NODE_ENV=production + ${pnpm}/bin/pnpm run build + + runHook postBuild + ''; + + # The install phase + installPhase = '' + runHook preInstall + mkdir -p $out/lib/rsshub + cp -r dist $out/lib/rsshub/ + cp -r node_modules $out/lib/rsshub/ + cp package.json $out/lib/rsshub/ + + mkdir -p $out/bin + cat > $out/bin/rsshub < { - const url = typeof input === 'string' ? input : input instanceof URL ? input.toString() : input.url; +let innertubePromise: Promise | undefined; - return fetch(url, { - method: input?.method, - ...init, +const getInnertube = () => { + if (!innertubePromise) { + // Lazy init to avoid network calls during import time (e.g. when building) + innertubePromise = Innertube.create({ + fetch: (input, init) => { + const url = typeof input === 'string' ? input : input instanceof URL ? input.toString() : input.url; + + return fetch(url, { + method: input?.method, + ...init, + }); + }, }); - }, -}); + } + return innertubePromise; +}; export const getChannelIdByUsername = (username: string) => cache.tryGet(`youtube:getChannelIdByUsername:${username}`, async () => { - const innertube = await innertubePromise; + const innertube = await getInnertube(); const navigationEndpoint = await innertube.resolveURL(`https://www.youtube.com/${username}`); return navigationEndpoint.payload.browseId; }); @@ -31,7 +39,7 @@ export const getDataByUsername = async ({ username, embed, filterShorts, isJsonF }; export const getDataByChannelId = async ({ channelId, embed, isJsonFeed }: { channelId: string; embed: boolean; filterShorts: boolean; isJsonFeed: boolean }): Promise => { - const innertube = await innertubePromise; + const innertube = await getInnertube(); const channel = await innertube.getChannel(channelId); const videos = await channel.getVideos(); const videoSubtitles = isJsonFeed ? await getSrtAttachmentBatch(videos.videos.filter((video) => 'video_id' in video).map((video) => video.video_id)) : {}; @@ -71,7 +79,7 @@ export const getDataByChannelId = async ({ channelId, embed, isJsonFeed }: { cha }; export const getDataByPlaylistId = async ({ playlistId, embed }: { playlistId: string; embed: boolean; isJsonFeed: boolean }): Promise => { - const innertube = await innertubePromise; + const innertube = await getInnertube(); const playlist = await innertube.getPlaylist(playlistId); const videos = await playlist.videos; diff --git a/lib/utils/got.test.ts b/lib/utils/got.test.ts index bc1d404ad1..4882fea817 100644 --- a/lib/utils/got.test.ts +++ b/lib/utils/got.test.ts @@ -53,7 +53,7 @@ describe('got', () => { }); it('buffer-get', async () => { - const response = await got.get('http://example.com', { + const response = await got.get('http://rsshub.test/headers', { responseType: 'buffer', }); expect(response.body instanceof Buffer).toBe(true); diff --git a/scripts/workflow/build-routes.ts b/scripts/workflow/build-routes.ts index 1bbb64cc59..3d6b32322e 100644 --- a/scripts/workflow/build-routes.ts +++ b/scripts/workflow/build-routes.ts @@ -11,16 +11,31 @@ import { getCurrentPath } from '../../lib/utils/helpers'; const __dirname = getCurrentPath(import.meta.url); -const foloAnalysis = await ( - await fetch('https://raw.githubusercontent.com/RSSNext/rsshub-docs/refs/heads/main/rsshub-analytics.json', { - headers: { - 'user-agent': config.trueUA, - }, - }) -).json(); -const foloAnalysisResult = foloAnalysis.data as Record; +type FoloAnalysis = Record; + +const loadFoloAnalysis = async (): Promise => { + try { + const response = await fetch('https://raw.githubusercontent.com/RSSNext/rsshub-docs/refs/heads/main/rsshub-analytics.json', { + headers: { + 'user-agent': config.trueUA, + }, + }); + + if (!response.ok) { + throw new Error(`Unexpected status ${response.status}`); + } + + const data = await response.json(); + return (data?.data as FoloAnalysis) || {}; + } catch (error) { + process.emitWarning(`Failed to fetch rsshub-analytics.json, continuing without popularity data. ${String(error)}`); + return {}; + } +}; + +const foloAnalysisResult = await loadFoloAnalysis(); const foloAnalysisTop100 = Object.entries(foloAnalysisResult) - .sort((a, b) => b[1].subscriptionCount - a[1].subscriptionCount) + .toSorted((a, b) => b[1].subscriptionCount - a[1].subscriptionCount) .slice(0, 150); const maintainers: Record = {};