implemented cloudflare clearance

2025-05-21 08:26:29 +08:00 · 2023-10-23 23:59:38 -07:00
parent f71535438b
commit 76c80be783
11 changed files with 335 additions and 51 deletions
--- a/assets/captcha.svg
+++ b/assets/captcha.svg
--- a/lib/main.dart
+++ b/lib/main.dart
@ -19,6 +19,8 @@ import 'package:openlib/state/state.dart'
        themeModeProvider,
        openPdfWithExternalAppProvider,
        openEpubWithExternalAppProvider,
+        userAgentProvider,
+        cookieProvider,
        dbProvider;

 void main() async {
@ -37,6 +39,9 @@ void main() async {
  bool openEpubwithExternalapp =
      await dataBase.getPreference('openEpubwithExternalApp');

+  String browserUserAgent = await dataBase.getBrowserOptions('userAgent');
+  String browserCookie = await dataBase.getBrowserOptions('cookie');
+
  if (Platform.isAndroid) {
    //[SystemChrome] Also change colors in settings page Theme colors if any change
    SystemChrome.setSystemUIOverlayStyle(SystemUiOverlayStyle(
@ -54,7 +59,9 @@ void main() async {
        openPdfWithExternalAppProvider
            .overrideWith((ref) => openPdfwithExternalapp),
        openEpubWithExternalAppProvider
-            .overrideWith((ref) => openEpubwithExternalapp)
+            .overrideWith((ref) => openEpubwithExternalapp),
+        userAgentProvider.overrideWith((ref) => browserUserAgent),
+        cookieProvider.overrideWith((ref) => browserCookie),
      ],
      child: const MyApp(),
    ),
--- a/lib/services/annas_archieve.dart
+++ b/lib/services/annas_archieve.dart
@ -1,6 +1,8 @@
 import 'package:dio/dio.dart';
+import 'dart:convert';
 import 'package:flutter/material.dart';
 import 'package:html/parser.dart' show parse;
+import 'package:http/http.dart' as http;

 class BookData {
  final String title;
@ -48,11 +50,11 @@ class BookInfoData extends BookData {
 }

 class AnnasArchieve {
-  String baseUrl = "https://annas-archive.se";
+  String baseUrl = "https://annas-archive.org";

  final Dio dio = Dio(BaseOptions(headers: {
-    'User-Agent':
-        'Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/7.0.4 Mobile/16B91 Safari/605.1.15'
+    "user-agent":
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/118.0",
  }));

  String getMd5(String url) {
@ -125,20 +127,29 @@ class AnnasArchieve {
    }
  }

-  Future<String?> _getMirrorLink(String url) async {
+  Future<String?> _getMirrorLink(
+      String url, String userAgent, String cookie) async {
    try {
-      final response = await dio.get(url);
-      var document = parse(response.toString());
+      // final response = await dio.get(url);
+      final response = await http.get(Uri.parse(url),
+          headers: {"Cookie": cookie, "User-Agent": userAgent});
+      if (response.statusCode == 403) {
+        throw jsonEncode({"code": "403", "url": url});
+      }
+      var document = parse(response.body.toString());
      var pTag = document.querySelector('p[class="mb-4"]');
      String? link = pTag?.querySelector('a')?.attributes['href'];
      return link;
    } catch (e) {
      // print(e);
+      if (e.toString().contains("403")) {
+        rethrow;
+      }
      return null;
    }
  }

-  Future<BookInfoData?> _bookInfoParser(resData, url) async {
+  Future<BookInfoData?> _bookInfoParser(resData, url, userAgent, cookie) async {
    var document = parse(resData.toString());
    var main = document.querySelector('main[class="main"]');
    var ul = main?.querySelectorAll('ul[class="mb-4"]');
@ -155,14 +166,17 @@ class AnnasArchieve {

      for (var element in anchorTags) {
        if (element.attributes['href']!.startsWith('https://')) {
-          if (element.attributes['href'] != null) {
+          if (element.attributes['href'] != null &&
+              element.attributes['href'].startsWith('https://1lib.sk') !=
+                  true) {
            mirrors.add(element.attributes['href']!);
          }
        } else if (element.attributes['href'] != null &&
            element.attributes['href']!.startsWith('/slow_download')) {
-          if (element.text.contains('Slow Partner Server #1') != true) {
-            String? url =
-                await _getMirrorLink('$baseUrl${element.attributes['href']!}');
+          if (element.text.contains('Slow Partner Server #1') == true ||
+              true == true) {
+            String? url = await _getMirrorLink(
+                '$baseUrl${element.attributes['href']!}', userAgent, cookie);
            if (url != null && url.isNotEmpty) {
              mirrors.add(url);
            }
@ -238,10 +252,14 @@ class AnnasArchieve {
    }
  }

-  Future<BookInfoData> bookInfo({required String url}) async {
+  Future<BookInfoData> bookInfo(
+      {required String url,
+      required String userAgent,
+      required String cookie}) async {
    try {
      final response = await dio.get(url);
-      BookInfoData? data = await _bookInfoParser(response.data, url);
+      BookInfoData? data =
+          await _bookInfoParser(response.data, url, userAgent, cookie);
      if (data != null) {
        return data;
      } else {
--- a/lib/services/database.dart
+++ b/lib/services/database.dart
@ -9,7 +9,7 @@ class Sqlite {

    Database dbInstance = await openDatabase(
      path,
-      version: 3,
+      version: 5,
      onCreate: (Database db, int version) async {
        await db.execute(
            'CREATE TABLE mybooks (id TEXT PRIMARY KEY, title TEXT,author TEXT,thumbnail TEXT,link TEXT,publisher TEXT,info TEXT,format TEXT,description TEXT)');
@ -18,6 +18,8 @@ class Sqlite {
        if (isMobile) {
          await db.execute(
              'CREATE TABLE bookposition (fileName TEXT PRIMARY KEY,position TEXT)');
+          await db.execute(
+              'CREATE TABLE browserOptions (name TEXT PRIMARY KEY,value TEXT)');
        }
      },
      onUpgrade: (db, oldVersion, newVersion) async {
@ -25,6 +27,8 @@ class Sqlite {
            where: 'name = ?', whereArgs: ['bookposition']);
        List<dynamic> isPreferenceTableExist = await db.query('sqlite_master',
            where: 'name = ?', whereArgs: ['preferences']);
+        List<dynamic> isbrowserOptionsExist = await db.query('sqlite_master',
+            where: 'name = ?', whereArgs: ['browserOptions']);
        if (isPreferenceTableExist.isEmpty) {
          await db.execute(
              'CREATE TABLE preferences (name TEXT PRIMARY KEY,value BOOLEAN)');
@ -33,6 +37,10 @@ class Sqlite {
          await db.execute(
              'CREATE TABLE bookposition (fileName TEXT PRIMARY KEY,position TEXT)');
        }
+        if (isMobile && isbrowserOptionsExist.isEmpty) {
+          await db.execute(
+              'CREATE TABLE browserOptions (name TEXT PRIMARY KEY,value TEXT)');
+        }
      },
    );
    return dbInstance;
@ -194,4 +202,25 @@ class MyLibraryDb {
      return false;
    }
  }
+
+  Future<void> setBrowserOptions(String name, String value) async {
+    await dbInstance.insert(
+      'browserOptions',
+      {'name': name, 'value': value},
+      conflictAlgorithm: ConflictAlgorithm.replace,
+    );
+  }
+
+  Future<String> getBrowserOptions(String name) async {
+    List<Map<String, dynamic>> data = await dbInstance
+        .query('browserOptions', where: 'name = ?', whereArgs: [name]);
+    List<dynamic> dataList = List.generate(data.length, (i) {
+      return {'name': data[i]['name'], 'value': data[i]['value']};
+    });
+    if (dataList.isNotEmpty) {
+      return dataList[0]['value'];
+    } else {
+      return "";
+    }
+  }
 }
--- a/lib/services/download_file.dart
+++ b/lib/services/download_file.dart
@ -1,4 +1,5 @@
 import 'package:dio/dio.dart';
+import 'package:http/http.dart' as http;
 import 'files.dart';

 Future<String> _getFilePath(String fileName) async {
@ -14,7 +15,7 @@ List<String> _reorderMirrors(List<String> mirrors) {
    if (element.contains('ipfs') == true) {
      ipfsMirrors.add(element);
    } else {
-      if (element.startsWith('https://annas-archive.se') != true &&
+      if (element.startsWith('https://annas-archive.org') != true &&
          element.startsWith('https://1lib.sk') != true) {
        httpsMirrors.add(element);
      }
@ -23,6 +24,22 @@ List<String> _reorderMirrors(List<String> mirrors) {
  return [...ipfsMirrors, ...httpsMirrors];
 }

+Future<String?> _getAliveMirror(List<String> mirrors) async {
+  for (var url in mirrors) {
+    try {
+      final response =
+          await http.head(Uri.parse(url)).timeout(const Duration(seconds: 2));
+      print(response.statusCode);
+      if (response.statusCode == 200) {
+        return url;
+      }
+    } catch (e) {
+      print("timeOut");
+    }
+  }
+  return null;
+}
+
 Future<void> downloadFile(
    {required List<String> mirrors,
    required String md5,
@ -34,34 +51,42 @@ Future<void> downloadFile(
  String path = await _getFilePath('$md5.$format');
  List<String> orderedMirrors = _reorderMirrors(mirrors);

+  String? workingMirror = await _getAliveMirror(orderedMirrors);
+  print(workingMirror);
+
  // print(path);
  // print(orderedMirrors);
+  // print(orderedMirrors[0]);

-  try {
-    CancelToken cancelToken = CancelToken();
+  if (workingMirror != null) {
+    try {
+      CancelToken cancelToken = CancelToken();

-    dio.download(
-      orderedMirrors[0],
-      path,
-      options: Options(headers: {
-        'Connection': 'Keep-Alive',
-        'User-Agent':
-            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36'
-      }),
-      onReceiveProgress: (rcv, total) {
-        onProgress(rcv, total);
-      },
-      deleteOnError: true,
-      cancelToken: cancelToken,
-    ).catchError((err) {
-      if (err.type != DioExceptionType.cancel) {
-        onDownlaodFailed();
-      }
-      throw err;
-    });
+      dio.download(
+        workingMirror,
+        path,
+        options: Options(headers: {
+          'Connection': 'Keep-Alive',
+          'User-Agent':
+              'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36'
+        }),
+        onReceiveProgress: (rcv, total) {
+          onProgress(rcv, total);
+        },
+        deleteOnError: true,
+        cancelToken: cancelToken,
+      ).catchError((err) {
+        if (err.type != DioExceptionType.cancel) {
+          onDownlaodFailed();
+        }
+        throw err;
+      });

-    cancelDownlaod(cancelToken);
-  } catch (e) {
+      cancelDownlaod(cancelToken);
+    } catch (e) {
+      onDownlaodFailed();
+    }
+  } else {
    onDownlaodFailed();
  }
 }
--- a/lib/state/state.dart
+++ b/lib/state/state.dart
@ -80,11 +80,19 @@ final searchProvider = FutureProvider.family
  return data;
 });

+final cookieProvider = StateProvider<String>((ref) => "");
+final userAgentProvider = StateProvider<String>((ref) => "");
+
 //Provider for Book Info
 final bookInfoProvider =
    FutureProvider.family<BookInfoData, String>((ref, url) async {
+  print(ref.read(userAgentProvider));
+  print(ref.read(cookieProvider));
  AnnasArchieve annasArchieve = AnnasArchieve();
-  BookInfoData data = await annasArchieve.bookInfo(url: url);
+  BookInfoData data = await annasArchieve.bookInfo(
+      url: url,
+      userAgent: ref.read(userAgentProvider),
+      cookie: ref.read(cookieProvider));
  return data;
 });

--- a/lib/ui/book_info_page.dart
+++ b/lib/ui/book_info_page.dart
@ -1,6 +1,8 @@
 import 'package:dio/dio.dart';
+import 'dart:convert';
 import 'package:flutter/material.dart';
 import 'package:flutter_riverpod/flutter_riverpod.dart';
+import 'package:flutter/scheduler.dart';

 import 'package:openlib/services/database.dart';
 import 'package:openlib/ui/components/error_widget.dart';
@ -21,6 +23,8 @@ import 'package:openlib/state/state.dart'
 import 'package:openlib/ui/components/book_info_widget.dart';
 import 'package:openlib/ui/components/file_buttons_widget.dart';
 import 'package:openlib/ui/components/snack_bar_widget.dart';
+import 'package:flutter_svg/svg.dart';
+import 'package:openlib/ui/webview_page.dart';

 class BookInfoPage extends ConsumerWidget {
  const BookInfoPage({Key? key, required this.url}) : super(key: key);
@ -37,19 +41,101 @@ class BookInfoPage extends ConsumerWidget {
        titleTextStyle: Theme.of(context).textTheme.displayLarge,
      ),
      body: bookInfo.when(
+        skipLoadingOnRefresh: false,
        data: (data) {
          return BookInfoWidget(
              data: data, child: ActionButtonWidget(data: data));
        },
        error: (err, _) {
-          return CustomErrorWidget(
-            error: err,
-            stackTrace: _,
-            onRefresh: () {
-              // ignore: unused_result
-              ref.refresh(bookInfoProvider(url));
-            },
-          );
+          // print(err);
+          if (err.toString().contains("403")) {
+            var errJson = jsonDecode(err.toString());
+            if (SchedulerBinding.instance.schedulerPhase ==
+                SchedulerPhase.persistentCallbacks) {
+              SchedulerBinding.instance.addPostFrameCallback((_) async {
+                await Future.delayed(const Duration(seconds: 2));
+                // ignore: use_build_context_synchronously
+                Navigator.push(context,
+                    MaterialPageRoute(builder: (BuildContext context) {
+                  return Webview(url: errJson["url"]);
+                })).then((value) {
+                  // print("got Cf Clearance");
+                  // ignore: unused_result
+                  ref.refresh(bookInfoProvider(url));
+                  // });
+                });
+              });
+            }
+            return Column(
+              mainAxisAlignment: MainAxisAlignment.center,
+              crossAxisAlignment: CrossAxisAlignment.stretch,
+              children: [
+                SizedBox(
+                  width: 210,
+                  child: SvgPicture.asset(
+                    'assets/captcha.svg',
+                    width: 210,
+                  ),
+                ),
+                const SizedBox(
+                  height: 30,
+                ),
+                Text(
+                  "Captcha required",
+                  textAlign: TextAlign.center,
+                  style: TextStyle(
+                    fontSize: 18,
+                    fontWeight: FontWeight.bold,
+                    color: Theme.of(context).textTheme.headlineMedium?.color,
+                    overflow: TextOverflow.ellipsis,
+                  ),
+                ),
+                Padding(
+                  padding: const EdgeInsets.all(8.0),
+                  child: Text(
+                    "you will be redirected to solve captcha",
+                    textAlign: TextAlign.center,
+                    style: TextStyle(
+                      fontSize: 13,
+                      fontWeight: FontWeight.bold,
+                      color: Theme.of(context).textTheme.headlineSmall?.color,
+                      overflow: TextOverflow.ellipsis,
+                    ),
+                  ),
+                ),
+                Padding(
+                  padding: const EdgeInsets.fromLTRB(30, 15, 30, 10),
+                  child: Container(
+                    width: double.infinity,
+                    decoration: BoxDecoration(
+                        color: const Color.fromARGB(255, 255, 186, 186),
+                        borderRadius: BorderRadius.circular(5)),
+                    child: const Padding(
+                      padding: EdgeInsets.all(10.0),
+                      child: Text(
+                        "If you have solved the captcha then you will be automatically redirected to the book page",
+                        textAlign: TextAlign.start,
+                        style: TextStyle(
+                          fontSize: 13,
+                          fontWeight: FontWeight.bold,
+                          color: Colors.black,
+                        ),
+                      ),
+                    ),
+                  ),
+                )
+              ],
+            );
+          } else {
+            return CustomErrorWidget(
+              error: err,
+              stackTrace: _,
+              onRefresh: () {
+                // ignore: unused_result
+                ref.refresh(bookInfoProvider(url));
+              },
+            );
+          }
        },
        loading: () {
          return Center(
--- a/lib/ui/search_page.dart
+++ b/lib/ui/search_page.dart
@ -11,7 +11,6 @@ import 'package:openlib/state/state.dart'
        typeValues,
        fileType,
        sortValues;
-
 import 'components/snack_bar_widget.dart';

 class SearchPage extends ConsumerWidget {
@ -212,7 +211,7 @@ class SearchPage extends ConsumerWidget {
                  },
                ),
              ),
-            )
+            ),
          ],
        ),
      ),
--- a/lib/ui/webview_page.dart
+++ b/lib/ui/webview_page.dart
@ -0,0 +1,67 @@
+import 'package:flutter/material.dart';
+import 'package:flutter_riverpod/flutter_riverpod.dart';
+import 'package:webview_flutter/webview_flutter.dart';
+import 'package:webview_cookie_manager/webview_cookie_manager.dart'
+    as cookiejar;
+
+import 'package:openlib/state/state.dart'
+    show cookieProvider, userAgentProvider, dbProvider;
+
+class Webview extends ConsumerStatefulWidget {
+  const Webview({Key? key, required this.url}) : super(key: key);
+  final String url;
+  @override
+  // ignore: library_private_types_in_public_api
+  _WebviewState createState() => _WebviewState();
+}
+
+class _WebviewState extends ConsumerState<Webview> {
+  WebViewController controller = WebViewController();
+
+  final cookieManager = cookiejar.WebviewCookieManager();
+  @override
+  Widget build(BuildContext context) {
+    return Scaffold(
+      appBar: AppBar(
+        automaticallyImplyLeading: false,
+        title: const Text("Solve Captcha"),
+      ),
+      body: SafeArea(
+        child: WebViewWidget(
+          controller: controller
+            ..setJavaScriptMode(JavaScriptMode.unrestricted)
+            ..setBackgroundColor(const Color(0x00000000))
+            ..loadRequest(Uri.parse(widget.url))
+            ..getUserAgent().then((value) {
+              ref.read(userAgentProvider.notifier).state = value!;
+              ref.read(dbProvider).setBrowserOptions('userAgent', value);
+            })
+            ..setNavigationDelegate(NavigationDelegate(
+              onPageStarted: (url) async {
+                var x = await controller.runJavaScriptReturningResult(
+                    "var xhr = new XMLHttpRequest();xhr.open('GET', window.location.href, false);xhr.send(null);xhr.status;");
+
+                if (x.toString().contains('200')) {
+                  final cookies = await cookieManager
+                      .getCookies("https://annas-archive.org");
+                  List<String> cookie = [];
+                  for (var element in cookies) {
+                    if (element.name == 'cf_clearance' ||
+                        element.name == 'cf_chl_2') {
+                      cookie.add(element.toString().split(';')[0]);
+                    }
+                  }
+                  ref.read(cookieProvider.notifier).state = cookie.join('; ');
+                  ref
+                      .read(dbProvider)
+                      .setBrowserOptions('cookie', cookie.join('; '));
+                  // ignore: use_build_context_synchronously
+                  Navigator.pop(context);
+                }
+              },
+            )),
+        ),
+      ),
+    );
+  }
+}
--- a/pubspec.lock
+++ b/pubspec.lock
@ -305,7 +305,7 @@ packages:
    source: hosted
    version: "0.15.4"
  http:
-    dependency: transitive
+    dependency: "direct main"
    description:
      name: http
      sha256: "759d1a329847dd0f39226c688d3e06a6b8679668e350e2891a6474f8b4bb8525"
@ -813,6 +813,46 @@ packages:
      url: "https://pub.dev"
    source: hosted
    version: "0.1.4-beta"
+  webview_cookie_manager:
+    dependency: "direct main"
+    description:
+      name: webview_cookie_manager
+      sha256: "425a9feac5cd2cb62a71da3dda5ac2eaf9ece5481ee8d79f3868dc5ba8223ad3"
+      url: "https://pub.dev"
+    source: hosted
+    version: "2.0.6"
+  webview_flutter:
+    dependency: "direct main"
+    description:
+      name: webview_flutter
+      sha256: c1ab9b81090705c6069197d9fdc1625e587b52b8d70cdde2339d177ad0dbb98e
+      url: "https://pub.dev"
+    source: hosted
+    version: "4.4.1"
+  webview_flutter_android:
+    dependency: transitive
+    description:
+      name: webview_flutter_android
+      sha256: b0cd33dd7d3dd8e5f664e11a19e17ba12c352647269921a3b568406b001f1dff
+      url: "https://pub.dev"
+    source: hosted
+    version: "3.12.0"
+  webview_flutter_platform_interface:
+    dependency: transitive
+    description:
+      name: webview_flutter_platform_interface
+      sha256: "6d9213c65f1060116757a7c473247c60f3f7f332cac33dc417c9e362a9a13e4f"
+      url: "https://pub.dev"
+    source: hosted
+    version: "2.6.0"
+  webview_flutter_wkwebview:
+    dependency: transitive
+    description:
+      name: webview_flutter_wkwebview
+      sha256: "30b9af6bdd457b44c08748b9190d23208b5165357cc2eb57914fee1366c42974"
+      url: "https://pub.dev"
+    source: hosted
+    version: "3.9.1"
  win32:
    dependency: transitive
    description:
--- a/pubspec.yaml
+++ b/pubspec.yaml
@ -47,6 +47,10 @@ dependencies:
  path_provider: ^2.0.15
  permission_handler: ^10.4.3
  open_file: ^3.3.2
+  # flutter_inappwebview: ^5.8.0
+  http: ^1.1.0
+  webview_flutter: ^4.4.1
+  webview_cookie_manager: ^2.0.6

  flutter_svg: ^2.0.7
  google_fonts: ^5.1.0