Bump version to 1.1.71, refactor source loaders to use iter_source_download_data

Author: farfarfun

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "funread"
-version = "1.1.70"
+version = "1.1.71"
 description = "一个用于管理和处理阅读源（Legado 阅读 APP 的书源和 RSS 源）的 Python 工具库"
 readme = "README.md"
 requires-python = ">=3.8"

src/funread/legado/manage/download/book.py

@@ -1,8 +1,9 @@
 """书源下载和管理模块"""
 
-from typing import Any, Dict, List
+from typing import Any, Dict
 
 from funread.legado.manage.download.base import DownloadSource
+from funread.legado.manage.source_download import iter_source_download_data
 from funread.legado.manage.utils import retain_zh_ch_dig
 
 
@@ -130,11 +131,6 @@ def format_toc(self):
         self.__format_base("ruleToc", map)
 
 
-BOOK_SOURCE_URLS: List[str] = [
-    "https://bitbucket.org/xiu2/yuedu/raw/master/shuyuan",
-]
-
-
 class BookSourceDownload(DownloadSource):
     """书源下载器，继承自 DownloadSource"""
 
@@ -143,9 +139,9 @@ def get_source_url_key(self) -> str:
         return "bookSourceUrl"
 
     def loader(self) -> None:
-        """加载公开书源列表"""
-        for url in BOOK_SOURCE_URLS:
-            self.add_sources(url)
+        """从 URL 记录迭代器中加载书源数据"""
+        for _, data in iter_source_download_data(source_type=self.cate1):
+            self.add_sources(data)
 
     def source_format(self, source: Dict[str, Any]) -> Dict[str, Any]:
         """

src/funread/legado/manage/download/rss.py

@@ -2,18 +2,10 @@
 
 from typing import Any, Dict
 
-import requests
-from funfake.headers import Headers
-from nltlog import getLogger
-from funutil.cache import disk_cache
-from tqdm import tqdm
-
 from funread.legado.manage.download.base import DownloadSource
+from funread.legado.manage.source_download import iter_source_download_data
 from funread.legado.manage.utils import retain_zh_ch_dig
 
-logger = getLogger("funread")
-faker = Headers()
-
 
 class RSSSourceFormat:
     """RSS 源格式化类，用于统一 RSS 源格式"""
@@ -85,32 +77,7 @@ def source_format(self, source: Dict[str, Any]) -> Dict[str, Any]:
         """
         return RSSSourceFormat(source).run()
 
-    def loader(self):
-        urls = [
-            "https://agit.ai/butterfly/yd/raw/branch/yd/迷迭订阅源.json",
-        ]
-        urls.extend(
-            [f"https://www.yckceo.com/yuedu/rsss/json/id/{_id}.json" for _id in range(0, 250)]
-        )
-        urls.extend(
-            [f"https://www.yckceo.com/yuedu/rss/json/id/{_id}.json" for _id in range(0, 500)]
-        )
-
-        cache_path = f"{self.path_rot}/../cache"
-        logger.info(f"cache_path:{cache_path}")
-
-        @disk_cache(cache_key="url", cache_dir=cache_path, expire=3600 * 24 * 7)
-        def load_data(url: str) -> dict:
-            try:
-                return requests.get(url, headers=faker.generate()).json()
-            except Exception as e:
-                # logger.error(f"error: {e},traceback: {traceback.format_exc()}")
-                return {}
-
-        for _url in tqdm(urls):
-            try:
-                data = load_data(_url)
-                self.persist_download_record(_url, data)
-                self.add_sources(data)
-            except Exception as e:
-                logger.info(f"error:{e}")
+    def loader(self) -> None:
+        """从 URL 记录迭代器中加载 RSS 源数据"""
+        for _, data in iter_source_download_data(source_type=self.cate1):
+            self.add_sources(data)

tests/test_download_base.py

@@ -1,5 +1,8 @@
 from pathlib import Path
 
+import funread.legado.manage.download.book as book_module
+import funread.legado.manage.download.rss as rss_module
+
 from funread.legado.manage.download.base import DownloadSource
 from funread.legado.manage.download.book import BookSourceDownload
 
@@ -52,3 +55,50 @@ def test_book_source_download_accepts_book_source_url(tmp_path: Path) -> None:
     assert added is True
     exported = next(source.export_sources(size=10))
     assert exported[0]["bookSourceUrl"].startswith("https://books.example.com/api#")
+
+
+def test_book_loader_reads_source_download_iterator(monkeypatch, tmp_path: Path) -> None:
+    source = book_module.BookSourceDownload(path=str(tmp_path), cate1="book")
+
+    monkeypatch.setattr(
+        book_module,
+        "iter_source_download_data",
+        lambda source_type: iter(
+            [
+                (
+                    object(),
+                    {
+                        "bookSourceUrl": "https://books.example.com/api/",
+                        "bookSourceName": "Example Book",
+                    },
+                )
+            ]
+        ),
+    )
+
+    source.loader()
+
+    exported = next(source.export_sources(size=10))
+    assert exported[0]["bookSourceUrl"].startswith("https://books.example.com/api#")
+
+
+def test_rss_loader_reads_source_download_iterator(monkeypatch, tmp_path: Path) -> None:
+    source = rss_module.RSSSourceDownload(path=str(tmp_path), cate1="rss")
+
+    monkeypatch.setattr(
+        rss_module,
+        "iter_source_download_data",
+        lambda source_type: iter(
+            [
+                (
+                    object(),
+                    {"sourceUrl": "https://rss.example.com/feed/", "sourceName": "Example RSS"},
+                )
+            ]
+        ),
+    )
+
+    source.loader()
+
+    exported = next(source.export_sources(size=10))
+    assert exported[0]["sourceUrl"].startswith("https://rss.example.com/feed#")