Improve batch upload handling for large files with splitting logic

Author: farfarfun

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "funread"
-version = "1.1.78"
+version = "1.1.79"
 description = "一个用于管理和处理阅读源（Legado 阅读 APP 的书源和 RSS 源）的 Python 工具库"
 readme = "README.md"
 requires-python = ">=3.8"

src/funread/legado/manage/download/generate.py

@@ -22,6 +22,8 @@
 DEFAULT_DIR_PATH = "funread/legado/snapshot/lasted"
 EXPORT_BATCH_SIZE = 500
 INITIAL_COUNTER = 1000
+MIN_UPLOAD_BATCH_SIZE = 20
+
 
 # 组织仓库列表
 ORG_REPOS = [
@@ -257,24 +259,46 @@ def _export_and_upload(self, path: str) -> None:
                 counter = INITIAL_COUNTER
                 for data in runner.export_sources(size=EXPORT_BATCH_SIZE):
                     if data:
-                        self._upload_batch(data, counter)
-                        counter += 1
+                        counter = self._upload_batch(data, counter)
         except Exception as e:
             logger.error(f"Failed to export and upload: {e}")
             raise
 
-    def _upload_batch(self, data: List[Dict[str, Any]], counter: int) -> None:
-        """上传数据批次"""
+    @staticmethod
+    def _is_file_too_large_error(error: Exception) -> bool:
+        """判断是否为远端文件过大错误"""
+        message = str(error).lower()
+        return "too large" in message or "422" in message
+
+    def _upload_single_batch(self, data: List[Dict[str, Any]], counter: int) -> None:
+        """上传单个数据批次"""
+        git_path = f"{self.dir_path}/progress-{counter}.json"
+        self.drive.upload_file(
+            content=json.dumps(data),
+            fid=self.dir_path,
+            filepath=None,
+            filename=f"progress-{counter}.json",
+        )
+        logger.info(f"Uploaded {len(data)} sources to {git_path}")
+
+    def _upload_batch(self, data: List[Dict[str, Any]], counter: int) -> int:
+        """上传数据批次，必要时自动拆分为更小文件"""
         try:
-            git_path = f"{self.dir_path}/progress-{counter}.json"
-            self.drive.upload_file(
-                content=json.dumps(data),
-                fid=self.dir_path,
-                filepath=None,
-                filename=f"progress-{counter}.json",
-            )
-            logger.info(f"Uploaded {len(data)} sources to {git_path}")
+            self._upload_single_batch(data, counter)
+            return counter + 1
         except Exception as e:
+            if self._is_file_too_large_error(e) and len(data) > MIN_UPLOAD_BATCH_SIZE:
+                split_size = max(len(data) // 2, MIN_UPLOAD_BATCH_SIZE)
+                logger.warning(
+                    f"Batch {counter} too large with {len(data)} sources, split into chunks of {split_size}"
+                )
+                next_counter = counter
+                for start in range(0, len(data), split_size):
+                    next_counter = self._upload_batch(
+                        data[start : start + split_size], next_counter
+                    )
+                return next_counter
+
             logger.error(f"Failed to upload batch {counter}: {e}")
             raise
 

tests/test_download_base.py

@@ -1,6 +1,7 @@
 from pathlib import Path
 
 import funread.legado.manage.download.book as book_module
+import funread.legado.manage.download.generate as generate_module
 import funread.legado.manage.download.rss as rss_module
 
 from funread.legado.manage.download.base import DownloadSource
@@ -103,3 +104,42 @@ def test_rss_loader_reads_source_download_iterator(monkeypatch, tmp_path: Path)
 
     exported = next(source.export_sources(size=10))
     assert exported[0]["sourceUrl"].startswith("https://rss.example.com/feed#")
+
+
+class _FakeDrive:
+    def __init__(self, fail_threshold=None):
+        self.fail_threshold = fail_threshold
+        self.calls = []
+
+    def upload_file(self, content, fid, filepath, filename):
+        payload = generate_module.json.loads(content)
+        self.calls.append((filename, len(payload)))
+        if self.fail_threshold is not None and len(payload) > self.fail_threshold:
+            raise RuntimeError("GitHub API返回422: Sorry, the file is too large to be processed.")
+
+
+def test_upload_batch_splits_large_payloads() -> None:
+    generator = generate_module.GenerateSourceType.__new__(generate_module.GenerateSourceType)
+    generator.dir_path = "funread/legado/snapshot/lasted/book"
+    generator.drive = _FakeDrive(fail_threshold=2)
+
+    next_counter = generator._upload_batch([{"i": i} for i in range(5)], 1000)
+
+    assert next_counter == 1003
+    assert generator.drive.calls == [
+        ("progress-1000.json", 5),
+        ("progress-1000.json", 2),
+        ("progress-1001.json", 2),
+        ("progress-1002.json", 1),
+    ]
+
+
+def test_upload_batch_increments_counter_on_success() -> None:
+    generator = generate_module.GenerateSourceType.__new__(generate_module.GenerateSourceType)
+    generator.dir_path = "funread/legado/snapshot/lasted/book"
+    generator.drive = _FakeDrive()
+
+    next_counter = generator._upload_batch([{"i": 1}, {"i": 2}], 1000)
+
+    assert next_counter == 1001
+    assert generator.drive.calls == [("progress-1000.json", 2)]