switch from list to set for deduplication

Author: Philip Johansson

src/murid/clients/calibre.py

@@ -56,7 +56,7 @@ def validate(self) -> None:
             logger.error("Error connecting to Calibre database: %s", e)
             raise CalibreError(f"Error connecting to Calibre database: {e}") from e
 
-    def get_books(self) -> list[Book]:
+    def get_books(self) -> set[Book]:
         """Retrieve a list of books from the Calibre database."""
         try:
             conn = self.connect(f"file:{self.db_path}?mode=ro", uri=True)
@@ -90,7 +90,7 @@ def get_books(self) -> list[Book]:
 
         rows = cursor.fetchall()
         conn.close()
-        books = [
+        books = {
             Book(
                 id=row["id"],
                 title=row["title"],
@@ -100,7 +100,7 @@ def get_books(self) -> list[Book]:
                 source="calibre",
             )
             for row in rows
-        ]
+        }
         return books
 
     def add_book(self, book: Book, path: str) -> None:

src/murid/clients/hardcover.py

@@ -150,15 +150,15 @@ def _extract_isbn(editions: list[dict] | None) -> list[str | None]:
 
         return isbns
 
-    def get_books(self) -> list[Book]:
+    def get_books(self) -> set[Book]:
         """Fetch the user's "Want to Read" books from the Hardcover API.
 
         Return them as a list of Book objects.
         """
         data = self.fetch_data()
         items = data.get("data", {}).get("user_books", [])
 
-        books: list[Book] = []
+        books: set[Book] = set()
 
         for item in items:
             book = item.get("book", {})
@@ -171,7 +171,7 @@ def get_books(self) -> list[Book]:
             isbn = self._extract_isbn(editions)
             series, series_number = self._extract_series_info(book)
 
-            books.append(
+            books.add(
                 Book(
                     id=book.get("id"),
                     title=book.get("title"),

src/murid/clients/myanonamouse.py

@@ -55,7 +55,7 @@ def _request(self, method: str, url: str, **kwargs) -> requests.Response:
             self._last_request_time = time.monotonic()
             return self.session.request(method, url, **kwargs)
 
-    def search(self, query: MyAnonamouseQuery) -> list[Torrent]:
+    def search(self, query: MyAnonamouseQuery) -> set[Torrent]:
         """Search for torrents on MyAnonamouse matching the specified criteria."""
 
         payload = {
@@ -89,17 +89,17 @@ def search(self, query: MyAnonamouseQuery) -> list[Torrent]:
             response.raise_for_status()
         except requests.RequestException as e:
             logger.error("Error searching MyAnonamouse: %s", e)
-            return []
+            return set()
 
         data = response.json()
 
         if "error" in data:
-            return []
+            return set()
 
         if "data" not in data:
             raise MAMError(f"Unexpected response: {data}")
 
-        return [self._parse_torrent(row) for row in data["data"][:100]]
+        return {self._parse_torrent(row) for row in data["data"][:100]}
 
     @staticmethod
     def _parse_torrent(data: dict[str, Any]) -> Torrent:
@@ -135,7 +135,7 @@ def _parse_torrent(data: dict[str, Any]) -> Torrent:
             raw=data,
         )
 
-    def search_ebook(self, title: str, author: str | None = None) -> list[Torrent]:
+    def search_ebook(self, title: str, author: str | None = None) -> set[Torrent]:
         """Search for ebooks on MyAnonamouse matching the specified title and optional author."""
         query = title
         if author:

src/murid/domain/book.py

@@ -17,3 +17,30 @@ class Book:
 
     def __str__(self):
         return f"{self.title} by {', '.join(self.authors)}"
+
+    def __hash__(self):
+        return hash(
+            (
+                self.title,
+                tuple(self.authors),
+                self.id,
+                self.source,
+                self.series,
+                self.series_number,
+                tuple(self.isbn),
+            )
+        )
+
+    def __eq__(self, other):
+        if not isinstance(other, Book):
+            return NotImplemented
+
+        return (
+            self.title == other.title
+            and self.authors == other.authors
+            and self.id == other.id
+            and self.isbn == other.isbn
+            and self.source == other.source
+            and self.series == other.series
+            and self.series_number == other.series_number
+        )

src/murid/domain/book_matcher.py

@@ -2,6 +2,7 @@
 
 import logging
 import re
+from typing import Iterable
 
 from rapidfuzz import fuzz
 
@@ -105,17 +106,17 @@ def best_match(self, book: Book, candidates: list[Book]) -> tuple[Book | None, f
 
     def match_books(
         self,
-        books_a: list[Book],
-        books_b: list[Book],
-    ) -> list[tuple[Book, Book, float]]:
+        books_a: Iterable[Book],
+        books_b: Iterable[Book],
+    ) -> set[tuple[Book, Book, float]]:
         """Match books from two lists and return a list of matched pairs and their similarity."""
 
-        matches = []
+        matches = set()
 
         for book_b in books_b:
             match, score = self.best_match(book_b, books_a)
             if match and score >= self.threshold:
-                matches.append((match, book_b, score))
+                matches.add((match, book_b, score))
             else:
                 logger.debug("No match for %s in calibre db. Best similarity: %.2f", book_b, score)
 

src/murid/domain/torrent.py

@@ -41,3 +41,28 @@ def download_url(self) -> str | None:
     def __str__(self) -> str:
         """Return a string representation of the torrent as a link to the torrents page."""
         return f"https://www.myanonamouse.net/t/{self.book.id}"
+
+    def __hash__(self) -> int:
+        if self.download_hash is not None:
+            return hash(self.download_hash)
+
+        return hash(
+            (
+                self.book,
+                self.language,
+                tuple(self.file_types or ()),
+            )
+        )
+
+    def __eq__(self, other) -> bool:
+        if not isinstance(other, Torrent):
+            return NotImplemented
+
+        if self.download_hash is not None and other.download_hash is not None:
+            return self.download_hash == other.download_hash
+
+        return (
+            self.book == other.book
+            and self.language == other.language
+            and self.file_types == other.file_types
+        )

src/murid/domain/torrent_selector.py

@@ -1,6 +1,7 @@
 """Class for selecting the best matching torrent for a given book"""
 
 import logging
+from typing import Iterable
 
 from .book import Book
 from .book_matcher import BookMatcher
@@ -23,14 +24,16 @@ def __init__(
             wanted_filetypes = {"epub", "mobi", "azw3"}
         self.wanted_filetypes = wanted_filetypes
 
-    def select(self, book: Book, torrents: list[Torrent], matcher: BookMatcher) -> Torrent | None:
+    def select(
+        self, book: Book, torrents: Iterable[Torrent], matcher: BookMatcher
+    ) -> Torrent | None:
         """Select the best matching torrent for the given book from the list of torrents"""
-        torrent_books = [
+        torrent_books = {
             t.book
             for t in torrents
             if (t.language is None or t.language in self.lang_codes)
             and self.wanted_filetypes.intersection(t.file_types or [])
-        ]
+        }
 
         if not torrent_books:
             logger.debug("No torrents for %s passed language and file type filters", book)

src/murid/services/service_factory.py

@@ -59,11 +59,11 @@ def calibre(self):
             logger.error("Error initializing Calibre: %s", e)
             raise SystemExit(1) from e
 
-    def hardcover(self) -> list[Hardcover]:
+    def hardcover(self) -> set[Hardcover]:
         """Create a list of Hardcover instances for each user specified in the configuration."""
         keys = self._load_config()["hardcover_api_keys"]
         with ThreadPoolExecutor(max_workers=min(10, len(keys))) as executor:
-            hardcover_clients = list(executor.map(Hardcover, keys))
+            hardcover_clients = set(executor.map(Hardcover, keys))
         return hardcover_clients
 
     def myanonamouse(self):

src/murid/services/sync_service.py

@@ -5,6 +5,7 @@
 from concurrent.futures import ThreadPoolExecutor
 from datetime import datetime
 from itertools import chain
+from typing import Iterable
 
 from ..clients.calibre import Calibre
 from ..clients.hardcover import Hardcover
@@ -64,43 +65,43 @@ def run(self) -> None:
         logger.info("Finished murid cycle")
 
     @staticmethod
-    def fetch_calibre_books(calibre: Calibre) -> list[Book]:
+    def fetch_calibre_books(calibre: Calibre) -> set[Book]:
         """Fetch books from the Calibre library."""
         books = calibre.get_books()
         logger.info("Feched %d books from Calibre database", len(books))
         return books
 
     @staticmethod
-    def fetch_hardcover_books(hardcover: list[Hardcover]) -> list[Book]:
+    def fetch_hardcover_books(hardcover: list[Hardcover]) -> set[Book]:
         """Fetch books from the Hardcover API."""
         with ThreadPoolExecutor(max_workers=min(10, len(hardcover))) as executor:
             results = executor.map(lambda client: client.get_books(), hardcover)
-            books = list(chain.from_iterable(results))
+            books = set(chain.from_iterable(results))
         logger.info("Fetched %d books from Hardcover API", len(books))
         return books
 
     @staticmethod
     def match_books(
-        calibre_books: list[Book],
-        hardcover_books: list[Book],
+        calibre_books: Iterable[Book],
+        hardcover_books: Iterable[Book],
         matcher: BookMatcher,
-    ) -> list[tuple[Book, Book, float]]:
+    ) -> set[tuple[Book, Book, float]]:
         """Match books from the Hardcover list against the Calibre library."""
         matches = matcher.match_books(calibre_books, hardcover_books)
         logger.debug("%d books already present in Calibre library", len(matches))
         return matches
 
     @staticmethod
     def process_books(
-        present_books: list[tuple[Book, Book, float]],
-        wanted_books: list[Book],
+        present_books: Iterable[tuple[Book, Book, float]],
+        wanted_books: Iterable[Book],
         torrent_discovery: TorrentDiscoveryService,
         matcher: BookMatcher,
-    ) -> list[tuple[bytes, Book]]:
+    ) -> set[tuple[bytes, Book]]:
         """Process the matched and unmatched books to find torrents for the missing ones."""
         matched_ids = {h.id for _, h, _ in present_books}
-        missing_books = [h for h in wanted_books if h.id not in matched_ids]
+        missing_books = {h for h in wanted_books if h.id not in matched_ids}
 
         if not missing_books:
-            return []
+            return set()
         return torrent_discovery.download_torrents(missing_books, matcher)

src/murid/services/torrent_discovery.py

@@ -2,6 +2,7 @@
 
 import logging
 from concurrent.futures import Future, ThreadPoolExecutor, as_completed
+from typing import Iterable
 
 from ..clients.myanonamouse import MyAnonamouse
 from ..domain.book import Book
@@ -26,7 +27,7 @@ def __init__(
         self.lang_codes = set(lang_codes)
         self.torrent_selector = torrent_selector
 
-    def find_torrents(self, book: Book) -> tuple[Book, list[Torrent]]:
+    def find_torrents(self, book: Book) -> tuple[Book, set[Torrent]]:
         """Find torrents for a given book."""
         mam_books = self.mam.search_ebook(book.title, book.authors[0] if book.authors else None)
 
@@ -71,21 +72,21 @@ def submit_downloads(
     def collect_downloads(
         self,
         download_futures: dict[Future, Book],
-    ) -> list[tuple[bytes, Book]]:
+    ) -> set[tuple[bytes, Book]]:
         """Collect the downloaded torrent data once the download futures complete."""
 
-        torrent_files = []
+        torrent_files = set()
         for future in as_completed(download_futures):
             book = download_futures[future]
             torrent_file = future.result()
             if torrent_file:
-                torrent_files.append((torrent_file, book))
+                torrent_files.add((torrent_file, book))
 
         return torrent_files
 
     def download_torrents(
-        self, books: list[Book], matcher: BookMatcher
-    ) -> list[tuple[bytes, Book]]:
+        self, books: Iterable[Book], matcher: BookMatcher
+    ) -> set[tuple[bytes, Book]]:
         """Find and download torrents for a list of books."""
         logger.info(
             "Searching for torrents for %d book%s", len(books), "s" if len(books) != 1 else ""
@@ -96,5 +97,5 @@ def download_torrents(
             download_futures = self.submit_downloads(executor, search_futures, matcher)
 
             if not download_futures:
-                return []
+                return set()
             return self.collect_downloads(download_futures)