feat: define search folder per newsletter #3

Author: LeonMusCoden

backend/alembic/versions/75ed3dbf1e16_add_search_folder_to_newsletter.py

@@ -0,0 +1,32 @@
+"""add search_folder to newsletter
+
+Revision ID: 75ed3dbf1e16
+Revises: 1ed7baeaf282
+Create Date: 2025-08-12 21:13:07.535625
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '75ed3dbf1e16'
+down_revision: Union[str, Sequence[str], None] = '1ed7baeaf282'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('newsletters', sa.Column('search_folder', sa.String(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('newsletters', 'search_folder')
+    # ### end Alembic commands ###

backend/app/crud/newsletters.py

@@ -64,6 +64,7 @@ def create_newsletter(db: Session, newsletter: NewsletterCreate):
         id=generate(size=10),
         name=newsletter.name,
         slug=newsletter.slug,
+        search_folder=newsletter.search_folder,
         extract_content=newsletter.extract_content,
         move_to_folder=newsletter.move_to_folder,
     )

backend/app/models/newsletters.py

@@ -12,6 +12,7 @@ class Newsletter(Base):
     id = Column(String, primary_key=True, index=True)
     slug = Column(String, unique=True, index=True, nullable=True)
     name = Column(String)
+    search_folder = Column(String, nullable=True)
     move_to_folder = Column(String, nullable=True)
     is_active = Column(Boolean, default=True)
     extract_content = Column(Boolean, default=False)

backend/app/schemas/newsletters.py

@@ -31,6 +31,7 @@ class NewsletterBase(BaseModel):
 
     name: str
     slug: str | None = None
+    search_folder: str | None = None
     move_to_folder: str | None = None
     extract_content: bool = False
 

backend/app/services/email_processor.py

@@ -34,20 +34,22 @@ def _is_configured(settings: Settings | None) -> bool:
     return True
 
 
-def _connect_to_imap(settings: Settings) -> imaplib.IMAP4_SSL | None:
+def _connect_to_imap(
+    settings: Settings, search_folder: str
+) -> imaplib.IMAP4_SSL | None:
     """Connect to the IMAP server and select the mailbox."""
     try:
         logger.info(f"Connecting to IMAP server: {settings.imap_server}")
         mail = imaplib.IMAP4_SSL(settings.imap_server)
         mail.login(settings.imap_username, settings.imap_password)
-        status, messages = mail.select(settings.search_folder)
+        status, messages = mail.select(search_folder)
         if status != "OK":
             logger.error(
-                f"Failed to select mailbox: {settings.search_folder}, status: {status}, messages: {messages}"
+                f"Failed to select mailbox: {search_folder}, status: {status}, messages: {messages}"
             )
             mail.logout()
             return None
-        logger.info(f"Selected mailbox: {settings.search_folder}")
+        logger.info(f"Selected mailbox: {search_folder}")
         return mail
     except Exception as e:
         logger.error(f"Failed to connect to IMAP server: {e}", exc_info=True)
@@ -235,26 +237,61 @@ def process_emails(db: Session) -> None:
     if not _is_configured(settings):
         return
 
-    newsletters = get_newsletters(db)
-    sender_map = {sender.email: nl for nl in newsletters for sender in nl.senders}
-    logger.info(f"Processing emails for {len(newsletters)} newsletters.")
+    all_newsletters = get_newsletters(db)
+    logger.info(f"Processing emails for {len(all_newsletters)} newsletters.")
 
-    mail = _connect_to_imap(settings)
-    if not mail:
-        return
+    # Group newsletters by search folder
+    folder_groups: dict[str, list[Newsletter]] = {}
+    for nl in all_newsletters:
+        folder = nl.search_folder or settings.search_folder
+        if folder not in folder_groups:
+            folder_groups[folder] = []
+        folder_groups[folder].append(nl)
 
-    try:
-        email_ids = _fetch_unread_email_ids(mail)
-        logger.info(f"Found {len(email_ids)} unseen emails.")
-        for num in email_ids:
-            _process_single_email(num, mail, db, sender_map, settings)
+    # If auto-adding is enabled, ensure the default search folder is always checked.
+    if settings.auto_add_new_senders and settings.search_folder not in folder_groups:
+        folder_groups[settings.search_folder] = []
 
-        if settings.move_to_folder:
-            logger.info("Expunging deleted emails")
-            mail.expunge()
+    for search_folder, newsletters_in_folder in folder_groups.items():
+        logger.info(
+            f"Processing folder '{search_folder}' for {len(newsletters_in_folder)} newsletters."
+        )
+        sender_map = {
+            sender.email: nl for nl in newsletters_in_folder for sender in nl.senders
+        }
+
+        mail = _connect_to_imap(settings, search_folder)
+        if not mail:
+            logger.warning(
+                f"Skipping folder '{search_folder}' due to connection issue."
+            )
+            continue
 
-    except Exception as e:
-        logger.error(f"Error processing emails: {e}", exc_info=True)
-    finally:
-        mail.logout()
-        logger.info("Email processing finished successfully.")
+        try:
+            email_ids = _fetch_unread_email_ids(mail)
+            logger.info(
+                f"Found {len(email_ids)} unseen emails in folder '{search_folder}'."
+            )
+            for num in email_ids:
+                _process_single_email(num, mail, db, sender_map, settings)
+
+            # Expunge logic needs to be carefully considered.
+            # If any newsletter in this folder group has a move_to_folder, we expunge.
+            # This is an approximation. A more robust solution might require per-email expunge.
+            should_expunge = any(
+                nl.move_to_folder or settings.move_to_folder
+                for nl in newsletters_in_folder
+            )
+            if should_expunge:
+                logger.info(f"Expunging deleted emails from '{search_folder}'")
+                mail.expunge()
+
+        except Exception as e:
+            logger.error(
+                f"Error processing emails in folder '{search_folder}': {e}",
+                exc_info=True,
+            )
+        finally:
+            mail.logout()
+
+    logger.info("Email processing finished successfully.")

backend/app/tests/test_crud.py

@@ -152,6 +152,33 @@ def test_create_newsletter_with_move_to_folder(db_session: Session):
     assert retrieved_newsletter.extract_content is True
 
 
+def test_create_newsletter_with_search_folder(db_session: Session):
+    """Test creating and updating a newsletter with the search_folder attribute."""
+    unique_email = f"sender_{uuid.uuid4()}@test.com"
+    newsletter_data = NewsletterCreate(
+        name="Test Newsletter with Search Folder",
+        sender_emails=[unique_email],
+        search_folder="CustomInbox",
+    )
+    newsletter = create_newsletter(db_session, newsletter_data)
+    retrieved_newsletter = get_newsletter_by_identifier(db_session, newsletter.id)
+
+    assert retrieved_newsletter.name == "Test Newsletter with Search Folder"
+    assert retrieved_newsletter.search_folder == "CustomInbox"
+
+    # Test updating the search_folder
+    from app.crud.newsletters import update_newsletter
+    from app.schemas.newsletters import NewsletterUpdate
+
+    update_data = NewsletterUpdate(
+        name=newsletter.name,
+        sender_emails=[unique_email],
+        search_folder="UpdatedCustomInbox",
+    )
+    updated_newsletter = update_newsletter(db_session, newsletter.id, update_data)
+    assert updated_newsletter.search_folder == "UpdatedCustomInbox"
+
+
 def test_get_newsletter_by_identifier(db_session: Session):
     """Test getting a single newsletter."""
     unique_email = f"sender_{uuid.uuid4()}@test.com"

backend/app/tests/test_email_processor.py

@@ -8,15 +8,15 @@
 from app.crud.settings import create_or_update_settings
 from app.models.newsletters import Newsletter
 from app.schemas.newsletters import NewsletterCreate
-from app.schemas.settings import SettingsCreate
-from app.services.email_processor import _process_single_email
+from app.schemas.settings import Settings, SettingsCreate
+from app.services.email_processor import _process_single_email, process_emails
 
 
 def _setup_test_email_processing(
     db_session: Session,
     newsletter_create_data: NewsletterCreate,
     settings_create_data: SettingsCreate,
-) -> tuple[MagicMock, Newsletter, SettingsCreate]:
+) -> tuple[MagicMock, Newsletter, Settings]:
     """Help to set up mocks and data for email processing tests."""
     settings = create_or_update_settings(db_session, settings_create_data)
     newsletter = create_newsletter(db_session, newsletter_create_data)
@@ -84,6 +84,74 @@ def test_process_single_email_with_global_move_folder(db_session: Session):
     mock_mail.store.assert_any_call("1", "+FLAGS", "\\Deleted")
 
 
+@patch("app.services.email_processor._connect_to_imap")
+def test_process_emails_uses_newsletter_search_folder(
+    mock_connect_to_imap,
+    db_session: Session,
+):
+    """Test that the per-newsletter search_folder is used, overriding the global setting."""
+    # 1. ARRANGE
+    settings_data = SettingsCreate(
+        imap_server="test.com",
+        imap_username="test",
+        imap_password="password",
+        search_folder="GlobalInbox",
+    )
+    create_or_update_settings(db_session, settings_data)
+
+    newsletter_data = NewsletterCreate(
+        name="Test Newsletter",
+        sender_emails=["test@example.com"],
+        search_folder="NewsletterInbox",
+    )
+    create_newsletter(db_session, newsletter_data)
+
+    # Mock the return of _connect_to_imap to avoid a real IMAP connection
+    mock_connect_to_imap.return_value = None
+
+    # 2. ACT
+    process_emails(db_session)
+
+    # 3. ASSERT
+    # Check that _connect_to_imap was called with the newsletter's specific folder
+    mock_connect_to_imap.assert_called_once()
+    call_args = mock_connect_to_imap.call_args[0]
+    assert call_args[1] == "NewsletterInbox"
+
+
+@patch("app.services.email_processor._connect_to_imap")
+def test_process_emails_uses_global_search_folder(
+    mock_connect_to_imap,
+    db_session: Session,
+):
+    """Test that the global search_folder is used when the per-newsletter one is not set."""
+    # 1. ARRANGE
+    settings_data = SettingsCreate(
+        imap_server="test.com",
+        imap_username="test",
+        imap_password="password",
+        search_folder="GlobalInbox",
+    )
+    create_or_update_settings(db_session, settings_data)
+
+    newsletter_data = NewsletterCreate(
+        name="Test Newsletter",
+        sender_emails=["test@example.com"],
+        search_folder=None,  # Explicitly not set
+    )
+    create_newsletter(db_session, newsletter_data)
+
+    mock_connect_to_imap.return_value = None
+
+    # 2. ACT
+    process_emails(db_session)
+
+    # 3. ASSERT
+    mock_connect_to_imap.assert_called_once()
+    call_args = mock_connect_to_imap.call_args[0]
+    assert call_args[1] == "GlobalInbox"
+
+
 @patch("app.services.email_processor._extract_and_clean_html")
 def test_process_single_email_with_content_extraction(
     mock_extract_clean,

frontend/src/components/letterfeed/NewsletterDialog.tsx

@@ -36,6 +36,7 @@ const getInitialState = (newsletter: Newsletter | null | undefined) => {
       name: newsletter.name,
       slug: newsletter.slug || "",
       emails: newsletter.senders.map((s) => s.email),
+      search_folder: newsletter.search_folder || "",
       move_to_folder: newsletter.move_to_folder || "",
       extract_content: newsletter.extract_content,
     }
@@ -44,6 +45,7 @@ const getInitialState = (newsletter: Newsletter | null | undefined) => {
     name: "",
     slug: "",
     emails: [""],
+    search_folder: "",
     move_to_folder: "",
     extract_content: false,
   }
@@ -89,6 +91,7 @@ export function NewsletterDialog({ newsletter, isOpen, folderOptions, onOpenChan
       name: formData.name,
       slug: formData.slug,
       sender_emails: formData.emails.filter((email) => email.trim()),
+      search_folder: formData.search_folder,
       move_to_folder: formData.move_to_folder,
       extract_content: formData.extract_content,
     }
@@ -148,6 +151,28 @@ export function NewsletterDialog({ newsletter, isOpen, folderOptions, onOpenChan
             />
           </div>
 
+          <div className="space-y-2">
+            <Label htmlFor="search_folder">Folder to Search</Label>
+            <Select
+              value={formData.search_folder || "None"}
+              onValueChange={(value) =>
+                setFormData((prev) => ({ ...prev, search_folder: value === "None" ? "" : value }))
+              }
+            >
+              <SelectTrigger>
+                <SelectValue placeholder="Select folder or leave empty" />
+              </SelectTrigger>
+              <SelectContent>
+                <SelectItem value="None">Default (use global setting)</SelectItem>
+                {folderOptions.map((folder) => (
+                  <SelectItem key={folder} value={folder}>
+                    {folder}
+                  </SelectItem>
+                ))}
+              </SelectContent>
+            </Select>
+          </div>
+
           <div className="space-y-2">
             <Label htmlFor="move_to_folder">Move To Folder</Label>
             <Select

frontend/src/components/letterfeed/__tests__/NewsletterDialog.test.tsx

@@ -23,6 +23,7 @@ const mockNewsletter: Newsletter = {
   extract_content: false,
   senders: [{ id: "1", email: "current@example.com" }],
   entries_count: 5,
+  search_folder: "",
   move_to_folder: "",
 }
 
@@ -62,6 +63,7 @@ describe("NewsletterDialog", () => {
           name: "My New Newsletter",
           slug: "my-new-newsletter",
           sender_emails: ["test@example.com"],
+          search_folder: "",
           move_to_folder: "",
           extract_content: false,
         })
@@ -109,6 +111,7 @@ describe("NewsletterDialog", () => {
           name: "Updated Name",
           slug: "existing-newsletter",
           sender_emails: ["current@example.com"],
+          search_folder: "",
           move_to_folder: "",
           extract_content: false,
         })

frontend/src/lib/api.ts

@@ -11,6 +11,7 @@ export interface Newsletter {
   name: string
   slug: string | null
   is_active: boolean
+  search_folder?: string | null
   move_to_folder?: string | null
   extract_content: boolean
   senders: { id: string; email: string }[]
@@ -21,6 +22,7 @@ export interface NewsletterCreate {
     name: string;
     slug?: string | null;
     sender_emails: string[];
+    search_folder?: string | null;
     move_to_folder?: string | null;
     extract_content: boolean;
 }
@@ -29,6 +31,7 @@ export interface NewsletterUpdate {
     name: string;
     slug?: string | null;
     sender_emails: string[];
+    search_folder?: string | null;
     move_to_folder?: string | null;
     extract_content: boolean;
 }