feat: warn about duplicate recipients in CSV uploads (#3319)

Surfaces a non-blocking warning on the bulk-send preview page when the
uploaded CSV contains the same recipient more than once. The warning is
case-insensitive, ignores leading/trailing whitespace, and (for SMS)
treats phone numbers in different formats as equivalent. Letters are
excluded because multiple recipients can legitimately share an address.

Changes:
- new get_warnings_for_csv helper in app/utils.py
- new download_duplicate_recipients route serves a CSV of duplicate
  rows so the sender can review them before sending; the response is
  built server-side from the original upload, so duplicates are visible
  only to the authenticated sender (privacy)
- ok.html renders the warning in a banner inside an aria-live=polite
  region, including a link to download the duplicates list
- EN/FR translation strings added to fr.csv (en.csv and messages.po
  files are regenerated by 'make babel')
- unit tests for get_warnings_for_csv

Depends on notifications-utils with duplicate-recipient detection
(see corresponding PR in notification-utils).

Refs: cds-snc/notification-planning#3319

Author: smcmurtry

app/main/views/send.py

@@ -70,6 +70,7 @@
     get_help_argument,
     get_limit_reset_time_et,
     get_template,
+    get_warnings_for_csv,
     should_skip_template_page,
     unicode_truncate,
     user_has_permissions,
@@ -771,8 +772,11 @@ def _check_messages(service_id, template_id, upload_id, preview_row, letters_as_
         template=template,
         errors=recipients.has_errors,
         row_errors=get_errors_for_csv(recipients, template.template_type),
+        row_warnings=get_warnings_for_csv(recipients, template.template_type),
         count_of_recipients=len(recipients),
         count_of_displayed_recipients=len(list(recipients.displayed_rows)),
+        count_of_duplicate_recipients=recipients.count_of_unique_duplicate_recipients,
+        count_of_duplicate_recipient_rows=recipients.count_of_duplicate_recipient_rows,
         original_file_name=request.args.get("original_file_name", ""),
         upload_id=upload_id,
         form=CsvUploadForm(),
@@ -919,6 +923,48 @@ def check_messages_preview(service_id, template_id, upload_id, filetype, row_ind
     return TemplatePreview.from_utils_template(template, filetype, page=page)
 
 
+@main.route(
+    "/services/<service_id>/<uuid:template_id>/check/<upload_id>/duplicates.csv",
+    methods=["GET"],
+)
+@user_has_permissions("send_messages", restrict_admin_usage=True)
+def download_duplicate_recipients(service_id, template_id, upload_id):
+    """Return a CSV of the rows whose recipient is a duplicate of an earlier
+    row in the same upload. The list is rendered server-side from the original
+    upload (which is scoped to this authenticated user) and is not persisted,
+    so duplicates are visible only to the authenticated sender (issue #3319).
+    """
+    contents = s3download(service_id, upload_id)
+    db_template = current_service.get_template_with_user_permission_or_403(template_id, current_user)
+    template = get_template(db_template, current_service)
+    recipients = RecipientCSV(
+        contents,
+        template=template,
+        template_type=template.template_type,
+        placeholders=template.placeholders,
+        max_rows=get_csv_max_rows(service_id),
+        international_sms=current_service.has_permission("international_sms"),
+        user_language=get_current_locale(current_app),
+    )
+
+    column_headers = recipients.column_headers
+    duplicate_rows = list(recipients.rows_with_duplicate_recipients)
+
+    rows = [["Row number", *column_headers]]
+    for row in duplicate_rows:
+        rows.append([str(row.index + 2), *(row[column].data or "" for column in column_headers)])
+
+    safe_filename = SanitiseASCII.encode(request.args.get("original_file_name", "duplicates.csv"))
+    return (
+        Spreadsheet.from_rows(rows).as_csv_data,
+        200,
+        {
+            "Content-Type": "text/csv; charset=utf-8",
+            "Content-Disposition": 'attachment; filename="duplicates-{}"'.format(safe_filename),
+        },
+    )
+
+
 @main.route(
     "/services/<service_id>/<uuid:template_id>/check.<filetype>",
     methods=["GET"],

app/templates/views/check/ok.html

@@ -6,6 +6,7 @@
 {% from "components/page-footer.html" import page_footer %}
 {% from "components/message-count-label.html" import message_count_label %}
 {% from "components/banner.html" import banner %}
+{% from "components/banner.html" import banner_wrapper %}
 {% from "components/remaining-messages-summary.html" import remaining_messages_summary with context %}
 {% set page_title = _('Review before sending') %}
 {% block service_page_title %}
@@ -23,6 +24,28 @@
         with_tick=True
       ) }}
   </section>
+
+  {# Issue #3319: non-blocking warning when the CSV contains duplicate recipients. #}
+  {% if row_warnings %}
+    <div class="mb-12 clear-both contain-floats" aria-live="polite" data-testid="duplicate-recipients-warning">
+      {% call banner_wrapper(type='warning') %}
+        <h2 class='banner-title' data-module="track-error" data-error-type="Duplicate recipients" data-error-label="{{ upload_id }}">
+          {{ row_warnings[0] }}
+        </h2>
+        {% for warning in row_warnings[1:] %}
+          <p class="mb-0">{{ warning }}</p>
+        {% endfor %}
+        {% if count_of_duplicate_recipients %}
+          <p class="mb-0">
+            <a href="{{ url_for('.download_duplicate_recipients', service_id=current_service.id, template_id=template.id, upload_id=upload_id, original_file_name=original_file_name) }}" download>
+              {{ _('Download a list of duplicate recipients') }}
+            </a>
+          </p>
+        {% endif %}
+      {% endcall %}
+    </div>
+  {% endif %}
+
   {% if not request.args.from_test %}
     <section class="clear-both contain-floats" id="ok-file">
       <h2 class="heading-medium">{{ _('Recipients') }}</h2>

app/translations/csv/fr.csv

@@ -2453,4 +2453,8 @@
 "Use ((unsub_link)) for one-click unsubscribe","Utilisez ((unsub_link)) pour le désabonnement en un clic"
 "Allow this key to manage templates (create, update, or delete)","Permettre à cette clé de gérer les gabarits (créer, mettre à jour ou supprimer)"
 "API key permissions","Permissions de la clé API"
-"Manage templates","Gestion des gabarits"
+"Manage templates","Gestion des gabarits"
+"1 recipient appears in your list more than once. They will receive the message multiple times.","1 destinataire figure plusieurs fois dans votre liste. Cette personne recevra le message plusieurs fois."
+"{} recipients appear in your list more than once. They will receive the message multiple times.","{} destinataires figurent plusieurs fois dans votre liste. Ces personnes recevront le message plusieurs fois."
+"{} rows are duplicates of an earlier row.","{} lignes sont des doublons d’une ligne précédente."
+"Download a list of duplicate recipients","Télécharger la liste des destinataires en double"

app/utils.py

@@ -231,6 +231,42 @@ def get_errors_for_csv(recipients, template_type):
     return errors
 
 
+def get_warnings_for_csv(recipients, template_type):
+    """
+    Return a list of non-blocking warning messages for a CSV upload.
+
+    Currently this surfaces duplicate-recipient warnings for issue #3319:
+    duplicate detection is case-insensitive, ignores leading/trailing
+    whitespace, and (for SMS) treats phone numbers in different formats as
+    equivalent. Letters are intentionally excluded because multiple recipients
+    can legitimately share an address.
+    """
+    warnings = []
+
+    if template_type == TemplateType.LETTER.value:
+        return warnings
+
+    if recipients.has_duplicate_recipients:
+        unique_duplicates = recipients.count_of_unique_duplicate_recipients
+        duplicate_rows = recipients.count_of_duplicate_recipient_rows
+        if unique_duplicates == 1:
+            warnings.append(
+                _("1 recipient appears in your list more than once. " "They will receive the message multiple times.")
+            )
+        else:
+            warnings.append(
+                _("{} recipients appear in your list more than once. " "They will receive the message multiple times.").format(
+                    unique_duplicates
+                )
+            )
+        # Provide a secondary line so that the count of *rows* affected is also visible
+        # (useful when one recipient appears many times).
+        if duplicate_rows != unique_duplicates:
+            warnings.append(_("{} rows are duplicates of an earlier row.").format(duplicate_rows))
+
+    return warnings
+
+
 def localize_and_format_csv_headers(column_headers: list) -> list:
     from app import get_current_locale
 

tests/app/main/test_warnings_for_csv.py

@@ -0,0 +1,87 @@
+from collections import namedtuple
+
+import pytest
+
+from app.utils import get_warnings_for_csv
+
+# A minimal stand-in for ``RecipientCSV`` covering only the attributes consumed
+# by ``get_warnings_for_csv``. Keeps these tests independent of the real
+# notifications-utils CSV parser.
+MockRecipients = namedtuple(
+    "MockRecipients",
+    [
+        "has_duplicate_recipients",
+        "count_of_unique_duplicate_recipients",
+        "count_of_duplicate_recipient_rows",
+        "template_type",
+    ],
+)
+
+
+@pytest.mark.parametrize(
+    "template_type",
+    ["sms", "email"],
+)
+def test_returns_empty_list_when_no_duplicates(app_, template_type):
+    recipients = MockRecipients(
+        has_duplicate_recipients=False,
+        count_of_unique_duplicate_recipients=0,
+        count_of_duplicate_recipient_rows=0,
+        template_type=template_type,
+    )
+    with app_.test_request_context():
+        assert get_warnings_for_csv(recipients, template_type) == []
+
+
+def test_returns_singular_warning_for_one_duplicate_recipient(app_):
+    recipients = MockRecipients(
+        has_duplicate_recipients=True,
+        count_of_unique_duplicate_recipients=1,
+        count_of_duplicate_recipient_rows=1,
+        template_type="email",
+    )
+    with app_.test_request_context():
+        warnings = get_warnings_for_csv(recipients, "email")
+    assert len(warnings) == 1
+    assert "1 recipient appears in your list more than once" in warnings[0]
+
+
+def test_returns_plural_warning_for_multiple_duplicate_recipients(app_):
+    recipients = MockRecipients(
+        has_duplicate_recipients=True,
+        count_of_unique_duplicate_recipients=3,
+        count_of_duplicate_recipient_rows=3,
+        template_type="email",
+    )
+    with app_.test_request_context():
+        warnings = get_warnings_for_csv(recipients, "email")
+    assert len(warnings) == 1
+    assert "3 recipients appear in your list more than once" in warnings[0]
+
+
+def test_includes_row_count_when_it_differs_from_unique_count(app_):
+    # one recipient appears 4 times → 1 unique duplicate, 3 duplicate rows
+    recipients = MockRecipients(
+        has_duplicate_recipients=True,
+        count_of_unique_duplicate_recipients=1,
+        count_of_duplicate_recipient_rows=3,
+        template_type="sms",
+    )
+    with app_.test_request_context():
+        warnings = get_warnings_for_csv(recipients, "sms")
+    assert len(warnings) == 2
+    assert "1 recipient appears in your list more than once" in warnings[0]
+    assert "3 rows are duplicates of an earlier row" in warnings[1]
+
+
+def test_letter_template_never_warns_about_duplicates(app_):
+    # Letters can legitimately share an address (e.g. roommates), so the
+    # warning would be misleading and is therefore suppressed.
+    recipients = MockRecipients(
+        has_duplicate_recipients=True,
+        count_of_unique_duplicate_recipients=2,
+        count_of_duplicate_recipient_rows=2,
+        template_type="letter",
+    )
+    with app_.test_request_context():
+        assert get_warnings_for_csv(recipients, "letter") == []