Experiment/cens integration

app/__init__.py

@@ -45,6 +45,7 @@
 from app import proxy_fix
 from app.articles.routing import gca_url_for
 from app.asset_fingerprinter import asset_fingerprinter
+from app.cens_client import cens_client
 from app.commands import setup_commands
 from app.config import configs
 from app.extensions import (
@@ -194,6 +195,7 @@ def get_locale():
         zendesk_client,
         redis_client,
         bounce_rate_client,
+        cens_client,
     ):
         client.init_app(application)
 

app/cens_client/__init__.py

@@ -0,0 +1,119 @@
+import logging
+from typing import Any, Dict, Optional
+
+import requests
+
+logger = logging.getLogger(__name__)
+
+
+class CensClient:
+    """Simple client for talking to CENS provider endpoints.
+
+    Usage:
+        client = CensClient()
+        client.init_app(app)
+
+    The client fetches a short-lived token for every request by calling
+    the provider `get-token` endpoint using credentials from app config.
+    """
+
+    def __init__(self, testing_mode: bool = False):
+        self.base_url: Optional[str] = None
+        self.client_id: Optional[str] = None
+        self.client_secret: Optional[str] = None
+        self.testing_mode = testing_mode
+
+    def init_app(self, app):
+        self.base_url = app.config.get("CENS_URL")
+        self.client_id = app.config.get("CENS_USER")
+        self.client_secret = app.config.get("CENS_SECRET")
+        # optional testing mode flag in app config
+        self.testing_mode = bool(app.config.get("CENS_TESTING_MODE", self.testing_mode))
+
+    def _form_url(self, path: str) -> str:
+        if not self.base_url:
+            raise RuntimeError("CENS client not configured with base URL")
+        return f"{self.base_url.rstrip('/')}/{path.lstrip('/')}"
+
+    def get_token(self) -> str:
+        """Request a token from the provider using client credentials.
+
+        Returns the token string on success, raises requests.HTTPError on failure.
+        """
+        if self.testing_mode:
+            # return a stable test token
+            return "test-token"
+
+        url = self._form_url("get-token")
+        data = {"clientId": self.client_id or "", "clientSecret": self.client_secret or ""}
+        headers = {"Content-Type": "application/x-www-form-urlencoded"}
+
+        resp = requests.post(url, data=data, headers=headers, timeout=10)
+        try:
+            resp.raise_for_status()
+        except requests.HTTPError:
+            logger.exception("Failed to fetch CENS token")
+            raise
+
+        json = resp.json()
+        token = json.get("token")
+        if not token:
+            raise RuntimeError("CENS token missing from response")
+        return token
+
+    def has_service(self, service_id: str) -> bool:
+        """Return True if provider has topics for given service_id."""
+        if self.testing_mode:
+            # in testing mode assume service has topics
+            return True
+
+        token = self.get_token()
+        url = self._form_url("provider/has-service")
+        data = {"serviceId": service_id}
+        headers = {
+            "Content-Type": "application/x-www-form-urlencoded",
+            "Authorization": f"Bearer {token}",
+        }
+
+        resp = requests.post(url, data=data, headers=headers, timeout=10)
+        resp.raise_for_status()
+        json = resp.json()
+        return bool(json.get("exists", False))
+
+    def get_subscribers_list(self, service_id: str) -> Dict[str, Any]:
+        """Return the raw JSON response listing subscribers for a service.
+
+        The response is expected to include keys like `name`, `service-id` and `rows`.
+        """
+        if self.testing_mode:
+            return {
+                "name": "CENS Topic 123",
+                "template_id": "bcd2d833-e087-4bb3-81d5-39da32fa361a",
+                "rows": [
+                    ["email address", "unsub_link"],
+                    [
+                        "andrew.leith+1@cds-snc.ca",
+                        "https://apps.canada.ca/cens2/subs/remove/{subsCode1}/F853e0212b92a127",
+                    ],
+                    [
+                        "andrew.leith+2@cds-snc.ca",
+                        "https://apps.canada.ca/cens2/subs/remove/{subsCode2}/F853e0212b92a127",
+                    ],
+                ],
+            }
+
+        token = self.get_token()
+        url = self._form_url("provider/get-subscribers")
+        data = {"serviceId": service_id}
+        headers = {
+            "Content-Type": "application/x-www-form-urlencoded",
+            "Authorization": f"Bearer {token}",
+        }
+
+        resp = requests.post(url, data=data, headers=headers, timeout=30)
+        resp.raise_for_status()
+        return resp.json()
+
+
+# convenience single client instance used by the app
+cens_client = CensClient()

app/config.py

@@ -46,6 +46,15 @@ class Config(object):
 
     BULK_SEND_AWS_BUCKET = os.getenv("BULK_SEND_AWS_BUCKET")
 
+    # CENS Experiment
+    CENS_INTEGRATED_SERVICES: List[str] = [id.strip() for id in os.getenv("CENS_INTEGRATED_SERVICES", "").split(",")]
+    CENS_INTEGRATED_TEMPLATES: List[str] = [id.strip() for id in os.getenv("CENS_INTEGRATED_TEMPLATES", "").split(",")]
+    CENS_URL = os.environ.get("CENS_URL")
+    CENS_USER = os.environ.get("CENS_USER")
+    CENS_SECRET = os.environ.get("CENS_SECRET")
+    # testing mode for local development to use canned CENS responses
+    CENS_TESTING_MODE = env.bool("CENS_TESTING_MODE", False)
+
     CHECK_PROXY_HEADER = False
     CONTACT_EMAIL = os.environ.get("CONTACT_EMAIL", "assistance+notification@cds-snc.ca")
     CSV_MAX_ROWS = env.int("CSV_MAX_ROWS", 50_000)

app/main/views/templates.py

@@ -1,3 +1,5 @@
+import csv
+import io
 import json
 from datetime import datetime, timedelta
 from string import ascii_uppercase
@@ -24,6 +26,7 @@
 from notifications_utils.recipients import first_column_headings
 
 from app import (
+    cens_client,
     current_service,
     get_current_locale,
     service_api_client,
@@ -58,6 +61,7 @@
     TemplateLists,
 )
 from app.notify_client.notification_counts_client import notification_counts_client
+from app.s3_client.s3_csv_client import s3upload
 from app.sample_template_utils import create_temporary_sample_template, get_sample_templates, get_sample_templates_by_type
 from app.template_previews import TemplatePreview, get_page_count_for_letter
 from app.utils import (
@@ -1298,6 +1302,12 @@ def get_human_readable_delta(from_time, until_time):
 def add_recipients(service_id, template_id):
     template = current_service.get_template_with_user_permission_or_403(template_id, current_user)
 
+    # if this template and service is in CENS_INTEGRATED_SERVICES and CENS_INTEGRATED_TEMPLATES, add a third option to radios
+    # Check if service and template are CENS integrated
+    is_cens_integrated = service_id in current_app.config.get("CENS_INTEGRATED_SERVICES", []) and str(
+        template_id
+    ) in current_app.config.get("CENS_INTEGRATED_TEMPLATES", [])
+
     if template["template_type"] == "email":
         form = AddEmailRecipientsForm()
         option_hints = {
@@ -1308,6 +1318,11 @@ def add_recipients(service_id, template_id):
             ),
             "one_recipient": Markup(_l("Enter their email address.")),
         }
+        # add third option if CENS integrated
+        if is_cens_integrated:
+            form.what_type.choices.insert(1, ("cens_recipient", _l("Many recipients (from CENS)")))
+            form.what_type.choices[0] = ("many_recipients", _l("Many recipients (CSV)"))
+            option_hints["cens_recipient"] = Markup(_l("Send to a mailing list stored in the CENS system."))
     else:
         form = AddSMSRecipientsForm()
         option_hints = {
@@ -1330,6 +1345,95 @@ def add_recipients(service_id, template_id):
                         template_id=template_id,
                     )
                 )
+            elif form.what_type.data == "cens_recipient":
+                # call cens and see if we can get the mailing lists
+                # CensClient.has_service
+                try:
+                    has_topics = cens_client.has_service(service_id)
+
+                    if has_topics:
+                        # use get_subscribers_list to get the mailing list JSON
+
+                        try:
+                            recipients = cens_client.get_subscribers_list(service_id)
+                        except Exception as e:
+                            recipients = None
+                            current_app.logger.error("Error fetching mailing lists from CENS: %s", str(e))
+
+                        if recipients:
+                            # Convert the CENS response rows into a CSV and upload to s3 uploads
+                            # The expected structure from CENS is: {"name": "<topic>", "service-id": <service-id>, "rows": [[col1, col2], ...]}
+                            try:
+                                rows = recipients.get("rows", [])
+
+                                # create CSV in memory
+                                buf = io.StringIO()
+                                writer = csv.writer(buf)
+
+                                # write each row
+                                for row in rows:
+                                    writer.writerow(row)
+
+                                csv_data = buf.getvalue().encode("utf-8")
+
+                                # s3upload expects a dict with 'data' and region
+                                upload_id = s3upload(service_id, {"data": csv_data}, current_app.config["AWS_REGION"])
+
+                                # save the original CENS response for later UI use
+                                session["cens_recipients"] = recipients
+
+                                # Redirect to the check endpoint the same way a user-uploaded CSV would
+                                return redirect(
+                                    url_for(
+                                        "main.check_messages",
+                                        service_id=service_id,
+                                        upload_id=upload_id,
+                                        template_id=template_id,
+                                        original_file_name=recipients["name"],
+                                    )
+                                )
+                            except Exception as e:
+                                current_app.logger.exception("Failed to convert or upload CENS recipients: %s", e)
+                                flash(
+                                    _l("There was a problem processing the mailing list from CENS. Please try again later."),
+                                    "error",
+                                )
+                                return redirect(
+                                    url_for(
+                                        "main.add_recipients",
+                                        service_id=service_id,
+                                        template_id=template_id,
+                                    )
+                                )
+                        else:
+                            flash(_l("No mailing lists found for this service in CENS."), "error")
+                            return redirect(
+                                url_for(
+                                    "main.add_recipients",
+                                    service_id=service_id,
+                                    template_id=template_id,
+                                )
+                            )
+                    else:
+                        flash(_l("No mailing lists found for this service in CENS."), "error")
+                        return redirect(
+                            url_for(
+                                "main.add_recipients",
+                                service_id=service_id,
+                                template_id=template_id,
+                            )
+                        )
+                # catch ANY error
+                except Exception as e:
+                    current_app.logger.error("Error connecting to CENS: %s", str(e))
+                    flash(_l("There was a problem connecting to CENS. Please try again later."), "error")
+                    return redirect(
+                        url_for(
+                            "main.add_recipients",
+                            service_id=service_id,
+                            template_id=template_id,
+                        )
+                    )
             elif form.validate_on_submit():
                 session["placeholders"] = {}
                 session["recipient"] = form.placeholder_value.data