From d0a884dc79815c5d7941cb4d9c6aeb630e1a642c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20U=C4=9EUR?= <39213991+alithethird@users.noreply.github.com> Date: Thu, 14 May 2026 08:25:19 +0300 Subject: [PATCH 01/46] fix(postfix): align map filenames with configurator output postfix-relay-operator was reading transport_maps, virtual_alias_maps and watching stale Juju-state-style filenames in inotify, while postfix-relay-configurator writes transport, virtual_alias, relay_access, etc. Align all three files to use the correct on-disk names. Also add an isolated integration test (test_configurator_maps.py) that deploys only postfix-relay + configurator and asserts sender_login_maps enforcement: authorized sender gets 250, spoofed sender gets 553. --- postfix-relay-operator/src/postfix.py | 13 +- .../templates/inotify-config-change.sh.tmpl | 14 +- .../templates/postfix_main_cf.tmpl | 2 +- tests/integration/test_configurator_maps.py | 262 ++++++++++++++++++ 4 files changed, 279 insertions(+), 12 deletions(-) create mode 100644 tests/integration/test_configurator_maps.py diff --git a/postfix-relay-operator/src/postfix.py b/postfix-relay-operator/src/postfix.py index 13b2fe5..ca27ca4 100644 --- a/postfix-relay-operator/src/postfix.py +++ b/postfix-relay-operator/src/postfix.py @@ -19,7 +19,7 @@ "/etc/postfix/access", "/etc/postfix/sender_login", "/etc/postfix/tls_policy", - "/etc/postfix/transport_maps", + "/etc/postfix/transport", "/etc/postfix/virtual_alias", ] @@ -228,7 +228,12 @@ def _parse_access_map(path: Path) -> dict[str, AccessMapValue]: def _parse_map(path: Path) -> dict[str, str]: if path.exists(): raw_content = path.read_text("utf-8") - return {line.split(" ")[0]: line.split(" ")[1] for line in raw_content.split("\n")} + result = {} + for line in raw_content.split("\n"): + parts = line.split(" ", 1) + if len(parts) == 2 and parts[0]: + result[parts[0]] = parts[1] + return result return {} @@ -301,7 +306,7 @@ def fetch_transport_maps() -> dict[str, str]: Returns: the transport maps. """ - return _parse_map(POSTFIX_CONF_DIRPATH / "transport_maps") + return _parse_map(POSTFIX_CONF_DIRPATH / "transport") def fetch_virtual_alias_maps() -> dict[str, str]: @@ -310,4 +315,4 @@ def fetch_virtual_alias_maps() -> dict[str, str]: Returns: the virtual alias maps. """ - return _parse_map(POSTFIX_CONF_DIRPATH / "virtual_alias_maps") + return _parse_map(POSTFIX_CONF_DIRPATH / "virtual_alias") diff --git a/postfix-relay-operator/templates/inotify-config-change.sh.tmpl b/postfix-relay-operator/templates/inotify-config-change.sh.tmpl index 0d381b9..5973b28 100644 --- a/postfix-relay-operator/templates/inotify-config-change.sh.tmpl +++ b/postfix-relay-operator/templates/inotify-config-change.sh.tmpl @@ -3,13 +3,13 @@ # The script is copied into the charm and executed as a service # shellcheck disable=SC2034 inotifywait -mr /etc/postfix \ - --include /etc/postfix/relay_access_sources \ - --include /etc/postfix/relay_recipient_maps \ - --include /etc/postfix/restrict_recipients \ - --include /etc/postfix/restrict_senders \ - --include /etc/postfix/sender_login_maps \ - --include /etc/postfix/transport_maps \ - --include /etc/postfix/virtual_alias_maps \ + --include /etc/postfix/relay_access \ + --include /etc/postfix/relay_recipient \ + --include /etc/postfix/restricted_recipients \ + --include /etc/postfix/restricted_senders \ + --include /etc/postfix/sender_login \ + --include /etc/postfix/transport \ + --include /etc/postfix/virtual_alias \ -e close_write -e create -e delete -e move --format '%w%f %e' | while read -r file event; do juju-exec {{unit_name}} JUJU_DISPATCH_PATH='hooks/config-changed' ./dispatch done diff --git a/postfix-relay-operator/templates/postfix_main_cf.tmpl b/postfix-relay-operator/templates/postfix_main_cf.tmpl index e10c91f..f762402 100644 --- a/postfix-relay-operator/templates/postfix_main_cf.tmpl +++ b/postfix-relay-operator/templates/postfix_main_cf.tmpl @@ -107,7 +107,7 @@ relay_domains = {{relay_domains}} {% endif %} {%- if transport_maps %} -transport_maps = hash:/etc/postfix/transport_maps +transport_maps = hash:/etc/postfix/transport {% endif %} {%- if virtual_alias_maps -%} virtual_alias_maps = {{virtual_alias_maps_type}}:/etc/postfix/virtual_alias diff --git a/tests/integration/test_configurator_maps.py b/tests/integration/test_configurator_maps.py new file mode 100644 index 0000000..83456d6 --- /dev/null +++ b/tests/integration/test_configurator_maps.py @@ -0,0 +1,262 @@ +# Copyright 2026 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Integration tests for postfix-relay-configurator sender_login_maps enforcement. + +These tests deploy only postfix-relay + postfix-relay-configurator (no Dovecot +or OpenDKIM required) to verify that the configurator correctly writes +sender_login maps and that postfix enforces them. +""" + +import base64 +import hashlib +import logging +import pathlib +import smtplib +import ssl +import typing +from collections.abc import Generator + +import jubilant +import pytest +import yaml + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# App / domain constants +# --------------------------------------------------------------------------- +POSTFIX_RELAY_APP = "postfix-relay-maps" +CONFIGURATOR_APP = "postfix-relay-configurator-maps" +SELF_SIGNED_APP = "self-signed-certificates" + +TEST_DOMAIN = "mailstack.internal" +SMTP_PORT = 587 + +AUTH_USER = "testuser" +AUTH_PASSWORD = "test-password" +AUTHORIZED_SENDER = f"authorized@{TEST_DOMAIN}" +SPOOFED_SENDER = f"spoofed@{TEST_DOMAIN}" +RECIPIENT = f"recipient@{TEST_DOMAIN}" + +_REPO_ROOT = pathlib.Path(__file__).resolve().parents[2] + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- +def _sha512_dovecot_password(password: str) -> str: + salt = b"mailtest" + digest = hashlib.sha512(password.encode() + salt).digest() + return "{SSHA512}" + base64.b64encode(digest + salt).decode() + + +def _integrate_once(juju: jubilant.Juju, endpoint_a: str, endpoint_b: str) -> None: + """Call ``juju integrate`` tolerating 'already related' errors.""" + try: + juju.integrate(endpoint_a, endpoint_b) + except Exception as exc: # noqa: BLE001 + msg = str(exc) + if "already exists" not in msg and "already related" not in msg: + raise + logger.debug("Relation %s ↔ %s already exists, skipping", endpoint_a, endpoint_b) + + +def _select_charm_file(pytestconfig: pytest.Config, marker: str) -> str: + charm_files: list[str] = pytestconfig.getoption("--charm-file", default=[]) + for path in charm_files: + if marker in pathlib.Path(path).name.lower(): + return path + use_existing = pytestconfig.getoption("--use-existing", default=False) + if use_existing: + return "" + provided = ", ".join(charm_files) if charm_files else "" + raise AssertionError( + f"Missing --charm-file matching '{marker}'. Provided: {provided}." + ) + + +# --------------------------------------------------------------------------- +# Session fixtures +# --------------------------------------------------------------------------- +@pytest.fixture(scope="module", name="maps_juju") +def maps_juju_fixture(request: pytest.FixtureRequest) -> Generator[jubilant.Juju, None, None]: + """Module-scoped Juju client in a temporary model for configurator map tests.""" + + def _show_debug_log(juju: jubilant.Juju) -> None: + if request.session.testsfailed: + log = juju.debug_log(limit=2000) + print(log, end="") + + use_existing = request.config.getoption("--use-existing", default=False) + if use_existing: + juju = jubilant.Juju() + juju.model_config({"automatically-retry-hooks": True}) + yield juju + _show_debug_log(juju) + return + + model = request.config.getoption("--model", default=None) + if model: + juju = jubilant.Juju(model=model) + juju.model_config({"automatically-retry-hooks": True}) + yield juju + _show_debug_log(juju) + return + + keep_models = typing.cast(bool, request.config.getoption("--keep-models", default=False)) + with jubilant.temp_model(keep=keep_models) as juju: + juju.wait_timeout = 15 * 60 + juju.model_config({"automatically-retry-hooks": True}) + yield juju + _show_debug_log(juju) + + +@pytest.fixture(scope="module", name="maps_stack") +def maps_stack_fixture( + maps_juju: jubilant.Juju, + pytestconfig: pytest.Config, +) -> typing.Dict[str, str]: + """Deploy postfix-relay + postfix-relay-configurator configured for sender_login enforcement. + + Returns a dict with ``postfix_relay_ip``. + """ + juju = maps_juju + + # --- self-signed-certificates (TLS for postfix-relay) --- + if not juju.status().apps.get(SELF_SIGNED_APP): + juju.deploy(SELF_SIGNED_APP, channel="latest/stable") + juju.wait( + lambda status: status.apps[SELF_SIGNED_APP].is_active, + error=jubilant.any_error, + timeout=10 * 60, + ) + + # --- postfix-relay --- + if not juju.status().apps.get(POSTFIX_RELAY_APP): + charm_path = _select_charm_file(pytestconfig, "postfix-relay") + if not charm_path.startswith(("./", "/")): + charm_path = f"./{charm_path}" + juju.deploy( + charm_path, + app=POSTFIX_RELAY_APP, + config={ + "relay_domains": f"- {TEST_DOMAIN}", + "enable_smtp_auth": "true", + "smtp_auth_users": yaml.dump( + [f"{AUTH_USER}:{_sha512_dovecot_password(AUTH_PASSWORD)}"] + ), + "enable_reject_unknown_sender_domain": "false", + }, + ) + _integrate_once( + juju, + f"{POSTFIX_RELAY_APP}:certificates", + f"{SELF_SIGNED_APP}:certificates", + ) + + # --- postfix-relay-configurator --- + if not juju.status().apps.get(CONFIGURATOR_APP): + charm_path = _select_charm_file(pytestconfig, "postfix-relay-configurator") + if not charm_path.startswith(("./", "/")): + charm_path = f"./{charm_path}" + juju.deploy( + charm_path, + app=CONFIGURATOR_APP, + config={ + "sender_login_maps": yaml.dump( + {AUTHORIZED_SENDER: AUTH_USER} + ), + }, + ) + _integrate_once( + juju, + f"{POSTFIX_RELAY_APP}:juju-info", + f"{CONFIGURATOR_APP}:juju-info", + ) + + # Wait for both to be active. + def _both_active(status: jubilant.Status) -> bool: + if not status.apps.get(POSTFIX_RELAY_APP): + return False + if not status.apps[POSTFIX_RELAY_APP].is_active: + return False + for unit in status.apps[POSTFIX_RELAY_APP].units.values(): + subs = unit.subordinates or {} + conf_subs = {k: v for k, v in subs.items() if CONFIGURATOR_APP in k} + if not conf_subs: + return False + for sub in conf_subs.values(): + if sub.workload_status.current != "active": + return False + return True + + juju.wait(_both_active, error=jubilant.any_error, timeout=15 * 60) + logger.info("postfix-relay + configurator active for maps tests") + + status = juju.status() + relay_ip = next(iter(status.apps[POSTFIX_RELAY_APP].units.values())).public_address + logger.info("postfix-relay IP: %s", relay_ip) + return {"postfix_relay_ip": relay_ip} + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- +class TestSenderLoginMapEnforcement: + """Verify that sender_login_maps written by the configurator are enforced by postfix.""" + + def test_sender_login_map_enforcement(self, maps_stack: typing.Dict[str, str]) -> None: + """Authenticated user can send from authorized address but not from a spoofed one.""" + relay_ip = maps_stack["postfix_relay_ip"] + + # --- Success case: send from authorized address --- + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + + with smtplib.SMTP(relay_ip, SMTP_PORT, timeout=30) as smtp: + smtp.ehlo() + smtp.starttls(context=ctx) + smtp.ehlo() + smtp.login(AUTH_USER, AUTH_PASSWORD) + smtp.sendmail( + from_addr=AUTHORIZED_SENDER, + to_addrs=[RECIPIENT], + msg=( + f"From: {AUTHORIZED_SENDER}\r\n" + f"To: {RECIPIENT}\r\n" + "Subject: test authorized sender\r\n" + "\r\n" + "This message should be accepted.\r\n" + ), + ) + logger.info("Success case: message from %s accepted", AUTHORIZED_SENDER) + + # --- Failure case: send from spoofed address --- + with smtplib.SMTP(relay_ip, SMTP_PORT, timeout=30) as smtp: + smtp.ehlo() + smtp.starttls(context=ctx) + smtp.ehlo() + smtp.login(AUTH_USER, AUTH_PASSWORD) + with pytest.raises(smtplib.SMTPSenderRefused) as exc_info: + smtp.sendmail( + from_addr=SPOOFED_SENDER, + to_addrs=[RECIPIENT], + msg=( + f"From: {SPOOFED_SENDER}\r\n" + f"To: {RECIPIENT}\r\n" + "Subject: test spoofed sender\r\n" + "\r\n" + "This message should be rejected.\r\n" + ), + ) + logger.info( + "Failure case: message from %s rejected with code %s", + SPOOFED_SENDER, + exc_info.value.smtp_code, + ) + assert exc_info.value.smtp_code == 553, ( + f"Expected 553 Sender address rejected, got {exc_info.value.smtp_code}: " + f"{exc_info.value.smtp_error}" + ) From 0a6ca6ce62b4e2462833fd85a13cc4e379219e6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20U=C4=9EUR?= <39213991+alithethird@users.noreply.github.com> Date: Thu, 14 May 2026 09:03:39 +0300 Subject: [PATCH 02/46] ci: add integration-tests-configurator-maps job Wire test_configurator_maps.py into CI using the same stack-integration tox env as integration-tests-global, mirroring that job's pattern (no working-directory). --- .github/workflows/integration_test.yaml | 73 +++++++++++++++++++++---- 1 file changed, 61 insertions(+), 12 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 2602e86..da98ae7 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -7,31 +7,80 @@ on: jobs: integration-tests-juju-3: - uses: - canonical/operator-workflows/.github/workflows/integration_test.yaml@main + strategy: + matrix: + charm: + - name: dovecot + working-directory: ./dovecot-charm + modules: | + [ + "test_action.py", + "test_config.py", + "test_ha.py", + "test_mail.py", + "test_observability.py", + "test_storage.py", + "test_tls.py" + ] + - name: opendkim-operator + working-directory: ./opendkim-operator + modules: '["test_charm.py"]' + pre-run-script: ./opendkim-operator/tests/integration/setup-integration-tests.sh + - name: postfix-relay-operator + working-directory: ./postfix-relay-operator + modules: '["test_charm.py"]' + pre-run-script: ./postfix-relay-operator/tests/integration/mailcatcher-installation.sh + name: Integration tests for ${{ matrix.charm.name }} + uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main secrets: inherit with: extra-arguments: '-x --log-format="%(asctime)s %(levelname)s %(message)s"' provider: lxd trivy-fs-enabled: false - trivy-image-config: "trivy.yaml" + juju-channel: 3/stable + self-hosted-runner: true + self-hosted-runner-label: "self-hosted-linux-amd64-noble-large" + charmcraft-channel: latest/edge + working-directory: ${{ matrix.charm.working-directory }} + modules: ${{ matrix.charm.modules }} + pre-run-script: ${{ matrix.charm.pre-run-script }} + with-uv: true + integration-tests-global: + uses: + canonical/operator-workflows/.github/workflows/integration_test.yaml@main + secrets: inherit + with: + test-tox-env: stack-integration + provider: lxd + trivy-fs-enabled: false self-hosted-runner: true self-hosted-runner-label: "self-hosted-linux-amd64-noble-large" - juju-channel: '3/stable' + juju-channel: 3/stable + modules: | + [ + "test_e2e.py" + ] with-uv: true - working-directory: dovecot-charm + integration-tests-configurator-maps: + uses: + canonical/operator-workflows/.github/workflows/integration_test.yaml@main + secrets: inherit + with: + test-tox-env: stack-integration + provider: lxd + trivy-fs-enabled: false + self-hosted-runner: true + self-hosted-runner-label: "self-hosted-linux-amd64-noble-large" + juju-channel: 3/stable modules: | [ - "test_action.py", - "test_config.py", - "test_ha.py", - "test_mail.py", - "test_observability.py", - "test_storage.py", - "test_tls.py" + "test_configurator_maps.py" ] + with-uv: true allure-report: if: ${{ !cancelled() && github.event_name == 'schedule' }} needs: - integration-tests-juju-3 + - integration-tests-global + - integration-tests-configurator-maps uses: canonical/operator-workflows/.github/workflows/allure_report.yaml@main From 15a121d276fed2d4bb9d54a1bcce54c6a5ac7733 Mon Sep 17 00:00:00 2001 From: Ali Ugur Date: Thu, 14 May 2026 11:34:14 +0300 Subject: [PATCH 03/46] fix(tests): Remove running e2e tests for this branch. --- .github/workflows/integration_test.yaml | 17 ----------------- .gitignore | 1 + 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index da98ae7..8144f8f 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -46,22 +46,6 @@ jobs: pre-run-script: ${{ matrix.charm.pre-run-script }} with-uv: true integration-tests-global: - uses: - canonical/operator-workflows/.github/workflows/integration_test.yaml@main - secrets: inherit - with: - test-tox-env: stack-integration - provider: lxd - trivy-fs-enabled: false - self-hosted-runner: true - self-hosted-runner-label: "self-hosted-linux-amd64-noble-large" - juju-channel: 3/stable - modules: | - [ - "test_e2e.py" - ] - with-uv: true - integration-tests-configurator-maps: uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main secrets: inherit @@ -82,5 +66,4 @@ jobs: needs: - integration-tests-juju-3 - integration-tests-global - - integration-tests-configurator-maps uses: canonical/operator-workflows/.github/workflows/allure_report.yaml@main diff --git a/.gitignore b/.gitignore index 94d847a..b3b3233 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ __pycache__/ */*.rock terraform/tests/.terraform .opencode/ +plans/ # BEGIN VALE WORKFLOW IGNORE .vale/styles/* From ae019e507a62e32ab11d8ece57f52ff82ed3f07c Mon Sep 17 00:00:00 2001 From: Ali Ugur Date: Thu, 14 May 2026 11:41:33 +0300 Subject: [PATCH 04/46] fix(tests): Remove running e2e tests for this branch. --- postfix-relay-operator/pyproject.toml | 1 + postfix-relay-operator/tests/unit/files/dovecot_config | 2 -- .../tests/unit/files/dovecot_config_auth_disabled | 2 -- postfix-relay-operator/tests/unit/files/dovecot_users | 3 --- .../tests/unit/files/policyd_spf_config_skip_addresses | 3 --- tests/integration/test_configurator_maps.py | 8 ++------ 6 files changed, 3 insertions(+), 16 deletions(-) diff --git a/postfix-relay-operator/pyproject.toml b/postfix-relay-operator/pyproject.toml index a267da4..b50406c 100644 --- a/postfix-relay-operator/pyproject.toml +++ b/postfix-relay-operator/pyproject.toml @@ -46,6 +46,7 @@ module = "tests.*" [tool.pytest.ini_options] minversion = "6.0" log_cli_level = "INFO" +pythonpath = ["src", "lib"] # Linting tools configuration [tool.ruff] diff --git a/postfix-relay-operator/tests/unit/files/dovecot_config b/postfix-relay-operator/tests/unit/files/dovecot_config index 288e24a..56f8f6a 100644 --- a/postfix-relay-operator/tests/unit/files/dovecot_config +++ b/postfix-relay-operator/tests/unit/files/dovecot_config @@ -1,5 +1,3 @@ -# Copyright 2026 Canonical Ltd. -# See LICENSE file for licensing details. auth_mechanisms = plain login auth_verbose = yes diff --git a/postfix-relay-operator/tests/unit/files/dovecot_config_auth_disabled b/postfix-relay-operator/tests/unit/files/dovecot_config_auth_disabled index 98f23df..8006342 100644 --- a/postfix-relay-operator/tests/unit/files/dovecot_config_auth_disabled +++ b/postfix-relay-operator/tests/unit/files/dovecot_config_auth_disabled @@ -1,4 +1,2 @@ -# Copyright 2026 Canonical Ltd. -# See LICENSE file for licensing details. ## DISABLED diff --git a/postfix-relay-operator/tests/unit/files/dovecot_users b/postfix-relay-operator/tests/unit/files/dovecot_users index 12cdce8..9902d74 100644 --- a/postfix-relay-operator/tests/unit/files/dovecot_users +++ b/postfix-relay-operator/tests/unit/files/dovecot_users @@ -1,5 +1,2 @@ -# Copyright 2026 Canonical Ltd. -# See LICENSE file for licensing details. - myuser1:$1$bPb0IPiM$kmrSMZkZvICKKHXu66daQ. myuser2:$6$3r//F36qLB/J8rUfIIndaDtkxeb5iR3gs1uBn9fNyJDD1 diff --git a/postfix-relay-operator/tests/unit/files/policyd_spf_config_skip_addresses b/postfix-relay-operator/tests/unit/files/policyd_spf_config_skip_addresses index b5172ed..284d920 100644 --- a/postfix-relay-operator/tests/unit/files/policyd_spf_config_skip_addresses +++ b/postfix-relay-operator/tests/unit/files/policyd_spf_config_skip_addresses @@ -1,6 +1,3 @@ -# Copyright 2026 Canonical Ltd. -# See LICENSE file for licensing details. - debugLevel = 1 TestOnly = 1 diff --git a/tests/integration/test_configurator_maps.py b/tests/integration/test_configurator_maps.py index 83456d6..17c22b8 100644 --- a/tests/integration/test_configurator_maps.py +++ b/tests/integration/test_configurator_maps.py @@ -71,9 +71,7 @@ def _select_charm_file(pytestconfig: pytest.Config, marker: str) -> str: if use_existing: return "" provided = ", ".join(charm_files) if charm_files else "" - raise AssertionError( - f"Missing --charm-file matching '{marker}'. Provided: {provided}." - ) + raise AssertionError(f"Missing --charm-file matching '{marker}'. Provided: {provided}.") # --------------------------------------------------------------------------- @@ -164,9 +162,7 @@ def maps_stack_fixture( charm_path, app=CONFIGURATOR_APP, config={ - "sender_login_maps": yaml.dump( - {AUTHORIZED_SENDER: AUTH_USER} - ), + "sender_login_maps": yaml.dump({AUTHORIZED_SENDER: AUTH_USER}), }, ) _integrate_once( From 7af8191dbedb2fe244750823e1625eadc182821e Mon Sep 17 00:00:00 2001 From: Ali Ugur Date: Thu, 14 May 2026 12:34:28 +0300 Subject: [PATCH 05/46] fix(tests): Restructure integration test. --- tests/integration/conftest.py | 215 ++++++++++++++++++++ tests/integration/test_configurator_maps.py | 168 +-------------- 2 files changed, 216 insertions(+), 167 deletions(-) create mode 100644 tests/integration/conftest.py diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py new file mode 100644 index 0000000..2e756bb --- /dev/null +++ b/tests/integration/conftest.py @@ -0,0 +1,215 @@ +# Copyright 2026 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Shared fixtures and configuration for integration tests.""" + +import base64 +import hashlib +import logging +import pathlib +import typing +from collections.abc import Generator + +import jubilant +import pytest +import yaml + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# App / domain constants +# --------------------------------------------------------------------------- +POSTFIX_RELAY_APP = "postfix-relay-maps" +CONFIGURATOR_APP = "postfix-relay-configurator-maps" +SELF_SIGNED_APP = "self-signed-certificates" + +TEST_DOMAIN = "mailstack.internal" +SMTP_PORT = 587 + + +# --------------------------------------------------------------------------- +# Pytest configuration +# --------------------------------------------------------------------------- +def pytest_addoption(parser: pytest.Parser) -> None: + """Add integration test command-line options.""" + parser.addoption( + "--charm-file", + action="append", + default=[], + help="Path to charm file (can be used multiple times)", + ) + parser.addoption( + "--use-existing", + action="store_true", + default=False, + help="Use existing model instead of creating a temporary one", + ) + parser.addoption( + "--model", + default=None, + help="Specific model name to use", + ) + parser.addoption( + "--keep-models", + action="store_true", + default=False, + help="Keep temporary models after tests complete", + ) + + +# --------------------------------------------------------------------------- +# Helper functions +# --------------------------------------------------------------------------- +def _sha512_dovecot_password(password: str) -> str: + """Generate a SSHA512 password hash compatible with dovecot.""" + salt = b"mailtest" + digest = hashlib.sha512(password.encode() + salt).digest() + return "{SSHA512}" + base64.b64encode(digest + salt).decode() + + +def _integrate_once(juju: jubilant.Juju, endpoint_a: str, endpoint_b: str) -> None: + """Call ``juju integrate`` tolerating 'already related' errors.""" + try: + juju.integrate(endpoint_a, endpoint_b) + except Exception as exc: # noqa: BLE001 + msg = str(exc) + if "already exists" not in msg and "already related" not in msg: + raise + logger.debug("Relation %s ↔ %s already exists, skipping", endpoint_a, endpoint_b) + + +def _select_charm_file(pytestconfig: pytest.Config, marker: str) -> str: + """Select charm file matching marker from --charm-file options.""" + charm_files: list[str] = pytestconfig.getoption("--charm-file", default=[]) + for path in charm_files: + if marker in pathlib.Path(path).name.lower(): + return path + use_existing = pytestconfig.getoption("--use-existing", default=False) + if use_existing: + return "" + provided = ", ".join(charm_files) if charm_files else "" + raise AssertionError(f"Missing --charm-file matching '{marker}'. Provided: {provided}.") + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- +@pytest.fixture(scope="module", name="maps_juju") +def maps_juju_fixture(request: pytest.FixtureRequest) -> Generator[jubilant.Juju, None, None]: + """Module-scoped Juju client in a temporary model for configurator map tests.""" + + def _show_debug_log(juju: jubilant.Juju) -> None: + if request.session.testsfailed: + log = juju.debug_log(limit=2000) + print(log, end="") + + use_existing = request.config.getoption("--use-existing", default=False) + if use_existing: + juju = jubilant.Juju() + juju.model_config({"automatically-retry-hooks": True}) + yield juju + _show_debug_log(juju) + return + + model = request.config.getoption("--model", default=None) + if model: + juju = jubilant.Juju(model=model) + juju.model_config({"automatically-retry-hooks": True}) + yield juju + _show_debug_log(juju) + return + + keep_models = typing.cast(bool, request.config.getoption("--keep-models", default=False)) + with jubilant.temp_model(keep=keep_models) as juju: + juju.wait_timeout = 15 * 60 + juju.model_config({"automatically-retry-hooks": True}) + yield juju + _show_debug_log(juju) + + +@pytest.fixture(scope="module", name="maps_stack") +def maps_stack_fixture( + maps_juju: jubilant.Juju, + pytestconfig: pytest.Config, +) -> typing.Dict[str, str]: + """Deploy postfix-relay + postfix-relay-configurator configured for sender_login enforcement. + + Returns a dict with ``postfix_relay_ip``. + """ + juju = maps_juju + + # --- self-signed-certificates (TLS for postfix-relay) --- + if not juju.status().apps.get(SELF_SIGNED_APP): + juju.deploy(SELF_SIGNED_APP, channel="latest/stable") + juju.wait( + lambda status: status.apps[SELF_SIGNED_APP].is_active, + error=jubilant.any_error, + timeout=10 * 60, + ) + + # --- postfix-relay --- + auth_password = "test-password" + if not juju.status().apps.get(POSTFIX_RELAY_APP): + charm_path = _select_charm_file(pytestconfig, "postfix-relay") + if not charm_path.startswith(("./", "/")): + charm_path = f"./{charm_path}" + juju.deploy( + charm_path, + app=POSTFIX_RELAY_APP, + config={ + "relay_domains": f"- {TEST_DOMAIN}", + "enable_smtp_auth": "true", + "smtp_auth_users": yaml.dump( + [f"testuser:{_sha512_dovecot_password(auth_password)}"] + ), + "enable_reject_unknown_sender_domain": "false", + }, + ) + _integrate_once( + juju, + f"{POSTFIX_RELAY_APP}:certificates", + f"{SELF_SIGNED_APP}:certificates", + ) + + # --- postfix-relay-configurator --- + authorized_sender = f"authorized@{TEST_DOMAIN}" + if not juju.status().apps.get(CONFIGURATOR_APP): + charm_path = _select_charm_file(pytestconfig, "postfix-relay-configurator") + if not charm_path.startswith(("./", "/")): + charm_path = f"./{charm_path}" + juju.deploy( + charm_path, + app=CONFIGURATOR_APP, + config={ + "sender_login_maps": yaml.dump({authorized_sender: "testuser"}), + }, + ) + _integrate_once( + juju, + f"{POSTFIX_RELAY_APP}:juju-info", + f"{CONFIGURATOR_APP}:juju-info", + ) + + # Wait for both to be active. + def _both_active(status: jubilant.Status) -> bool: + if not status.apps.get(POSTFIX_RELAY_APP): + return False + if not status.apps[POSTFIX_RELAY_APP].is_active: + return False + for unit in status.apps[POSTFIX_RELAY_APP].units.values(): + subs = unit.subordinates or {} + conf_subs = {k: v for k, v in subs.items() if CONFIGURATOR_APP in k} + if not conf_subs: + return False + for sub in conf_subs.values(): + if sub.workload_status.current != "active": + return False + return True + + juju.wait(_both_active, error=jubilant.any_error, timeout=15 * 60) + logger.info("postfix-relay + configurator active for maps tests") + + status = juju.status() + relay_ip = next(iter(status.apps[POSTFIX_RELAY_APP].units.values())).public_address + logger.info("postfix-relay IP: %s", relay_ip) + return {"postfix_relay_ip": relay_ip} diff --git a/tests/integration/test_configurator_maps.py b/tests/integration/test_configurator_maps.py index 17c22b8..6f4c78f 100644 --- a/tests/integration/test_configurator_maps.py +++ b/tests/integration/test_configurator_maps.py @@ -8,28 +8,18 @@ sender_login maps and that postfix enforces them. """ -import base64 -import hashlib import logging -import pathlib import smtplib import ssl import typing -from collections.abc import Generator -import jubilant import pytest -import yaml logger = logging.getLogger(__name__) # --------------------------------------------------------------------------- -# App / domain constants +# Test-specific constants # --------------------------------------------------------------------------- -POSTFIX_RELAY_APP = "postfix-relay-maps" -CONFIGURATOR_APP = "postfix-relay-configurator-maps" -SELF_SIGNED_APP = "self-signed-certificates" - TEST_DOMAIN = "mailstack.internal" SMTP_PORT = 587 @@ -39,162 +29,6 @@ SPOOFED_SENDER = f"spoofed@{TEST_DOMAIN}" RECIPIENT = f"recipient@{TEST_DOMAIN}" -_REPO_ROOT = pathlib.Path(__file__).resolve().parents[2] - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- -def _sha512_dovecot_password(password: str) -> str: - salt = b"mailtest" - digest = hashlib.sha512(password.encode() + salt).digest() - return "{SSHA512}" + base64.b64encode(digest + salt).decode() - - -def _integrate_once(juju: jubilant.Juju, endpoint_a: str, endpoint_b: str) -> None: - """Call ``juju integrate`` tolerating 'already related' errors.""" - try: - juju.integrate(endpoint_a, endpoint_b) - except Exception as exc: # noqa: BLE001 - msg = str(exc) - if "already exists" not in msg and "already related" not in msg: - raise - logger.debug("Relation %s ↔ %s already exists, skipping", endpoint_a, endpoint_b) - - -def _select_charm_file(pytestconfig: pytest.Config, marker: str) -> str: - charm_files: list[str] = pytestconfig.getoption("--charm-file", default=[]) - for path in charm_files: - if marker in pathlib.Path(path).name.lower(): - return path - use_existing = pytestconfig.getoption("--use-existing", default=False) - if use_existing: - return "" - provided = ", ".join(charm_files) if charm_files else "" - raise AssertionError(f"Missing --charm-file matching '{marker}'. Provided: {provided}.") - - -# --------------------------------------------------------------------------- -# Session fixtures -# --------------------------------------------------------------------------- -@pytest.fixture(scope="module", name="maps_juju") -def maps_juju_fixture(request: pytest.FixtureRequest) -> Generator[jubilant.Juju, None, None]: - """Module-scoped Juju client in a temporary model for configurator map tests.""" - - def _show_debug_log(juju: jubilant.Juju) -> None: - if request.session.testsfailed: - log = juju.debug_log(limit=2000) - print(log, end="") - - use_existing = request.config.getoption("--use-existing", default=False) - if use_existing: - juju = jubilant.Juju() - juju.model_config({"automatically-retry-hooks": True}) - yield juju - _show_debug_log(juju) - return - - model = request.config.getoption("--model", default=None) - if model: - juju = jubilant.Juju(model=model) - juju.model_config({"automatically-retry-hooks": True}) - yield juju - _show_debug_log(juju) - return - - keep_models = typing.cast(bool, request.config.getoption("--keep-models", default=False)) - with jubilant.temp_model(keep=keep_models) as juju: - juju.wait_timeout = 15 * 60 - juju.model_config({"automatically-retry-hooks": True}) - yield juju - _show_debug_log(juju) - - -@pytest.fixture(scope="module", name="maps_stack") -def maps_stack_fixture( - maps_juju: jubilant.Juju, - pytestconfig: pytest.Config, -) -> typing.Dict[str, str]: - """Deploy postfix-relay + postfix-relay-configurator configured for sender_login enforcement. - - Returns a dict with ``postfix_relay_ip``. - """ - juju = maps_juju - - # --- self-signed-certificates (TLS for postfix-relay) --- - if not juju.status().apps.get(SELF_SIGNED_APP): - juju.deploy(SELF_SIGNED_APP, channel="latest/stable") - juju.wait( - lambda status: status.apps[SELF_SIGNED_APP].is_active, - error=jubilant.any_error, - timeout=10 * 60, - ) - - # --- postfix-relay --- - if not juju.status().apps.get(POSTFIX_RELAY_APP): - charm_path = _select_charm_file(pytestconfig, "postfix-relay") - if not charm_path.startswith(("./", "/")): - charm_path = f"./{charm_path}" - juju.deploy( - charm_path, - app=POSTFIX_RELAY_APP, - config={ - "relay_domains": f"- {TEST_DOMAIN}", - "enable_smtp_auth": "true", - "smtp_auth_users": yaml.dump( - [f"{AUTH_USER}:{_sha512_dovecot_password(AUTH_PASSWORD)}"] - ), - "enable_reject_unknown_sender_domain": "false", - }, - ) - _integrate_once( - juju, - f"{POSTFIX_RELAY_APP}:certificates", - f"{SELF_SIGNED_APP}:certificates", - ) - - # --- postfix-relay-configurator --- - if not juju.status().apps.get(CONFIGURATOR_APP): - charm_path = _select_charm_file(pytestconfig, "postfix-relay-configurator") - if not charm_path.startswith(("./", "/")): - charm_path = f"./{charm_path}" - juju.deploy( - charm_path, - app=CONFIGURATOR_APP, - config={ - "sender_login_maps": yaml.dump({AUTHORIZED_SENDER: AUTH_USER}), - }, - ) - _integrate_once( - juju, - f"{POSTFIX_RELAY_APP}:juju-info", - f"{CONFIGURATOR_APP}:juju-info", - ) - - # Wait for both to be active. - def _both_active(status: jubilant.Status) -> bool: - if not status.apps.get(POSTFIX_RELAY_APP): - return False - if not status.apps[POSTFIX_RELAY_APP].is_active: - return False - for unit in status.apps[POSTFIX_RELAY_APP].units.values(): - subs = unit.subordinates or {} - conf_subs = {k: v for k, v in subs.items() if CONFIGURATOR_APP in k} - if not conf_subs: - return False - for sub in conf_subs.values(): - if sub.workload_status.current != "active": - return False - return True - - juju.wait(_both_active, error=jubilant.any_error, timeout=15 * 60) - logger.info("postfix-relay + configurator active for maps tests") - - status = juju.status() - relay_ip = next(iter(status.apps[POSTFIX_RELAY_APP].units.values())).public_address - logger.info("postfix-relay IP: %s", relay_ip) - return {"postfix_relay_ip": relay_ip} - # --------------------------------------------------------------------------- # Tests From 5168fa793a10a91cf5c3703719a10ae11e51b2ec Mon Sep 17 00:00:00 2001 From: Ali Ugur Date: Thu, 14 May 2026 12:41:39 +0300 Subject: [PATCH 06/46] ci: add stack-integration environment for cross-charm integration tests --- tox.toml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tox.toml b/tox.toml index 2fc231a..a09de19 100644 --- a/tox.toml +++ b/tox.toml @@ -3,7 +3,7 @@ skipsdist = true skip_missing_interpreters = true -envlist = [ "lint", "unit", "static", "coverage-report" ] +envlist = [ "lint", "unit", "static", "coverage-report", "stack-integration" ] requires = [ "tox>=4.21" ] no_package = true @@ -145,3 +145,19 @@ dependency_groups = [ "lint" ] src_path = "{toxinidir}/src/" tst_path = "{toxinidir}/tests/" all_path = [ "{toxinidir}/src/", "{toxinidir}/tests/" ] + +[env.stack-integration] +description = "Run cross-charm integration tests (postfix-relay-configurator)" +commands = [ + [ + "pytest", + "-v", + "--tb", + "native", + "tests/integration/test_configurator_maps.py", + "--log-cli-level=INFO", + "-s", + { replace = "posargs", extend = "true" }, + ], +] +dependency_groups = [ "integration" ] From 012a6e1f6f8e2886a314012aaaf0f4d6a8422d0b Mon Sep 17 00:00:00 2001 From: Ali Ugur Date: Thu, 14 May 2026 13:47:24 +0300 Subject: [PATCH 07/46] fix: licence and integration test --- .licenserc.yaml | 1 + tests/integration/conftest.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.licenserc.yaml b/.licenserc.yaml index f6dd8a9..6f6b8f3 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -25,6 +25,7 @@ header: - 'postfix-relay-operator/templates/*' - 'postfix-relay-operator/files/**' - 'postfix-relay-operator/lib/**' + - 'postfix-relay-operator/tests/unit/files/**' - 'pyproject.toml' - 'zap_rules.tsv' - '**/*.md.j2' diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 2e756bb..3f47c16 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -150,7 +150,7 @@ def maps_stack_fixture( # --- postfix-relay --- auth_password = "test-password" if not juju.status().apps.get(POSTFIX_RELAY_APP): - charm_path = _select_charm_file(pytestconfig, "postfix-relay") + charm_path = _select_charm_file(pytestconfig, "postfix-relay_") if not charm_path.startswith(("./", "/")): charm_path = f"./{charm_path}" juju.deploy( @@ -174,7 +174,7 @@ def maps_stack_fixture( # --- postfix-relay-configurator --- authorized_sender = f"authorized@{TEST_DOMAIN}" if not juju.status().apps.get(CONFIGURATOR_APP): - charm_path = _select_charm_file(pytestconfig, "postfix-relay-configurator") + charm_path = _select_charm_file(pytestconfig, "postfix-relay-configurator_") if not charm_path.startswith(("./", "/")): charm_path = f"./{charm_path}" juju.deploy( From 71ae1638f39471a33f92adfad325f77b9f24bc2a Mon Sep 17 00:00:00 2001 From: Ali Ugur Date: Thu, 14 May 2026 15:07:54 +0300 Subject: [PATCH 08/46] fix: infinite loop --- postfix-relay-operator/src/charm.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/postfix-relay-operator/src/charm.py b/postfix-relay-operator/src/charm.py index b051f39..6cea1a7 100755 --- a/postfix-relay-operator/src/charm.py +++ b/postfix-relay-operator/src/charm.py @@ -250,8 +250,17 @@ def _configure_relay(self, charm_state: State) -> None: @staticmethod def _apply_postfix_maps(postfix_maps: list[postfix.PostfixMap]) -> None: logger.info("Applying postfix maps") + changed = False for postfix_map in postfix_maps: + path = Path(postfix_map.path) + existing = path.read_text("utf-8") if path.exists() else None + if existing == postfix_map.content: + continue utils.write_file(postfix_map.content, postfix_map.path) + changed = True + if not changed: + logger.debug("Postfix map files unchanged, skipping postmap") + return for map_file in postfix.POSTFIX_MAP_FILES: if Path(map_file).exists(): subprocess.check_call(["postmap", f"hash:{map_file}"]) # nosec From 652f8c1bdda95cfed745fcf0902c8d80159a472b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 14 May 2026 12:32:30 +0000 Subject: [PATCH 09/46] Fix postmap recompilation for externally updated map files Agent-Logs-Url: https://github.com/canonical/mailserver-operators/sessions/939e98de-faff-46a9-af5b-c3b7c87657de Co-authored-by: alithethird <39213991+alithethird@users.noreply.github.com> --- postfix-relay-operator/src/charm.py | 20 +++++++--- .../tests/unit/test_charm.py | 38 +++++++++++++++++++ 2 files changed, 52 insertions(+), 6 deletions(-) diff --git a/postfix-relay-operator/src/charm.py b/postfix-relay-operator/src/charm.py index 6cea1a7..668522e 100755 --- a/postfix-relay-operator/src/charm.py +++ b/postfix-relay-operator/src/charm.py @@ -250,20 +250,28 @@ def _configure_relay(self, charm_state: State) -> None: @staticmethod def _apply_postfix_maps(postfix_maps: list[postfix.PostfixMap]) -> None: logger.info("Applying postfix maps") - changed = False for postfix_map in postfix_maps: path = Path(postfix_map.path) existing = path.read_text("utf-8") if path.exists() else None if existing == postfix_map.content: continue utils.write_file(postfix_map.content, postfix_map.path) - changed = True - if not changed: + + maps_to_compile: list[str] = [] + for map_file in postfix.POSTFIX_MAP_FILES: + source = Path(map_file) + if not source.exists(): + continue + database = source.with_suffix(f"{source.suffix}.db") + if not database.exists() or source.stat().st_mtime_ns > database.stat().st_mtime_ns: + maps_to_compile.append(map_file) + + if not maps_to_compile: logger.debug("Postfix map files unchanged, skipping postmap") return - for map_file in postfix.POSTFIX_MAP_FILES: - if Path(map_file).exists(): - subprocess.check_call(["postmap", f"hash:{map_file}"]) # nosec + + for map_file in maps_to_compile: + subprocess.check_call(["postmap", f"hash:{map_file}"]) # nosec @staticmethod def _calculate_offset(seed: str, length: int = 2) -> int: diff --git a/postfix-relay-operator/tests/unit/test_charm.py b/postfix-relay-operator/tests/unit/test_charm.py index 8814f6f..6fe49f3 100644 --- a/postfix-relay-operator/tests/unit/test_charm.py +++ b/postfix-relay-operator/tests/unit/test_charm.py @@ -407,3 +407,41 @@ def test_configure_policyd_spf( assert investigated_call not in write_file_mock.mock_calls assert out.unit_status == ops.testing.ActiveStatus() + + +def test_apply_postfix_maps_postmaps_when_external_map_is_newer( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """ + arrange: Internal maps are unchanged but an external map source is newer than its .db. + act: Apply postfix maps. + assert: postmap is run for the updated external map. + """ + managed_map = tmp_path / "access" + managed_map.write_text("example.com OK\n", encoding="utf-8") + managed_map_db = tmp_path / "access.db" + managed_map_db.write_text("current", encoding="utf-8") + external_map = tmp_path / "sender_login" + external_map_db = tmp_path / "sender_login.db" + external_map_db.write_text("stale", encoding="utf-8") + external_map.write_text("authorized@mailstack.internal testuser\n", encoding="utf-8") + + check_call_mock = Mock() + write_file_mock = Mock() + monkeypatch.setattr(charm.subprocess, "check_call", check_call_mock) + monkeypatch.setattr(charm.utils, "write_file", write_file_mock) + monkeypatch.setattr(charm.postfix, "POSTFIX_MAP_FILES", [str(managed_map), str(external_map)]) + + charm.PostfixRelayCharm._apply_postfix_maps( + [ + charm.postfix.PostfixMap( + type=state.PostfixLookupTableType.HASH, + path=managed_map, + content="example.com OK\n", + ) + ] + ) + + write_file_mock.assert_not_called() + check_call_mock.assert_called_once_with(["postmap", f"hash:{external_map}"]) From ff760fae6d378d912f09dd563a65faff0a585302 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 14 May 2026 12:33:56 +0000 Subject: [PATCH 10/46] Refine postfix map db path handling Agent-Logs-Url: https://github.com/canonical/mailserver-operators/sessions/939e98de-faff-46a9-af5b-c3b7c87657de Co-authored-by: alithethird <39213991+alithethird@users.noreply.github.com> --- postfix-relay-operator/src/charm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postfix-relay-operator/src/charm.py b/postfix-relay-operator/src/charm.py index 668522e..8231ae2 100755 --- a/postfix-relay-operator/src/charm.py +++ b/postfix-relay-operator/src/charm.py @@ -262,7 +262,7 @@ def _apply_postfix_maps(postfix_maps: list[postfix.PostfixMap]) -> None: source = Path(map_file) if not source.exists(): continue - database = source.with_suffix(f"{source.suffix}.db") + database = source.parent / f"{source.name}.db" if not database.exists() or source.stat().st_mtime_ns > database.stat().st_mtime_ns: maps_to_compile.append(map_file) From bace91773755ae780b348419b2a1a89e5794dc95 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 14 May 2026 13:10:30 +0000 Subject: [PATCH 11/46] fix(postfix): recompile maps when source and db mtimes match Agent-Logs-Url: https://github.com/canonical/mailserver-operators/sessions/34ad0510-ff34-48ca-97c5-af2bfff494a0 Co-authored-by: alithethird <39213991+alithethird@users.noreply.github.com> --- postfix-relay-operator/src/charm.py | 2 +- postfix-relay-operator/tests/unit/test_charm.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/postfix-relay-operator/src/charm.py b/postfix-relay-operator/src/charm.py index 8231ae2..063ecc5 100755 --- a/postfix-relay-operator/src/charm.py +++ b/postfix-relay-operator/src/charm.py @@ -263,7 +263,7 @@ def _apply_postfix_maps(postfix_maps: list[postfix.PostfixMap]) -> None: if not source.exists(): continue database = source.parent / f"{source.name}.db" - if not database.exists() or source.stat().st_mtime_ns > database.stat().st_mtime_ns: + if not database.exists() or source.stat().st_mtime_ns >= database.stat().st_mtime_ns: maps_to_compile.append(map_file) if not maps_to_compile: diff --git a/postfix-relay-operator/tests/unit/test_charm.py b/postfix-relay-operator/tests/unit/test_charm.py index 6fe49f3..7fa92bb 100644 --- a/postfix-relay-operator/tests/unit/test_charm.py +++ b/postfix-relay-operator/tests/unit/test_charm.py @@ -3,6 +3,7 @@ """Unit tests for the Postfix relay charm.""" +import os from pathlib import Path from unittest.mock import ANY, Mock, call, patch @@ -409,12 +410,12 @@ def test_configure_policyd_spf( assert out.unit_status == ops.testing.ActiveStatus() -def test_apply_postfix_maps_postmaps_when_external_map_is_newer( +def test_apply_postfix_maps_postmaps_when_external_map_is_not_older( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: """ - arrange: Internal maps are unchanged but an external map source is newer than its .db. + arrange: Internal maps are unchanged and an external map source is not older than its .db. act: Apply postfix maps. assert: postmap is run for the updated external map. """ @@ -426,6 +427,9 @@ def test_apply_postfix_maps_postmaps_when_external_map_is_newer( external_map_db = tmp_path / "sender_login.db" external_map_db.write_text("stale", encoding="utf-8") external_map.write_text("authorized@mailstack.internal testuser\n", encoding="utf-8") + same_timestamp = external_map.stat().st_mtime_ns + external_map_db.touch() + os.utime(external_map_db, ns=(same_timestamp, same_timestamp)) check_call_mock = Mock() write_file_mock = Mock() From aeb6e031e2410b48d61b441e628d2fd252e29427 Mon Sep 17 00:00:00 2001 From: Ali Ugur Date: Fri, 15 May 2026 10:35:12 +0300 Subject: [PATCH 12/46] fix(postfix): update sender login mismatch handling and improve tests --- postfix-relay-operator/src/postfix.py | 9 ++-- .../tests/unit/test_postfix.py | 53 ++++++++++++++++--- tests/integration/test_configurator_maps.py | 16 ++++-- 3 files changed, 62 insertions(+), 16 deletions(-) diff --git a/postfix-relay-operator/src/postfix.py b/postfix-relay-operator/src/postfix.py index ca27ca4..5943202 100644 --- a/postfix-relay-operator/src/postfix.py +++ b/postfix-relay-operator/src/postfix.py @@ -30,10 +30,6 @@ def _smtpd_relay_restrictions(charm_state: State) -> list[str]: smtpd_relay_restrictions.append("check_client_access cidr:/etc/postfix/relay_access") if charm_state.enable_smtp_auth: - if charm_state.sender_login_maps: - smtpd_relay_restrictions.append("reject_known_sender_login_mismatch") - if charm_state.restrict_senders: - smtpd_relay_restrictions.append("reject_sender_login_mismatch") smtpd_relay_restrictions.append("permit_sasl_authenticated") smtpd_relay_restrictions.append("defer_unauth_destination") @@ -53,6 +49,10 @@ def smtpd_sender_restrictions(charm_state: State) -> list[str]: if charm_state.enable_reject_unknown_sender_domain: restrictions.append("reject_unknown_sender_domain") restrictions.append("check_sender_access hash:/etc/postfix/access") + if charm_state.enable_smtp_auth and ( + charm_state.sender_login_maps or charm_state.restrict_senders + ): + restrictions.append("reject_sender_login_mismatch") if charm_state.restrict_sender_access: restrictions.append("reject") @@ -109,6 +109,7 @@ def construct_postfix_config_params( # pylint: disable=too-many-arguments "hostname": hostname, "connection_limit": charm_state.connection_limit, "enable_rate_limits": charm_state.enable_rate_limits, + "enable_sender_login_map": bool(charm_state.sender_login_maps), "enable_smtp_auth": charm_state.enable_smtp_auth, "enable_spf": charm_state.enable_spf, "header_checks": bool(charm_state.header_checks), diff --git a/postfix-relay-operator/tests/unit/test_postfix.py b/postfix-relay-operator/tests/unit/test_postfix.py index 4055324..1dcad3e 100644 --- a/postfix-relay-operator/tests/unit/test_postfix.py +++ b/postfix-relay-operator/tests/unit/test_postfix.py @@ -61,7 +61,6 @@ {}, [ "permit_mynetworks", - "reject_known_sender_login_mismatch", "permit_sasl_authenticated", "defer_unauth_destination", ], @@ -74,7 +73,6 @@ {"sender": state.AccessMapValue.OK}, [ "permit_mynetworks", - "reject_sender_login_mismatch", "permit_sasl_authenticated", "defer_unauth_destination", ], @@ -87,8 +85,6 @@ {"sender": state.AccessMapValue.OK}, [ "permit_mynetworks", - "reject_known_sender_login_mismatch", - "reject_sender_login_mismatch", "permit_sasl_authenticated", "defer_unauth_destination", ], @@ -139,17 +135,21 @@ def test_smtpd_relay_restrictions( @pytest.mark.parametrize( - ("enable_reject_unknown_sender", "restrict_sender_access", "expected"), + ("enable_reject_unknown_sender", "restrict_sender_access", "sender_login_maps", "restrict_senders", "expected"), [ pytest.param( False, None, + {}, + {}, ["check_sender_access hash:/etc/postfix/access"], id="neither_enabled", ), pytest.param( True, None, + {}, + {}, [ "reject_unknown_sender_domain", "check_sender_access hash:/etc/postfix/access", @@ -159,6 +159,8 @@ def test_smtpd_relay_restrictions( pytest.param( False, "- example.com", + {}, + {}, [ "check_sender_access hash:/etc/postfix/access", "reject", @@ -168,6 +170,8 @@ def test_smtpd_relay_restrictions( pytest.param( True, "- example.com", + {}, + {}, [ "reject_unknown_sender_domain", "check_sender_access hash:/etc/postfix/access", @@ -175,11 +179,46 @@ def test_smtpd_relay_restrictions( ], id="both_enabled", ), + pytest.param( + False, + None, + {"group@example.com": "group"}, + {}, + [ + "check_sender_access hash:/etc/postfix/access", + "reject_sender_login_mismatch", + ], + id="sender_login_maps_enabled", + ), + pytest.param( + False, + None, + {}, + {"sender": state.AccessMapValue.OK}, + [ + "check_sender_access hash:/etc/postfix/access", + "reject_sender_login_mismatch", + ], + id="restrict_senders_enabled", + ), + pytest.param( + False, + None, + {"group@example.com": "group"}, + {"sender": state.AccessMapValue.OK}, + [ + "check_sender_access hash:/etc/postfix/access", + "reject_sender_login_mismatch", + ], + id="sender_login_maps_and_restrict_senders", + ), ], ) def test_smtpd_sender_restrictions( enable_reject_unknown_sender: bool, restrict_sender_access: str, + sender_login_maps: dict[str, str], + restrict_senders: dict[str, state.AccessMapValue], expected: list[str], ) -> None: """ @@ -202,9 +241,9 @@ def test_smtpd_sender_restrictions( config=charm_config, relay_access_sources={}, restrict_recipients={}, - restrict_senders={}, + restrict_senders=restrict_senders, relay_recipient_maps={}, - sender_login_maps={}, + sender_login_maps=sender_login_maps, transport_maps={}, virtual_alias_maps={}, ) diff --git a/tests/integration/test_configurator_maps.py b/tests/integration/test_configurator_maps.py index 6f4c78f..5468bc0 100644 --- a/tests/integration/test_configurator_maps.py +++ b/tests/integration/test_configurator_maps.py @@ -69,7 +69,7 @@ def test_sender_login_map_enforcement(self, maps_stack: typing.Dict[str, str]) - smtp.starttls(context=ctx) smtp.ehlo() smtp.login(AUTH_USER, AUTH_PASSWORD) - with pytest.raises(smtplib.SMTPSenderRefused) as exc_info: + with pytest.raises(smtplib.SMTPRecipientsRefused) as exc_info: smtp.sendmail( from_addr=SPOOFED_SENDER, to_addrs=[RECIPIENT], @@ -81,12 +81,18 @@ def test_sender_login_map_enforcement(self, maps_stack: typing.Dict[str, str]) - "This message should be rejected.\r\n" ), ) + # Postfix defers sender restriction checks to RCPT TO (smtpd_delay_reject=yes), + # so the 553 "Sender address rejected" comes back as SMTPRecipientsRefused. + recipients_errors = exc_info.value.recipients + assert RECIPIENT in recipients_errors, ( + f"Expected rejection for {RECIPIENT}, got: {recipients_errors}" + ) + smtp_code, smtp_error = recipients_errors[RECIPIENT] logger.info( "Failure case: message from %s rejected with code %s", SPOOFED_SENDER, - exc_info.value.smtp_code, + smtp_code, ) - assert exc_info.value.smtp_code == 553, ( - f"Expected 553 Sender address rejected, got {exc_info.value.smtp_code}: " - f"{exc_info.value.smtp_error}" + assert smtp_code == 553, ( + f"Expected 553 Sender address rejected, got {smtp_code}: {smtp_error}" ) From 755637a621ebfdfc426772b13e9408d38a601793 Mon Sep 17 00:00:00 2001 From: Ali Ugur Date: Fri, 15 May 2026 12:37:57 +0300 Subject: [PATCH 13/46] fix(postfix): update fixture names and constants for clarity --- tests/integration/conftest.py | 17 +++++++---------- tests/integration/test_configurator_maps.py | 4 ++-- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 3f47c16..c003eaa 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -19,8 +19,8 @@ # --------------------------------------------------------------------------- # App / domain constants # --------------------------------------------------------------------------- -POSTFIX_RELAY_APP = "postfix-relay-maps" -CONFIGURATOR_APP = "postfix-relay-configurator-maps" +POSTFIX_RELAY_APP = "postfix-relay" +CONFIGURATOR_APP = "postfix-relay-configurator" SELF_SIGNED_APP = "self-signed-certificates" TEST_DOMAIN = "mailstack.internal" @@ -94,8 +94,8 @@ def _select_charm_file(pytestconfig: pytest.Config, marker: str) -> str: # --------------------------------------------------------------------------- # Fixtures # --------------------------------------------------------------------------- -@pytest.fixture(scope="module", name="maps_juju") -def maps_juju_fixture(request: pytest.FixtureRequest) -> Generator[jubilant.Juju, None, None]: +@pytest.fixture(scope="module", name="juju") +def juju_fixture(request: pytest.FixtureRequest) -> Generator[jubilant.Juju, None, None]: """Module-scoped Juju client in a temporary model for configurator map tests.""" def _show_debug_log(juju: jubilant.Juju) -> None: @@ -127,18 +127,15 @@ def _show_debug_log(juju: jubilant.Juju) -> None: _show_debug_log(juju) -@pytest.fixture(scope="module", name="maps_stack") -def maps_stack_fixture( - maps_juju: jubilant.Juju, +@pytest.fixture(scope="module", name="postfix_stack") +def postfix_stack_fixture( + juju: jubilant.Juju, pytestconfig: pytest.Config, ) -> typing.Dict[str, str]: """Deploy postfix-relay + postfix-relay-configurator configured for sender_login enforcement. Returns a dict with ``postfix_relay_ip``. """ - juju = maps_juju - - # --- self-signed-certificates (TLS for postfix-relay) --- if not juju.status().apps.get(SELF_SIGNED_APP): juju.deploy(SELF_SIGNED_APP, channel="latest/stable") juju.wait( diff --git a/tests/integration/test_configurator_maps.py b/tests/integration/test_configurator_maps.py index 5468bc0..d24194c 100644 --- a/tests/integration/test_configurator_maps.py +++ b/tests/integration/test_configurator_maps.py @@ -36,9 +36,9 @@ class TestSenderLoginMapEnforcement: """Verify that sender_login_maps written by the configurator are enforced by postfix.""" - def test_sender_login_map_enforcement(self, maps_stack: typing.Dict[str, str]) -> None: + def test_sender_login_map_enforcement(self, postfix_stack: typing.Dict[str, str]) -> None: """Authenticated user can send from authorized address but not from a spoofed one.""" - relay_ip = maps_stack["postfix_relay_ip"] + relay_ip = postfix_stack["postfix_relay_ip"] # --- Success case: send from authorized address --- ctx = ssl.create_default_context() From cd36284e857c212c8cfb66e47528b8f1b7bae834 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20U=C4=9EUR?= <39213991+alithethird@users.noreply.github.com> Date: Mon, 18 May 2026 15:16:06 +0300 Subject: [PATCH 14/46] Apply suggestions from code review Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- postfix-relay-operator/src/postfix.py | 6 ++---- postfix-relay-operator/tests/unit/test_postfix.py | 8 +++++++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/postfix-relay-operator/src/postfix.py b/postfix-relay-operator/src/postfix.py index 5943202..0910215 100644 --- a/postfix-relay-operator/src/postfix.py +++ b/postfix-relay-operator/src/postfix.py @@ -48,11 +48,9 @@ def smtpd_sender_restrictions(charm_state: State) -> list[str]: restrictions = [] if charm_state.enable_reject_unknown_sender_domain: restrictions.append("reject_unknown_sender_domain") - restrictions.append("check_sender_access hash:/etc/postfix/access") - if charm_state.enable_smtp_auth and ( - charm_state.sender_login_maps or charm_state.restrict_senders - ): + if charm_state.enable_smtp_auth and charm_state.sender_login_maps: restrictions.append("reject_sender_login_mismatch") + restrictions.append("check_sender_access hash:/etc/postfix/access") if charm_state.restrict_sender_access: restrictions.append("reject") diff --git a/postfix-relay-operator/tests/unit/test_postfix.py b/postfix-relay-operator/tests/unit/test_postfix.py index 1dcad3e..22720aa 100644 --- a/postfix-relay-operator/tests/unit/test_postfix.py +++ b/postfix-relay-operator/tests/unit/test_postfix.py @@ -135,7 +135,13 @@ def test_smtpd_relay_restrictions( @pytest.mark.parametrize( - ("enable_reject_unknown_sender", "restrict_sender_access", "sender_login_maps", "restrict_senders", "expected"), + ( + "enable_reject_unknown_sender", + "restrict_sender_access", + "sender_login_maps", + "restrict_senders", + "expected", + ), [ pytest.param( False, From b32fc0cb2723fca7a43856f7dd7a2f1aa3594e4d Mon Sep 17 00:00:00 2001 From: Ali Ugur Date: Tue, 19 May 2026 07:55:32 +0300 Subject: [PATCH 15/46] =?UTF-8?q?feat(dovecot):=20wire=20internal=20postfi?= =?UTF-8?q?x=20=E2=80=93=20LMTP=20socket,=20virtual=20domains,=20port=2025?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - setup_procmail() now accepts mailname and configures: - virtual_mailbox_domains for the charm's primary domain - virtual_transport = lmtp:unix:private/dovecot-lmtp - smtpd_reject_unlisted_recipient = no - inet_interfaces = all - dovecot.conf: expose LMTP Unix listener at /var/spool/postfix/private/dovecot-lmtp (mode 0600, owned by postfix) so Postfix can deliver via LMTP - charm.py: open TCP port 25 so postfix-relay can forward mail to Dovecot - Unit tests updated to reflect new setup_procmail(mailname) signature and port 25 --- dovecot-charm/src/charm.py | 3 ++- dovecot-charm/src/dovecot_setup.py | 25 ++++++++++++++++------- dovecot-charm/templates/dovecot.conf.tmpl | 8 ++++++++ dovecot-charm/tests/unit/test_charm.py | 4 ++-- dovecot-charm/tests/unit/testing.py | 4 ++-- 5 files changed, 32 insertions(+), 12 deletions(-) diff --git a/dovecot-charm/src/charm.py b/dovecot-charm/src/charm.py index baf5853..6cf8471 100644 --- a/dovecot-charm/src/charm.py +++ b/dovecot-charm/src/charm.py @@ -192,7 +192,7 @@ def _reconcile(self, event): try: self._dovecot_setup.setup_tls(dovecot_config) self._dovecot_setup.setup_dovecot(dovecot_config) - self._dovecot_setup.setup_procmail() + self._dovecot_setup.setup_procmail(dovecot_config.mailname) except ConfigurationError as e: self.unit.status = BlockedStatus(str(e)) return @@ -219,6 +219,7 @@ def _install(self): def _open_ports(self): """Open mail ports (TLS-only: plaintext 143/110 are not exposed).""" + self.unit.open_port("tcp", 25) self.unit.open_port("tcp", 993) self.unit.open_port("tcp", 995) self.unit.open_port("tcp", 4190) diff --git a/dovecot-charm/src/dovecot_setup.py b/dovecot-charm/src/dovecot_setup.py index 4956ec6..ac6af7d 100644 --- a/dovecot-charm/src/dovecot_setup.py +++ b/dovecot-charm/src/dovecot_setup.py @@ -127,9 +127,12 @@ def _validate_dovecot_config(self) -> bool: logger.exception(f"Failed to validate dovecot configuration: {e}") return False - def setup_procmail(self) -> None: + def setup_procmail(self, mailname: str) -> None: """Render procmail config and configure Postfix to use it. + Args: + mailname: The mail domain this unit accepts mail for. + Raises: ConfigurationError: If postfix configuration fails. """ @@ -144,13 +147,21 @@ def setup_procmail(self) -> None: contents = template.render(template_context) host.write_file(PROCMAILRC_TARGET, contents, perms=0o644) + postconf_settings = [ + 'mailbox_command=/usr/bin/procmail -a "$EXTENSION"', + f"virtual_mailbox_domains = {mailname}", + "virtual_transport = lmtp:unix:private/dovecot-lmtp", + "smtpd_reject_unlisted_recipient = no", + "inet_interfaces = all", + ] try: - subprocess.run( - ["/usr/sbin/postconf", "-e", 'mailbox_command=/usr/bin/procmail -a "$EXTENSION"'], - check=True, - capture_output=True, - text=True, - ) + for setting in postconf_settings: + subprocess.run( + ["/usr/sbin/postconf", "-e", setting], + check=True, + capture_output=True, + text=True, + ) systemd.service_reload("postfix", restart_on_failure=True) except subprocess.CalledProcessError as e: logger.exception(f"Failed to configure postfix: {e}") diff --git a/dovecot-charm/templates/dovecot.conf.tmpl b/dovecot-charm/templates/dovecot.conf.tmpl index 433c148..4a6f28f 100644 --- a/dovecot-charm/templates/dovecot.conf.tmpl +++ b/dovecot-charm/templates/dovecot.conf.tmpl @@ -41,6 +41,14 @@ protocol lmtp { mail_plugins = $mail_plugins sieve } +service lmtp { + unix_listener /var/spool/postfix/private/dovecot-lmtp { + mode = 0600 + user = postfix + group = postfix + } +} + protocol imap { # Maximum number of IMAP connections allowed for a user from each IP address. mail_max_userip_connections = 30 diff --git a/dovecot-charm/tests/unit/test_charm.py b/dovecot-charm/tests/unit/test_charm.py index 9470b73..5e6fd69 100644 --- a/dovecot-charm/tests/unit/test_charm.py +++ b/dovecot-charm/tests/unit/test_charm.py @@ -52,7 +52,7 @@ def test_reconcile_sets_active_on_success(ctx, base_state): def test_reconcile_opens_mail_ports(ctx, base_state): """All required IMAP/POP3/Sieve/metrics ports must be opened.""" state_out = ctx.run(ctx.on.config_changed(), base_state) - expected = {ops.testing.TCPPort(p) for p in [993, 995, 4190]} + expected = {ops.testing.TCPPort(p) for p in [25, 993, 995, 4190]} assert state_out.opened_ports == expected @@ -74,7 +74,7 @@ def test_reconcile_blocks_when_procmail_setup_fails(ctx, base_state): """Charm must be Blocked when setup_procmail raises ConfigurationError.""" class _FailingSetup(NoOpDovecotSetup): - def setup_procmail(self): + def setup_procmail(self, mailname: str): raise ConfigurationError("Failed to configure postfix: error") with patch.object(DovecotTestCharm, "_dovecot_setup", _FailingSetup()): diff --git a/dovecot-charm/tests/unit/testing.py b/dovecot-charm/tests/unit/testing.py index 1b74386..fc88ceb 100644 --- a/dovecot-charm/tests/unit/testing.py +++ b/dovecot-charm/tests/unit/testing.py @@ -83,7 +83,7 @@ def setup_tls(self, dovecot_config): def setup_dovecot(self, dovecot_config): pass - def setup_procmail(self): + def setup_procmail(self, mailname: str): pass @@ -162,7 +162,7 @@ def is_installed(self) -> bool: def setup_dovecot(self, dovecot_config): pass - def setup_procmail(self): + def setup_procmail(self, mailname: str): pass From 2bf8bcce4d67cab705bd95a48c60fe3286bfbf18 Mon Sep 17 00:00:00 2001 From: Ali Ugur Date: Tue, 19 May 2026 07:56:01 +0300 Subject: [PATCH 16/46] chore(dovecot): add explanatory comments for postfix/dovecot wiring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - charm.py: document that port 25 is for postfix-relay → LMTP delivery - dovecot_setup.py: explain why mailbox_command vs virtual_transport/virtual_mailbox_domains are used for different user classes --- dovecot-charm/src/charm.py | 2 ++ dovecot-charm/src/dovecot_setup.py | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/dovecot-charm/src/charm.py b/dovecot-charm/src/charm.py index 6cf8471..62a3cf7 100644 --- a/dovecot-charm/src/charm.py +++ b/dovecot-charm/src/charm.py @@ -219,6 +219,8 @@ def _install(self): def _open_ports(self): """Open mail ports (TLS-only: plaintext 143/110 are not exposed).""" + # Port 25 accepts SMTP from postfix-relay, which forwards to Dovecot via + # the LMTP Unix socket for final delivery into the user mailbox. self.unit.open_port("tcp", 25) self.unit.open_port("tcp", 993) self.unit.open_port("tcp", 995) diff --git a/dovecot-charm/src/dovecot_setup.py b/dovecot-charm/src/dovecot_setup.py index ac6af7d..85441c4 100644 --- a/dovecot-charm/src/dovecot_setup.py +++ b/dovecot-charm/src/dovecot_setup.py @@ -148,7 +148,12 @@ def setup_procmail(self, mailname: str) -> None: host.write_file(PROCMAILRC_TARGET, contents, perms=0o644) postconf_settings = [ + # mailbox_command applies only to the Postfix *local* delivery agent and is + # used here for local system users not covered by virtual_mailbox_domains. 'mailbox_command=/usr/bin/procmail -a "$EXTENSION"', + # virtual_mailbox_domains + virtual_transport route mail for the charm's + # primary domain directly to Dovecot via the LMTP Unix socket, bypassing + # the local delivery agent (and therefore mailbox_command) for that domain. f"virtual_mailbox_domains = {mailname}", "virtual_transport = lmtp:unix:private/dovecot-lmtp", "smtpd_reject_unlisted_recipient = no", From 8a69fac528748e9cbef7addb4c30a48c754227e8 Mon Sep 17 00:00:00 2001 From: Ali Ugur Date: Tue, 19 May 2026 15:56:54 +0300 Subject: [PATCH 17/46] test(dovecot): send mail via SMTP/Postfix in integration tests Now that the charm wires Postfix into Dovecot via the LMTP Unix socket and opens port 25, integration tests should exercise that same path instead of injecting mail locally. Changes: - conftest.py: add MAILNAME = 'example.com' constant so tests can construct correct recipient addresses without hardcoding the string in each file. - test_mail.py: replace echo '...' | mail -s '{subject}' ubuntu@localhost with a Python smtplib.SMTP connection to port 25 of the unit. Postfix matches 'ubuntu@example.com' against virtual_mailbox_domains, then forwards to Dovecot via lmtp:unix:private/dovecot-lmtp. Also moved juju.status() call before the send (needed for unit_ip), and added mail directory creation so LMTP can deliver immediately. - test_ha.py: same replacement in both test_force_sync_action and test_auto_sync. Primary unit IP is resolved from juju.status() just before the send; secondary IP resolution is unchanged (still after sync). _send_mail_via_smtp() helper added alongside the imports. --- dovecot-charm/tests/integration/conftest.py | 3 + dovecot-charm/tests/integration/test_ha.py | 50 +++++++++++++-- dovecot-charm/tests/integration/test_mail.py | 65 ++++++++++++++++---- 3 files changed, 103 insertions(+), 15 deletions(-) diff --git a/dovecot-charm/tests/integration/conftest.py b/dovecot-charm/tests/integration/conftest.py index 0980485..d9d9611 100644 --- a/dovecot-charm/tests/integration/conftest.py +++ b/dovecot-charm/tests/integration/conftest.py @@ -11,6 +11,9 @@ logger = logging.getLogger(__name__) APP_NAME = "dovecot" +# Charm mailname — must match the value passed in deploy config so tests can +# construct the correct SMTP recipient addresses (@example.com). +MAILNAME = "example.com" @pytest.fixture(scope="session", name="juju") diff --git a/dovecot-charm/tests/integration/test_ha.py b/dovecot-charm/tests/integration/test_ha.py index c06266f..b3bbb18 100644 --- a/dovecot-charm/tests/integration/test_ha.py +++ b/dovecot-charm/tests/integration/test_ha.py @@ -4,14 +4,40 @@ import contextlib import imaplib import logging +import smtplib import ssl import time +from email.message import EmailMessage from secrets import token_hex from typing import cast import jubilant import pytest +from conftest import MAILNAME + + +def _send_mail_via_smtp( + host: str, + sender: str, + recipient: str, + subject: str, + body: str, +) -> None: + """Send a plain-text e-mail through the unit's Postfix SMTP listener on port 25. + + Postfix routes delivery for MAILNAME addresses via the LMTP Unix socket + (virtual_transport = lmtp:unix:private/dovecot-lmtp), so mail lands directly + in the Dovecot mail store — the same store that dsync replicates to the secondary. + """ + msg = EmailMessage() + msg["Subject"] = subject + msg["From"] = sender + msg["To"] = recipient + msg.set_content(body) + with smtplib.SMTP(host, 25, timeout=30) as smtp: + smtp.send_message(msg) + def _check_mail_via_imap(unit_ip: str, user: str, password: str, subject: str) -> bool: """Poll IMAP on unit_ip until the email with the given subject is found.""" @@ -177,10 +203,18 @@ def test_force_sync_action(juju: jubilant.Juju, dovecot_charm_dual_unit: str): juju.exec(f"rm -rf /srv/mail/{user}", unit=unit) _setup_mail_user(juju, primary, secondary, user, password) - # Send email on primary + # Send email on primary via SMTP so Postfix routes it through the LMTP socket + # into Dovecot's mail store (the same store dsync replicates). subject = f"Force Sync Test {token_hex(4)}" logging.info(f"Sending test email on primary with subject: {subject}") - juju.exec(f"echo 'test body' | mail -s '{subject}' {user}@localhost", unit=primary) + primary_ip = juju.status().apps[dovecot_charm_dual_unit].units[primary].public_address + _send_mail_via_smtp( + host=primary_ip, + sender=f"{user}@{MAILNAME}", + recipient=f"{user}@{MAILNAME}", + subject=subject, + body="test body", + ) # Run force-sync on primary logging.info("Running force-sync action on primary...") @@ -225,10 +259,18 @@ def test_auto_sync(juju: jubilant.Juju, dovecot_charm_dual_unit: str): juju.exec(f"rm -rf /srv/mail/{user}", unit=unit) _setup_mail_user(juju, primary, secondary, user, password) - # Send email on primary + # Send email on primary via SMTP so Postfix routes it through the LMTP socket + # into Dovecot's mail store (the same store dsync replicates). subject = f"Auto Sync Test {token_hex(4)}" logging.info(f"Sending test email on primary with subject: {subject}") - juju.exec(f"echo 'test body' | mail -s '{subject}' {user}@localhost", unit=primary) + primary_ip = juju.status().apps[dovecot_charm_dual_unit].units[primary].public_address + _send_mail_via_smtp( + host=primary_ip, + sender=f"{user}@{MAILNAME}", + recipient=f"{user}@{MAILNAME}", + subject=subject, + body="test body", + ) previous_sync_mtime = _get_last_sync_mtime(juju, primary) previous_timer_count = _get_sync_timer_run_count(juju, primary) diff --git a/dovecot-charm/tests/integration/test_mail.py b/dovecot-charm/tests/integration/test_mail.py index 315772e..77b1ff8 100644 --- a/dovecot-charm/tests/integration/test_mail.py +++ b/dovecot-charm/tests/integration/test_mail.py @@ -1,19 +1,53 @@ # Copyright 2026 Canonical Ltd. # See LICENSE file for licensing details. +"""Integration tests for end-to-end mail delivery via Postfix → LMTP → Dovecot.""" + import contextlib import imaplib import logging +import smtplib import ssl import time +from email.message import EmailMessage from secrets import token_hex import jubilant import pytest +from conftest import MAILNAME + + +def _send_mail_via_smtp( + host: str, + sender: str, + recipient: str, + subject: str, + body: str, +) -> None: + """Send a plain-text e-mail through the unit's Postfix SMTP listener on port 25. + + Postfix routes delivery for MAILNAME addresses via the LMTP Unix socket + (virtual_transport = lmtp:unix:private/dovecot-lmtp), so mail lands directly + in the Dovecot mail store without touching the local delivery agent / procmail. + """ + msg = EmailMessage() + msg["Subject"] = subject + msg["From"] = sender + msg["To"] = recipient + msg.set_content(body) + with smtplib.SMTP(host, 25, timeout=30) as smtp: + smtp.send_message(msg) + def test_mail_workflow(juju: jubilant.Juju, dovecot_charm: str): - """Test end-to-end mail delivery and IMAP retrieval.""" + """Test end-to-end mail delivery via Postfix LMTP → Dovecot and IMAP retrieval. + + Mail is submitted over SMTP on port 25. Postfix matches the recipient domain + against virtual_mailbox_domains and forwards it to Dovecot via the LMTP Unix + socket (virtual_transport = lmtp:unix:private/dovecot-lmtp). The test then + verifies the message is retrievable over IMAPS. + """ unit_name = f"{dovecot_charm}/0" logging.info(f"Updating primary-unit config to {unit_name}...") juju.config(dovecot_charm, {"primary-unit": unit_name}) @@ -21,21 +55,30 @@ def test_mail_workflow(juju: jubilant.Juju, dovecot_charm: str): password = token_hex(8) logging.info("Configuring user 'ubuntu'...") - juju.exec("usermod -aG mail ubuntu", unit=unit_name) juju.exec(f"echo 'ubuntu:{password}' | chpasswd", unit=unit_name) + # Ensure the Dovecot mail directory exists so the LMTP delivery can write + # immediately without waiting for Dovecot to auto-create it. + juju.exec( + "install -d -m 0700 -o ubuntu -g mail /srv/mail/ubuntu", + unit=unit_name, + ) - logging.info("Sending test email...") - subject = "Mail Verification" - cmd = f"echo 'This is the body' | mail -s '{subject}' ubuntu@localhost" - juju.exec(cmd, unit=unit_name) - - logging.info("Verifying via IMAP...") + # Resolve the unit IP before sending so we can reuse it for the IMAP check. status = juju.status() unit_ip = status.apps[dovecot_charm].units[unit_name].public_address - logging.info(f"Connecting to IMAP at {unit_ip}:993") + subject = "Mail Verification" + logging.info(f"Sending test email via SMTP to {unit_ip}:25 ...") + _send_mail_via_smtp( + host=unit_ip, + sender=f"test@{MAILNAME}", + recipient=f"ubuntu@{MAILNAME}", + subject=subject, + body="This is the body", + ) + logging.info(f"Verifying via IMAP at {unit_ip}:993 ...") context = ssl.create_default_context() context.check_hostname = False context.verify_mode = ssl.CERT_NONE @@ -51,7 +94,7 @@ def test_mail_workflow(juju: jubilant.Juju, dovecot_charm: str): if data and data[0]: logging.info(f"Email found successfully via IMAP! IDs: {data[0]}") email_found = True - break # Test passed + break else: logging.info("Email not found yet...") except (imaplib.IMAP4.error, OSError) as e: @@ -66,4 +109,4 @@ def test_mail_workflow(juju: jubilant.Juju, dovecot_charm: str): time.sleep(3) if not email_found: - pytest.fail("Failed to verify email via IMAP.") + pytest.fail("Failed to verify email delivery via IMAP.") From 8a3e30cf3b9d02cab7cc366754b97cdcf3800194 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20U=C4=9EUR?= <39213991+alithethird@users.noreply.github.com> Date: Wed, 20 May 2026 08:00:38 +0300 Subject: [PATCH 18/46] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- tox.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.toml b/tox.toml index a09de19..36d8653 100644 --- a/tox.toml +++ b/tox.toml @@ -3,7 +3,7 @@ skipsdist = true skip_missing_interpreters = true -envlist = [ "lint", "unit", "static", "coverage-report", "stack-integration" ] +envlist = [ "lint", "unit", "static", "coverage-report" ] requires = [ "tox>=4.21" ] no_package = true From ca50c60e6e3917bb58e5790e57b04941f7d09304 Mon Sep 17 00:00:00 2001 From: Ali Ugur Date: Wed, 20 May 2026 14:20:35 +0300 Subject: [PATCH 19/46] fix(postfix): refactor integration test setup and add failure case for sender login map enforcement --- .github/workflows/integration_test.yaml | 21 ++++---- tests/integration/conftest.py | 59 +++------------------ tests/integration/helpers.py | 52 ++++++++++++++++++ tests/integration/test_configurator_maps.py | 10 +++- tox.toml | 20 +------ 5 files changed, 80 insertions(+), 82 deletions(-) create mode 100644 tests/integration/helpers.py diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 8144f8f..b554961 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -34,32 +34,31 @@ jobs: uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main secrets: inherit with: + charmcraft-channel: latest/edge extra-arguments: '-x --log-format="%(asctime)s %(levelname)s %(message)s"' - provider: lxd - trivy-fs-enabled: false juju-channel: 3/stable - self-hosted-runner: true - self-hosted-runner-label: "self-hosted-linux-amd64-noble-large" - charmcraft-channel: latest/edge - working-directory: ${{ matrix.charm.working-directory }} modules: ${{ matrix.charm.modules }} pre-run-script: ${{ matrix.charm.pre-run-script }} + provider: lxd + self-hosted-runner: true + self-hosted-runner-label: "self-hosted-linux-amd64-noble-large" + trivy-fs-enabled: false with-uv: true + working-directory: ${{ matrix.charm.working-directory }} integration-tests-global: uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main secrets: inherit with: - test-tox-env: stack-integration - provider: lxd - trivy-fs-enabled: false - self-hosted-runner: true - self-hosted-runner-label: "self-hosted-linux-amd64-noble-large" juju-channel: 3/stable modules: | [ "test_configurator_maps.py" ] + provider: lxd + self-hosted-runner: true + self-hosted-runner-label: "self-hosted-linux-amd64-noble-large" + trivy-fs-enabled: false with-uv: true allure-report: if: ${{ !cancelled() && github.event_name == 'schedule' }} diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index c003eaa..f82e6c5 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -3,10 +3,7 @@ """Shared fixtures and configuration for integration tests.""" -import base64 -import hashlib import logging -import pathlib import typing from collections.abc import Generator @@ -14,11 +11,10 @@ import pytest import yaml +from helpers import integrate_once, select_charm_file, sha512_dovecot_password + logger = logging.getLogger(__name__) -# --------------------------------------------------------------------------- -# App / domain constants -# --------------------------------------------------------------------------- POSTFIX_RELAY_APP = "postfix-relay" CONFIGURATOR_APP = "postfix-relay-configurator" SELF_SIGNED_APP = "self-signed-certificates" @@ -27,9 +23,6 @@ SMTP_PORT = 587 -# --------------------------------------------------------------------------- -# Pytest configuration -# --------------------------------------------------------------------------- def pytest_addoption(parser: pytest.Parser) -> None: """Add integration test command-line options.""" parser.addoption( @@ -56,44 +49,6 @@ def pytest_addoption(parser: pytest.Parser) -> None: help="Keep temporary models after tests complete", ) - -# --------------------------------------------------------------------------- -# Helper functions -# --------------------------------------------------------------------------- -def _sha512_dovecot_password(password: str) -> str: - """Generate a SSHA512 password hash compatible with dovecot.""" - salt = b"mailtest" - digest = hashlib.sha512(password.encode() + salt).digest() - return "{SSHA512}" + base64.b64encode(digest + salt).decode() - - -def _integrate_once(juju: jubilant.Juju, endpoint_a: str, endpoint_b: str) -> None: - """Call ``juju integrate`` tolerating 'already related' errors.""" - try: - juju.integrate(endpoint_a, endpoint_b) - except Exception as exc: # noqa: BLE001 - msg = str(exc) - if "already exists" not in msg and "already related" not in msg: - raise - logger.debug("Relation %s ↔ %s already exists, skipping", endpoint_a, endpoint_b) - - -def _select_charm_file(pytestconfig: pytest.Config, marker: str) -> str: - """Select charm file matching marker from --charm-file options.""" - charm_files: list[str] = pytestconfig.getoption("--charm-file", default=[]) - for path in charm_files: - if marker in pathlib.Path(path).name.lower(): - return path - use_existing = pytestconfig.getoption("--use-existing", default=False) - if use_existing: - return "" - provided = ", ".join(charm_files) if charm_files else "" - raise AssertionError(f"Missing --charm-file matching '{marker}'. Provided: {provided}.") - - -# --------------------------------------------------------------------------- -# Fixtures -# --------------------------------------------------------------------------- @pytest.fixture(scope="module", name="juju") def juju_fixture(request: pytest.FixtureRequest) -> Generator[jubilant.Juju, None, None]: """Module-scoped Juju client in a temporary model for configurator map tests.""" @@ -147,7 +102,7 @@ def postfix_stack_fixture( # --- postfix-relay --- auth_password = "test-password" if not juju.status().apps.get(POSTFIX_RELAY_APP): - charm_path = _select_charm_file(pytestconfig, "postfix-relay_") + charm_path = select_charm_file(pytestconfig, "postfix-relay_") if not charm_path.startswith(("./", "/")): charm_path = f"./{charm_path}" juju.deploy( @@ -157,12 +112,12 @@ def postfix_stack_fixture( "relay_domains": f"- {TEST_DOMAIN}", "enable_smtp_auth": "true", "smtp_auth_users": yaml.dump( - [f"testuser:{_sha512_dovecot_password(auth_password)}"] + [f"testuser:{sha512_dovecot_password(auth_password)}"] ), "enable_reject_unknown_sender_domain": "false", }, ) - _integrate_once( + integrate_once( juju, f"{POSTFIX_RELAY_APP}:certificates", f"{SELF_SIGNED_APP}:certificates", @@ -171,7 +126,7 @@ def postfix_stack_fixture( # --- postfix-relay-configurator --- authorized_sender = f"authorized@{TEST_DOMAIN}" if not juju.status().apps.get(CONFIGURATOR_APP): - charm_path = _select_charm_file(pytestconfig, "postfix-relay-configurator_") + charm_path = select_charm_file(pytestconfig, "postfix-relay-configurator_") if not charm_path.startswith(("./", "/")): charm_path = f"./{charm_path}" juju.deploy( @@ -181,7 +136,7 @@ def postfix_stack_fixture( "sender_login_maps": yaml.dump({authorized_sender: "testuser"}), }, ) - _integrate_once( + integrate_once( juju, f"{POSTFIX_RELAY_APP}:juju-info", f"{CONFIGURATOR_APP}:juju-info", diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py new file mode 100644 index 0000000..e8c0937 --- /dev/null +++ b/tests/integration/helpers.py @@ -0,0 +1,52 @@ +# Copyright 2026 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Shared helpers for integration tests.""" + +import base64 +import hashlib +import logging +import pathlib + +import jubilant +import pytest + +logger = logging.getLogger(__name__) + +POSTFIX_RELAY_APP = "postfix-relay" +CONFIGURATOR_APP = "postfix-relay-configurator" +SELF_SIGNED_APP = "self-signed-certificates" + +TEST_DOMAIN = "mailstack.internal" +SMTP_PORT = 587 + + +def sha512_dovecot_password(password: str) -> str: + """Generate a SSHA512 password hash compatible with dovecot.""" + salt = b"mailtest" + digest = hashlib.sha512(password.encode() + salt).digest() + return "{SSHA512}" + base64.b64encode(digest + salt).decode() + + +def integrate_once(juju: jubilant.Juju, endpoint_a: str, endpoint_b: str) -> None: + """Call ``juju integrate`` tolerating 'already related' errors.""" + try: + juju.integrate(endpoint_a, endpoint_b) + except Exception as exc: # noqa: BLE001 + msg = str(exc) + if "already exists" not in msg and "already related" not in msg: + raise + logger.debug("Relation %s ↔ %s already exists, skipping", endpoint_a, endpoint_b) + + +def select_charm_file(pytestconfig: pytest.Config, marker: str) -> str: + """Select charm file matching marker from --charm-file options.""" + charm_files: list[str] = pytestconfig.getoption("--charm-file", default=[]) + for path in charm_files: + if marker in pathlib.Path(path).name.lower(): + return path + use_existing = pytestconfig.getoption("--use-existing", default=False) + if use_existing: + return "" + provided = ", ".join(charm_files) if charm_files else "" + raise AssertionError(f"Missing --charm-file matching '{marker}'. Provided: {provided}.") diff --git a/tests/integration/test_configurator_maps.py b/tests/integration/test_configurator_maps.py index d24194c..ff4681c 100644 --- a/tests/integration/test_configurator_maps.py +++ b/tests/integration/test_configurator_maps.py @@ -37,7 +37,7 @@ class TestSenderLoginMapEnforcement: """Verify that sender_login_maps written by the configurator are enforced by postfix.""" def test_sender_login_map_enforcement(self, postfix_stack: typing.Dict[str, str]) -> None: - """Authenticated user can send from authorized address but not from a spoofed one.""" + """Authenticated user can send from authorized address.""" relay_ip = postfix_stack["postfix_relay_ip"] # --- Success case: send from authorized address --- @@ -63,6 +63,14 @@ def test_sender_login_map_enforcement(self, postfix_stack: typing.Dict[str, str] ) logger.info("Success case: message from %s accepted", AUTHORIZED_SENDER) + def test_sender_login_map_enforcement_failure(self, postfix_stack: typing.Dict[str, str]) -> None: + """Spoofed user cannot send from an unauthorized address.""" + relay_ip = postfix_stack["postfix_relay_ip"] + + # --- Success case: send from authorized address --- + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE # --- Failure case: send from spoofed address --- with smtplib.SMTP(relay_ip, SMTP_PORT, timeout=30) as smtp: smtp.ehlo() diff --git a/tox.toml b/tox.toml index 36d8653..419aafb 100644 --- a/tox.toml +++ b/tox.toml @@ -110,14 +110,14 @@ commands = [ [ "bandit", "-c", "{toxinidir}/pyproject.toml", "-r", "{[vars]src_p dependency_groups = [ "static" ] [env.integration] -description = "Run integration tests" +description = "Run cross-charm integration tests" commands = [ [ "pytest", "-v", "--tb", "native", - "--ignore={[vars]tst_path}unit", + "tests/integration/", "--log-cli-level=INFO", "-s", { replace = "posargs", extend = "true" }, @@ -145,19 +145,3 @@ dependency_groups = [ "lint" ] src_path = "{toxinidir}/src/" tst_path = "{toxinidir}/tests/" all_path = [ "{toxinidir}/src/", "{toxinidir}/tests/" ] - -[env.stack-integration] -description = "Run cross-charm integration tests (postfix-relay-configurator)" -commands = [ - [ - "pytest", - "-v", - "--tb", - "native", - "tests/integration/test_configurator_maps.py", - "--log-cli-level=INFO", - "-s", - { replace = "posargs", extend = "true" }, - ], -] -dependency_groups = [ "integration" ] From 212c5b700df7a763ef623c818cfbd2ae0f48eee3 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 20 May 2026 12:35:13 +0000 Subject: [PATCH 20/46] fix: add pull-requests write permission to unit-tests job The check-libraries action uses the GITHUB_TOKEN to set PR labels (e.g. "Libraries: OK"). The pull_request trigger does not grant pull-requests:write by default, causing the action to fail with "Resource not accessible by integration". Adding the explicit permission fixes this. Agent-Logs-Url: https://github.com/canonical/mailserver-operators/sessions/4514e1b6-704f-4782-a10c-197c809fa515 Co-authored-by: alithethird <39213991+alithethird@users.noreply.github.com> --- .github/workflows/test.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 2e39ece..cf41c98 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -7,6 +7,8 @@ jobs: unit-tests: uses: canonical/operator-workflows/.github/workflows/test.yaml@main secrets: inherit + permissions: + pull-requests: write with: self-hosted-runner: false with-uv: true From aea7a8962cc03a1b6d81c056ef931b871e9e5e2d Mon Sep 17 00:00:00 2001 From: Ali Ugur Date: Wed, 20 May 2026 15:49:33 +0300 Subject: [PATCH 21/46] fix: lint --- dovecot-charm/tests/integration/test_ha.py | 1 - dovecot-charm/tests/integration/test_mail.py | 1 - 2 files changed, 2 deletions(-) diff --git a/dovecot-charm/tests/integration/test_ha.py b/dovecot-charm/tests/integration/test_ha.py index b3bbb18..dd1a188 100644 --- a/dovecot-charm/tests/integration/test_ha.py +++ b/dovecot-charm/tests/integration/test_ha.py @@ -13,7 +13,6 @@ import jubilant import pytest - from conftest import MAILNAME diff --git a/dovecot-charm/tests/integration/test_mail.py b/dovecot-charm/tests/integration/test_mail.py index 77b1ff8..c1a02ff 100644 --- a/dovecot-charm/tests/integration/test_mail.py +++ b/dovecot-charm/tests/integration/test_mail.py @@ -14,7 +14,6 @@ import jubilant import pytest - from conftest import MAILNAME From d5313c089362a5de99c03267f017672d637ce9ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20U=C4=9EUR?= <39213991+alithethird@users.noreply.github.com> Date: Thu, 21 May 2026 10:18:06 +0300 Subject: [PATCH 22/46] Apply suggestions from code review Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- dovecot-charm/src/charm.py | 14 +++++++++++--- dovecot-charm/tests/unit/test_charm.py | 2 +- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/dovecot-charm/src/charm.py b/dovecot-charm/src/charm.py index 62a3cf7..bbbee18 100644 --- a/dovecot-charm/src/charm.py +++ b/dovecot-charm/src/charm.py @@ -218,9 +218,17 @@ def _install(self): self.unit.status = MaintenanceStatus("Charm installation done") def _open_ports(self): - """Open mail ports (TLS-only: plaintext 143/110 are not exposed).""" - # Port 25 accepts SMTP from postfix-relay, which forwards to Dovecot via - # the LMTP Unix socket for final delivery into the user mailbox. + """Open mail ports. + + Exposes TLS-wrapped IMAP/POP3 listener ports (993/995) while leaving + plaintext IMAP/POP3 ports (143/110) closed. Also exposes SMTP on TCP/25 + intentionally for mail relay traffic; port 25 is standard SMTP (not + implicit TLS), and STARTTLS is expected when supported by the peer. + """ + # Port 25 intentionally accepts standard SMTP from postfix-relay. This + # is not an implicit-TLS port; peers should negotiate STARTTLS when + # available before Postfix forwards mail to Dovecot via the LMTP Unix + # socket for final delivery into the user mailbox. self.unit.open_port("tcp", 25) self.unit.open_port("tcp", 993) self.unit.open_port("tcp", 995) diff --git a/dovecot-charm/tests/unit/test_charm.py b/dovecot-charm/tests/unit/test_charm.py index 5e6fd69..c044f94 100644 --- a/dovecot-charm/tests/unit/test_charm.py +++ b/dovecot-charm/tests/unit/test_charm.py @@ -50,7 +50,7 @@ def test_reconcile_sets_active_on_success(ctx, base_state): def test_reconcile_opens_mail_ports(ctx, base_state): - """All required IMAP/POP3/Sieve/metrics ports must be opened.""" + """All required SMTP/IMAP/POP3/Sieve/metrics ports must be opened.""" state_out = ctx.run(ctx.on.config_changed(), base_state) expected = {ops.testing.TCPPort(p) for p in [25, 993, 995, 4190]} assert state_out.opened_ports == expected From 89e66d4cf71a45d839d193cb23830ee9a5af1956 Mon Sep 17 00:00:00 2001 From: Ali Ugur Date: Thu, 21 May 2026 10:29:17 +0300 Subject: [PATCH 23/46] chore: extract helper functions from tests --- dovecot-charm/tests/integration/helpers.py | 175 ++++++++++++++++ dovecot-charm/tests/integration/test_ha.py | 206 +++---------------- dovecot-charm/tests/integration/test_mail.py | 27 +-- tests/integration/conftest.py | 60 +----- tests/integration/helpers.py | 45 ++++ 5 files changed, 254 insertions(+), 259 deletions(-) create mode 100644 dovecot-charm/tests/integration/helpers.py create mode 100644 tests/integration/helpers.py diff --git a/dovecot-charm/tests/integration/helpers.py b/dovecot-charm/tests/integration/helpers.py new file mode 100644 index 0000000..417e7ed --- /dev/null +++ b/dovecot-charm/tests/integration/helpers.py @@ -0,0 +1,175 @@ +# Copyright 2026 Canonical Ltd. +# See LICENSE file for licensing details. + +import contextlib +import imaplib +import logging +import smtplib +import ssl +import time +from email.message import EmailMessage + +import jubilant + + +def send_mail_via_smtp( + host: str, + sender: str, + recipient: str, + subject: str, + body: str, +) -> None: + """Send a plain-text e-mail through the unit's Postfix SMTP listener on port 25. + + Postfix routes delivery for MAILNAME addresses via the LMTP Unix socket + (virtual_transport = lmtp:unix:private/dovecot-lmtp), so mail lands directly + in the Dovecot mail store — the same store that dsync replicates to the secondary. + """ + msg = EmailMessage() + msg["Subject"] = subject + msg["From"] = sender + msg["To"] = recipient + msg.set_content(body) + with smtplib.SMTP(host, 25, timeout=30) as smtp: + smtp.send_message(msg) + + +def check_mail_via_imap(unit_ip: str, user: str, password: str, subject: str) -> bool: + """Poll IMAP on unit_ip until the email with the given subject is found.""" + context = ssl.create_default_context() + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + + for attempt in range(20): + mail = None + try: + mail = imaplib.IMAP4_SSL(unit_ip, port=993, ssl_context=context) + mail.login(user, password) + mail.select("inbox") + _, data = mail.search(None, f'(HEADER Subject "{subject}")') + if data and data[0]: + logging.info(f"Email found via IMAP on {unit_ip}. IDs: {data[0]}") + return True + logging.info(f"Email not found yet on {unit_ip} (attempt {attempt + 1})...") + except (imaplib.IMAP4.error, OSError) as e: + logging.warning(f"IMAP attempt {attempt + 1} on {unit_ip} failed: {e}. Retrying...") + finally: + if mail is not None: + with contextlib.suppress(imaplib.IMAP4.error, OSError): + mail.close() + with contextlib.suppress(imaplib.IMAP4.error, OSError): + mail.logout() + time.sleep(3) + + return False + + +def setup_mail_user( + juju: jubilant.Juju, + primary: str, + secondary: str, + user: str, + password: str, +): + """Create a mail user on both units. + + The system account and password are created on both units so PAM auth works + on the secondary after sync. The Maildir is only initialised on the primary + so that dsync can replicate it to the secondary without GUID conflicts. + """ + for unit in (primary, secondary): + juju.exec( + ( + f"id -u {user} >/dev/null 2>&1 || " + f"useradd -M -d /srv/mail/{user} -s /usr/sbin/nologin {user}" + ), + unit=unit, + ) + juju.exec(f"echo '{user}:{password}' | chpasswd", unit=unit) + juju.exec(f"usermod -aG mail {user}", unit=unit) + + # Maildir only on primary — dsync creates it on the secondary during the + # first sync. Pre-initialising it on the secondary would give INBOX a + # different GUID and cause doveadm backup to fail with + # "mailbox_delete failed: INBOX can't be deleted". + juju.exec( + ( + f"install -d -m 0700 -o {user} -g mail /srv/mail/{user} && " + f"doveadm mailbox create -u {user} INBOX 2>/dev/null || true" + ), + unit=primary, + ) + + +def get_last_sync_mtime(juju: jubilant.Juju, unit: str) -> int | None: + """Return /srv/mail/.last-dsync mtime epoch on unit, or None if missing.""" + output = juju.exec( + "stat -c %Y /srv/mail/.last-dsync 2>/dev/null || true", unit=unit + ).stdout.strip() + return int(output) if output.isdigit() else None + + +def get_sync_timer_run_count(juju: jubilant.Juju, unit: str) -> int: + """Return count of sync-to-secondary service invocations from the journal.""" + output = juju.exec( + "journalctl -u sync-to-secondary.service --no-pager -q 2>/dev/null | wc -l || true", + unit=unit, + ).stdout.strip() + return int(output) if output.isdigit() else 0 + + +def get_sync_log_content(juju: jubilant.Juju, unit: str, lines: int = 20) -> str: + """Return last N lines from the sync-to-secondary service journal for debugging.""" + output = juju.exec( + f"journalctl -u sync-to-secondary.service --no-pager -n {lines} 2>/dev/null || echo 'No journal entries for sync-to-secondary'", + unit=unit, + ).stdout + return output + + +def get_timer_status(juju: jubilant.Juju, unit: str) -> str | None: + """Return systemctl show output for the sync-to-secondary timer, or None if absent.""" + result = juju.exec( + "systemctl show sync-to-secondary.timer --property=ActiveState,LastTriggerUSec 2>/dev/null || true", + unit=unit, + ).stdout.strip() + return result if result else None + + +def wait_for_sync_trigger( + juju: jubilant.Juju, + unit: str, + previous_mtime: int | None, + previous_timer_count: int, + timeout: int = 4 * 60, + poll_interval: int = 5, +) -> int: + """Wait until /srv/mail/.last-dsync mtime advances, indicating a completed sync. + + The sync script touches .last-dsync only at the very end, so this is a + reliable end-of-sync marker. Journal timer count is checked only to log + that the timer appears to have fired while we continue waiting for + .last-dsync to be updated. + """ + deadline = time.time() + timeout + timer_fired = False + while time.time() < deadline: + current_mtime = get_last_sync_mtime(juju, unit) + if current_mtime is not None and ( + previous_mtime is None or current_mtime > previous_mtime + ): + return current_mtime + + current_timer_count = get_sync_timer_run_count(juju, unit) + if current_timer_count > previous_timer_count and not timer_fired: + logging.info( + "Timer fired (journal count increased); waiting for .last-dsync to update..." + ) + timer_fired = True + + time.sleep(poll_interval) + + raise AssertionError( + "Timed out waiting for sync trigger on " + f"{unit}; previous mtime={previous_mtime}, previous timer count={previous_timer_count}" + ) diff --git a/dovecot-charm/tests/integration/test_ha.py b/dovecot-charm/tests/integration/test_ha.py index dd1a188..fb42336 100644 --- a/dovecot-charm/tests/integration/test_ha.py +++ b/dovecot-charm/tests/integration/test_ha.py @@ -1,182 +1,24 @@ # Copyright 2026 Canonical Ltd. # See LICENSE file for licensing details. -import contextlib -import imaplib import logging -import smtplib -import ssl import time -from email.message import EmailMessage from secrets import token_hex from typing import cast import jubilant import pytest from conftest import MAILNAME - - -def _send_mail_via_smtp( - host: str, - sender: str, - recipient: str, - subject: str, - body: str, -) -> None: - """Send a plain-text e-mail through the unit's Postfix SMTP listener on port 25. - - Postfix routes delivery for MAILNAME addresses via the LMTP Unix socket - (virtual_transport = lmtp:unix:private/dovecot-lmtp), so mail lands directly - in the Dovecot mail store — the same store that dsync replicates to the secondary. - """ - msg = EmailMessage() - msg["Subject"] = subject - msg["From"] = sender - msg["To"] = recipient - msg.set_content(body) - with smtplib.SMTP(host, 25, timeout=30) as smtp: - smtp.send_message(msg) - - -def _check_mail_via_imap(unit_ip: str, user: str, password: str, subject: str) -> bool: - """Poll IMAP on unit_ip until the email with the given subject is found.""" - context = ssl.create_default_context() - context.check_hostname = False - context.verify_mode = ssl.CERT_NONE - - for attempt in range(20): - mail = None - try: - mail = imaplib.IMAP4_SSL(unit_ip, port=993, ssl_context=context) - mail.login(user, password) - mail.select("inbox") - _, data = mail.search(None, f'(HEADER Subject "{subject}")') - if data and data[0]: - logging.info(f"Email found via IMAP on {unit_ip}. IDs: {data[0]}") - return True - logging.info(f"Email not found yet on {unit_ip} (attempt {attempt + 1})...") - except (imaplib.IMAP4.error, OSError) as e: - logging.warning(f"IMAP attempt {attempt + 1} on {unit_ip} failed: {e}. Retrying...") - finally: - if mail is not None: - with contextlib.suppress(imaplib.IMAP4.error, OSError): - mail.close() - with contextlib.suppress(imaplib.IMAP4.error, OSError): - mail.logout() - time.sleep(3) - - return False - - -def _setup_mail_user( - juju: jubilant.Juju, - primary: str, - secondary: str, - user: str, - password: str, -): - """Create a mail user on both units. - - The system account and password are created on both units so PAM auth works - on the secondary after sync. The Maildir is only initialised on the primary - so that dsync can replicate it to the secondary without GUID conflicts. - """ - for unit in (primary, secondary): - juju.exec( - ( - f"id -u {user} >/dev/null 2>&1 || " - f"useradd -M -d /srv/mail/{user} -s /usr/sbin/nologin {user}" - ), - unit=unit, - ) - juju.exec(f"echo '{user}:{password}' | chpasswd", unit=unit) - juju.exec(f"usermod -aG mail {user}", unit=unit) - - # Maildir only on primary — dsync creates it on the secondary during the - # first sync. Pre-initialising it on the secondary would give INBOX a - # different GUID and cause doveadm backup to fail with - # "mailbox_delete failed: INBOX can't be deleted". - juju.exec( - ( - f"install -d -m 0700 -o {user} -g mail /srv/mail/{user} && " - f"doveadm mailbox create -u {user} INBOX 2>/dev/null || true" - ), - unit=primary, - ) - - -def _get_last_sync_mtime(juju: jubilant.Juju, unit: str) -> int | None: - """Return /srv/mail/.last-dsync mtime epoch on unit, or None if missing.""" - output = juju.exec( - "stat -c %Y /srv/mail/.last-dsync 2>/dev/null || true", unit=unit - ).stdout.strip() - return int(output) if output.isdigit() else None - - -def _get_sync_timer_run_count(juju: jubilant.Juju, unit: str) -> int: - """Return count of sync-to-secondary service invocations from the journal.""" - output = juju.exec( - "journalctl -u sync-to-secondary.service --no-pager -q 2>/dev/null | wc -l || true", - unit=unit, - ).stdout.strip() - return int(output) if output.isdigit() else 0 - - -def _get_sync_log_content(juju: jubilant.Juju, unit: str, lines: int = 20) -> str: - """Return last N lines from the sync-to-secondary service journal for debugging.""" - output = juju.exec( - f"journalctl -u sync-to-secondary.service --no-pager -n {lines} 2>/dev/null || echo 'No journal entries for sync-to-secondary'", - unit=unit, - ).stdout - return output - - -def _get_timer_status(juju: jubilant.Juju, unit: str) -> str | None: - """Return systemctl show output for the sync-to-secondary timer, or None if absent.""" - result = juju.exec( - "systemctl show sync-to-secondary.timer --property=ActiveState,LastTriggerUSec 2>/dev/null || true", - unit=unit, - ).stdout.strip() - return result if result else None - - -def _wait_for_sync_trigger( - juju: jubilant.Juju, - unit: str, - previous_mtime: int | None, - previous_timer_count: int, - timeout: int = 4 * 60, - poll_interval: int = 5, -) -> int: - """Wait until /srv/mail/.last-dsync mtime advances, indicating a completed sync. - - The sync script touches .last-dsync only at the very end, so this is a - reliable end-of-sync marker. Journal timer count is checked only to log - that the timer appears to have fired while we continue waiting for - .last-dsync to be updated. - """ - deadline = time.time() + timeout - timer_fired = False - while time.time() < deadline: - current_mtime = _get_last_sync_mtime(juju, unit) - if current_mtime is not None and ( - previous_mtime is None or current_mtime > previous_mtime - ): - return current_mtime - - current_timer_count = _get_sync_timer_run_count(juju, unit) - if current_timer_count > previous_timer_count and not timer_fired: - logging.info( - "Timer fired (journal count increased); waiting for .last-dsync to update..." - ) - timer_fired = True - - time.sleep(poll_interval) - - raise AssertionError( - "Timed out waiting for sync trigger on " - f"{unit}; previous mtime={previous_mtime}, previous timer count={previous_timer_count}" - ) +from helpers import ( + check_mail_via_imap, + get_last_sync_mtime, + get_sync_log_content, + get_sync_timer_run_count, + get_timer_status, + send_mail_via_smtp, + setup_mail_user, + wait_for_sync_trigger, +) def test_force_sync_action(juju: jubilant.Juju, dovecot_charm_dual_unit: str): @@ -200,14 +42,14 @@ def test_force_sync_action(juju: jubilant.Juju, dovecot_charm_dual_unit: str): password = token_hex(8) for unit in (primary, secondary): juju.exec(f"rm -rf /srv/mail/{user}", unit=unit) - _setup_mail_user(juju, primary, secondary, user, password) + setup_mail_user(juju, primary, secondary, user, password) # Send email on primary via SMTP so Postfix routes it through the LMTP socket # into Dovecot's mail store (the same store dsync replicates). subject = f"Force Sync Test {token_hex(4)}" logging.info(f"Sending test email on primary with subject: {subject}") primary_ip = juju.status().apps[dovecot_charm_dual_unit].units[primary].public_address - _send_mail_via_smtp( + send_mail_via_smtp( host=primary_ip, sender=f"{user}@{MAILNAME}", recipient=f"{user}@{MAILNAME}", @@ -224,7 +66,7 @@ def test_force_sync_action(juju: jubilant.Juju, dovecot_charm_dual_unit: str): # Verify email arrived on secondary via IMAP secondary_ip = juju.status().apps[dovecot_charm_dual_unit].units[secondary].public_address logging.info(f"Checking for email on secondary via IMAP at {secondary_ip}:993...") - assert _check_mail_via_imap(secondary_ip, user, password, subject), ( + assert check_mail_via_imap(secondary_ip, user, password, subject), ( f"Email with subject '{subject}' not found on secondary after force-sync" ) @@ -256,14 +98,14 @@ def test_auto_sync(juju: jubilant.Juju, dovecot_charm_dual_unit: str): password = token_hex(8) for unit in (primary, secondary): juju.exec(f"rm -rf /srv/mail/{user}", unit=unit) - _setup_mail_user(juju, primary, secondary, user, password) + setup_mail_user(juju, primary, secondary, user, password) # Send email on primary via SMTP so Postfix routes it through the LMTP socket # into Dovecot's mail store (the same store dsync replicates). subject = f"Auto Sync Test {token_hex(4)}" logging.info(f"Sending test email on primary with subject: {subject}") primary_ip = juju.status().apps[dovecot_charm_dual_unit].units[primary].public_address - _send_mail_via_smtp( + send_mail_via_smtp( host=primary_ip, sender=f"{user}@{MAILNAME}", recipient=f"{user}@{MAILNAME}", @@ -271,8 +113,8 @@ def test_auto_sync(juju: jubilant.Juju, dovecot_charm_dual_unit: str): body="test body", ) - previous_sync_mtime = _get_last_sync_mtime(juju, primary) - previous_timer_count = _get_sync_timer_run_count(juju, primary) + previous_sync_mtime = get_last_sync_mtime(juju, primary) + previous_timer_count = get_sync_timer_run_count(juju, primary) try: # Lower sync schedule to every minute, wait for reconcile @@ -280,26 +122,26 @@ def test_auto_sync(juju: jubilant.Juju, dovecot_charm_dual_unit: str): juju.config(dovecot_charm_dual_unit, {"sync-schedule": "*:*"}) juju.wait(jubilant.all_active, timeout=5 * 60) - logging.info(f"Timer status after config change:\n{_get_timer_status(juju, primary)}") + logging.info(f"Timer status after config change:\n{get_timer_status(juju, primary)}") logging.info("Waiting for first timer-triggered sync signal on primary...") - _wait_for_sync_trigger(juju, primary, previous_sync_mtime, previous_timer_count) + wait_for_sync_trigger(juju, primary, previous_sync_mtime, previous_timer_count) # Verify email arrived on secondary via IMAP secondary_ip = juju.status().apps[dovecot_charm_dual_unit].units[secondary].public_address logging.info(f"Checking for email on secondary via IMAP at {secondary_ip}:993...") - synced = _check_mail_via_imap(secondary_ip, user, password, subject) + synced = check_mail_via_imap(secondary_ip, user, password, subject) if not synced: logging.info("Email not found after first timer sync.") - logging.info(f"Sync log on primary:\n{_get_sync_log_content(juju, primary)}") + logging.info(f"Sync log on primary:\n{get_sync_log_content(juju, primary)}") logging.info("Timer status:") - logging.info(_get_timer_status(juju, primary)) + logging.info(get_timer_status(juju, primary)) logging.info("Trying manual sync as fallback to verify sync mechanism works...") juju.exec("/usr/local/bin/sync-to-secondary.sh", unit=primary) time.sleep(15) - synced = _check_mail_via_imap(secondary_ip, user, password, subject) + synced = check_mail_via_imap(secondary_ip, user, password, subject) if not synced: logging.info("Manual sync also failed. Checking sync log after manual run:") - logging.info(f"Sync log:\n{_get_sync_log_content(juju, primary, lines=30)}") + logging.info(f"Sync log:\n{get_sync_log_content(juju, primary, lines=30)}") assert synced, f"Email with subject '{subject}' not found on secondary after auto-sync" finally: diff --git a/dovecot-charm/tests/integration/test_mail.py b/dovecot-charm/tests/integration/test_mail.py index c1a02ff..f036fcd 100644 --- a/dovecot-charm/tests/integration/test_mail.py +++ b/dovecot-charm/tests/integration/test_mail.py @@ -6,37 +6,14 @@ import contextlib import imaplib import logging -import smtplib import ssl import time -from email.message import EmailMessage from secrets import token_hex import jubilant import pytest from conftest import MAILNAME - - -def _send_mail_via_smtp( - host: str, - sender: str, - recipient: str, - subject: str, - body: str, -) -> None: - """Send a plain-text e-mail through the unit's Postfix SMTP listener on port 25. - - Postfix routes delivery for MAILNAME addresses via the LMTP Unix socket - (virtual_transport = lmtp:unix:private/dovecot-lmtp), so mail lands directly - in the Dovecot mail store without touching the local delivery agent / procmail. - """ - msg = EmailMessage() - msg["Subject"] = subject - msg["From"] = sender - msg["To"] = recipient - msg.set_content(body) - with smtplib.SMTP(host, 25, timeout=30) as smtp: - smtp.send_message(msg) +from helpers import send_mail_via_smtp def test_mail_workflow(juju: jubilant.Juju, dovecot_charm: str): @@ -69,7 +46,7 @@ def test_mail_workflow(juju: jubilant.Juju, dovecot_charm: str): subject = "Mail Verification" logging.info(f"Sending test email via SMTP to {unit_ip}:25 ...") - _send_mail_via_smtp( + send_mail_via_smtp( host=unit_ip, sender=f"test@{MAILNAME}", recipient=f"ubuntu@{MAILNAME}", diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index c003eaa..e354bfd 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -3,10 +3,7 @@ """Shared fixtures and configuration for integration tests.""" -import base64 -import hashlib import logging -import pathlib import typing from collections.abc import Generator @@ -14,11 +11,11 @@ import pytest import yaml +from helpers import integrate_once, select_charm_file, sha512_dovecot_password + logger = logging.getLogger(__name__) -# --------------------------------------------------------------------------- -# App / domain constants -# --------------------------------------------------------------------------- + POSTFIX_RELAY_APP = "postfix-relay" CONFIGURATOR_APP = "postfix-relay-configurator" SELF_SIGNED_APP = "self-signed-certificates" @@ -27,9 +24,6 @@ SMTP_PORT = 587 -# --------------------------------------------------------------------------- -# Pytest configuration -# --------------------------------------------------------------------------- def pytest_addoption(parser: pytest.Parser) -> None: """Add integration test command-line options.""" parser.addoption( @@ -56,44 +50,6 @@ def pytest_addoption(parser: pytest.Parser) -> None: help="Keep temporary models after tests complete", ) - -# --------------------------------------------------------------------------- -# Helper functions -# --------------------------------------------------------------------------- -def _sha512_dovecot_password(password: str) -> str: - """Generate a SSHA512 password hash compatible with dovecot.""" - salt = b"mailtest" - digest = hashlib.sha512(password.encode() + salt).digest() - return "{SSHA512}" + base64.b64encode(digest + salt).decode() - - -def _integrate_once(juju: jubilant.Juju, endpoint_a: str, endpoint_b: str) -> None: - """Call ``juju integrate`` tolerating 'already related' errors.""" - try: - juju.integrate(endpoint_a, endpoint_b) - except Exception as exc: # noqa: BLE001 - msg = str(exc) - if "already exists" not in msg and "already related" not in msg: - raise - logger.debug("Relation %s ↔ %s already exists, skipping", endpoint_a, endpoint_b) - - -def _select_charm_file(pytestconfig: pytest.Config, marker: str) -> str: - """Select charm file matching marker from --charm-file options.""" - charm_files: list[str] = pytestconfig.getoption("--charm-file", default=[]) - for path in charm_files: - if marker in pathlib.Path(path).name.lower(): - return path - use_existing = pytestconfig.getoption("--use-existing", default=False) - if use_existing: - return "" - provided = ", ".join(charm_files) if charm_files else "" - raise AssertionError(f"Missing --charm-file matching '{marker}'. Provided: {provided}.") - - -# --------------------------------------------------------------------------- -# Fixtures -# --------------------------------------------------------------------------- @pytest.fixture(scope="module", name="juju") def juju_fixture(request: pytest.FixtureRequest) -> Generator[jubilant.Juju, None, None]: """Module-scoped Juju client in a temporary model for configurator map tests.""" @@ -147,7 +103,7 @@ def postfix_stack_fixture( # --- postfix-relay --- auth_password = "test-password" if not juju.status().apps.get(POSTFIX_RELAY_APP): - charm_path = _select_charm_file(pytestconfig, "postfix-relay_") + charm_path = select_charm_file(pytestconfig, "postfix-relay_") if not charm_path.startswith(("./", "/")): charm_path = f"./{charm_path}" juju.deploy( @@ -157,12 +113,12 @@ def postfix_stack_fixture( "relay_domains": f"- {TEST_DOMAIN}", "enable_smtp_auth": "true", "smtp_auth_users": yaml.dump( - [f"testuser:{_sha512_dovecot_password(auth_password)}"] + [f"testuser:{sha512_dovecot_password(auth_password)}"] ), "enable_reject_unknown_sender_domain": "false", }, ) - _integrate_once( + integrate_once( juju, f"{POSTFIX_RELAY_APP}:certificates", f"{SELF_SIGNED_APP}:certificates", @@ -171,7 +127,7 @@ def postfix_stack_fixture( # --- postfix-relay-configurator --- authorized_sender = f"authorized@{TEST_DOMAIN}" if not juju.status().apps.get(CONFIGURATOR_APP): - charm_path = _select_charm_file(pytestconfig, "postfix-relay-configurator_") + charm_path = select_charm_file(pytestconfig, "postfix-relay-configurator_") if not charm_path.startswith(("./", "/")): charm_path = f"./{charm_path}" juju.deploy( @@ -181,7 +137,7 @@ def postfix_stack_fixture( "sender_login_maps": yaml.dump({authorized_sender: "testuser"}), }, ) - _integrate_once( + integrate_once( juju, f"{POSTFIX_RELAY_APP}:juju-info", f"{CONFIGURATOR_APP}:juju-info", diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py new file mode 100644 index 0000000..bb52738 --- /dev/null +++ b/tests/integration/helpers.py @@ -0,0 +1,45 @@ +# Copyright 2026 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Shared helpers for integration tests.""" + +import base64 +import hashlib +import logging +import pathlib + +import jubilant +import pytest + +logger = logging.getLogger(__name__) + +def sha512_dovecot_password(password: str) -> str: + """Generate a SSHA512 password hash compatible with dovecot.""" + salt = b"mailtest" + digest = hashlib.sha512(password.encode() + salt).digest() + return "{SSHA512}" + base64.b64encode(digest + salt).decode() + + +def integrate_once(juju: jubilant.Juju, endpoint_a: str, endpoint_b: str) -> None: + """Call ``juju integrate`` tolerating 'already related' errors.""" + try: + juju.integrate(endpoint_a, endpoint_b) + except Exception as exc: # noqa: BLE001 + msg = str(exc) + if "already exists" not in msg and "already related" not in msg: + raise + logger.debug("Relation %s ↔ %s already exists, skipping", endpoint_a, endpoint_b) + + +def select_charm_file(pytestconfig: pytest.Config, marker: str) -> str: + """Select charm file matching marker from --charm-file options.""" + charm_files: list[str] = pytestconfig.getoption("--charm-file", default=[]) + for path in charm_files: + if marker in pathlib.Path(path).name.lower(): + return path + use_existing = pytestconfig.getoption("--use-existing", default=False) + if use_existing: + return "" + provided = ", ".join(charm_files) if charm_files else "" + raise AssertionError(f"Missing --charm-file matching '{marker}'. Provided: {provided}.") + From abaab0d8de304a0e0ac29e36e17e40d004ee3c6a Mon Sep 17 00:00:00 2001 From: Ali Ugur Date: Thu, 21 May 2026 12:11:18 +0300 Subject: [PATCH 24/46] test: add full-stack integration suite --- pyproject.toml | 6 + tests/integration/conftest.py | 540 ++++++++++++++++++- tests/integration/test_full_stack.py | 447 +++++++++++++++ tests/integration/test_whole_email_system.py | 128 +++++ tox.toml | 19 + uv.lock | 125 +++++ 6 files changed, 1248 insertions(+), 17 deletions(-) create mode 100644 tests/integration/test_full_stack.py create mode 100644 tests/integration/test_whole_email_system.py diff --git a/pyproject.toml b/pyproject.toml index 314f373..652baff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,8 +47,11 @@ static = [ integration = [ "allure-pytest>=2.8.18", "allure-pytest-collection-report @ git+https://github.com/canonical/data-platform-workflows@v24.0.0#subdirectory=python/pytest_plugins/allure_pytest_collection_report", + "cryptography>=42.0", "jubilant==1.9.0", "pytest", + "pyyaml>=6.0", + "requests>=2.31", ] [tool.uv] @@ -168,6 +171,9 @@ pythonpath = [ "dovecot-charm/lib", "dovecot-charm/src", ] +markers = [ + "abort_on_fail: abort remaining tests if this test fails", +] [tool.ruff.mccabe] max-complexity = 10 diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index e354bfd..17e87c7 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,58 +1,104 @@ # Copyright 2026 Canonical Ltd. # See LICENSE file for licensing details. -"""Shared fixtures and configuration for integration tests.""" +"""Fixtures for the full-stack mailserver integration tests. +Topology +-------- + ┌─────────────────────┐ + test runner ──587──► │ postfix-relay │ ◄─milter─► opendkim + (smtplib) │ + configurator │ (DKIM sign) + └──────────┬──────────┘ + │ LMTP :24 + ▼ + dovecot (IMAP) + │ + ◄──993──── │ + test runner (imaplib) │ + verifies DKIM-Signature header │ + and subject in delivered mail ┘ + +TLS for postfix-relay is provided by self-signed-certificates (CharmHub). +""" + +import base64 import logging +import pathlib +import socket import typing from collections.abc import Generator import jubilant import pytest import yaml +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa -from helpers import integrate_once, select_charm_file, sha512_dovecot_password +from helpers import select_charm_file, sha512_dovecot_password logger = logging.getLogger(__name__) +# --------------------------------------------------------------------------- +# Charm / app names +# --------------------------------------------------------------------------- +DOVECOT_APP = "dovecot-charm" POSTFIX_RELAY_APP = "postfix-relay" +OPENDKIM_APP = "opendkim" CONFIGURATOR_APP = "postfix-relay-configurator" SELF_SIGNED_APP = "self-signed-certificates" +# Domain used throughout the test suite TEST_DOMAIN = "mailstack.internal" SMTP_PORT = 587 +# parents[0]=tests/integration, parents[1]=tests, parents[2]=mailserver-operators/ +_REPO_ROOT = pathlib.Path(__file__).resolve().parents[2] +OPENDKIM_SNAP_DIR = _REPO_ROOT / "opendkim-snap" + +# --------------------------------------------------------------------------- +# pytest CLI options +# --------------------------------------------------------------------------- def pytest_addoption(parser: pytest.Parser) -> None: - """Add integration test command-line options.""" + """Register extra CLI options consumed by the integration suite.""" parser.addoption( "--charm-file", action="append", default=[], - help="Path to charm file (can be used multiple times)", + help=( + "Path to a pre-built .charm file. " + "Pass this option multiple times (one per charm)." + ), ) parser.addoption( - "--use-existing", + "--keep-models", action="store_true", default=False, - help="Use existing model instead of creating a temporary one", + help="Keep Juju models after tests complete (useful for debugging).", ) parser.addoption( "--model", + action="store", default=None, - help="Specific model name to use", + help="Use an existing Juju model by name instead of creating a temp model.", ) parser.addoption( - "--keep-models", + "--use-existing", action="store_true", default=False, - help="Keep temporary models after tests complete", + help="Attach to the current Juju model without deploying anything new.", ) -@pytest.fixture(scope="module", name="juju") -def juju_fixture(request: pytest.FixtureRequest) -> Generator[jubilant.Juju, None, None]: - """Module-scoped Juju client in a temporary model for configurator map tests.""" + +# --------------------------------------------------------------------------- +# Juju session fixture +# --------------------------------------------------------------------------- +@pytest.fixture(scope="session", name="juju") +def juju_fixture( + request: pytest.FixtureRequest, +) -> Generator[jubilant.Juju, None, None]: + """Session-scoped Juju client pointing at a temporary (or named) model.""" def _show_debug_log(juju: jubilant.Juju) -> None: if request.session.testsfailed: @@ -67,7 +113,7 @@ def _show_debug_log(juju: jubilant.Juju) -> None: _show_debug_log(juju) return - model = request.config.getoption("--model", default=None) + model = request.config.getoption("--model") if model: juju = jubilant.Juju(model=model) juju.model_config({"automatically-retry-hooks": True}) @@ -75,20 +121,466 @@ def _show_debug_log(juju: jubilant.Juju) -> None: _show_debug_log(juju) return - keep_models = typing.cast(bool, request.config.getoption("--keep-models", default=False)) + keep_models = typing.cast(bool, request.config.getoption("--keep-models")) with jubilant.temp_model(keep=keep_models) as juju: juju.wait_timeout = 15 * 60 juju.model_config({"automatically-retry-hooks": True}) yield juju _show_debug_log(juju) + return + + +# --------------------------------------------------------------------------- +# Machine IP (test runner) +# --------------------------------------------------------------------------- +@pytest.fixture(scope="session", name="machine_ip_address") +def machine_ip_address_fixture() -> str: + """IP address of the machine running the tests. + + Used to configure /etc/hosts on Juju units so TEST_DOMAIN resolves to + the test runner (where mailcatcher or similar sinks may be listening), + and also to tell postfix-relay where to forward mail. + """ + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s.connect(("8.8.8.8", 80)) + ip = s.getsockname()[0] + s.close() + logger.info("Test runner IP: %s", ip) + return ip + + +# --------------------------------------------------------------------------- +# Charm-file fixtures +# --------------------------------------------------------------------------- +def _select_charm_file_for_app(pytestconfig: pytest.Config, app_name: str) -> str: + """Return a charm file path matching *app_name* from repeated --charm-file args.""" + charm_files = typing.cast(list[str], pytestconfig.getoption("--charm-file")) + use_existing = pytestconfig.getoption("--use-existing", default=False) + + # Match more specific names first to avoid postfix-relay matching configurator. + match_order = ( + (CONFIGURATOR_APP, "postfix-relay-configurator"), + (POSTFIX_RELAY_APP, "postfix-relay"), + (DOVECOT_APP, "dovecot-charm"), + (OPENDKIM_APP, "opendkim"), + ) + + app_to_path: dict[str, str] = {} + for path in charm_files: + name = pathlib.Path(path).name.lower() + for app, marker in match_order: + if marker in name: + app_to_path[app] = path + break + + selected = app_to_path.get(app_name) + if selected: + return selected + + if use_existing: + # In --use-existing mode, deployment may be skipped if apps already exist. + return "" + + provided = ", ".join(charm_files) if charm_files else "" + raise AssertionError( + f"Missing --charm-file for {app_name}. Provided: {provided}. " + "Pass one --charm-file per charm artifact." + ) + + +@pytest.fixture(scope="session", name="dovecot_charm_file") +def dovecot_charm_file_fixture(pytestconfig: pytest.Config) -> str: + """Absolute path to the pre-built dovecot-charm .charm file.""" + return _select_charm_file_for_app(pytestconfig, DOVECOT_APP) + + +@pytest.fixture(scope="session", name="postfix_relay_charm_file") +def postfix_relay_charm_file_fixture(pytestconfig: pytest.Config) -> str: + """Absolute path to the pre-built postfix-relay .charm file.""" + return _select_charm_file_for_app(pytestconfig, POSTFIX_RELAY_APP) + + +@pytest.fixture(scope="session", name="opendkim_charm_file") +def opendkim_charm_file_fixture(pytestconfig: pytest.Config) -> str: + """Absolute path to the pre-built opendkim .charm file.""" + return _select_charm_file_for_app(pytestconfig, OPENDKIM_APP) + + +@pytest.fixture(scope="session", name="configurator_charm_file") +def configurator_charm_file_fixture(pytestconfig: pytest.Config) -> str: + """Absolute path to the pre-built postfix-relay-configurator .charm file.""" + return _select_charm_file_for_app(pytestconfig, CONFIGURATOR_APP) + + +# --------------------------------------------------------------------------- +# DKIM key generation helper +# --------------------------------------------------------------------------- +def generate_dkim_keypair(domain: str, selector: str) -> typing.Tuple[str, str]: + """Generate a DKIM keypair using the Python cryptography library. + + Args: + domain: The signing domain (e.g. ``mailstack.internal``). + selector: The DKIM selector (e.g. ``default``). + + Returns: + A ``(txt_record, private_key_pem)`` tuple where ``txt_record`` is a + DNS TXT record string and ``private_key_pem`` is a PEM-encoded RSA + private key. + """ + private_key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + private_key_pem = private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption(), + ).decode() + + pub_der = private_key.public_key().public_bytes( + encoding=serialization.Encoding.DER, + format=serialization.PublicFormat.SubjectPublicKeyInfo, + ) + pub_b64 = base64.b64encode(pub_der).decode() + txt_record = ( + f'{selector}._domainkey\tIN\tTXT\t( "v=DKIM1; h=sha256; k=rsa; "\n' + f'\t"p={pub_b64}" )\n' + f"; ----- DKIM key {selector} for {domain}\n" + ) + return txt_record, private_key_pem + + +# --------------------------------------------------------------------------- +# Deploy: self-signed-certificates (TLS provider for postfix-relay / dovecot) +# --------------------------------------------------------------------------- +@pytest.fixture(scope="session", name="self_signed_app") +def deploy_self_signed_certs_fixture(juju: jubilant.Juju) -> str: + """Deploy self-signed-certificates from CharmHub.""" + if not juju.status().apps.get(SELF_SIGNED_APP): + juju.deploy(SELF_SIGNED_APP, channel="latest/stable") + juju.wait( + lambda status: status.apps[SELF_SIGNED_APP].is_active, + error=jubilant.any_error, + timeout=10 * 60, + ) + logger.info("self-signed-certificates is active") + return SELF_SIGNED_APP + + +# --------------------------------------------------------------------------- +# Deploy: opendkim +# --------------------------------------------------------------------------- +@pytest.fixture(scope="session", name="opendkim_app") +def deploy_opendkim_fixture( + opendkim_charm_file: str, + juju: jubilant.Juju, +) -> str: + """Deploy opendkim and optionally replace the store snap with a local build.""" + if not juju.status().apps.get(OPENDKIM_APP): + charm_path = ( + opendkim_charm_file + if opendkim_charm_file.startswith(("./", "/")) + else f"./{opendkim_charm_file}" + ) + juju.deploy(charm_path, OPENDKIM_APP) + # Charm starts blocked (not yet configured) or waiting for milter relation. + juju.wait( + lambda status: ( + status.apps[OPENDKIM_APP].is_blocked + or status.apps[OPENDKIM_APP].app_status.current == "waiting" + ), + timeout=10 * 60, + ) + + _replace_opendkim_snap(juju, OPENDKIM_APP) + return OPENDKIM_APP + + +def _replace_opendkim_snap(juju: jubilant.Juju, app_name: str) -> None: + """Replace the store-installed opendkim snap with a locally-built one if present.""" + snap_files = sorted(OPENDKIM_SNAP_DIR.glob("opendkim_*.snap")) + if not snap_files: + logger.warning( + "No locally-built opendkim snap found in %s — using store version", + OPENDKIM_SNAP_DIR, + ) + return + + snap_path = snap_files[-1] + snap_name = snap_path.name + logger.info("Replacing opendkim snap with local build: %s", snap_path) + + status = juju.status() + for unit_name in status.apps[app_name].units: + juju.scp(snap_path, f"{unit_name}:/tmp/{snap_name}") + juju.exec( + "sudo", + "snap", + "install", + "--dangerous", + f"/tmp/{snap_name}", # nosec B108 + unit=unit_name, + ) + logger.info("Installed local opendkim snap on %s", unit_name) + + +# --------------------------------------------------------------------------- +# Deploy: dovecot +# --------------------------------------------------------------------------- +@pytest.fixture(scope="session", name="dovecot_app") +def deploy_dovecot_fixture( + dovecot_charm_file: str, + self_signed_app: str, + juju: jubilant.Juju, +) -> str: + """Deploy dovecot and wire up TLS.""" + if not juju.status().apps.get(DOVECOT_APP): + charm_path = ( + dovecot_charm_file + if dovecot_charm_file.startswith(("./", "/")) + else f"./{dovecot_charm_file}" + ) + juju.deploy( + charm_path, + app=DOVECOT_APP, + config={ + "mailname": TEST_DOMAIN, + "postmaster-address": f"postmaster@{TEST_DOMAIN}", + "primary-unit": f"{DOVECOT_APP}/0", + "manage-luks": "false", + }, + constraints={"virt-type": "virtual-machine"}, + trust=True, + ) + + # Relate to TLS provider if not already related. + _integrate_once(juju, f"{DOVECOT_APP}:certificates", f"{self_signed_app}:certificates") + + juju.wait( + lambda status: status.apps[DOVECOT_APP].is_active, + error=jubilant.any_error, + timeout=15 * 60, + ) + logger.info("dovecot is active") + return DOVECOT_APP +# --------------------------------------------------------------------------- +# Deploy: postfix-relay +# --------------------------------------------------------------------------- +@pytest.fixture(scope="session", name="postfix_relay_app") +def deploy_postfix_relay_fixture( + postfix_relay_charm_file: str, + self_signed_app: str, + opendkim_app: str, + juju: jubilant.Juju, +) -> str: + """Deploy postfix-relay and integrate with TLS provider and opendkim milter.""" + if not juju.status().apps.get(POSTFIX_RELAY_APP): + charm_path = ( + postfix_relay_charm_file + if postfix_relay_charm_file.startswith(("./", "/")) + else f"./{postfix_relay_charm_file}" + ) + juju.deploy( + charm_path, + app=POSTFIX_RELAY_APP, + config={ + "relay_domains": f"- {TEST_DOMAIN}", + "enable_smtp_auth": "true", + "enable_reject_unknown_sender_domain": "false", + }, + ) + + _integrate_once(juju, f"{POSTFIX_RELAY_APP}:certificates", f"{self_signed_app}:certificates") + _integrate_once(juju, f"{POSTFIX_RELAY_APP}:milter", f"{opendkim_app}:milter") + + juju.wait( + lambda status: ( + status.apps[POSTFIX_RELAY_APP].is_active + and status.apps[opendkim_app].app_status.current in {"blocked", "active"} + ), + timeout=15 * 60, + ) + logger.info("postfix-relay is active (opendkim is blocked or already active)") + return POSTFIX_RELAY_APP + + +# --------------------------------------------------------------------------- +# Deploy: postfix-relay-configurator (subordinate) +# --------------------------------------------------------------------------- +@pytest.fixture(scope="session", name="configurator_app") +def deploy_configurator_fixture( + configurator_charm_file: str, + postfix_relay_app: str, + dovecot_app: str, + juju: jubilant.Juju, +) -> str: + """Deploy the postfix-relay-configurator subordinate and configure LMTP routing. + + The configurator's ``transport_maps`` is set to route mail for TEST_DOMAIN + to dovecot's LMTP port (24) so that postfix-relay delivers locally to dovecot. + """ + # Resolve dovecot's IP after it is active. + status = juju.status() + dovecot_unit = next(iter(status.apps[dovecot_app].units.values())) + dovecot_ip = dovecot_unit.public_address + logger.info("Routing %s → lmtp:inet:%s:24", TEST_DOMAIN, dovecot_ip) + + transport_maps = yaml.dump({TEST_DOMAIN: f"lmtp:inet:{dovecot_ip}:24"}) + + if not juju.status().apps.get(CONFIGURATOR_APP): + charm_path = ( + configurator_charm_file + if configurator_charm_file.startswith(("./", "/")) + else f"./{configurator_charm_file}" + ) + juju.deploy( + charm_path, + app=CONFIGURATOR_APP, + config={"transport_maps": transport_maps}, + ) + + _integrate_once(juju, f"{POSTFIX_RELAY_APP}:juju-info", f"{CONFIGURATOR_APP}:juju-info") + + def _configurator_active(status: jubilant.Status) -> bool: + """Return True once the configurator subordinate is active on all relay units.""" + if not status.apps.get(CONFIGURATOR_APP): + return False + relay_units = status.apps[POSTFIX_RELAY_APP].units + for unit in relay_units.values(): + subs = unit.subordinates or {} + conf_subs = {k: v for k, v in subs.items() if CONFIGURATOR_APP in k} + if not conf_subs: + return False + for sub in conf_subs.values(): + if sub.workload_status.current != "active": + return False + return status.apps[POSTFIX_RELAY_APP].is_active + + juju.wait( + _configurator_active, + error=jubilant.any_error, + timeout=10 * 60, + ) + logger.info("postfix-relay-configurator subordinate is active") + return CONFIGURATOR_APP + + +# --------------------------------------------------------------------------- +# Configure opendkim with DKIM keys for TEST_DOMAIN +# --------------------------------------------------------------------------- +@pytest.fixture(scope="session", name="opendkim_configured") +def configure_opendkim_fixture( + opendkim_app: str, + postfix_relay_app: str, + machine_ip_address: str, + juju: jubilant.Juju, +) -> str: + """Generate a DKIM keypair and configure opendkim for TEST_DOMAIN. + + Returns the opendkim app name once the app is active. + """ + import json # noqa: PLC0415 — local import to avoid top-level cost when not needed + + selector = "default" + keyname = f"{TEST_DOMAIN.replace('.', '-')}-{selector}" + _, private_key = generate_dkim_keypair(domain=TEST_DOMAIN, selector=selector) + + # Store private key as a Juju secret. + try: + secret_id = juju.add_secret("mailstack-dkim-secret", {keyname: private_key}) + except jubilant.CLIError as exc: + if "already exists" in exc.stderr: + secret_info = juju.show_secret("mailstack-dkim-secret") + secret_id = secret_info.uri + juju.update_secret(secret_id, {keyname: private_key}) + else: + logger.error("Failed to add secret: %s %s", exc.stderr, exc.stdout) + raise + + juju.cli("grant-secret", secret_id, opendkim_app) + + keytable = [ + [ + f"{selector}._domainkey.{TEST_DOMAIN}", + f"{TEST_DOMAIN}:{selector}:/etc/dkimkeys/{keyname}.private", + ] + ] + signingtable = [[f"*@{TEST_DOMAIN}", f"{selector}._domainkey.{TEST_DOMAIN}"]] + juju.config( + opendkim_app, + { + "keytable": json.dumps(keytable), + "signingtable": json.dumps(signingtable), + "private-keys": secret_id, + "mode": "s", + }, + ) + + # Inject TEST_DOMAIN → test-runner IP in /etc/hosts on the postfix-relay unit + # so that Postfix can resolve the domain when it looks up the MX / transport. + status = juju.status() + relay_unit = next(iter(status.apps[postfix_relay_app].units.values())) + juju.exec( + machine=relay_unit.machine, + command=f"echo '{machine_ip_address} {TEST_DOMAIN}' | sudo tee -a /etc/hosts", + ) + + juju.wait( + lambda status: jubilant.all_active(status, opendkim_app, postfix_relay_app), + timeout=5 * 60, + delay=5, + ) + logger.info("opendkim configured and active with DKIM keys for %s", TEST_DOMAIN) + return opendkim_app + + +# --------------------------------------------------------------------------- +# Full stack fixture — depends on everything +# --------------------------------------------------------------------------- +@pytest.fixture(scope="session", name="mail_stack") +def mail_stack_fixture( + juju: jubilant.Juju, + dovecot_app: str, + postfix_relay_app: str, + opendkim_configured: str, + configurator_app: str, +) -> typing.Dict[str, str]: + """Ensure the complete mail stack is up and return a dict of app names and IPs. + + Returns a mapping with keys: + ``dovecot_app``, ``postfix_relay_app``, ``opendkim_app``, + ``configurator_app``, ``dovecot_ip``, ``postfix_relay_ip``. + """ + juju.wait( + lambda status: jubilant.all_active( + status, dovecot_app, postfix_relay_app, opendkim_configured, SELF_SIGNED_APP + ), + timeout=5 * 60, + ) + + status = juju.status() + dovecot_ip = next(iter(status.apps[dovecot_app].units.values())).public_address + relay_ip = next(iter(status.apps[postfix_relay_app].units.values())).public_address + + logger.info("Mail stack ready — dovecot: %s, postfix-relay: %s", dovecot_ip, relay_ip) + return { + "dovecot_app": dovecot_app, + "postfix_relay_app": postfix_relay_app, + "opendkim_app": opendkim_configured, + "configurator_app": configurator_app, + "dovecot_ip": dovecot_ip, + "postfix_relay_ip": relay_ip, + } + + +# --------------------------------------------------------------------------- +# postfix_stack fixture — for configurator-maps integration tests +# --------------------------------------------------------------------------- @pytest.fixture(scope="module", name="postfix_stack") def postfix_stack_fixture( juju: jubilant.Juju, pytestconfig: pytest.Config, ) -> typing.Dict[str, str]: - """Deploy postfix-relay + postfix-relay-configurator configured for sender_login enforcement. + """Deploy postfix-relay + postfix-relay-configurator for sender_login enforcement tests. Returns a dict with ``postfix_relay_ip``. """ @@ -118,7 +610,7 @@ def postfix_stack_fixture( "enable_reject_unknown_sender_domain": "false", }, ) - integrate_once( + _integrate_once( juju, f"{POSTFIX_RELAY_APP}:certificates", f"{SELF_SIGNED_APP}:certificates", @@ -137,7 +629,7 @@ def postfix_stack_fixture( "sender_login_maps": yaml.dump({authorized_sender: "testuser"}), }, ) - integrate_once( + _integrate_once( juju, f"{POSTFIX_RELAY_APP}:juju-info", f"{CONFIGURATOR_APP}:juju-info", @@ -166,3 +658,17 @@ def _both_active(status: jubilant.Status) -> bool: relay_ip = next(iter(status.apps[POSTFIX_RELAY_APP].units.values())).public_address logger.info("postfix-relay IP: %s", relay_ip) return {"postfix_relay_ip": relay_ip} + + +# --------------------------------------------------------------------------- +# Helper: idempotent juju integrate +# --------------------------------------------------------------------------- +def _integrate_once(juju: jubilant.Juju, endpoint_a: str, endpoint_b: str) -> None: + """Call ``juju integrate`` tolerating 'already related' errors.""" + try: + juju.integrate(endpoint_a, endpoint_b) + except Exception as exc: # noqa: BLE001 + msg = str(exc) + if "already exists" not in msg and "already related" not in msg: + raise + logger.debug("Relation %s ↔ %s already exists, skipping", endpoint_a, endpoint_b) diff --git a/tests/integration/test_full_stack.py b/tests/integration/test_full_stack.py new file mode 100644 index 0000000..d004f91 --- /dev/null +++ b/tests/integration/test_full_stack.py @@ -0,0 +1,447 @@ +# Copyright 2026 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Full-stack integration tests for the mailserver operators monorepo. + +These tests exercise the complete mail path: + + test runner ──SMTP AUTH/STARTTLS (port 587)──► postfix-relay + │ + milter ──► opendkim (DKIM sign) + │ + LMTP :24 ──► dovecot + │ + test runner ◄──IMAP SSL (port 993)────────────────────┘ + +All four charms must be pre-built and passed via repeated CLI options: + --charm-file= + --charm-file= + --charm-file= + --charm-file= + +A self-signed-certificates charm is pulled from CharmHub to provide TLS +for postfix-relay and dovecot. +""" + +import base64 +import contextlib +import email +import hashlib +import imaplib +import logging +import os +import smtplib +import ssl +import time +import typing + +import jubilant +import pytest +import requests + +# App name constants — kept in sync with conftest.py. +DOVECOT_APP = "dovecot-charm" +POSTFIX_RELAY_APP = "postfix-relay" +OPENDKIM_APP = "opendkim" +CONFIGURATOR_APP = "postfix-relay-configurator" +SELF_SIGNED_APP = "self-signed-certificates" +TEST_DOMAIN = "mailstack.internal" + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Test constants +# --------------------------------------------------------------------------- +IMAP_PORT = 993 +SMTP_SUBMISSION_PORT = 587 +METRICS_PORT = 9103 + +TEST_USER = "testuser" +TEST_PASSWORD = "TestP@ssw0rd!" # nosec B105 + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- +def _sha512_dovecot(password: str, salt: bytes | None = None) -> str: + """Return a Dovecot-compatible SSHA512 hash for *password*. + + This is the same algorithm used by postfix-relay's SMTP AUTH Dovecot + backend to validate credentials. + """ + if salt is None: + salt = os.urandom(8) + digest = hashlib.sha512(password.encode() + salt).digest() + return "{SSHA512}" + base64.b64encode(digest + salt).decode() + + +def _wait_for_imap_message( + host: str, + username: str, + password: str, + subject: str, + *, + retries: int = 20, + delay: float = 3.0, +) -> bytes: + """Poll dovecot via IMAP4_SSL until a message with *subject* arrives. + + Args: + host: Dovecot unit's public IP address. + username: IMAP login name. + password: IMAP login password. + subject: Expected Subject header value to search for. + retries: Number of polling attempts before giving up. + delay: Seconds to wait between attempts. + + Returns: + The raw RFC822 bytes of the first matching message. + + Raises: + pytest.fail: If the message is not found within *retries* attempts. + """ + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + + for attempt in range(retries): + try: + conn = imaplib.IMAP4_SSL(host, port=IMAP_PORT, ssl_context=ctx) + try: + conn.login(username, password) + conn.select("inbox") + _, data = conn.search(None, f'(HEADER Subject "{subject}")') + if data and data[0]: + msg_ids = data[0].split() + _, msg_data = conn.fetch(msg_ids[-1], "(RFC822)") + raw: bytes = msg_data[0][1] # type: ignore[index] + logger.info("Message found (attempt %d): %d bytes", attempt + 1, len(raw)) + return raw + logger.debug("Message not yet in INBOX (attempt %d/%d)", attempt + 1, retries) + finally: + with contextlib.suppress(Exception): + conn.close() + with contextlib.suppress(Exception): + conn.logout() + except Exception as exc: # noqa: BLE001 + logger.warning("IMAP attempt %d failed: %s", attempt + 1, exc) + + time.sleep(delay) + + pytest.fail(f"Message with subject '{subject}' never arrived in dovecot INBOX.") + + +def _setup_dovecot_user( + juju: jubilant.Juju, + dovecot_app: str, + username: str, + password: str, +) -> None: + """Create / update an OS user on the dovecot unit and add them to the mail group.""" + status = juju.status() + unit_name = next(iter(status.apps[dovecot_app].units)) + # Create user if missing, then set password and add to mail group. + juju.exec( + f"id -u {username} &>/dev/null || sudo useradd -m {username}", + unit=unit_name, + ) + juju.exec(f"sudo usermod -aG mail {username}", unit=unit_name) + juju.exec(f"echo '{username}:{password}' | sudo chpasswd", unit=unit_name) + logger.info("Dovecot OS user '%s' configured on %s", username, unit_name) + + +# --------------------------------------------------------------------------- +# Test 1: All apps reach active status +# --------------------------------------------------------------------------- +@pytest.mark.abort_on_fail +def test_stack_is_active(juju: jubilant.Juju, mail_stack: typing.Dict[str, str]) -> None: + """ + arrange: Deploy the full mail stack (postfix-relay, opendkim, dovecot, + postfix-relay-configurator, self-signed-certificates). + act: Wait for all applications to reach active status (done in fixtures). + assert: Every application in the stack reports active. + """ + status = juju.status() + + for app_name in ( + DOVECOT_APP, + POSTFIX_RELAY_APP, + OPENDKIM_APP, + SELF_SIGNED_APP, + ): + app = status.apps.get(app_name) + assert app is not None, f"Application '{app_name}' not found in model" + assert app.is_active, ( + f"Application '{app_name}' is not active: " + f"{app.app_status.current!r} — {app.app_status.message!r}" + ) + logger.info("✓ %s is active", app_name) + + # The configurator is a subordinate — its units live inside postfix-relay units. + relay_units = status.apps[POSTFIX_RELAY_APP].units + for unit_name, unit in relay_units.items(): + subordinates = unit.subordinates or {} + assert any(CONFIGURATOR_APP in sub_name for sub_name in subordinates), ( + f"postfix-relay-configurator subordinate not found on unit {unit_name}" + ) + logger.info("✓ %s subordinate present on all postfix-relay units", CONFIGURATOR_APP) + + +# --------------------------------------------------------------------------- +# Test 2: End-to-end send → DKIM sign → deliver → IMAP retrieve +# --------------------------------------------------------------------------- +@pytest.mark.abort_on_fail +def test_send_and_receive_with_dkim( + juju: jubilant.Juju, + mail_stack: typing.Dict[str, str], +) -> None: + """ + arrange: Full mail stack is active. A local OS user exists on dovecot. + postfix-relay has SMTP AUTH enabled with the test user's credentials. + opendkim is configured to sign mail from TEST_DOMAIN. + The configurator routes TEST_DOMAIN mail to dovecot via LMTP. + act: Send an email via SMTP AUTH (port 587 + STARTTLS) to + testuser@mailstack.internal through postfix-relay. + assert: + - The message is delivered to the dovecot mailbox. + - The raw RFC822 message contains a DKIM-Signature header. + - The Subject and From headers match what was sent. + """ + relay_ip = mail_stack["postfix_relay_ip"] + dovecot_ip = mail_stack["dovecot_ip"] + + # 1. Create OS user on dovecot for IMAP login. + _setup_dovecot_user(juju, DOVECOT_APP, TEST_USER, TEST_PASSWORD) + + # 2. Configure SMTP AUTH credentials on postfix-relay. + import yaml # noqa: PLC0415 + + hashed = _sha512_dovecot(TEST_PASSWORD) + auth_users_yaml = yaml.dump([f"{TEST_USER}:{hashed}"]) + juju.config( + POSTFIX_RELAY_APP, + { + "enable_smtp_auth": "true", + "smtp_auth_users": auth_users_yaml, + }, + ) + juju.wait( + lambda status: status.apps[POSTFIX_RELAY_APP].is_active, + error=jubilant.any_error, + timeout=5 * 60, + ) + + subject = f"Full-stack DKIM test {int(time.time())}" + from_addr = f"sender@{TEST_DOMAIN}" + to_addr = f"{TEST_USER}@{TEST_DOMAIN}" + body = ( + f"Subject: {subject}\r\n" + f"From: {from_addr}\r\n" + f"To: {to_addr}\r\n" + f"\r\n" + f"This message was sent through the full mailserver stack.\r\n" + ) + + # 3. Send via postfix-relay with SMTP AUTH + STARTTLS. + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + + with smtplib.SMTP(relay_ip, SMTP_SUBMISSION_PORT, timeout=30) as server: + server.set_debuglevel(1) + server.ehlo() + server.starttls(context=ctx) + server.ehlo() + server.login(TEST_USER, TEST_PASSWORD) + server.sendmail(from_addr, [to_addr], body) + logger.info("Message submitted to postfix-relay at %s:%d", relay_ip, SMTP_SUBMISSION_PORT) + + # 4. Poll dovecot IMAP until the message lands in the inbox. + raw_message = _wait_for_imap_message( + dovecot_ip, + TEST_USER, + TEST_PASSWORD, + subject, + ) + + # 5. Parse and assert. + msg = email.message_from_bytes(raw_message) + logger.info("Received headers: %s", dict(msg.items())) + + assert "DKIM-Signature" in msg, ( + "DKIM-Signature header missing — opendkim did not sign the message.\n" + f"Headers present: {list(msg.keys())}" + ) + assert msg["Subject"] == subject, ( + f"Subject mismatch: expected {subject!r}, got {msg['Subject']!r}" + ) + assert TEST_DOMAIN in msg.get("DKIM-Signature", ""), ( + f"DKIM-Signature does not reference domain {TEST_DOMAIN!r}.\n" + f"DKIM-Signature: {msg.get('DKIM-Signature')}" + ) + logger.info("✓ DKIM-signed message delivered and verified via IMAP") + + +# --------------------------------------------------------------------------- +# Test 3: Unauthenticated SMTP is rejected on port 587 +# --------------------------------------------------------------------------- +@pytest.mark.abort_on_fail +def test_unauthenticated_smtp_rejected( + juju: jubilant.Juju, + mail_stack: typing.Dict[str, str], +) -> None: + """ + arrange: postfix-relay has SMTP AUTH enabled (enable_smtp_auth=true). + act: Attempt to relay mail on port 587 to an external domain without + providing AUTH credentials. + assert: The server refuses the recipient (SMTPRecipientsRefused or + SMTPSenderRefused), indicating that unauthenticated relay to + external destinations is blocked. + """ + relay_ip = mail_stack["postfix_relay_ip"] + + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + + # Send to a domain that is NOT in relay_domains — this should be deferred/rejected + # for unauthenticated clients by the defer_unauth_destination restriction. + with pytest.raises((smtplib.SMTPRecipientsRefused, smtplib.SMTPSenderRefused)): + with smtplib.SMTP(relay_ip, SMTP_SUBMISSION_PORT, timeout=15) as server: + server.ehlo() + server.starttls(context=ctx) + server.ehlo() + # No server.login() — deliberately unauthenticated + server.sendmail( + from_addr=f"attacker@{TEST_DOMAIN}", + to_addrs=["victim@external.example.com"], + msg="Subject: Spam\r\n\r\nShouldBeRejected", + ) + + logger.info("✓ Unauthenticated relay correctly refused on port %d", SMTP_SUBMISSION_PORT) + + +# --------------------------------------------------------------------------- +# Test 4: opendkim blocks when given an invalid key path +# --------------------------------------------------------------------------- +def test_dkim_invalid_key_blocks_opendkim( + juju: jubilant.Juju, + mail_stack: typing.Dict[str, str], +) -> None: + """ + arrange: opendkim is active with a valid DKIM key configuration. + act: Reconfigure the keytable to point to a non-existent key file. + assert: opendkim transitions to blocked with a configuration error message. + act (restore): Restore the original valid configuration. + assert: opendkim returns to active. + """ + import json # noqa: PLC0415 + + opendkim_app = mail_stack["opendkim_app"] + + selector = "default" + keyname = f"{TEST_DOMAIN.replace('.', '-')}-{selector}" + + # Save the current (valid) config so we can restore it. + current_config = juju.config(opendkim_app) + valid_keytable = current_config.get("keytable", "") + valid_signingtable = current_config.get("signingtable", "") + + # Apply a broken keytable pointing to a non-existent file. + broken_keytable = json.dumps( + [ + [ + f"{selector}._domainkey.{TEST_DOMAIN}", + f"{TEST_DOMAIN}:{selector}:/etc/dkimkeys/DOESNOTEXIST.private", + ] + ] + ) + juju.config(opendkim_app, {"keytable": broken_keytable}) + + juju.wait( + lambda status: status.apps[opendkim_app].is_blocked, + timeout=3 * 60, + delay=5, + ) + status = juju.status() + blocked_message = status.apps[opendkim_app].app_status.message + assert "opendkim" in blocked_message.lower() or "configuration" in blocked_message.lower(), ( + f"Expected a configuration-related blocked message, got: {blocked_message!r}" + ) + logger.info("✓ opendkim blocked with message: %s", blocked_message) + + # Restore the valid configuration. + juju.config(opendkim_app, {"keytable": valid_keytable}) + juju.wait( + lambda status: jubilant.all_active(status, opendkim_app, POSTFIX_RELAY_APP), + timeout=3 * 60, + delay=5, + ) + logger.info("✓ opendkim restored to active after valid keytable re-applied") + + +# --------------------------------------------------------------------------- +# Test 5: Metrics endpoints are reachable on all charms +# --------------------------------------------------------------------------- +def test_metrics_endpoints( + juju: jubilant.Juju, + mail_stack: typing.Dict[str, str], +) -> None: + """ + arrange: Full mail stack is active. + act: Scrape the Telegraf metrics endpoint on postfix-relay and opendkim. + assert: Each endpoint responds with HTTP 200 and contains expected metric names. + """ + relay_ip = mail_stack["postfix_relay_ip"] + + status = juju.status() + opendkim_ip = next(iter(status.apps[OPENDKIM_APP].units.values())).public_address + + expected_relay_metrics = [ + "cpu_usage_idle", + "postfix_queue_length", + "procstat_lookup_running", + ] + expected_opendkim_metrics = ["cpu_usage_idle", "procstat_lookup_running"] + + for ip, expected in ( + (relay_ip, expected_relay_metrics), + (opendkim_ip, expected_opendkim_metrics), + ): + url = f"http://{ip}:{METRICS_PORT}/metrics" + resp = requests.get(url, timeout=10) + assert resp.status_code == 200, f"Metrics endpoint {url} returned {resp.status_code}" + for metric in expected: + assert metric in resp.text, ( + f"Expected metric {metric!r} not found in response from {url}" + ) + logger.info("✓ Metrics OK at %s", url) + + +# --------------------------------------------------------------------------- +# Test 6: TLS certificate is presented by postfix-relay on port 587 +# --------------------------------------------------------------------------- +def test_tls_certificate_presented( + juju: jubilant.Juju, + mail_stack: typing.Dict[str, str], +) -> None: + """ + arrange: postfix-relay is related to self-signed-certificates. + act: Initiate a STARTTLS handshake on port 587. + assert: A TLS certificate is presented by the server. + """ + import ssl as _ssl # noqa: PLC0415 + + relay_ip = mail_stack["postfix_relay_ip"] + + ctx = _ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = _ssl.CERT_NONE + + with smtplib.SMTP(relay_ip, SMTP_SUBMISSION_PORT, timeout=15) as server: + server.ehlo() + server.starttls(context=ctx) + peer_cert = typing.cast(_ssl.SSLSocket, server.sock).getpeercert(binary_form=True) + + assert peer_cert, "No TLS certificate was presented by postfix-relay on STARTTLS" + logger.info("✓ TLS certificate presented by postfix-relay (%d bytes)", len(peer_cert)) diff --git a/tests/integration/test_whole_email_system.py b/tests/integration/test_whole_email_system.py new file mode 100644 index 0000000..565a5ff --- /dev/null +++ b/tests/integration/test_whole_email_system.py @@ -0,0 +1,128 @@ +# Copyright 2026 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Single end-to-end test for the full mail system.""" + +import base64 +import contextlib +import email +import hashlib +import imaplib +import os +import smtplib +import ssl +import time +from typing import Dict + +import jubilant +import pytest +import yaml + +TEST_DOMAIN = "mailstack.internal" +TEST_USER = "e2euser" +MAILBOX_USER = f"{TEST_USER}@{TEST_DOMAIN}" +TEST_PASSWORD = "E2eP@ssw0rd!" # nosec B105 +SMTP_SUBMISSION_PORT = 587 +IMAP_PORT = 993 + + +def _sha512_dovecot(password: str, salt: bytes | None = None) -> str: + if salt is None: + salt = os.urandom(8) + digest = hashlib.sha512(password.encode() + salt).digest() + return "{SSHA512}" + base64.b64encode(digest + salt).decode() + + +def _setup_dovecot_user(juju: jubilant.Juju, username: str, password: str) -> None: + status = juju.status() + unit_name = next(iter(status.apps["dovecot-charm"].units)) + juju.exec(f"id -u {username} &>/dev/null || sudo useradd -m {username}", unit=unit_name) + juju.exec( + f"id -u {MAILBOX_USER} &>/dev/null || sudo useradd --badname -m {MAILBOX_USER}", + unit=unit_name, + ) + juju.exec(f"sudo usermod -aG mail {username}", unit=unit_name) + juju.exec(f"sudo usermod -aG mail {MAILBOX_USER}", unit=unit_name) + juju.exec(f"echo '{username}:{password}' | sudo chpasswd", unit=unit_name) + juju.exec(f"echo '{MAILBOX_USER}:{password}' | sudo chpasswd", unit=unit_name) + + +def _wait_for_subject(host: str, username: str, password: str, subject: str) -> bytes: + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + + deadline = time.time() + 120 + while time.time() < deadline: + try: + conn = imaplib.IMAP4_SSL(host, port=IMAP_PORT, ssl_context=ctx) + try: + conn.login(username, password) + conn.select("inbox") + _, data = conn.search(None, f'(HEADER Subject "{subject}")') + if data and data[0]: + msg_id = data[0].split()[-1] + _, msg_data = conn.fetch(msg_id, "(RFC822)") + return msg_data[0][1] # type: ignore[index] + finally: + with contextlib.suppress(Exception): + conn.close() + with contextlib.suppress(Exception): + conn.logout() + except Exception: + pass + time.sleep(3) + + pytest.fail(f"Message with subject '{subject}' did not arrive in IMAP inbox") + + +@pytest.mark.abort_on_fail +def test_whole_email_system_e2e(juju: jubilant.Juju, mail_stack: Dict[str, str]) -> None: + """Verify SMTP AUTH -> DKIM signing -> LMTP delivery -> IMAP retrieval.""" + relay_ip = mail_stack["postfix_relay_ip"] + dovecot_ip = mail_stack["dovecot_ip"] + + _setup_dovecot_user(juju, TEST_USER, TEST_PASSWORD) + + smtp_auth_users = yaml.dump([f"{TEST_USER}:{_sha512_dovecot(TEST_PASSWORD)}"]) + juju.config( + "postfix-relay", + { + "enable_smtp_auth": "true", + "smtp_auth_users": smtp_auth_users, + }, + ) + juju.wait( + lambda status: status.apps["postfix-relay"].is_active, + error=jubilant.any_error, + timeout=5 * 60, + ) + + subject = f"Whole system e2e {int(time.time())}" + from_addr = f"sender@{TEST_DOMAIN}" + to_addr = MAILBOX_USER + message = ( + f"Subject: {subject}\r\n" + f"From: {from_addr}\r\n" + f"To: {to_addr}\r\n" + "\r\n" + "full system integration test\r\n" + ) + + tls_ctx = ssl.create_default_context() + tls_ctx.check_hostname = False + tls_ctx.verify_mode = ssl.CERT_NONE + + with smtplib.SMTP(relay_ip, SMTP_SUBMISSION_PORT, timeout=30) as server: + server.ehlo() + server.starttls(context=tls_ctx) + server.ehlo() + server.login(TEST_USER, TEST_PASSWORD) + server.sendmail(from_addr, [to_addr], message) + + raw_message = _wait_for_subject(dovecot_ip, MAILBOX_USER, TEST_PASSWORD, subject) + parsed = email.message_from_bytes(raw_message) + + assert parsed["Subject"] == subject + assert "DKIM-Signature" in parsed + assert TEST_DOMAIN in parsed.get("DKIM-Signature", "") diff --git a/tox.toml b/tox.toml index 419aafb..e3f75a4 100644 --- a/tox.toml +++ b/tox.toml @@ -125,6 +125,25 @@ commands = [ ] dependency_groups = [ "integration" ] +[env.stack-integration] +description = "Run full-stack mailserver integration tests (all charms)" +commands = [ + [ + "pytest", + "-v", + "--tb", "short", + "--log-cli-level=INFO", + "-s", + "{toxinidir}/tests/integration", + { replace = "posargs", extend = true }, + ], +] +dependency_groups = [ "integration" ] + +[env.stack-integration.setenv] +PYTHONPATH = "{toxinidir}/tests/integration" +PY_COLORS = "1" + [env.lint-fix] description = "Apply coding style standards to code" commands = [ diff --git a/uv.lock b/uv.lock index c14df9b..ba5e550 100644 --- a/uv.lock +++ b/uv.lock @@ -70,6 +70,63 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/70/7d/9bc192684cea499815ff478dfcdc13835ddf401365057044fb721ec6bddb/certifi-2025.11.12-py3-none-any.whl", hash = "sha256:97de8790030bbd5c2d96b7ec782fc2f7820ef8dba6db909ccf95449f2d062d4b", size = 159438, upload-time = "2025-11-12T02:54:49.735Z" }, ] +[[package]] +name = "cffi" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser", marker = "implementation_name != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588, upload-time = "2025-09-08T23:24:04.541Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ea/47/4f61023ea636104d4f16ab488e268b93008c3d0bb76893b1b31db1f96802/cffi-2.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d", size = 185271, upload-time = "2025-09-08T23:22:44.795Z" }, + { url = "https://files.pythonhosted.org/packages/df/a2/781b623f57358e360d62cdd7a8c681f074a71d445418a776eef0aadb4ab4/cffi-2.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c", size = 181048, upload-time = "2025-09-08T23:22:45.938Z" }, + { url = "https://files.pythonhosted.org/packages/ff/df/a4f0fbd47331ceeba3d37c2e51e9dfc9722498becbeec2bd8bc856c9538a/cffi-2.0.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe", size = 212529, upload-time = "2025-09-08T23:22:47.349Z" }, + { url = "https://files.pythonhosted.org/packages/d5/72/12b5f8d3865bf0f87cf1404d8c374e7487dcf097a1c91c436e72e6badd83/cffi-2.0.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062", size = 220097, upload-time = "2025-09-08T23:22:48.677Z" }, + { url = "https://files.pythonhosted.org/packages/c2/95/7a135d52a50dfa7c882ab0ac17e8dc11cec9d55d2c18dda414c051c5e69e/cffi-2.0.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e", size = 207983, upload-time = "2025-09-08T23:22:50.06Z" }, + { url = "https://files.pythonhosted.org/packages/3a/c8/15cb9ada8895957ea171c62dc78ff3e99159ee7adb13c0123c001a2546c1/cffi-2.0.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037", size = 206519, upload-time = "2025-09-08T23:22:51.364Z" }, + { url = "https://files.pythonhosted.org/packages/78/2d/7fa73dfa841b5ac06c7b8855cfc18622132e365f5b81d02230333ff26e9e/cffi-2.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba", size = 219572, upload-time = "2025-09-08T23:22:52.902Z" }, + { url = "https://files.pythonhosted.org/packages/07/e0/267e57e387b4ca276b90f0434ff88b2c2241ad72b16d31836adddfd6031b/cffi-2.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94", size = 222963, upload-time = "2025-09-08T23:22:54.518Z" }, + { url = "https://files.pythonhosted.org/packages/b6/75/1f2747525e06f53efbd878f4d03bac5b859cbc11c633d0fb81432d98a795/cffi-2.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187", size = 221361, upload-time = "2025-09-08T23:22:55.867Z" }, + { url = "https://files.pythonhosted.org/packages/7b/2b/2b6435f76bfeb6bbf055596976da087377ede68df465419d192acf00c437/cffi-2.0.0-cp312-cp312-win32.whl", hash = "sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18", size = 172932, upload-time = "2025-09-08T23:22:57.188Z" }, + { url = "https://files.pythonhosted.org/packages/f8/ed/13bd4418627013bec4ed6e54283b1959cf6db888048c7cf4b4c3b5b36002/cffi-2.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5", size = 183557, upload-time = "2025-09-08T23:22:58.351Z" }, + { url = "https://files.pythonhosted.org/packages/95/31/9f7f93ad2f8eff1dbc1c3656d7ca5bfd8fb52c9d786b4dcf19b2d02217fa/cffi-2.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6", size = 177762, upload-time = "2025-09-08T23:22:59.668Z" }, + { url = "https://files.pythonhosted.org/packages/4b/8d/a0a47a0c9e413a658623d014e91e74a50cdd2c423f7ccfd44086ef767f90/cffi-2.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:00bdf7acc5f795150faa6957054fbbca2439db2f775ce831222b66f192f03beb", size = 185230, upload-time = "2025-09-08T23:23:00.879Z" }, + { url = "https://files.pythonhosted.org/packages/4a/d2/a6c0296814556c68ee32009d9c2ad4f85f2707cdecfd7727951ec228005d/cffi-2.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:45d5e886156860dc35862657e1494b9bae8dfa63bf56796f2fb56e1679fc0bca", size = 181043, upload-time = "2025-09-08T23:23:02.231Z" }, + { url = "https://files.pythonhosted.org/packages/b0/1e/d22cc63332bd59b06481ceaac49d6c507598642e2230f201649058a7e704/cffi-2.0.0-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b", size = 212446, upload-time = "2025-09-08T23:23:03.472Z" }, + { url = "https://files.pythonhosted.org/packages/a9/f5/a2c23eb03b61a0b8747f211eb716446c826ad66818ddc7810cc2cc19b3f2/cffi-2.0.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d48a880098c96020b02d5a1f7d9251308510ce8858940e6fa99ece33f610838b", size = 220101, upload-time = "2025-09-08T23:23:04.792Z" }, + { url = "https://files.pythonhosted.org/packages/f2/7f/e6647792fc5850d634695bc0e6ab4111ae88e89981d35ac269956605feba/cffi-2.0.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f93fd8e5c8c0a4aa1f424d6173f14a892044054871c771f8566e4008eaa359d2", size = 207948, upload-time = "2025-09-08T23:23:06.127Z" }, + { url = "https://files.pythonhosted.org/packages/cb/1e/a5a1bd6f1fb30f22573f76533de12a00bf274abcdc55c8edab639078abb6/cffi-2.0.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:dd4f05f54a52fb558f1ba9f528228066954fee3ebe629fc1660d874d040ae5a3", size = 206422, upload-time = "2025-09-08T23:23:07.753Z" }, + { url = "https://files.pythonhosted.org/packages/98/df/0a1755e750013a2081e863e7cd37e0cdd02664372c754e5560099eb7aa44/cffi-2.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c8d3b5532fc71b7a77c09192b4a5a200ea992702734a2e9279a37f2478236f26", size = 219499, upload-time = "2025-09-08T23:23:09.648Z" }, + { url = "https://files.pythonhosted.org/packages/50/e1/a969e687fcf9ea58e6e2a928ad5e2dd88cc12f6f0ab477e9971f2309b57c/cffi-2.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d9b29c1f0ae438d5ee9acb31cadee00a58c46cc9c0b2f9038c6b0b3470877a8c", size = 222928, upload-time = "2025-09-08T23:23:10.928Z" }, + { url = "https://files.pythonhosted.org/packages/36/54/0362578dd2c9e557a28ac77698ed67323ed5b9775ca9d3fe73fe191bb5d8/cffi-2.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6d50360be4546678fc1b79ffe7a66265e28667840010348dd69a314145807a1b", size = 221302, upload-time = "2025-09-08T23:23:12.42Z" }, + { url = "https://files.pythonhosted.org/packages/eb/6d/bf9bda840d5f1dfdbf0feca87fbdb64a918a69bca42cfa0ba7b137c48cb8/cffi-2.0.0-cp313-cp313-win32.whl", hash = "sha256:74a03b9698e198d47562765773b4a8309919089150a0bb17d829ad7b44b60d27", size = 172909, upload-time = "2025-09-08T23:23:14.32Z" }, + { url = "https://files.pythonhosted.org/packages/37/18/6519e1ee6f5a1e579e04b9ddb6f1676c17368a7aba48299c3759bbc3c8b3/cffi-2.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:19f705ada2530c1167abacb171925dd886168931e0a7b78f5bffcae5c6b5be75", size = 183402, upload-time = "2025-09-08T23:23:15.535Z" }, + { url = "https://files.pythonhosted.org/packages/cb/0e/02ceeec9a7d6ee63bb596121c2c8e9b3a9e150936f4fbef6ca1943e6137c/cffi-2.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:256f80b80ca3853f90c21b23ee78cd008713787b1b1e93eae9f3d6a7134abd91", size = 177780, upload-time = "2025-09-08T23:23:16.761Z" }, + { url = "https://files.pythonhosted.org/packages/92/c4/3ce07396253a83250ee98564f8d7e9789fab8e58858f35d07a9a2c78de9f/cffi-2.0.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:fc33c5141b55ed366cfaad382df24fe7dcbc686de5be719b207bb248e3053dc5", size = 185320, upload-time = "2025-09-08T23:23:18.087Z" }, + { url = "https://files.pythonhosted.org/packages/59/dd/27e9fa567a23931c838c6b02d0764611c62290062a6d4e8ff7863daf9730/cffi-2.0.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c654de545946e0db659b3400168c9ad31b5d29593291482c43e3564effbcee13", size = 181487, upload-time = "2025-09-08T23:23:19.622Z" }, + { url = "https://files.pythonhosted.org/packages/d6/43/0e822876f87ea8a4ef95442c3d766a06a51fc5298823f884ef87aaad168c/cffi-2.0.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:24b6f81f1983e6df8db3adc38562c83f7d4a0c36162885ec7f7b77c7dcbec97b", size = 220049, upload-time = "2025-09-08T23:23:20.853Z" }, + { url = "https://files.pythonhosted.org/packages/b4/89/76799151d9c2d2d1ead63c2429da9ea9d7aac304603de0c6e8764e6e8e70/cffi-2.0.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:12873ca6cb9b0f0d3a0da705d6086fe911591737a59f28b7936bdfed27c0d47c", size = 207793, upload-time = "2025-09-08T23:23:22.08Z" }, + { url = "https://files.pythonhosted.org/packages/bb/dd/3465b14bb9e24ee24cb88c9e3730f6de63111fffe513492bf8c808a3547e/cffi-2.0.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:d9b97165e8aed9272a6bb17c01e3cc5871a594a446ebedc996e2397a1c1ea8ef", size = 206300, upload-time = "2025-09-08T23:23:23.314Z" }, + { url = "https://files.pythonhosted.org/packages/47/d9/d83e293854571c877a92da46fdec39158f8d7e68da75bf73581225d28e90/cffi-2.0.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:afb8db5439b81cf9c9d0c80404b60c3cc9c3add93e114dcae767f1477cb53775", size = 219244, upload-time = "2025-09-08T23:23:24.541Z" }, + { url = "https://files.pythonhosted.org/packages/2b/0f/1f177e3683aead2bb00f7679a16451d302c436b5cbf2505f0ea8146ef59e/cffi-2.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:737fe7d37e1a1bffe70bd5754ea763a62a066dc5913ca57e957824b72a85e205", size = 222828, upload-time = "2025-09-08T23:23:26.143Z" }, + { url = "https://files.pythonhosted.org/packages/c6/0f/cafacebd4b040e3119dcb32fed8bdef8dfe94da653155f9d0b9dc660166e/cffi-2.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:38100abb9d1b1435bc4cc340bb4489635dc2f0da7456590877030c9b3d40b0c1", size = 220926, upload-time = "2025-09-08T23:23:27.873Z" }, + { url = "https://files.pythonhosted.org/packages/3e/aa/df335faa45b395396fcbc03de2dfcab242cd61a9900e914fe682a59170b1/cffi-2.0.0-cp314-cp314-win32.whl", hash = "sha256:087067fa8953339c723661eda6b54bc98c5625757ea62e95eb4898ad5e776e9f", size = 175328, upload-time = "2025-09-08T23:23:44.61Z" }, + { url = "https://files.pythonhosted.org/packages/bb/92/882c2d30831744296ce713f0feb4c1cd30f346ef747b530b5318715cc367/cffi-2.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:203a48d1fb583fc7d78a4c6655692963b860a417c0528492a6bc21f1aaefab25", size = 185650, upload-time = "2025-09-08T23:23:45.848Z" }, + { url = "https://files.pythonhosted.org/packages/9f/2c/98ece204b9d35a7366b5b2c6539c350313ca13932143e79dc133ba757104/cffi-2.0.0-cp314-cp314-win_arm64.whl", hash = "sha256:dbd5c7a25a7cb98f5ca55d258b103a2054f859a46ae11aaf23134f9cc0d356ad", size = 180687, upload-time = "2025-09-08T23:23:47.105Z" }, + { url = "https://files.pythonhosted.org/packages/3e/61/c768e4d548bfa607abcda77423448df8c471f25dbe64fb2ef6d555eae006/cffi-2.0.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:9a67fc9e8eb39039280526379fb3a70023d77caec1852002b4da7e8b270c4dd9", size = 188773, upload-time = "2025-09-08T23:23:29.347Z" }, + { url = "https://files.pythonhosted.org/packages/2c/ea/5f76bce7cf6fcd0ab1a1058b5af899bfbef198bea4d5686da88471ea0336/cffi-2.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7a66c7204d8869299919db4d5069a82f1561581af12b11b3c9f48c584eb8743d", size = 185013, upload-time = "2025-09-08T23:23:30.63Z" }, + { url = "https://files.pythonhosted.org/packages/be/b4/c56878d0d1755cf9caa54ba71e5d049479c52f9e4afc230f06822162ab2f/cffi-2.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7cc09976e8b56f8cebd752f7113ad07752461f48a58cbba644139015ac24954c", size = 221593, upload-time = "2025-09-08T23:23:31.91Z" }, + { url = "https://files.pythonhosted.org/packages/e0/0d/eb704606dfe8033e7128df5e90fee946bbcb64a04fcdaa97321309004000/cffi-2.0.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:92b68146a71df78564e4ef48af17551a5ddd142e5190cdf2c5624d0c3ff5b2e8", size = 209354, upload-time = "2025-09-08T23:23:33.214Z" }, + { url = "https://files.pythonhosted.org/packages/d8/19/3c435d727b368ca475fb8742ab97c9cb13a0de600ce86f62eab7fa3eea60/cffi-2.0.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b1e74d11748e7e98e2f426ab176d4ed720a64412b6a15054378afdb71e0f37dc", size = 208480, upload-time = "2025-09-08T23:23:34.495Z" }, + { url = "https://files.pythonhosted.org/packages/d0/44/681604464ed9541673e486521497406fadcc15b5217c3e326b061696899a/cffi-2.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:28a3a209b96630bca57cce802da70c266eb08c6e97e5afd61a75611ee6c64592", size = 221584, upload-time = "2025-09-08T23:23:36.096Z" }, + { url = "https://files.pythonhosted.org/packages/25/8e/342a504ff018a2825d395d44d63a767dd8ebc927ebda557fecdaca3ac33a/cffi-2.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7553fb2090d71822f02c629afe6042c299edf91ba1bf94951165613553984512", size = 224443, upload-time = "2025-09-08T23:23:37.328Z" }, + { url = "https://files.pythonhosted.org/packages/e1/5e/b666bacbbc60fbf415ba9988324a132c9a7a0448a9a8f125074671c0f2c3/cffi-2.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6c6c373cfc5c83a975506110d17457138c8c63016b563cc9ed6e056a82f13ce4", size = 223437, upload-time = "2025-09-08T23:23:38.945Z" }, + { url = "https://files.pythonhosted.org/packages/a0/1d/ec1a60bd1a10daa292d3cd6bb0b359a81607154fb8165f3ec95fe003b85c/cffi-2.0.0-cp314-cp314t-win32.whl", hash = "sha256:1fc9ea04857caf665289b7a75923f2c6ed559b8298a1b8c49e59f7dd95c8481e", size = 180487, upload-time = "2025-09-08T23:23:40.423Z" }, + { url = "https://files.pythonhosted.org/packages/bf/41/4c1168c74fac325c0c8156f04b6749c8b6a8f405bbf91413ba088359f60d/cffi-2.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:d68b6cef7827e8641e8ef16f4494edda8b36104d79773a334beaa1e3521430f6", size = 191726, upload-time = "2025-09-08T23:23:41.742Z" }, + { url = "https://files.pythonhosted.org/packages/ae/3a/dbeec9d1ee0844c679f6bb5d6ad4e9f198b1224f4e7a32825f47f6192b0c/cffi-2.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9", size = 184195, upload-time = "2025-09-08T23:23:43.004Z" }, +] + [[package]] name = "charset-normalizer" version = "3.4.4" @@ -219,6 +276,59 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/19/8f/92bdd27b067204b99f396a1414d6342122f3e2663459baf787108a6b8b84/coverage-7.11.3-py3-none-any.whl", hash = "sha256:351511ae28e2509c8d8cae5311577ea7dd511ab8e746ffc8814a0896c3d33fbe", size = 208478, upload-time = "2025-11-10T00:13:14.908Z" }, ] +[[package]] +name = "cryptography" +version = "47.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ef/b2/7ffa7fe8207a8c42147ffe70c3e360b228160c1d85dc3faff16aaa3244c0/cryptography-47.0.0.tar.gz", hash = "sha256:9f8e55fe4e63613a5e1cc5819030f27b97742d720203a087802ce4ce9ceb52bb", size = 830863, upload-time = "2026-04-24T19:54:57.056Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a4/98/40dfe932134bdcae4f6ab5927c87488754bf9eb79297d7e0070b78dd58e9/cryptography-47.0.0-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:160ad728f128972d362e714054f6ba0067cab7fb350c5202a9ae8ae4ce3ef1a0", size = 7912214, upload-time = "2026-04-24T19:53:03.864Z" }, + { url = "https://files.pythonhosted.org/packages/34/c6/2733531243fba725f58611b918056b277692f1033373dcc8bd01af1c05d4/cryptography-47.0.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b9a8943e359b7615db1a3ba587994618e094ff3d6fa5a390c73d079ce18b3973", size = 4644617, upload-time = "2026-04-24T19:53:06.909Z" }, + { url = "https://files.pythonhosted.org/packages/00/e3/b27be1a670a9b87f855d211cf0e1174a5d721216b7616bd52d8581d912ed/cryptography-47.0.0-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f5c15764f261394b22aef6b00252f5195f46f2ca300bec57149474e2538b31f8", size = 4668186, upload-time = "2026-04-24T19:53:09.053Z" }, + { url = "https://files.pythonhosted.org/packages/81/b9/8443cfe5d17d482d348cee7048acf502bb89a51b6382f06240fd290d4ca3/cryptography-47.0.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:9c59ab0e0fa3a180a5a9c59f3a5abe3ef90d474bc56d7fadfbe80359491b615b", size = 4651244, upload-time = "2026-04-24T19:53:11.217Z" }, + { url = "https://files.pythonhosted.org/packages/5d/5e/13ed0cdd0eb88ba159d6dd5ebfece8cb901dbcf1ae5ac4072e28b55d3153/cryptography-47.0.0-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:34b4358b925a5ea3e14384ca781a2c0ef7ac219b57bb9eacc4457078e2b19f92", size = 5252906, upload-time = "2026-04-24T19:53:13.532Z" }, + { url = "https://files.pythonhosted.org/packages/64/16/ed058e1df0f33d440217cd120d41d5dda9dd215a80b8187f68483185af82/cryptography-47.0.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:0024b87d47ae2399165a6bfb20d24888881eeab83ae2566d62467c5ff0030ce7", size = 4701842, upload-time = "2026-04-24T19:53:15.618Z" }, + { url = "https://files.pythonhosted.org/packages/02/e0/3d30986b30fdbd9e969abbdf8ba00ed0618615144341faeb57f395a084fe/cryptography-47.0.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:1e47422b5557bb82d3fff997e8d92cff4e28b9789576984f08c248d2b3535d93", size = 4289313, upload-time = "2026-04-24T19:53:17.755Z" }, + { url = "https://files.pythonhosted.org/packages/df/fd/32db38e3ad0cb331f0691cb4c7a8a6f176f679124dee746b3af6633db4d9/cryptography-47.0.0-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:6f29f36582e6151d9686235e586dd35bb67491f024767d10b842e520dc6a07ac", size = 4650964, upload-time = "2026-04-24T19:53:20.062Z" }, + { url = "https://files.pythonhosted.org/packages/86/53/5395d944dfd48cb1f67917f533c609c34347185ef15eb4308024c876f274/cryptography-47.0.0-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:a9b761f012a943b7de0e828843c5688d0de94a0578d44d6c85a1bae32f87791f", size = 5207817, upload-time = "2026-04-24T19:53:22.498Z" }, + { url = "https://files.pythonhosted.org/packages/34/4f/e5711b28e1901f7d480a2b1b688b645aa4c77c73f10731ed17e7f7db3f0d/cryptography-47.0.0-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:4e1de79e047e25d6e9f8cea71c86b4a53aced64134f0f003bbcbf3655fd172c8", size = 4701544, upload-time = "2026-04-24T19:53:24.356Z" }, + { url = "https://files.pythonhosted.org/packages/22/22/c8ddc25de3010fc8da447648f5a092c40e7a8fadf01dd6d255d9c0b9373d/cryptography-47.0.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:ef6b3634087f18d2155b1e8ce264e5345a753da2c5fa9815e7d41315c90f8318", size = 4783536, upload-time = "2026-04-24T19:53:26.665Z" }, + { url = "https://files.pythonhosted.org/packages/66/b6/d4a68f4ea999c6d89e8498579cba1c5fcba4276284de7773b17e4fa69293/cryptography-47.0.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:11dbb9f50a0f1bb9757b3d8c27c1101780efb8f0bdecfb12439c22a74d64c001", size = 4926106, upload-time = "2026-04-24T19:53:28.686Z" }, + { url = "https://files.pythonhosted.org/packages/54/ed/5f524db1fade9c013aa618e1c99c6ed05e8ffc9ceee6cda22fed22dda3f4/cryptography-47.0.0-cp311-abi3-win32.whl", hash = "sha256:7fda2f02c9015db3f42bb8a22324a454516ed10a8c29ca6ece6cdbb5efe2a203", size = 3258581, upload-time = "2026-04-24T19:53:31.058Z" }, + { url = "https://files.pythonhosted.org/packages/b2/dc/1b901990b174786569029f67542b3edf72ac068b6c3c8683c17e6a2f5363/cryptography-47.0.0-cp311-abi3-win_amd64.whl", hash = "sha256:f5c3296dab66202f1b18a91fa266be93d6aa0c2806ea3d67762c69f60adc71aa", size = 3775309, upload-time = "2026-04-24T19:53:33.054Z" }, + { url = "https://files.pythonhosted.org/packages/14/88/7aa18ad9c11bc87689affa5ce4368d884b517502d75739d475fc6f4a03c7/cryptography-47.0.0-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:be12cb6a204f77ed968bcefe68086eb061695b540a3dd05edac507a3111b25f0", size = 7904299, upload-time = "2026-04-24T19:53:35.003Z" }, + { url = "https://files.pythonhosted.org/packages/07/55/c18f75724544872f234678fdedc871391722cb34a2aee19faa9f63100bb2/cryptography-47.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2ebd84adf0728c039a3be2700289378e1c164afc6748df1a5ed456767bef9ba7", size = 4631180, upload-time = "2026-04-24T19:53:37.517Z" }, + { url = "https://files.pythonhosted.org/packages/ee/65/31a5cc0eaca99cec5bafffe155d407115d96136bb161e8b49e0ef73f09a7/cryptography-47.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7f68d6fbc7fbbcfb0939fea72c3b96a9f9a6edfc0e1b1d29778a2066030418b1", size = 4653529, upload-time = "2026-04-24T19:53:39.775Z" }, + { url = "https://files.pythonhosted.org/packages/e5/bc/641c0519a495f3bfd0421b48d7cd325c4336578523ccd76ea322b6c29c7a/cryptography-47.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:6651d32eff255423503aa276739da98c30f26c40cbeffcc6048e0d54ef704c0c", size = 4638570, upload-time = "2026-04-24T19:53:42.129Z" }, + { url = "https://files.pythonhosted.org/packages/2b/f2/300327b0a47f6dc94dd8b71b57052aefe178bb51745073d73d80604f11ab/cryptography-47.0.0-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:3fb8fa48075fad7193f2e5496135c6a76ac4b2aa5a38433df0a539296b377829", size = 5238019, upload-time = "2026-04-24T19:53:44.577Z" }, + { url = "https://files.pythonhosted.org/packages/e9/5a/5b5cf994391d4bf9d9c7efd4c66aabe4d95227256627f8fea6cff7dfadbd/cryptography-47.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:11438c7518132d95f354fa01a4aa2f806d172a061a7bed18cf18cbdacdb204d7", size = 4686832, upload-time = "2026-04-24T19:53:47.015Z" }, + { url = "https://files.pythonhosted.org/packages/dc/2c/ae950e28fd6475c852fc21a44db3e6b5bcc1261d1e370f2b6e42fa800fef/cryptography-47.0.0-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:8c1a736bbb3288005796c3f7ccb9453360d7fed483b13b9f468aea5171432923", size = 4269301, upload-time = "2026-04-24T19:53:48.97Z" }, + { url = "https://files.pythonhosted.org/packages/67/fb/6a39782e150ffe5cc1b0018cb6ddc48bf7ca62b498d7539ffc8a758e977d/cryptography-47.0.0-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:f1557695e5c2b86e204f6ce9470497848634100787935ab7adc5397c54abd7ab", size = 4638110, upload-time = "2026-04-24T19:53:51.011Z" }, + { url = "https://files.pythonhosted.org/packages/8e/d7/0b3c71090a76e5c203164a47688b697635ece006dcd2499ab3a4dbd3f0bd/cryptography-47.0.0-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:f9a034b642b960767fb343766ae5ba6ad653f2e890ddd82955aef288ffea8736", size = 5194988, upload-time = "2026-04-24T19:53:52.962Z" }, + { url = "https://files.pythonhosted.org/packages/63/33/63a961498a9df51721ab578c5a2622661411fc520e00bd83b0cc64eb20c4/cryptography-47.0.0-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:b1c76fca783aa7698eb21eb14f9c4aa09452248ee54a627d125025a43f83e7a7", size = 4686563, upload-time = "2026-04-24T19:53:55.274Z" }, + { url = "https://files.pythonhosted.org/packages/b7/bf/5ee5b145248f92250de86145d1c1d6edebbd57a7fe7caa4dedb5d4cf06a1/cryptography-47.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:4f7722c97826770bab8ae92959a2e7b20a5e9e9bf4deae68fd86c3ca457bab52", size = 4770094, upload-time = "2026-04-24T19:53:57.753Z" }, + { url = "https://files.pythonhosted.org/packages/92/43/21d220b2da5d517773894dacdcdb5c682c28d3fffce65548cb06e87d5501/cryptography-47.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:09f6d7bf6724f8db8b32f11eccf23efc8e759924bc5603800335cf8859a3ddbd", size = 4913811, upload-time = "2026-04-24T19:54:00.236Z" }, + { url = "https://files.pythonhosted.org/packages/31/98/dc4ad376ac5f1a1a7d4a83f7b0c6f2bcad36b5d2d8f30aeb482d3a7d9582/cryptography-47.0.0-cp314-cp314t-win32.whl", hash = "sha256:6eebcaf0df1d21ce1f90605c9b432dd2c4f4ab665ac29a40d5e3fc68f51b5e63", size = 3237158, upload-time = "2026-04-24T19:54:02.606Z" }, + { url = "https://files.pythonhosted.org/packages/bc/da/97f62d18306b5133468bc3f8cc73a3111e8cdc8cf8d3e69474d6e5fd2d1b/cryptography-47.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:51c9313e90bd1690ec5a75ed047c27c0b8e6c570029712943d6116ef9a90620b", size = 3758706, upload-time = "2026-04-24T19:54:04.433Z" }, + { url = "https://files.pythonhosted.org/packages/e0/34/a4fae8ae7c3bc227460c9ae43f56abf1b911da0ec29e0ebac53bb0a4b6b7/cryptography-47.0.0-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:14432c8a9bcb37009784f9594a62fae211a2ae9543e96c92b2a8e4c3cd5cd0c4", size = 7904072, upload-time = "2026-04-24T19:54:06.411Z" }, + { url = "https://files.pythonhosted.org/packages/01/64/d7b1e54fdb69f22d24a64bb3e88dc718b31c7fb10ef0b9691a3cf7eeea6e/cryptography-47.0.0-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:07efe86201817e7d3c18781ca9770bc0db04e1e48c994be384e4602bc38f8f27", size = 4635767, upload-time = "2026-04-24T19:54:08.519Z" }, + { url = "https://files.pythonhosted.org/packages/8b/7b/cca826391fb2a94efdcdfe4631eb69306ee1cff0b22f664a412c90713877/cryptography-47.0.0-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2b45761c6ec22b7c726d6a829558777e32d0f1c8be7c3f3480f9c912d5ee8a10", size = 4654350, upload-time = "2026-04-24T19:54:10.795Z" }, + { url = "https://files.pythonhosted.org/packages/4c/65/4b57bcc823f42a991627c51c2f68c9fd6eb1393c1756aac876cba2accae2/cryptography-47.0.0-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:edd4da498015da5b9f26d38d3bfc2e90257bfa9cbed1f6767c282a0025ae649b", size = 4643394, upload-time = "2026-04-24T19:54:13.275Z" }, + { url = "https://files.pythonhosted.org/packages/f4/c4/2c5fbeea70adbbca2bbae865e1d605d6a4a7f8dbd9d33eaf69645087f06c/cryptography-47.0.0-cp38-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:9af828c0d5a65c70ec729cd7495a4bf1a67ecb66417b8f02ff125ab8a6326a74", size = 5225777, upload-time = "2026-04-24T19:54:15.18Z" }, + { url = "https://files.pythonhosted.org/packages/7e/b8/ac57107ef32749d2b244e36069bb688792a363aaaa3acc9e3cf84c130315/cryptography-47.0.0-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:256d07c78a04d6b276f5df935a9923275f53bd1522f214447fdf365494e2d515", size = 4688771, upload-time = "2026-04-24T19:54:17.835Z" }, + { url = "https://files.pythonhosted.org/packages/56/fc/9f1de22ff8be99d991f240a46863c52d475404c408886c5a38d2b5c3bb26/cryptography-47.0.0-cp38-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:5d0e362ff51041b0c0d219cc7d6924d7b8996f57ce5712bdcef71eb3c65a59cc", size = 4270753, upload-time = "2026-04-24T19:54:19.963Z" }, + { url = "https://files.pythonhosted.org/packages/00/68/d70c852797aa68e8e48d12e5a87170c43f67bb4a59403627259dd57d15de/cryptography-47.0.0-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:1581aef4219f7ca2849d0250edaa3866212fb74bf5667284f46aa92f9e65c1ca", size = 4642911, upload-time = "2026-04-24T19:54:21.818Z" }, + { url = "https://files.pythonhosted.org/packages/a5/51/661cbee74f594c5d97ff82d34f10d5551c085ca4668645f4606ebd22bd5d/cryptography-47.0.0-cp38-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:a49a3eb5341b9503fa3000a9a0db033161db90d47285291f53c2a9d2cd1b7f76", size = 5181411, upload-time = "2026-04-24T19:54:24.376Z" }, + { url = "https://files.pythonhosted.org/packages/94/87/f2b6c374a82cf076cfa1416992ac8e8ec94d79facc37aec87c1a5cb72352/cryptography-47.0.0-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:2207a498b03275d0051589e326b79d4cf59985c99031b05bb292ac52631c37fe", size = 4688262, upload-time = "2026-04-24T19:54:26.946Z" }, + { url = "https://files.pythonhosted.org/packages/14/e2/8b7462f4acf21ec509616f0245018bb197194ab0b65c2ea21a0bdd53c0eb/cryptography-47.0.0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:7a02675e2fabd0c0fc04c868b8781863cbf1967691543c22f5470500ff840b31", size = 4775506, upload-time = "2026-04-24T19:54:28.926Z" }, + { url = "https://files.pythonhosted.org/packages/70/75/158e494e4c08dc05e039da5bb48553826bd26c23930cf8d3cd5f21fa8921/cryptography-47.0.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:80887c5cbd1774683cb126f0ab4184567f080071d5acf62205acb354b4b753b7", size = 4912060, upload-time = "2026-04-24T19:54:30.869Z" }, + { url = "https://files.pythonhosted.org/packages/06/bd/0a9d3edbf5eadbac926d7b9b3cd0c4be584eeeae4a003d24d9eda4affbbd/cryptography-47.0.0-cp38-abi3-win32.whl", hash = "sha256:ed67ea4e0cfb5faa5bc7ecb6e2b8838f3807a03758eec239d6c21c8769355310", size = 3248487, upload-time = "2026-04-24T19:54:33.494Z" }, + { url = "https://files.pythonhosted.org/packages/60/80/5681af756d0da3a599b7bdb586fac5a1540f1bcefd2717a20e611ddade45/cryptography-47.0.0-cp38-abi3-win_amd64.whl", hash = "sha256:835d2d7f47cdc53b3224e90810fb1d36ca94ea29cc1801fb4c1bc43876735769", size = 3755737, upload-time = "2026-04-24T19:54:35.408Z" }, +] + [[package]] name = "flake8" version = "7.3.0" @@ -294,8 +404,11 @@ fmt = [ integration = [ { name = "allure-pytest" }, { name = "allure-pytest-collection-report" }, + { name = "cryptography" }, { name = "jubilant" }, { name = "pytest" }, + { name = "pyyaml" }, + { name = "requests" }, ] lint = [ { name = "codespell" }, @@ -329,8 +442,11 @@ fmt = [{ name = "ruff" }] integration = [ { name = "allure-pytest", specifier = ">=2.8.18" }, { name = "allure-pytest-collection-report", git = "https://github.com/canonical/data-platform-workflows?subdirectory=python%2Fpytest_plugins%2Fallure_pytest_collection_report&rev=v24.0.0" }, + { name = "cryptography", specifier = ">=42.0" }, { name = "jubilant", specifier = "==1.9.0" }, { name = "pytest" }, + { name = "pyyaml", specifier = ">=6.0" }, + { name = "requests", specifier = ">=2.31" }, ] lint = [ { name = "codespell" }, @@ -515,6 +631,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d7/27/a58ddaf8c588a3ef080db9d0b7e0b97215cee3a45df74f3a94dbbf5c893a/pycodestyle-2.14.0-py2.py3-none-any.whl", hash = "sha256:dd6bf7cb4ee77f8e016f9c8e74a35ddd9f67e1d5fd4184d86c3b98e07099f42d", size = 31594, upload-time = "2025-06-20T18:49:47.491Z" }, ] +[[package]] +name = "pycparser" +version = "3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/7d/92392ff7815c21062bea51aa7b87d45576f649f16458d78b7cf94b9ab2e6/pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29", size = 103492, upload-time = "2026-01-21T14:26:51.89Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", size = 48172, upload-time = "2026-01-21T14:26:50.693Z" }, +] + [[package]] name = "pyflakes" version = "3.4.0" From cf0d406531d09e8cc3b1ccc9d7d726141321dc7b Mon Sep 17 00:00:00 2001 From: Ali Ugur Date: Thu, 21 May 2026 12:11:19 +0300 Subject: [PATCH 25/46] feat: enhance CI workflows with integration tests for multiple charms and add snap publishing --- .github/workflows/integration_test.yaml | 17 ++++++++ .github/workflows/publish_charm.yaml | 15 +++++++ .github/workflows/publish_snap.yaml | 56 +++++++++++++++++++++++++ .github/workflows/test.yaml | 27 +++++++++++- 4 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/publish_snap.yaml diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index b554961..e012073 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -60,9 +60,26 @@ jobs: self-hosted-runner-label: "self-hosted-linux-amd64-noble-large" trivy-fs-enabled: false with-uv: true + integration-tests-stack: + uses: + canonical/operator-workflows/.github/workflows/integration_test.yaml@main + secrets: inherit + with: + use-canonical-k8s: true + provider: lxd + self-hosted-runner: true + self-hosted-runner-label: "self-hosted-linux-amd64-noble-large" + juju-channel: 3/stable + modules: | + [ + "test_whole_email_system.py" + ] + with-uv: true + pre-run-script: ./tests/integration/setup-integration-tests.sh allure-report: if: ${{ !cancelled() && github.event_name == 'schedule' }} needs: - integration-tests-juju-3 - integration-tests-global + - integration-tests-stack uses: canonical/operator-workflows/.github/workflows/allure_report.yaml@main diff --git a/.github/workflows/publish_charm.yaml b/.github/workflows/publish_charm.yaml index fc9a31d..c79a00d 100644 --- a/.github/workflows/publish_charm.yaml +++ b/.github/workflows/publish_charm.yaml @@ -17,6 +17,21 @@ jobs: channel: "2.3/edge", tag-prefix: "dovecot", }, + { + working-directory: "./opendkim-operator", + channel: "2/edge", + tag-prefix: "opendkim-operator", + }, + { + working-directory: "./postfix-relay-operator", + channel: "3/edge", + tag-prefix: "postfix-relay", + }, + { + working-directory: "./postfix-relay-configurator-operator", + channel: "3/edge", + tag-prefix: "postfix-relay-configurator", + }, ] uses: canonical/operator-workflows/.github/workflows/publish_charm.yaml@main secrets: inherit diff --git a/.github/workflows/publish_snap.yaml b/.github/workflows/publish_snap.yaml new file mode 100644 index 0000000..0c2b381 --- /dev/null +++ b/.github/workflows/publish_snap.yaml @@ -0,0 +1,56 @@ +# Copyright 2026 Canonical Ltd. +# See LICENSE file for licensing details. + +name: Release snap to edge and promote + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + workflow_dispatch: + inputs: + promote-to: + description: Channel to promote the snap to (e.g. latest/candidate) + required: false + type: string + push: + branches: + - main + paths: + - opendkim-snap/** + +jobs: + build: + name: Build snap + uses: canonical/data-platform-workflows/.github/workflows/build_snap.yaml@v49 + with: + path-to-snap-project-directory: ./opendkim-snap + + release-edge: + name: Release snap to edge + needs: build + uses: canonical/data-platform-workflows/.github/workflows/release_snap.yaml@v49 + with: + channel: latest/edge + artifact-prefix: ${{ needs.build.outputs.artifact-prefix }} + path-to-snap-project-directory: ./opendkim-snap + secrets: + snap-store-token: ${{ secrets.SNAP_STORE_TOKEN }} + permissions: + contents: write + + promote: + name: Promote snap + if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.promote-to != '' }} + needs: [build, release-edge] + uses: canonical/data-platform-workflows/.github/workflows/release_snap.yaml@v49 + with: + channel: ${{ github.event.inputs.promote-to }} + artifact-prefix: ${{ needs.build.outputs.artifact-prefix }} + path-to-snap-project-directory: ./opendkim-snap + create-git-tags: false + secrets: + snap-store-token: ${{ secrets.SNAP_STORE_TOKEN }} + permissions: + contents: write \ No newline at end of file diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index cf41c98..79fae63 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -5,6 +5,17 @@ on: jobs: unit-tests: + strategy: + matrix: + charm: + - name: dovecot + working-directory: ./dovecot-charm + - name: opendkim-operator + working-directory: ./opendkim-operator + - name: postfix-relay-operator + working-directory: ./postfix-relay-operator + - name: postfix-relay-configurator-operator + working-directory: ./postfix-relay-configurator-operator uses: canonical/operator-workflows/.github/workflows/test.yaml@main secrets: inherit permissions: @@ -12,6 +23,20 @@ jobs: with: self-hosted-runner: false with-uv: true - working-directory: dovecot-charm + working-directory: ${{ matrix.charm.working-directory }} runs-on-base: ubuntu-24.04 python-version: 3.12 + + snap-tests: + name: Snap spread tests + runs-on: ubuntu-24.04 + steps: + - name: Checkout + uses: actions/checkout@v6 + - name: Set up LXD + uses: canonical/setup-lxd@main + - name: Install snapcraft + run: sudo snap install snapcraft --classic + - name: Run spread tests + run: CI=1 snapcraft test + working-directory: opendkim-snap From 7cf7c6091d28621defc36a4173db24b66179caa9 Mon Sep 17 00:00:00 2001 From: Ali Ugur Date: Thu, 21 May 2026 12:11:20 +0300 Subject: [PATCH 26/46] feat: update promote charm workflow to use charm selection and resolve channels Co-authored-by: Copilot --- .github/workflows/promote_charm.yaml | 44 +++++++++++++++++++++------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/.github/workflows/promote_charm.yaml b/.github/workflows/promote_charm.yaml index 66649de..00e4756 100644 --- a/.github/workflows/promote_charm.yaml +++ b/.github/workflows/promote_charm.yaml @@ -3,24 +3,48 @@ name: Promote charm on: workflow_dispatch: inputs: - origin-channel: + charm: type: choice - description: 'Origin Channel' + description: 'Charm to promote' options: - - latest/edge - destination-channel: - type: choice - description: 'Destination Channel' - options: - - latest/stable + - dovecot-charm + - opendkim-operator + - postfix-relay-operator + - postfix-relay-configurator-operator secrets: CHARMHUB_TOKEN: required: true jobs: + resolve-channels: + runs-on: ubuntu-latest + outputs: + origin-channel: ${{ steps.set-channels.outputs.origin-channel }} + destination-channel: ${{ steps.set-channels.outputs.destination-channel }} + steps: + - name: Set channels + id: set-channels + run: | + case "${{ github.event.inputs.charm }}" in + dovecot-charm) + echo "origin-channel=2.3/edge" >> "$GITHUB_OUTPUT" + echo "destination-channel=2.3/stable" >> "$GITHUB_OUTPUT" + ;; + opendkim-operator) + echo "origin-channel=2/edge" >> "$GITHUB_OUTPUT" + echo "destination-channel=2/stable" >> "$GITHUB_OUTPUT" + ;; + postfix-relay-operator|postfix-relay-configurator-operator) + echo "origin-channel=3/edge" >> "$GITHUB_OUTPUT" + echo "destination-channel=3/stable" >> "$GITHUB_OUTPUT" + ;; + esac + promote-charm: + needs: resolve-channels uses: canonical/operator-workflows/.github/workflows/promote_charm.yaml@main with: - origin-channel: ${{ github.event.inputs.origin-channel }} - destination-channel: ${{ github.event.inputs.destination-channel }} + origin-channel: ${{ needs.resolve-channels.outputs.origin-channel }} + destination-channel: ${{ needs.resolve-channels.outputs.destination-channel }} + working-directory: ${{ github.event.inputs.charm }} secrets: inherit From 30c26e5d538315f49d5aa23e95b79aeb2a41d40a Mon Sep 17 00:00:00 2001 From: Ali Ugur Date: Thu, 21 May 2026 12:11:20 +0300 Subject: [PATCH 27/46] feat: add end-to-end integration test for full mail system and clean up options in conftest.py --- tests/integration/conftest.py | 8 +- ...test_whole_email_system.py => test_e2e.py} | 106 ++--- tests/integration/test_full_stack.py | 447 ------------------ 3 files changed, 54 insertions(+), 507 deletions(-) rename tests/integration/{test_whole_email_system.py => test_e2e.py} (96%) delete mode 100644 tests/integration/test_full_stack.py diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 17e87c7..76fb753 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -66,10 +66,7 @@ def pytest_addoption(parser: pytest.Parser) -> None: "--charm-file", action="append", default=[], - help=( - "Path to a pre-built .charm file. " - "Pass this option multiple times (one per charm)." - ), + help=("Path to a pre-built .charm file. Pass this option multiple times (one per charm)."), ) parser.addoption( "--keep-models", @@ -91,9 +88,6 @@ def pytest_addoption(parser: pytest.Parser) -> None: ) -# --------------------------------------------------------------------------- -# Juju session fixture -# --------------------------------------------------------------------------- @pytest.fixture(scope="session", name="juju") def juju_fixture( request: pytest.FixtureRequest, diff --git a/tests/integration/test_whole_email_system.py b/tests/integration/test_e2e.py similarity index 96% rename from tests/integration/test_whole_email_system.py rename to tests/integration/test_e2e.py index 565a5ff..2f8e22f 100644 --- a/tests/integration/test_whole_email_system.py +++ b/tests/integration/test_e2e.py @@ -12,6 +12,7 @@ import smtplib import ssl import time +from secrets import token_hex from typing import Dict import jubilant @@ -21,63 +22,12 @@ TEST_DOMAIN = "mailstack.internal" TEST_USER = "e2euser" MAILBOX_USER = f"{TEST_USER}@{TEST_DOMAIN}" -TEST_PASSWORD = "E2eP@ssw0rd!" # nosec B105 +TEST_PASSWORD = token_hex(16) SMTP_SUBMISSION_PORT = 587 IMAP_PORT = 993 -def _sha512_dovecot(password: str, salt: bytes | None = None) -> str: - if salt is None: - salt = os.urandom(8) - digest = hashlib.sha512(password.encode() + salt).digest() - return "{SSHA512}" + base64.b64encode(digest + salt).decode() - - -def _setup_dovecot_user(juju: jubilant.Juju, username: str, password: str) -> None: - status = juju.status() - unit_name = next(iter(status.apps["dovecot-charm"].units)) - juju.exec(f"id -u {username} &>/dev/null || sudo useradd -m {username}", unit=unit_name) - juju.exec( - f"id -u {MAILBOX_USER} &>/dev/null || sudo useradd --badname -m {MAILBOX_USER}", - unit=unit_name, - ) - juju.exec(f"sudo usermod -aG mail {username}", unit=unit_name) - juju.exec(f"sudo usermod -aG mail {MAILBOX_USER}", unit=unit_name) - juju.exec(f"echo '{username}:{password}' | sudo chpasswd", unit=unit_name) - juju.exec(f"echo '{MAILBOX_USER}:{password}' | sudo chpasswd", unit=unit_name) - - -def _wait_for_subject(host: str, username: str, password: str, subject: str) -> bytes: - ctx = ssl.create_default_context() - ctx.check_hostname = False - ctx.verify_mode = ssl.CERT_NONE - - deadline = time.time() + 120 - while time.time() < deadline: - try: - conn = imaplib.IMAP4_SSL(host, port=IMAP_PORT, ssl_context=ctx) - try: - conn.login(username, password) - conn.select("inbox") - _, data = conn.search(None, f'(HEADER Subject "{subject}")') - if data and data[0]: - msg_id = data[0].split()[-1] - _, msg_data = conn.fetch(msg_id, "(RFC822)") - return msg_data[0][1] # type: ignore[index] - finally: - with contextlib.suppress(Exception): - conn.close() - with contextlib.suppress(Exception): - conn.logout() - except Exception: - pass - time.sleep(3) - - pytest.fail(f"Message with subject '{subject}' did not arrive in IMAP inbox") - - -@pytest.mark.abort_on_fail -def test_whole_email_system_e2e(juju: jubilant.Juju, mail_stack: Dict[str, str]) -> None: +def test_e2e(juju: jubilant.Juju, mail_stack: Dict[str, str]) -> None: """Verify SMTP AUTH -> DKIM signing -> LMTP delivery -> IMAP retrieval.""" relay_ip = mail_stack["postfix_relay_ip"] dovecot_ip = mail_stack["dovecot_ip"] @@ -126,3 +76,53 @@ def test_whole_email_system_e2e(juju: jubilant.Juju, mail_stack: Dict[str, str]) assert parsed["Subject"] == subject assert "DKIM-Signature" in parsed assert TEST_DOMAIN in parsed.get("DKIM-Signature", "") + + +def _sha512_dovecot(password: str, salt: bytes | None = None) -> str: + if salt is None: + salt = os.urandom(8) + digest = hashlib.sha512(password.encode() + salt).digest() + return "{SSHA512}" + base64.b64encode(digest + salt).decode() + + +def _setup_dovecot_user(juju: jubilant.Juju, username: str, password: str) -> None: + status = juju.status() + unit_name = next(iter(status.apps["dovecot-charm"].units)) + juju.exec(f"id -u {username} &>/dev/null || sudo useradd -m {username}", unit=unit_name) + juju.exec( + f"id -u {MAILBOX_USER} &>/dev/null || sudo useradd --badname -m {MAILBOX_USER}", + unit=unit_name, + ) + juju.exec(f"sudo usermod -aG mail {username}", unit=unit_name) + juju.exec(f"sudo usermod -aG mail {MAILBOX_USER}", unit=unit_name) + juju.exec(f"echo '{username}:{password}' | sudo chpasswd", unit=unit_name) + juju.exec(f"echo '{MAILBOX_USER}:{password}' | sudo chpasswd", unit=unit_name) + + +def _wait_for_subject(host: str, username: str, password: str, subject: str) -> bytes: + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + + deadline = time.time() + 120 + while time.time() < deadline: + try: + conn = imaplib.IMAP4_SSL(host, port=IMAP_PORT, ssl_context=ctx) + try: + conn.login(username, password) + conn.select("inbox") + _, data = conn.search(None, f'(HEADER Subject "{subject}")') + if data and data[0]: + msg_id = data[0].split()[-1] + _, msg_data = conn.fetch(msg_id, "(RFC822)") + return msg_data[0][1] # type: ignore[index] + finally: + with contextlib.suppress(Exception): + conn.close() + with contextlib.suppress(Exception): + conn.logout() + except Exception: + pass + time.sleep(3) + + pytest.fail(f"Message with subject '{subject}' did not arrive in IMAP inbox") diff --git a/tests/integration/test_full_stack.py b/tests/integration/test_full_stack.py deleted file mode 100644 index d004f91..0000000 --- a/tests/integration/test_full_stack.py +++ /dev/null @@ -1,447 +0,0 @@ -# Copyright 2026 Canonical Ltd. -# See LICENSE file for licensing details. - -"""Full-stack integration tests for the mailserver operators monorepo. - -These tests exercise the complete mail path: - - test runner ──SMTP AUTH/STARTTLS (port 587)──► postfix-relay - │ - milter ──► opendkim (DKIM sign) - │ - LMTP :24 ──► dovecot - │ - test runner ◄──IMAP SSL (port 993)────────────────────┘ - -All four charms must be pre-built and passed via repeated CLI options: - --charm-file= - --charm-file= - --charm-file= - --charm-file= - -A self-signed-certificates charm is pulled from CharmHub to provide TLS -for postfix-relay and dovecot. -""" - -import base64 -import contextlib -import email -import hashlib -import imaplib -import logging -import os -import smtplib -import ssl -import time -import typing - -import jubilant -import pytest -import requests - -# App name constants — kept in sync with conftest.py. -DOVECOT_APP = "dovecot-charm" -POSTFIX_RELAY_APP = "postfix-relay" -OPENDKIM_APP = "opendkim" -CONFIGURATOR_APP = "postfix-relay-configurator" -SELF_SIGNED_APP = "self-signed-certificates" -TEST_DOMAIN = "mailstack.internal" - -logger = logging.getLogger(__name__) - -# --------------------------------------------------------------------------- -# Test constants -# --------------------------------------------------------------------------- -IMAP_PORT = 993 -SMTP_SUBMISSION_PORT = 587 -METRICS_PORT = 9103 - -TEST_USER = "testuser" -TEST_PASSWORD = "TestP@ssw0rd!" # nosec B105 - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- -def _sha512_dovecot(password: str, salt: bytes | None = None) -> str: - """Return a Dovecot-compatible SSHA512 hash for *password*. - - This is the same algorithm used by postfix-relay's SMTP AUTH Dovecot - backend to validate credentials. - """ - if salt is None: - salt = os.urandom(8) - digest = hashlib.sha512(password.encode() + salt).digest() - return "{SSHA512}" + base64.b64encode(digest + salt).decode() - - -def _wait_for_imap_message( - host: str, - username: str, - password: str, - subject: str, - *, - retries: int = 20, - delay: float = 3.0, -) -> bytes: - """Poll dovecot via IMAP4_SSL until a message with *subject* arrives. - - Args: - host: Dovecot unit's public IP address. - username: IMAP login name. - password: IMAP login password. - subject: Expected Subject header value to search for. - retries: Number of polling attempts before giving up. - delay: Seconds to wait between attempts. - - Returns: - The raw RFC822 bytes of the first matching message. - - Raises: - pytest.fail: If the message is not found within *retries* attempts. - """ - ctx = ssl.create_default_context() - ctx.check_hostname = False - ctx.verify_mode = ssl.CERT_NONE - - for attempt in range(retries): - try: - conn = imaplib.IMAP4_SSL(host, port=IMAP_PORT, ssl_context=ctx) - try: - conn.login(username, password) - conn.select("inbox") - _, data = conn.search(None, f'(HEADER Subject "{subject}")') - if data and data[0]: - msg_ids = data[0].split() - _, msg_data = conn.fetch(msg_ids[-1], "(RFC822)") - raw: bytes = msg_data[0][1] # type: ignore[index] - logger.info("Message found (attempt %d): %d bytes", attempt + 1, len(raw)) - return raw - logger.debug("Message not yet in INBOX (attempt %d/%d)", attempt + 1, retries) - finally: - with contextlib.suppress(Exception): - conn.close() - with contextlib.suppress(Exception): - conn.logout() - except Exception as exc: # noqa: BLE001 - logger.warning("IMAP attempt %d failed: %s", attempt + 1, exc) - - time.sleep(delay) - - pytest.fail(f"Message with subject '{subject}' never arrived in dovecot INBOX.") - - -def _setup_dovecot_user( - juju: jubilant.Juju, - dovecot_app: str, - username: str, - password: str, -) -> None: - """Create / update an OS user on the dovecot unit and add them to the mail group.""" - status = juju.status() - unit_name = next(iter(status.apps[dovecot_app].units)) - # Create user if missing, then set password and add to mail group. - juju.exec( - f"id -u {username} &>/dev/null || sudo useradd -m {username}", - unit=unit_name, - ) - juju.exec(f"sudo usermod -aG mail {username}", unit=unit_name) - juju.exec(f"echo '{username}:{password}' | sudo chpasswd", unit=unit_name) - logger.info("Dovecot OS user '%s' configured on %s", username, unit_name) - - -# --------------------------------------------------------------------------- -# Test 1: All apps reach active status -# --------------------------------------------------------------------------- -@pytest.mark.abort_on_fail -def test_stack_is_active(juju: jubilant.Juju, mail_stack: typing.Dict[str, str]) -> None: - """ - arrange: Deploy the full mail stack (postfix-relay, opendkim, dovecot, - postfix-relay-configurator, self-signed-certificates). - act: Wait for all applications to reach active status (done in fixtures). - assert: Every application in the stack reports active. - """ - status = juju.status() - - for app_name in ( - DOVECOT_APP, - POSTFIX_RELAY_APP, - OPENDKIM_APP, - SELF_SIGNED_APP, - ): - app = status.apps.get(app_name) - assert app is not None, f"Application '{app_name}' not found in model" - assert app.is_active, ( - f"Application '{app_name}' is not active: " - f"{app.app_status.current!r} — {app.app_status.message!r}" - ) - logger.info("✓ %s is active", app_name) - - # The configurator is a subordinate — its units live inside postfix-relay units. - relay_units = status.apps[POSTFIX_RELAY_APP].units - for unit_name, unit in relay_units.items(): - subordinates = unit.subordinates or {} - assert any(CONFIGURATOR_APP in sub_name for sub_name in subordinates), ( - f"postfix-relay-configurator subordinate not found on unit {unit_name}" - ) - logger.info("✓ %s subordinate present on all postfix-relay units", CONFIGURATOR_APP) - - -# --------------------------------------------------------------------------- -# Test 2: End-to-end send → DKIM sign → deliver → IMAP retrieve -# --------------------------------------------------------------------------- -@pytest.mark.abort_on_fail -def test_send_and_receive_with_dkim( - juju: jubilant.Juju, - mail_stack: typing.Dict[str, str], -) -> None: - """ - arrange: Full mail stack is active. A local OS user exists on dovecot. - postfix-relay has SMTP AUTH enabled with the test user's credentials. - opendkim is configured to sign mail from TEST_DOMAIN. - The configurator routes TEST_DOMAIN mail to dovecot via LMTP. - act: Send an email via SMTP AUTH (port 587 + STARTTLS) to - testuser@mailstack.internal through postfix-relay. - assert: - - The message is delivered to the dovecot mailbox. - - The raw RFC822 message contains a DKIM-Signature header. - - The Subject and From headers match what was sent. - """ - relay_ip = mail_stack["postfix_relay_ip"] - dovecot_ip = mail_stack["dovecot_ip"] - - # 1. Create OS user on dovecot for IMAP login. - _setup_dovecot_user(juju, DOVECOT_APP, TEST_USER, TEST_PASSWORD) - - # 2. Configure SMTP AUTH credentials on postfix-relay. - import yaml # noqa: PLC0415 - - hashed = _sha512_dovecot(TEST_PASSWORD) - auth_users_yaml = yaml.dump([f"{TEST_USER}:{hashed}"]) - juju.config( - POSTFIX_RELAY_APP, - { - "enable_smtp_auth": "true", - "smtp_auth_users": auth_users_yaml, - }, - ) - juju.wait( - lambda status: status.apps[POSTFIX_RELAY_APP].is_active, - error=jubilant.any_error, - timeout=5 * 60, - ) - - subject = f"Full-stack DKIM test {int(time.time())}" - from_addr = f"sender@{TEST_DOMAIN}" - to_addr = f"{TEST_USER}@{TEST_DOMAIN}" - body = ( - f"Subject: {subject}\r\n" - f"From: {from_addr}\r\n" - f"To: {to_addr}\r\n" - f"\r\n" - f"This message was sent through the full mailserver stack.\r\n" - ) - - # 3. Send via postfix-relay with SMTP AUTH + STARTTLS. - ctx = ssl.create_default_context() - ctx.check_hostname = False - ctx.verify_mode = ssl.CERT_NONE - - with smtplib.SMTP(relay_ip, SMTP_SUBMISSION_PORT, timeout=30) as server: - server.set_debuglevel(1) - server.ehlo() - server.starttls(context=ctx) - server.ehlo() - server.login(TEST_USER, TEST_PASSWORD) - server.sendmail(from_addr, [to_addr], body) - logger.info("Message submitted to postfix-relay at %s:%d", relay_ip, SMTP_SUBMISSION_PORT) - - # 4. Poll dovecot IMAP until the message lands in the inbox. - raw_message = _wait_for_imap_message( - dovecot_ip, - TEST_USER, - TEST_PASSWORD, - subject, - ) - - # 5. Parse and assert. - msg = email.message_from_bytes(raw_message) - logger.info("Received headers: %s", dict(msg.items())) - - assert "DKIM-Signature" in msg, ( - "DKIM-Signature header missing — opendkim did not sign the message.\n" - f"Headers present: {list(msg.keys())}" - ) - assert msg["Subject"] == subject, ( - f"Subject mismatch: expected {subject!r}, got {msg['Subject']!r}" - ) - assert TEST_DOMAIN in msg.get("DKIM-Signature", ""), ( - f"DKIM-Signature does not reference domain {TEST_DOMAIN!r}.\n" - f"DKIM-Signature: {msg.get('DKIM-Signature')}" - ) - logger.info("✓ DKIM-signed message delivered and verified via IMAP") - - -# --------------------------------------------------------------------------- -# Test 3: Unauthenticated SMTP is rejected on port 587 -# --------------------------------------------------------------------------- -@pytest.mark.abort_on_fail -def test_unauthenticated_smtp_rejected( - juju: jubilant.Juju, - mail_stack: typing.Dict[str, str], -) -> None: - """ - arrange: postfix-relay has SMTP AUTH enabled (enable_smtp_auth=true). - act: Attempt to relay mail on port 587 to an external domain without - providing AUTH credentials. - assert: The server refuses the recipient (SMTPRecipientsRefused or - SMTPSenderRefused), indicating that unauthenticated relay to - external destinations is blocked. - """ - relay_ip = mail_stack["postfix_relay_ip"] - - ctx = ssl.create_default_context() - ctx.check_hostname = False - ctx.verify_mode = ssl.CERT_NONE - - # Send to a domain that is NOT in relay_domains — this should be deferred/rejected - # for unauthenticated clients by the defer_unauth_destination restriction. - with pytest.raises((smtplib.SMTPRecipientsRefused, smtplib.SMTPSenderRefused)): - with smtplib.SMTP(relay_ip, SMTP_SUBMISSION_PORT, timeout=15) as server: - server.ehlo() - server.starttls(context=ctx) - server.ehlo() - # No server.login() — deliberately unauthenticated - server.sendmail( - from_addr=f"attacker@{TEST_DOMAIN}", - to_addrs=["victim@external.example.com"], - msg="Subject: Spam\r\n\r\nShouldBeRejected", - ) - - logger.info("✓ Unauthenticated relay correctly refused on port %d", SMTP_SUBMISSION_PORT) - - -# --------------------------------------------------------------------------- -# Test 4: opendkim blocks when given an invalid key path -# --------------------------------------------------------------------------- -def test_dkim_invalid_key_blocks_opendkim( - juju: jubilant.Juju, - mail_stack: typing.Dict[str, str], -) -> None: - """ - arrange: opendkim is active with a valid DKIM key configuration. - act: Reconfigure the keytable to point to a non-existent key file. - assert: opendkim transitions to blocked with a configuration error message. - act (restore): Restore the original valid configuration. - assert: opendkim returns to active. - """ - import json # noqa: PLC0415 - - opendkim_app = mail_stack["opendkim_app"] - - selector = "default" - keyname = f"{TEST_DOMAIN.replace('.', '-')}-{selector}" - - # Save the current (valid) config so we can restore it. - current_config = juju.config(opendkim_app) - valid_keytable = current_config.get("keytable", "") - valid_signingtable = current_config.get("signingtable", "") - - # Apply a broken keytable pointing to a non-existent file. - broken_keytable = json.dumps( - [ - [ - f"{selector}._domainkey.{TEST_DOMAIN}", - f"{TEST_DOMAIN}:{selector}:/etc/dkimkeys/DOESNOTEXIST.private", - ] - ] - ) - juju.config(opendkim_app, {"keytable": broken_keytable}) - - juju.wait( - lambda status: status.apps[opendkim_app].is_blocked, - timeout=3 * 60, - delay=5, - ) - status = juju.status() - blocked_message = status.apps[opendkim_app].app_status.message - assert "opendkim" in blocked_message.lower() or "configuration" in blocked_message.lower(), ( - f"Expected a configuration-related blocked message, got: {blocked_message!r}" - ) - logger.info("✓ opendkim blocked with message: %s", blocked_message) - - # Restore the valid configuration. - juju.config(opendkim_app, {"keytable": valid_keytable}) - juju.wait( - lambda status: jubilant.all_active(status, opendkim_app, POSTFIX_RELAY_APP), - timeout=3 * 60, - delay=5, - ) - logger.info("✓ opendkim restored to active after valid keytable re-applied") - - -# --------------------------------------------------------------------------- -# Test 5: Metrics endpoints are reachable on all charms -# --------------------------------------------------------------------------- -def test_metrics_endpoints( - juju: jubilant.Juju, - mail_stack: typing.Dict[str, str], -) -> None: - """ - arrange: Full mail stack is active. - act: Scrape the Telegraf metrics endpoint on postfix-relay and opendkim. - assert: Each endpoint responds with HTTP 200 and contains expected metric names. - """ - relay_ip = mail_stack["postfix_relay_ip"] - - status = juju.status() - opendkim_ip = next(iter(status.apps[OPENDKIM_APP].units.values())).public_address - - expected_relay_metrics = [ - "cpu_usage_idle", - "postfix_queue_length", - "procstat_lookup_running", - ] - expected_opendkim_metrics = ["cpu_usage_idle", "procstat_lookup_running"] - - for ip, expected in ( - (relay_ip, expected_relay_metrics), - (opendkim_ip, expected_opendkim_metrics), - ): - url = f"http://{ip}:{METRICS_PORT}/metrics" - resp = requests.get(url, timeout=10) - assert resp.status_code == 200, f"Metrics endpoint {url} returned {resp.status_code}" - for metric in expected: - assert metric in resp.text, ( - f"Expected metric {metric!r} not found in response from {url}" - ) - logger.info("✓ Metrics OK at %s", url) - - -# --------------------------------------------------------------------------- -# Test 6: TLS certificate is presented by postfix-relay on port 587 -# --------------------------------------------------------------------------- -def test_tls_certificate_presented( - juju: jubilant.Juju, - mail_stack: typing.Dict[str, str], -) -> None: - """ - arrange: postfix-relay is related to self-signed-certificates. - act: Initiate a STARTTLS handshake on port 587. - assert: A TLS certificate is presented by the server. - """ - import ssl as _ssl # noqa: PLC0415 - - relay_ip = mail_stack["postfix_relay_ip"] - - ctx = _ssl.create_default_context() - ctx.check_hostname = False - ctx.verify_mode = _ssl.CERT_NONE - - with smtplib.SMTP(relay_ip, SMTP_SUBMISSION_PORT, timeout=15) as server: - server.ehlo() - server.starttls(context=ctx) - peer_cert = typing.cast(_ssl.SSLSocket, server.sock).getpeercert(binary_form=True) - - assert peer_cert, "No TLS certificate was presented by postfix-relay on STARTTLS" - logger.info("✓ TLS certificate presented by postfix-relay (%d bytes)", len(peer_cert)) From e443bfb2463228db94eeb272da854dbf86eee908 Mon Sep 17 00:00:00 2001 From: Ali Ugur Date: Thu, 21 May 2026 12:11:21 +0300 Subject: [PATCH 28/46] fix: use stack-integration tox env for global tests --- .github/workflows/integration_test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index e012073..f88a612 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -65,6 +65,7 @@ jobs: canonical/operator-workflows/.github/workflows/integration_test.yaml@main secrets: inherit with: + test-tox-env: stack-integration use-canonical-k8s: true provider: lxd self-hosted-runner: true From 9851f674c03f2134866a4ec9d0178a7a34a18721 Mon Sep 17 00:00:00 2001 From: Ali Ugur Date: Thu, 21 May 2026 12:11:22 +0300 Subject: [PATCH 29/46] chore: remove copyright comments from configuration and test files --- .licenserc.yaml | 2 ++ opendkim-operator/tests/unit/files/base_keytable | 3 --- opendkim-operator/tests/unit/files/base_opendkim.conf | 3 --- opendkim-operator/tests/unit/files/base_signingtable | 3 --- opendkim-operator/tests/unit/files/logrotate | 3 --- opendkim-operator/tests/unit/files/logrotate_frequency | 3 --- opendkim-operator/tests/unit/files/logrotate_retention | 3 --- .../tests/unit/files/logrotate_retention_no_dateext | 3 --- .../tests/unit/files/sv_trusted_sources_opendkim.conf | 3 --- .../tests/unit/files/verify_trusted_sources_opendkim.conf | 3 --- 10 files changed, 2 insertions(+), 27 deletions(-) diff --git a/.licenserc.yaml b/.licenserc.yaml index 6f6b8f3..baec2e8 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -33,6 +33,8 @@ header: - 'docs/release-notes/template/*.yaml' - 'uv.lock' - 'opendkim-snap/snap/local/opendkim.conf' + - 'opendkim-operator/templates/opendkim.conf.j2' + - 'opendkim-operator/tests/unit/files/**' - '**/templates/**' - '**/lib/**' comment: on-failure diff --git a/opendkim-operator/tests/unit/files/base_keytable b/opendkim-operator/tests/unit/files/base_keytable index 6327a7b..f7817bc 100644 --- a/opendkim-operator/tests/unit/files/base_keytable +++ b/opendkim-operator/tests/unit/files/base_keytable @@ -1,5 +1,2 @@ -# Copyright 2026 Canonical Ltd. -# See LICENSE file for licensing details. - selector._domainkey.example.com example.com:selector:/etc/dkimkeys/key1.private selector._domainkey.other.example.com other.example.com:selector:/etc/dkimkeys/key2.private \ No newline at end of file diff --git a/opendkim-operator/tests/unit/files/base_opendkim.conf b/opendkim-operator/tests/unit/files/base_opendkim.conf index c5bb264..6557dfd 100644 --- a/opendkim-operator/tests/unit/files/base_opendkim.conf +++ b/opendkim-operator/tests/unit/files/base_opendkim.conf @@ -1,6 +1,3 @@ -# Copyright 2026 Canonical Ltd. -# See LICENSE file for licensing details. - # This file is Juju managed - do not edit by hand # Socket inet:8892 diff --git a/opendkim-operator/tests/unit/files/base_signingtable b/opendkim-operator/tests/unit/files/base_signingtable index cfe7cde..f5b924d 100644 --- a/opendkim-operator/tests/unit/files/base_signingtable +++ b/opendkim-operator/tests/unit/files/base_signingtable @@ -1,5 +1,2 @@ -# Copyright 2026 Canonical Ltd. -# See LICENSE file for licensing details. - *@example.com selector._domainkey.example.com *@other.example.com selector._domainkey.other.example.com \ No newline at end of file diff --git a/opendkim-operator/tests/unit/files/logrotate b/opendkim-operator/tests/unit/files/logrotate index be92317..8e1c0c9 100644 --- a/opendkim-operator/tests/unit/files/logrotate +++ b/opendkim-operator/tests/unit/files/logrotate @@ -1,6 +1,3 @@ -# Copyright 2026 Canonical Ltd. -# See LICENSE file for licensing details. - var/log/syslog { rotate 7 diff --git a/opendkim-operator/tests/unit/files/logrotate_frequency b/opendkim-operator/tests/unit/files/logrotate_frequency index bf08389..1338147 100644 --- a/opendkim-operator/tests/unit/files/logrotate_frequency +++ b/opendkim-operator/tests/unit/files/logrotate_frequency @@ -1,6 +1,3 @@ -# Copyright 2026 Canonical Ltd. -# See LICENSE file for licensing details. - var/log/syslog { rotate 7 diff --git a/opendkim-operator/tests/unit/files/logrotate_retention b/opendkim-operator/tests/unit/files/logrotate_retention index 98208b9..58697be 100644 --- a/opendkim-operator/tests/unit/files/logrotate_retention +++ b/opendkim-operator/tests/unit/files/logrotate_retention @@ -1,6 +1,3 @@ -# Copyright 2026 Canonical Ltd. -# See LICENSE file for licensing details. - var/log/syslog { dateext diff --git a/opendkim-operator/tests/unit/files/logrotate_retention_no_dateext b/opendkim-operator/tests/unit/files/logrotate_retention_no_dateext index bac61bf..7545e2f 100644 --- a/opendkim-operator/tests/unit/files/logrotate_retention_no_dateext +++ b/opendkim-operator/tests/unit/files/logrotate_retention_no_dateext @@ -1,6 +1,3 @@ -# Copyright 2026 Canonical Ltd. -# See LICENSE file for licensing details. - var/log/syslog { rotate 30 diff --git a/opendkim-operator/tests/unit/files/sv_trusted_sources_opendkim.conf b/opendkim-operator/tests/unit/files/sv_trusted_sources_opendkim.conf index c623dfe..5730ed2 100644 --- a/opendkim-operator/tests/unit/files/sv_trusted_sources_opendkim.conf +++ b/opendkim-operator/tests/unit/files/sv_trusted_sources_opendkim.conf @@ -1,6 +1,3 @@ -# Copyright 2026 Canonical Ltd. -# See LICENSE file for licensing details. - # This file is Juju managed - do not edit by hand # Socket inet:8892 diff --git a/opendkim-operator/tests/unit/files/verify_trusted_sources_opendkim.conf b/opendkim-operator/tests/unit/files/verify_trusted_sources_opendkim.conf index 78e5195..2abcdbd 100644 --- a/opendkim-operator/tests/unit/files/verify_trusted_sources_opendkim.conf +++ b/opendkim-operator/tests/unit/files/verify_trusted_sources_opendkim.conf @@ -1,6 +1,3 @@ -# Copyright 2026 Canonical Ltd. -# See LICENSE file for licensing details. - # This file is Juju managed - do not edit by hand # Socket inet:8892 From 220fee0a217ae648af0749cc0a8cea5da33ee3d3 Mon Sep 17 00:00:00 2001 From: Ali Ugur Date: Thu, 21 May 2026 12:11:23 +0300 Subject: [PATCH 30/46] feat: update tox configuration for charm integration tests and linting --- tox.toml | 138 ++++++++++++++++++------------------------------------- 1 file changed, 44 insertions(+), 94 deletions(-) diff --git a/tox.toml b/tox.toml index e3f75a4..26dd028 100644 --- a/tox.toml +++ b/tox.toml @@ -17,111 +17,68 @@ PYTHONBREAKPOINT = "ipdb.set_trace" PY_COLORS = "1" [env.fmt] -description = "Apply coding style standards to code" +description = "Apply coding style standards to all charms" +allowlist_externals = [ "tox" ] commands = [ - [ - "ruff", - "check", - "--fix", - "--select", - "I", - { replace = "ref", of = [ - "vars", - "all_path", - ], extend = true }, - ], - [ - "ruff", - "format", - { replace = "ref", of = [ - "vars", - "all_path", - ], extend = true }, - ], + [ "tox", "-c", "{toxinidir}/dovecot-charm/tox.toml", "-e", "fmt", { replace = "posargs", extend = true } ], + [ "tox", "-c", "{toxinidir}/opendkim-operator/tox.toml", "-e", "fmt", { replace = "posargs", extend = true } ], + [ "tox", "-c", "{toxinidir}/postfix-relay-operator/tox.ini", "-e", "fmt", { replace = "posargs", extend = true } ], + [ "tox", "-c", "{toxinidir}/postfix-relay-configurator-operator/tox.ini", "-e", "fmt", { replace = "posargs", extend = true } ], ] dependency_groups = [ "fmt" ] [env.lint] -description = "Check code against coding style standards" +description = "Run lint checks for all charms" +allowlist_externals = [ "tox" ] commands = [ - [ - "codespell", - "{toxinidir}", - ], - [ - "ruff", - "format", - "--check", - "--diff", - { replace = "ref", of = [ - "vars", - "all_path", - ], extend = true }, - ], - [ - "ruff", - "check", - { replace = "ref", of = [ - "vars", - "all_path", - ], extend = true }, - ], - [ - "mypy", - { replace = "ref", of = [ - "vars", - "all_path", - ], extend = true }, - ], + [ "tox", "-c", "{toxinidir}/dovecot-charm/tox.toml", "-e", "lint", { replace = "posargs", extend = true } ], + [ "tox", "-c", "{toxinidir}/opendkim-operator/tox.toml", "-e", "lint", { replace = "posargs", extend = true } ], + [ "tox", "-c", "{toxinidir}/postfix-relay-operator/tox.ini", "-e", "lint", { replace = "posargs", extend = true } ], + [ "tox", "-c", "{toxinidir}/postfix-relay-configurator-operator/tox.ini", "-e", "lint", { replace = "posargs", extend = true } ], ] dependency_groups = [ "lint" ] [env.unit] -description = "Run unit tests" +description = "Run all charm unit tests" +allowlist_externals = [ "tox" ] commands = [ - [ - "coverage", - "run", - "--source={[vars]src_path}", - "-m", - "pytest", - "--ignore={[vars]tst_path}integration", - "-v", - "--tb", - "native", - "-s", - { replace = "posargs", extend = "true" }, - ], - [ - "coverage", - "report", - ], + [ "tox", "-c", "{toxinidir}/dovecot-charm/tox.toml", "-e", "unit", { replace = "posargs", extend = true } ], + [ "tox", "-c", "{toxinidir}/opendkim-operator/tox.toml", "-e", "unit", { replace = "posargs", extend = true } ], + [ "tox", "-c", "{toxinidir}/postfix-relay-operator/tox.ini", "-e", "unit", { replace = "posargs", extend = true } ], + [ "tox", "-c", "{toxinidir}/postfix-relay-configurator-operator/tox.ini", "-e", "unit", { replace = "posargs", extend = true } ], ] dependency_groups = [ "unit" ] [env.coverage-report] -description = "Create test coverage report" -commands = [ [ "coverage", "report" ] ] +description = "Create coverage reports for all charms" +allowlist_externals = [ "tox" ] +commands = [ + [ "tox", "-c", "{toxinidir}/dovecot-charm/tox.toml", "-e", "coverage-report", { replace = "posargs", extend = true } ], + [ "tox", "-c", "{toxinidir}/opendkim-operator/tox.toml", "-e", "coverage-report", { replace = "posargs", extend = true } ], + [ "tox", "-c", "{toxinidir}/postfix-relay-operator/tox.ini", "-e", "coverage-report", { replace = "posargs", extend = true } ], + [ "tox", "-c", "{toxinidir}/postfix-relay-configurator-operator/tox.ini", "-e", "coverage-report", { replace = "posargs", extend = true } ], +] dependency_groups = [ "coverage-report" ] [env.static] -description = "Run static analysis tests" -commands = [ [ "bandit", "-c", "{toxinidir}/pyproject.toml", "-r", "{[vars]src_path}", "{[vars]tst_path}" ] ] +description = "Run static analysis tests for all charms" +allowlist_externals = [ "tox" ] +commands = [ + [ "tox", "-c", "{toxinidir}/dovecot-charm/tox.toml", "-e", "static", { replace = "posargs", extend = true } ], + [ "tox", "-c", "{toxinidir}/opendkim-operator/tox.toml", "-e", "static", { replace = "posargs", extend = true } ], + [ "tox", "-c", "{toxinidir}/postfix-relay-operator/tox.ini", "-e", "static", { replace = "posargs", extend = true } ], + [ "tox", "-c", "{toxinidir}/postfix-relay-configurator-operator/tox.ini", "-e", "static", { replace = "posargs", extend = true } ], +] dependency_groups = [ "static" ] [env.integration] -description = "Run cross-charm integration tests" +description = "Run integration tests for all charms" +allowlist_externals = [ "tox" ] commands = [ - [ - "pytest", - "-v", - "--tb", - "native", - "tests/integration/", - "--log-cli-level=INFO", - "-s", - { replace = "posargs", extend = "true" }, - ], + [ "tox", "-c", "{toxinidir}/dovecot-charm/tox.toml", "-e", "integration", { replace = "posargs", extend = true } ], + [ "tox", "-c", "{toxinidir}/opendkim-operator/tox.toml", "-e", "integration", { replace = "posargs", extend = true } ], + [ "tox", "-c", "{toxinidir}/postfix-relay-operator/tox.ini", "-e", "integration", { replace = "posargs", extend = true } ], + [ "tox", "-c", "{toxinidir}/postfix-relay-configurator-operator/tox.ini", "-e", "integration", { replace = "posargs", extend = true } ], ] dependency_groups = [ "integration" ] @@ -145,18 +102,11 @@ PYTHONPATH = "{toxinidir}/tests/integration" PY_COLORS = "1" [env.lint-fix] -description = "Apply coding style standards to code" +description = "Apply lint fixes for all charms" +allowlist_externals = [ "tox" ] commands = [ - [ - "ruff", - "check", - "--fix", - "--fix-only", - { replace = "ref", of = [ - "vars", - "all_path", - ], extend = true }, - ], + [ "tox", "-c", "{toxinidir}/dovecot-charm/tox.toml", "-e", "lint-fix", { replace = "posargs", extend = true } ], + [ "tox", "-c", "{toxinidir}/opendkim-operator/tox.toml", "-e", "lint-fix", { replace = "posargs", extend = true } ], ] dependency_groups = [ "lint" ] From 0537841cae0727543e86ead0e5121dd12c320d51 Mon Sep 17 00:00:00 2001 From: Ali Ugur Date: Thu, 21 May 2026 12:11:24 +0300 Subject: [PATCH 31/46] feat: update integration test module to use end-to-end test --- .github/workflows/integration_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index f88a612..b0af16d 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -73,7 +73,7 @@ jobs: juju-channel: 3/stable modules: | [ - "test_whole_email_system.py" + "test_e2e.py" ] with-uv: true pre-run-script: ./tests/integration/setup-integration-tests.sh From d564d2c837c3bbb0f6bcac16934148ac6a57389a Mon Sep 17 00:00:00 2001 From: Ali Ugur Date: Thu, 21 May 2026 12:11:24 +0300 Subject: [PATCH 32/46] fix: unit test --- opendkim-operator/tests/unit/test_charm.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/opendkim-operator/tests/unit/test_charm.py b/opendkim-operator/tests/unit/test_charm.py index 1605c5d..a016c4d 100644 --- a/opendkim-operator/tests/unit/test_charm.py +++ b/opendkim-operator/tests/unit/test_charm.py @@ -90,28 +90,28 @@ def test_install(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): pytest.param( "*wrongyaml", "*wrongyaml", - {}, + {"thekey": "PRIVATEKEY"}, ["wrong signingtable", "wrong keytable"], id="Wrong YAML formats", ), pytest.param( "signingtable", "keytable", - {}, + {"thekey": "PRIVATEKEY"}, ["wrong", " signingtable,keytable."], id="Wrong YAML config options", ), pytest.param( json.dumps([["valid", "valid"]]), "keytable", - {}, + {"thekey": "PRIVATEKEY"}, ["wrong", " keytable."], id="Wrong YAML config options", ), pytest.param( "signingtable", json.dumps([["*@example.com", "selector._domainkey.example.com"]]), - {}, + {"thekey": "PRIVATEKEY"}, ["wrong", " signingtable."], id="Wrong YAML config options", ), From b188f321a345f07019da379fc17e9b31230f5a7a Mon Sep 17 00:00:00 2001 From: Ali Ugur Date: Thu, 21 May 2026 12:11:25 +0300 Subject: [PATCH 33/46] fix: update integration test configuration for stack-integration --- .github/workflows/integration_test.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index b0af16d..2527dac 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -66,8 +66,8 @@ jobs: secrets: inherit with: test-tox-env: stack-integration - use-canonical-k8s: true provider: lxd + trivy-fs-enabled: false self-hosted-runner: true self-hosted-runner-label: "self-hosted-linux-amd64-noble-large" juju-channel: 3/stable @@ -76,7 +76,6 @@ jobs: "test_e2e.py" ] with-uv: true - pre-run-script: ./tests/integration/setup-integration-tests.sh allure-report: if: ${{ !cancelled() && github.event_name == 'schedule' }} needs: From 8295fcf54de0b839458a66f1630ba0b0dc6c4a4b Mon Sep 17 00:00:00 2001 From: Ali Ugur Date: Thu, 21 May 2026 12:11:26 +0300 Subject: [PATCH 34/46] fix: update Dovecot charm name in integration tests --- tests/integration/conftest.py | 6 +++--- tests/integration/test_e2e.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 76fb753..2d05f84 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -42,7 +42,7 @@ # --------------------------------------------------------------------------- # Charm / app names # --------------------------------------------------------------------------- -DOVECOT_APP = "dovecot-charm" +DOVECOT_APP = "dovecot" POSTFIX_RELAY_APP = "postfix-relay" OPENDKIM_APP = "opendkim" CONFIGURATOR_APP = "postfix-relay-configurator" @@ -155,7 +155,7 @@ def _select_charm_file_for_app(pytestconfig: pytest.Config, app_name: str) -> st match_order = ( (CONFIGURATOR_APP, "postfix-relay-configurator"), (POSTFIX_RELAY_APP, "postfix-relay"), - (DOVECOT_APP, "dovecot-charm"), + (DOVECOT_APP, "dovecot"), (OPENDKIM_APP, "opendkim"), ) @@ -184,7 +184,7 @@ def _select_charm_file_for_app(pytestconfig: pytest.Config, app_name: str) -> st @pytest.fixture(scope="session", name="dovecot_charm_file") def dovecot_charm_file_fixture(pytestconfig: pytest.Config) -> str: - """Absolute path to the pre-built dovecot-charm .charm file.""" + """Absolute path to the pre-built dovecot .charm file.""" return _select_charm_file_for_app(pytestconfig, DOVECOT_APP) diff --git a/tests/integration/test_e2e.py b/tests/integration/test_e2e.py index 2f8e22f..6632f07 100644 --- a/tests/integration/test_e2e.py +++ b/tests/integration/test_e2e.py @@ -87,7 +87,7 @@ def _sha512_dovecot(password: str, salt: bytes | None = None) -> str: def _setup_dovecot_user(juju: jubilant.Juju, username: str, password: str) -> None: status = juju.status() - unit_name = next(iter(status.apps["dovecot-charm"].units)) + unit_name = next(iter(status.apps["dovecot"].units)) juju.exec(f"id -u {username} &>/dev/null || sudo useradd -m {username}", unit=unit_name) juju.exec( f"id -u {MAILBOX_USER} &>/dev/null || sudo useradd --badname -m {MAILBOX_USER}", From 83d6291e63372e53ef2e8595902272a599f5d06d Mon Sep 17 00:00:00 2001 From: Ali Ugur Date: Thu, 21 May 2026 12:11:27 +0300 Subject: [PATCH 35/46] feat: enhance configurator fixture with SMTP authentication and transport maps --- tests/integration/conftest.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 2d05f84..9a458ad 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -22,6 +22,7 @@ """ import base64 +import hashlib import logging import pathlib import socket @@ -51,6 +52,8 @@ # Domain used throughout the test suite TEST_DOMAIN = "mailstack.internal" SMTP_PORT = 587 +E2E_SMTP_USER = "e2euser" +E2E_SMTP_PASSWORD = "e2e-password" # parents[0]=tests/integration, parents[1]=tests, parents[2]=mailserver-operators/ _REPO_ROOT = pathlib.Path(__file__).resolve().parents[2] @@ -379,6 +382,9 @@ def deploy_postfix_relay_fixture( config={ "relay_domains": f"- {TEST_DOMAIN}", "enable_smtp_auth": "true", + "smtp_auth_users": yaml.dump( + [f"{E2E_SMTP_USER}:{_sha512_dovecot_password(E2E_SMTP_PASSWORD)}"] + ), "enable_reject_unknown_sender_domain": "false", }, ) @@ -418,7 +424,16 @@ def deploy_configurator_fixture( dovecot_ip = dovecot_unit.public_address logger.info("Routing %s → lmtp:inet:%s:24", TEST_DOMAIN, dovecot_ip) - transport_maps = yaml.dump({TEST_DOMAIN: f"lmtp:inet:{dovecot_ip}:24"}) + configurator_config = { + "relay_access_sources": yaml.dump({"192.0.2.0/24": "OK"}), + "relay_recipient_maps": yaml.dump( + {f"noreply@{TEST_DOMAIN}": f"postmaster@{TEST_DOMAIN}"} + ), + "restrict_recipients": yaml.dump({"blocked-recipient@example.invalid": "REJECT"}), + "restrict_senders": yaml.dump({"blocked-sender@example.invalid": "REJECT"}), + "sender_login_maps": yaml.dump({"auth-only@example.invalid": "nobody"}), + "transport_maps": yaml.dump({TEST_DOMAIN: f"lmtp:inet:{dovecot_ip}:24"}), + } if not juju.status().apps.get(CONFIGURATOR_APP): charm_path = ( @@ -429,8 +444,10 @@ def deploy_configurator_fixture( juju.deploy( charm_path, app=CONFIGURATOR_APP, - config={"transport_maps": transport_maps}, + config=configurator_config, ) + else: + juju.config(CONFIGURATOR_APP, configurator_config) _integrate_once(juju, f"{POSTFIX_RELAY_APP}:juju-info", f"{CONFIGURATOR_APP}:juju-info") @@ -666,3 +683,9 @@ def _integrate_once(juju: jubilant.Juju, endpoint_a: str, endpoint_b: str) -> No if "already exists" not in msg and "already related" not in msg: raise logger.debug("Relation %s ↔ %s already exists, skipping", endpoint_a, endpoint_b) + + +def _sha512_dovecot_password(password: str) -> str: + salt = b"mailtest" + digest = hashlib.sha512(password.encode() + salt).digest() + return "{SSHA512}" + base64.b64encode(digest + salt).decode() From f2b5e9ff78e3bafc1457e140ac59a1166b2bd77e Mon Sep 17 00:00:00 2001 From: Ali Ugur Date: Thu, 21 May 2026 12:11:27 +0300 Subject: [PATCH 36/46] feat: add LUKS key management for Dovecot deployment --- tests/integration/conftest.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 9a458ad..f54e67c 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -25,6 +25,7 @@ import hashlib import logging import pathlib +from secrets import token_hex import socket import typing from collections.abc import Generator @@ -328,12 +329,15 @@ def deploy_dovecot_fixture( juju: jubilant.Juju, ) -> str: """Deploy dovecot and wire up TLS.""" + luks_key = token_hex(16) + if not juju.status().apps.get(DOVECOT_APP): charm_path = ( dovecot_charm_file if dovecot_charm_file.startswith(("./", "/")) else f"./{dovecot_charm_file}" ) + secret_id = juju.cli("add-secret", "dovecot-luks-key", f"key={luks_key}").strip() juju.deploy( charm_path, app=DOVECOT_APP, @@ -341,11 +345,13 @@ def deploy_dovecot_fixture( "mailname": TEST_DOMAIN, "postmaster-address": f"postmaster@{TEST_DOMAIN}", "primary-unit": f"{DOVECOT_APP}/0", - "manage-luks": "false", + "luks-auto-provisioning": True, + "luks-key": secret_id, }, constraints={"virt-type": "virtual-machine"}, trust=True, ) + juju.cli("grant-secret", "dovecot-luks-key", DOVECOT_APP) # Relate to TLS provider if not already related. _integrate_once(juju, f"{DOVECOT_APP}:certificates", f"{self_signed_app}:certificates") From 3c914ae1d0820822495e65a77e7385a1c8f6c184 Mon Sep 17 00:00:00 2001 From: Ali Ugur Date: Thu, 21 May 2026 12:11:28 +0300 Subject: [PATCH 37/46] fix: use authenticated sender in e2e test --- tests/integration/test_e2e.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_e2e.py b/tests/integration/test_e2e.py index 6632f07..ec19e0a 100644 --- a/tests/integration/test_e2e.py +++ b/tests/integration/test_e2e.py @@ -49,7 +49,7 @@ def test_e2e(juju: jubilant.Juju, mail_stack: Dict[str, str]) -> None: ) subject = f"Whole system e2e {int(time.time())}" - from_addr = f"sender@{TEST_DOMAIN}" + from_addr = MAILBOX_USER to_addr = MAILBOX_USER message = ( f"Subject: {subject}\r\n" From d6be089012898430c6cc60fea3705aa816289540 Mon Sep 17 00:00:00 2001 From: Ali Ugur Date: Thu, 21 May 2026 12:11:29 +0300 Subject: [PATCH 38/46] feat(dovecot): add mail-user provisioning action Add a create-mail-user charm action and switch e2e user setup to call it, so user provisioning is idempotent and no longer relies on ad-hoc shell commands in tests. --- dovecot-charm/charmcraft.yaml | 14 +++++ dovecot-charm/src/charm.py | 81 ++++++++++++++++++++++++++ dovecot-charm/tests/unit/test_charm.py | 81 ++++++++++++++++++++++++++ tests/integration/test_e2e.py | 28 ++++----- 4 files changed, 187 insertions(+), 17 deletions(-) diff --git a/dovecot-charm/charmcraft.yaml b/dovecot-charm/charmcraft.yaml index 444d260..1fde8bf 100644 --- a/dovecot-charm/charmcraft.yaml +++ b/dovecot-charm/charmcraft.yaml @@ -99,6 +99,20 @@ config: type: string actions: + create-mail-user: + description: Create or update local mail users used for SMTP and mailbox login. + params: + username: + type: string + description: Primary system username to create/update. + password: + type: string + description: Plaintext password to set for created users. + mailbox-user: + type: string + description: Optional mailbox username (for example user@example.com). + default: "" + required: [username, password] clear-queue: description: Forcibly remove messages from the Postfix mail queue. params: diff --git a/dovecot-charm/src/charm.py b/dovecot-charm/src/charm.py index bbbee18..1ff273a 100644 --- a/dovecot-charm/src/charm.py +++ b/dovecot-charm/src/charm.py @@ -11,6 +11,7 @@ import typing from functools import cached_property from pathlib import Path +from pwd import getpwnam import jinja2 import ops @@ -61,6 +62,7 @@ def __init__(self, *args): self.framework.observe(self.on.config_changed, self._reconcile) self.framework.observe(self.on.upgrade_charm, self._on_install) self.framework.observe(self.on.clear_queue_action, self._on_clear_queue_action) + self.framework.observe(self.on.create_mail_user_action, self._on_create_mail_user_action) self.framework.observe(self.on.gdpr_archive_action, self._on_gdpr_archive) self.framework.observe(self.on.gdpr_delete_action, self._on_gdpr_delete) self.framework.observe(self.on.gdpr_takeout_action, self._on_gdpr_takeout) @@ -234,6 +236,85 @@ def _open_ports(self): self.unit.open_port("tcp", 995) self.unit.open_port("tcp", 4190) + def _on_create_mail_user_action(self, event): + """Create or update local mail users for integration and operations workflows.""" + username = str(event.params.get("username", "")).strip() + password = str(event.params.get("password", "")) + mailbox_user = str(event.params.get("mailbox-user", "")).strip() + + if not username: + event.fail("Parameter 'username' is required.") + return + if not password: + event.fail("Parameter 'password' is required.") + return + + users_to_manage = [username] + if mailbox_user and mailbox_user != username: + users_to_manage.append(mailbox_user) + + created_users: list[str] = [] + updated_users: list[str] = [] + + try: + for user in users_to_manage: + if self._system_user_exists(user): + updated_users.append(user) + else: + self._create_system_user(user) + created_users.append(user) + self._ensure_user_in_mail_group(user) + self._set_system_user_password(user, password) + except (subprocess.CalledProcessError, KeyError, FileNotFoundError) as exc: + event.fail(f"Failed to manage users: {exc}") + return + + event.set_results( + { + "status": "success", + "created": ",".join(created_users), + "updated": ",".join(updated_users), + } + ) + + @staticmethod + def _system_user_exists(username: str) -> bool: + """Return whether a local system user exists.""" + try: + getpwnam(username) + return True + except KeyError: + return False + + @staticmethod + def _create_system_user(username: str) -> None: + """Create a local system user, allowing mailbox-style names if needed.""" + command = ["/usr/sbin/useradd", "-m", username] + if "@" in username: + command.insert(1, "--badname") + subprocess.run(command, check=True, capture_output=True, text=True) + + @staticmethod + def _ensure_user_in_mail_group(username: str) -> None: + """Ensure the user is a member of the mail group.""" + subprocess.run( + ["/usr/sbin/usermod", "-aG", "mail", username], + check=True, + capture_output=True, + text=True, + ) + + @staticmethod + def _set_system_user_password(username: str, password: str) -> None: + """Set the password for the local system user.""" + subprocess.run( + ["/usr/sbin/chpasswd"], + check=True, + capture_output=True, + text=True, + input=f"{username}:{password}", + ) + def _on_clear_queue_action(self, event): """Handle the clear-queue action.""" queue_to_clear = event.params.get("queue", "deferred") diff --git a/dovecot-charm/tests/unit/test_charm.py b/dovecot-charm/tests/unit/test_charm.py index c044f94..e038b30 100644 --- a/dovecot-charm/tests/unit/test_charm.py +++ b/dovecot-charm/tests/unit/test_charm.py @@ -180,6 +180,87 @@ def test_clear_queue_failure(ctx, base_state): assert "postsuper" in exc_info.value.message +def test_create_mail_user_action_creates_primary_and_mailbox_user(ctx, base_state): + """create-mail-user creates missing users, groups and passwords.""" + with ( + patch( + "charm.getpwnam", + side_effect=[KeyError("e2euser"), KeyError("e2euser@example.com")], + ), + patch("charm.subprocess.run", return_value=MagicMock(returncode=0)), + ): + ctx.run( + ctx.on.action( + "create-mail-user", + params={ + "username": "e2euser", + "password": "secret", + "mailbox-user": "e2euser@example.com", + }, + ), + base_state, + ) + + assert ctx.action_results["status"] == "success" + assert ctx.action_results["created"] == "e2euser,e2euser@example.com" + assert ctx.action_results["updated"] == "" + + +def test_create_mail_user_action_updates_existing_user(ctx, base_state): + """create-mail-user updates password/group for existing users.""" + existing_user = object() + with ( + patch("charm.getpwnam", return_value=existing_user), + patch("charm.subprocess.run", return_value=MagicMock(returncode=0)), + ): + ctx.run( + ctx.on.action( + "create-mail-user", + params={ + "username": "e2euser", + "password": "secret", + }, + ), + base_state, + ) + + assert ctx.action_results["status"] == "success" + assert ctx.action_results["created"] == "" + assert ctx.action_results["updated"] == "e2euser" + + +def test_create_mail_user_action_requires_username(ctx, base_state): + """create-mail-user fails fast when username is missing.""" + with pytest.raises(ops.testing.ActionFailed) as exc_info: + ctx.run( + ctx.on.action( + "create-mail-user", + params={ + "username": "", + "password": "secret", + }, + ), + base_state, + ) + assert "username" in exc_info.value.message + + +def test_create_mail_user_action_requires_password(ctx, base_state): + """create-mail-user fails fast when password is missing.""" + with pytest.raises(ops.testing.ActionFailed) as exc_info: + ctx.run( + ctx.on.action( + "create-mail-user", + params={ + "username": "e2euser", + "password": "", + }, + ), + base_state, + ) + assert "password" in exc_info.value.message + + def test_force_sync_success(ctx, base_state): """force-sync succeeds when this unit is primary and a secondary exists.""" mock_result = MagicMock(stdout="ok", stderr="") diff --git a/tests/integration/test_e2e.py b/tests/integration/test_e2e.py index ec19e0a..27f83da 100644 --- a/tests/integration/test_e2e.py +++ b/tests/integration/test_e2e.py @@ -32,7 +32,17 @@ def test_e2e(juju: jubilant.Juju, mail_stack: Dict[str, str]) -> None: relay_ip = mail_stack["postfix_relay_ip"] dovecot_ip = mail_stack["dovecot_ip"] - _setup_dovecot_user(juju, TEST_USER, TEST_PASSWORD) + dovecot_unit = f"{mail_stack['dovecot_app']}/0" + action_result = juju.run( + dovecot_unit, + "create-mail-user", + params={ + "username": TEST_USER, + "password": TEST_PASSWORD, + "mailbox-user": MAILBOX_USER, + }, + ) + assert action_result.status == "completed" smtp_auth_users = yaml.dump([f"{TEST_USER}:{_sha512_dovecot(TEST_PASSWORD)}"]) juju.config( @@ -83,22 +93,6 @@ def _sha512_dovecot(password: str, salt: bytes | None = None) -> str: salt = os.urandom(8) digest = hashlib.sha512(password.encode() + salt).digest() return "{SSHA512}" + base64.b64encode(digest + salt).decode() - - -def _setup_dovecot_user(juju: jubilant.Juju, username: str, password: str) -> None: - status = juju.status() - unit_name = next(iter(status.apps["dovecot"].units)) - juju.exec(f"id -u {username} &>/dev/null || sudo useradd -m {username}", unit=unit_name) - juju.exec( - f"id -u {MAILBOX_USER} &>/dev/null || sudo useradd --badname -m {MAILBOX_USER}", - unit=unit_name, - ) - juju.exec(f"sudo usermod -aG mail {username}", unit=unit_name) - juju.exec(f"sudo usermod -aG mail {MAILBOX_USER}", unit=unit_name) - juju.exec(f"echo '{username}:{password}' | sudo chpasswd", unit=unit_name) - juju.exec(f"echo '{MAILBOX_USER}:{password}' | sudo chpasswd", unit=unit_name) - - def _wait_for_subject(host: str, username: str, password: str, subject: str) -> bytes: ctx = ssl.create_default_context() ctx.check_hostname = False From 2726e04ef917c25b5584fc14113f601e66654385 Mon Sep 17 00:00:00 2001 From: Ali Ugur Date: Thu, 21 May 2026 12:11:30 +0300 Subject: [PATCH 39/46] fix(test): align smtp auth user with sender Postfix enforces sender ownership against the authenticated SASL identity. Use the mailbox user for smtp_auth_users and SMTP login so the e2e flow matches sender_login checks. --- tests/integration/test_e2e.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_e2e.py b/tests/integration/test_e2e.py index 27f83da..9b4d37b 100644 --- a/tests/integration/test_e2e.py +++ b/tests/integration/test_e2e.py @@ -44,7 +44,7 @@ def test_e2e(juju: jubilant.Juju, mail_stack: Dict[str, str]) -> None: ) assert action_result.status == "completed" - smtp_auth_users = yaml.dump([f"{TEST_USER}:{_sha512_dovecot(TEST_PASSWORD)}"]) + smtp_auth_users = yaml.dump([f"{MAILBOX_USER}:{_sha512_dovecot(TEST_PASSWORD)}"]) juju.config( "postfix-relay", { @@ -77,7 +77,7 @@ def test_e2e(juju: jubilant.Juju, mail_stack: Dict[str, str]) -> None: server.ehlo() server.starttls(context=tls_ctx) server.ehlo() - server.login(TEST_USER, TEST_PASSWORD) + server.login(MAILBOX_USER, TEST_PASSWORD) server.sendmail(from_addr, [to_addr], message) raw_message = _wait_for_subject(dovecot_ip, MAILBOX_USER, TEST_PASSWORD, subject) From 2465aff99fe6827478c21b1e43b306be02dc5fe2 Mon Sep 17 00:00:00 2001 From: Ali Ugur Date: Thu, 21 May 2026 12:11:31 +0300 Subject: [PATCH 40/46] fix(test): allow e2e mailbox sender login mapping The configurator fixture enforces sender_login_maps, which rejected the e2e mailbox sender. Add a sender_login_maps entry for e2euser@mailstack.internal so Postfix sender ownership checks pass in test_e2e. --- tests/integration/conftest.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index f54e67c..d6ae9f5 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -437,7 +437,12 @@ def deploy_configurator_fixture( ), "restrict_recipients": yaml.dump({"blocked-recipient@example.invalid": "REJECT"}), "restrict_senders": yaml.dump({"blocked-sender@example.invalid": "REJECT"}), - "sender_login_maps": yaml.dump({"auth-only@example.invalid": "nobody"}), + "sender_login_maps": yaml.dump( + { + "auth-only@example.invalid": "nobody", + f"{E2E_SMTP_USER}@{TEST_DOMAIN}": f"{E2E_SMTP_USER}@{TEST_DOMAIN}", + } + ), "transport_maps": yaml.dump({TEST_DOMAIN: f"lmtp:inet:{dovecot_ip}:24"}), } From e0a9e784a440ead2b2002274223bcf00b5df9e65 Mon Sep 17 00:00:00 2001 From: Ali Ugur Date: Thu, 21 May 2026 12:11:31 +0300 Subject: [PATCH 41/46] refactor(tests): improve password handling and clean up integration test configurations --- dovecot-charm/tests/unit/test_charm.py | 9 +- tests/integration/conftest.py | 195 ++++++++++++------------- tox.toml | 13 -- 3 files changed, 95 insertions(+), 122 deletions(-) diff --git a/dovecot-charm/tests/unit/test_charm.py b/dovecot-charm/tests/unit/test_charm.py index e038b30..f344b07 100644 --- a/dovecot-charm/tests/unit/test_charm.py +++ b/dovecot-charm/tests/unit/test_charm.py @@ -3,6 +3,7 @@ import dataclasses import json +import secrets from pathlib import Path from subprocess import CalledProcessError # nosec from unittest.mock import MagicMock, patch @@ -194,7 +195,7 @@ def test_create_mail_user_action_creates_primary_and_mailbox_user(ctx, base_stat "create-mail-user", params={ "username": "e2euser", - "password": "secret", + "password": secrets.token_hex(8), "mailbox-user": "e2euser@example.com", }, ), @@ -218,7 +219,7 @@ def test_create_mail_user_action_updates_existing_user(ctx, base_state): "create-mail-user", params={ "username": "e2euser", - "password": "secret", + "password": secrets.token_hex(8), }, ), base_state, @@ -237,7 +238,7 @@ def test_create_mail_user_action_requires_username(ctx, base_state): "create-mail-user", params={ "username": "", - "password": "secret", + "password": secrets.token_hex(8), }, ), base_state, @@ -253,7 +254,7 @@ def test_create_mail_user_action_requires_password(ctx, base_state): "create-mail-user", params={ "username": "e2euser", - "password": "", + "password": "", # nosec B105 }, ), base_state, diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index d6ae9f5..f07508c 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -29,6 +29,7 @@ import socket import typing from collections.abc import Generator +import json import jubilant import pytest @@ -36,7 +37,7 @@ from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric import rsa -from helpers import select_charm_file, sha512_dovecot_password +from helpers import select_charm_file logger = logging.getLogger(__name__) @@ -262,6 +263,84 @@ def deploy_self_signed_certs_fixture(juju: jubilant.Juju) -> str: return SELF_SIGNED_APP +@pytest.fixture(scope="module", name="postfix_stack") +def postfix_stack_fixture( + juju: jubilant.Juju, + pytestconfig: pytest.Config, + self_signed_app: str, +) -> typing.Dict[str, str]: + """Deploy postfix-relay + postfix-relay-configurator configured for sender_login enforcement. + + Returns a dict with ``postfix_relay_ip``. + """ + # --- postfix-relay --- + auth_password = "test-password" + if not juju.status().apps.get(POSTFIX_RELAY_APP): + charm_path = select_charm_file(pytestconfig, "postfix-relay_") + if not charm_path.startswith(("./", "/")): + charm_path = f"./{charm_path}" + juju.deploy( + charm_path, + app=POSTFIX_RELAY_APP, + config={ + "relay_domains": f"- {TEST_DOMAIN}", + "enable_smtp_auth": "true", + "smtp_auth_users": yaml.dump( + [f"testuser:{_sha512_dovecot_password(auth_password)}"] + ), + "enable_reject_unknown_sender_domain": "false", + }, + ) + _integrate_once( + juju, + f"{POSTFIX_RELAY_APP}:certificates", + f"{self_signed_app}:certificates", + ) + + # --- postfix-relay-configurator --- + authorized_sender = f"authorized@{TEST_DOMAIN}" + if not juju.status().apps.get(CONFIGURATOR_APP): + charm_path = select_charm_file(pytestconfig, "postfix-relay-configurator_") + if not charm_path.startswith(("./", "/")): + charm_path = f"./{charm_path}" + juju.deploy( + charm_path, + app=CONFIGURATOR_APP, + config={ + "sender_login_maps": yaml.dump({authorized_sender: "testuser"}), + }, + ) + _integrate_once( + juju, + f"{POSTFIX_RELAY_APP}:juju-info", + f"{CONFIGURATOR_APP}:juju-info", + ) + + # Wait for both to be active. + def _both_active(status: jubilant.Status) -> bool: + if not status.apps.get(POSTFIX_RELAY_APP): + return False + if not status.apps[POSTFIX_RELAY_APP].is_active: + return False + for unit in status.apps[POSTFIX_RELAY_APP].units.values(): + subs = unit.subordinates or {} + conf_subs = {k: v for k, v in subs.items() if CONFIGURATOR_APP in k} + if not conf_subs: + return False + for sub in conf_subs.values(): + if sub.workload_status.current != "active": + return False + return True + + juju.wait(_both_active, error=jubilant.any_error, timeout=15 * 60) + logger.info("postfix-relay + configurator active for maps tests") + + status = juju.status() + relay_ip = next(iter(status.apps[POSTFIX_RELAY_APP].units.values())).public_address + logger.info("postfix-relay IP: %s", relay_ip) + return {"postfix_relay_ip": relay_ip} + + # --------------------------------------------------------------------------- # Deploy: opendkim # --------------------------------------------------------------------------- @@ -374,6 +453,7 @@ def deploy_postfix_relay_fixture( self_signed_app: str, opendkim_app: str, juju: jubilant.Juju, + pytestconfig: pytest.Config, ) -> str: """Deploy postfix-relay and integrate with TLS provider and opendkim milter.""" if not juju.status().apps.get(POSTFIX_RELAY_APP): @@ -460,14 +540,15 @@ def deploy_configurator_fixture( else: juju.config(CONFIGURATOR_APP, configurator_config) - _integrate_once(juju, f"{POSTFIX_RELAY_APP}:juju-info", f"{CONFIGURATOR_APP}:juju-info") + _integrate_once(juju, f"{postfix_relay_app}:juju-info", f"{CONFIGURATOR_APP}:juju-info") - def _configurator_active(status: jubilant.Status) -> bool: - """Return True once the configurator subordinate is active on all relay units.""" - if not status.apps.get(CONFIGURATOR_APP): + # Wait for both to be active. + def _both_active(status: jubilant.Status) -> bool: + if not status.apps.get(postfix_relay_app): + return False + if not status.apps[POSTFIX_RELAY_APP].is_active: return False - relay_units = status.apps[POSTFIX_RELAY_APP].units - for unit in relay_units.values(): + for unit in status.apps[POSTFIX_RELAY_APP].units.values(): subs = unit.subordinates or {} conf_subs = {k: v for k, v in subs.items() if CONFIGURATOR_APP in k} if not conf_subs: @@ -475,13 +556,9 @@ def _configurator_active(status: jubilant.Status) -> bool: for sub in conf_subs.values(): if sub.workload_status.current != "active": return False - return status.apps[POSTFIX_RELAY_APP].is_active + return True - juju.wait( - _configurator_active, - error=jubilant.any_error, - timeout=10 * 60, - ) + juju.wait(_both_active, error=jubilant.any_error, timeout=15 * 60) logger.info("postfix-relay-configurator subordinate is active") return CONFIGURATOR_APP @@ -500,7 +577,6 @@ def configure_opendkim_fixture( Returns the opendkim app name once the app is active. """ - import json # noqa: PLC0415 — local import to avoid top-level cost when not needed selector = "default" keyname = f"{TEST_DOMAIN.replace('.', '-')}-{selector}" @@ -594,97 +670,6 @@ def mail_stack_fixture( } -# --------------------------------------------------------------------------- -# postfix_stack fixture — for configurator-maps integration tests -# --------------------------------------------------------------------------- -@pytest.fixture(scope="module", name="postfix_stack") -def postfix_stack_fixture( - juju: jubilant.Juju, - pytestconfig: pytest.Config, -) -> typing.Dict[str, str]: - """Deploy postfix-relay + postfix-relay-configurator for sender_login enforcement tests. - - Returns a dict with ``postfix_relay_ip``. - """ - if not juju.status().apps.get(SELF_SIGNED_APP): - juju.deploy(SELF_SIGNED_APP, channel="latest/stable") - juju.wait( - lambda status: status.apps[SELF_SIGNED_APP].is_active, - error=jubilant.any_error, - timeout=10 * 60, - ) - - # --- postfix-relay --- - auth_password = "test-password" - if not juju.status().apps.get(POSTFIX_RELAY_APP): - charm_path = select_charm_file(pytestconfig, "postfix-relay_") - if not charm_path.startswith(("./", "/")): - charm_path = f"./{charm_path}" - juju.deploy( - charm_path, - app=POSTFIX_RELAY_APP, - config={ - "relay_domains": f"- {TEST_DOMAIN}", - "enable_smtp_auth": "true", - "smtp_auth_users": yaml.dump( - [f"testuser:{sha512_dovecot_password(auth_password)}"] - ), - "enable_reject_unknown_sender_domain": "false", - }, - ) - _integrate_once( - juju, - f"{POSTFIX_RELAY_APP}:certificates", - f"{SELF_SIGNED_APP}:certificates", - ) - - # --- postfix-relay-configurator --- - authorized_sender = f"authorized@{TEST_DOMAIN}" - if not juju.status().apps.get(CONFIGURATOR_APP): - charm_path = select_charm_file(pytestconfig, "postfix-relay-configurator_") - if not charm_path.startswith(("./", "/")): - charm_path = f"./{charm_path}" - juju.deploy( - charm_path, - app=CONFIGURATOR_APP, - config={ - "sender_login_maps": yaml.dump({authorized_sender: "testuser"}), - }, - ) - _integrate_once( - juju, - f"{POSTFIX_RELAY_APP}:juju-info", - f"{CONFIGURATOR_APP}:juju-info", - ) - - # Wait for both to be active. - def _both_active(status: jubilant.Status) -> bool: - if not status.apps.get(POSTFIX_RELAY_APP): - return False - if not status.apps[POSTFIX_RELAY_APP].is_active: - return False - for unit in status.apps[POSTFIX_RELAY_APP].units.values(): - subs = unit.subordinates or {} - conf_subs = {k: v for k, v in subs.items() if CONFIGURATOR_APP in k} - if not conf_subs: - return False - for sub in conf_subs.values(): - if sub.workload_status.current != "active": - return False - return True - - juju.wait(_both_active, error=jubilant.any_error, timeout=15 * 60) - logger.info("postfix-relay + configurator active for maps tests") - - status = juju.status() - relay_ip = next(iter(status.apps[POSTFIX_RELAY_APP].units.values())).public_address - logger.info("postfix-relay IP: %s", relay_ip) - return {"postfix_relay_ip": relay_ip} - - -# --------------------------------------------------------------------------- -# Helper: idempotent juju integrate -# --------------------------------------------------------------------------- def _integrate_once(juju: jubilant.Juju, endpoint_a: str, endpoint_b: str) -> None: """Call ``juju integrate`` tolerating 'already related' errors.""" try: diff --git a/tox.toml b/tox.toml index 26dd028..fffe60d 100644 --- a/tox.toml +++ b/tox.toml @@ -97,19 +97,6 @@ commands = [ ] dependency_groups = [ "integration" ] -[env.stack-integration.setenv] -PYTHONPATH = "{toxinidir}/tests/integration" -PY_COLORS = "1" - -[env.lint-fix] -description = "Apply lint fixes for all charms" -allowlist_externals = [ "tox" ] -commands = [ - [ "tox", "-c", "{toxinidir}/dovecot-charm/tox.toml", "-e", "lint-fix", { replace = "posargs", extend = true } ], - [ "tox", "-c", "{toxinidir}/opendkim-operator/tox.toml", "-e", "lint-fix", { replace = "posargs", extend = true } ], -] -dependency_groups = [ "lint" ] - [vars] src_path = "{toxinidir}/src/" tst_path = "{toxinidir}/tests/" From 6867c637e5c67069c6b19993571970d4c9f28763 Mon Sep 17 00:00:00 2001 From: Ali Ugur Date: Thu, 21 May 2026 12:11:32 +0300 Subject: [PATCH 42/46] fix(tests): update integration test configurations and change Juju fixture scope --- .github/workflows/integration_test.yaml | 3 ++ tests/integration/conftest.py | 59 +++++++++++++++++-------- 2 files changed, 43 insertions(+), 19 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 2527dac..ded9ba9 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -50,6 +50,7 @@ jobs: canonical/operator-workflows/.github/workflows/integration_test.yaml@main secrets: inherit with: + extra-arguments: '-x --log-format="%(asctime)s %(levelname)s %(message)s"' juju-channel: 3/stable modules: | [ @@ -65,12 +66,14 @@ jobs: canonical/operator-workflows/.github/workflows/integration_test.yaml@main secrets: inherit with: + extra-arguments: '-x --log-format="%(asctime)s %(levelname)s %(message)s"' test-tox-env: stack-integration provider: lxd trivy-fs-enabled: false self-hosted-runner: true self-hosted-runner-label: "self-hosted-linux-amd64-noble-large" juju-channel: 3/stable + charmcraft-channel: latest/edge modules: | [ "test_e2e.py" diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index f07508c..7e74b35 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -37,7 +37,6 @@ from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric import rsa -from helpers import select_charm_file logger = logging.getLogger(__name__) @@ -93,6 +92,44 @@ def pytest_addoption(parser: pytest.Parser) -> None: ) + +# --------------------------------------------------------------------------- +# Helper functions +# --------------------------------------------------------------------------- +def _sha512_dovecot_password(password: str) -> str: + """Generate a SSHA512 password hash compatible with dovecot.""" + salt = b"mailtest" + digest = hashlib.sha512(password.encode() + salt).digest() + return "{SSHA512}" + base64.b64encode(digest + salt).decode() + + +def _integrate_once(juju: jubilant.Juju, endpoint_a: str, endpoint_b: str) -> None: + """Call ``juju integrate`` tolerating 'already related' errors.""" + try: + juju.integrate(endpoint_a, endpoint_b) + except Exception as exc: # noqa: BLE001 + msg = str(exc) + if "already exists" not in msg and "already related" not in msg: + raise + logger.debug("Relation %s ↔ %s already exists, skipping", endpoint_a, endpoint_b) + + +def _select_charm_file(pytestconfig: pytest.Config, marker: str) -> str: + """Select charm file matching marker from --charm-file options.""" + charm_files: list[str] = pytestconfig.getoption("--charm-file", default=[]) + for path in charm_files: + if marker in pathlib.Path(path).name.lower(): + return path + use_existing = pytestconfig.getoption("--use-existing", default=False) + if use_existing: + return "" + provided = ", ".join(charm_files) if charm_files else "" + raise AssertionError(f"Missing --charm-file matching '{marker}'. Provided: {provided}.") + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- @pytest.fixture(scope="session", name="juju") def juju_fixture( request: pytest.FixtureRequest, @@ -276,7 +313,7 @@ def postfix_stack_fixture( # --- postfix-relay --- auth_password = "test-password" if not juju.status().apps.get(POSTFIX_RELAY_APP): - charm_path = select_charm_file(pytestconfig, "postfix-relay_") + charm_path = _select_charm_file(pytestconfig, "postfix-relay_") if not charm_path.startswith(("./", "/")): charm_path = f"./{charm_path}" juju.deploy( @@ -300,7 +337,7 @@ def postfix_stack_fixture( # --- postfix-relay-configurator --- authorized_sender = f"authorized@{TEST_DOMAIN}" if not juju.status().apps.get(CONFIGURATOR_APP): - charm_path = select_charm_file(pytestconfig, "postfix-relay-configurator_") + charm_path = _select_charm_file(pytestconfig, "postfix-relay-configurator_") if not charm_path.startswith(("./", "/")): charm_path = f"./{charm_path}" juju.deploy( @@ -669,19 +706,3 @@ def mail_stack_fixture( "postfix_relay_ip": relay_ip, } - -def _integrate_once(juju: jubilant.Juju, endpoint_a: str, endpoint_b: str) -> None: - """Call ``juju integrate`` tolerating 'already related' errors.""" - try: - juju.integrate(endpoint_a, endpoint_b) - except Exception as exc: # noqa: BLE001 - msg = str(exc) - if "already exists" not in msg and "already related" not in msg: - raise - logger.debug("Relation %s ↔ %s already exists, skipping", endpoint_a, endpoint_b) - - -def _sha512_dovecot_password(password: str) -> str: - salt = b"mailtest" - digest = hashlib.sha512(password.encode() + salt).digest() - return "{SSHA512}" + base64.b64encode(digest + salt).decode() From ff0e999ca073687fa11823d15a19081d09bf4b1d Mon Sep 17 00:00:00 2001 From: Ali Ugur Date: Thu, 21 May 2026 12:11:33 +0300 Subject: [PATCH 43/46] feat(tests): enhance integration tests and update procmail configuration --- dovecot-charm/src/charm.py | 6 +- tests/integration/conftest.py | 166 ++++++++------------ tests/integration/test_configurator_maps.py | 12 +- tests/integration/test_e2e.py | 19 +-- 4 files changed, 80 insertions(+), 123 deletions(-) diff --git a/dovecot-charm/src/charm.py b/dovecot-charm/src/charm.py index 1ff273a..3e7e5f7 100644 --- a/dovecot-charm/src/charm.py +++ b/dovecot-charm/src/charm.py @@ -227,10 +227,8 @@ def _open_ports(self): intentionally for mail relay traffic; port 25 is standard SMTP (not implicit TLS), and STARTTLS is expected when supported by the peer. """ - # Port 25 intentionally accepts standard SMTP from postfix-relay. This - # is not an implicit-TLS port; peers should negotiate STARTTLS when - # available before Postfix forwards mail to Dovecot via the LMTP Unix - # socket for final delivery into the user mailbox. + # Port 25 accepts SMTP from postfix-relay, which forwards to Dovecot via + # the LMTP Unix socket for final delivery into the user mailbox. self.unit.open_port("tcp", 25) self.unit.open_port("tcp", 993) self.unit.open_port("tcp", 995) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 7e74b35..2e668ff 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -52,15 +52,15 @@ # Domain used throughout the test suite TEST_DOMAIN = "mailstack.internal" -SMTP_PORT = 587 -E2E_SMTP_USER = "e2euser" -E2E_SMTP_PASSWORD = "e2e-password" +TEST_SMTP_USER = "e2euser" +TEST_SMTP_PASSWORD = token_hex(16) # parents[0]=tests/integration, parents[1]=tests, parents[2]=mailserver-operators/ _REPO_ROOT = pathlib.Path(__file__).resolve().parents[2] OPENDKIM_SNAP_DIR = _REPO_ROOT / "opendkim-snap" - +SMTP_PORT = 587 +AUTHORIZED_SENDER = f"authorized@{TEST_DOMAIN}" # --------------------------------------------------------------------------- # pytest CLI options # --------------------------------------------------------------------------- @@ -131,22 +131,14 @@ def _select_charm_file(pytestconfig: pytest.Config, marker: str) -> str: # Fixtures # --------------------------------------------------------------------------- @pytest.fixture(scope="session", name="juju") -def juju_fixture( - request: pytest.FixtureRequest, -) -> Generator[jubilant.Juju, None, None]: - """Session-scoped Juju client pointing at a temporary (or named) model.""" - - def _show_debug_log(juju: jubilant.Juju) -> None: - if request.session.testsfailed: - log = juju.debug_log(limit=2000) - print(log, end="") - +def juju_fixture(request: pytest.FixtureRequest) -> Generator[jubilant.Juju, None, None]: + """Session-scoped Juju client in a temporary model for integration tests.""" + logging.getLogger('jubilant.wait').setLevel(logging.WARNING) use_existing = request.config.getoption("--use-existing", default=False) if use_existing: juju = jubilant.Juju() juju.model_config({"automatically-retry-hooks": True}) yield juju - _show_debug_log(juju) return model = request.config.getoption("--model") @@ -154,7 +146,6 @@ def _show_debug_log(juju: jubilant.Juju) -> None: juju = jubilant.Juju(model=model) juju.model_config({"automatically-retry-hooks": True}) yield juju - _show_debug_log(juju) return keep_models = typing.cast(bool, request.config.getoption("--keep-models")) @@ -162,7 +153,6 @@ def _show_debug_log(juju: jubilant.Juju) -> None: juju.wait_timeout = 15 * 60 juju.model_config({"automatically-retry-hooks": True}) yield juju - _show_debug_log(juju) return @@ -300,57 +290,20 @@ def deploy_self_signed_certs_fixture(juju: jubilant.Juju) -> str: return SELF_SIGNED_APP -@pytest.fixture(scope="module", name="postfix_stack") +@pytest.fixture(scope="session", name="postfix_stack") def postfix_stack_fixture( juju: jubilant.Juju, - pytestconfig: pytest.Config, - self_signed_app: str, + postfix_relay_app: str, + postfix_relay_configurator_app: str, ) -> typing.Dict[str, str]: """Deploy postfix-relay + postfix-relay-configurator configured for sender_login enforcement. Returns a dict with ``postfix_relay_ip``. """ - # --- postfix-relay --- - auth_password = "test-password" - if not juju.status().apps.get(POSTFIX_RELAY_APP): - charm_path = _select_charm_file(pytestconfig, "postfix-relay_") - if not charm_path.startswith(("./", "/")): - charm_path = f"./{charm_path}" - juju.deploy( - charm_path, - app=POSTFIX_RELAY_APP, - config={ - "relay_domains": f"- {TEST_DOMAIN}", - "enable_smtp_auth": "true", - "smtp_auth_users": yaml.dump( - [f"testuser:{_sha512_dovecot_password(auth_password)}"] - ), - "enable_reject_unknown_sender_domain": "false", - }, - ) _integrate_once( juju, - f"{POSTFIX_RELAY_APP}:certificates", - f"{self_signed_app}:certificates", - ) - - # --- postfix-relay-configurator --- - authorized_sender = f"authorized@{TEST_DOMAIN}" - if not juju.status().apps.get(CONFIGURATOR_APP): - charm_path = _select_charm_file(pytestconfig, "postfix-relay-configurator_") - if not charm_path.startswith(("./", "/")): - charm_path = f"./{charm_path}" - juju.deploy( - charm_path, - app=CONFIGURATOR_APP, - config={ - "sender_login_maps": yaml.dump({authorized_sender: "testuser"}), - }, - ) - _integrate_once( - juju, - f"{POSTFIX_RELAY_APP}:juju-info", - f"{CONFIGURATOR_APP}:juju-info", + f"{postfix_relay_app}:juju-info", + f"{postfix_relay_configurator_app}:juju-info", ) # Wait for both to be active. @@ -375,7 +328,7 @@ def _both_active(status: jubilant.Status) -> bool: status = juju.status() relay_ip = next(iter(status.apps[POSTFIX_RELAY_APP].units.values())).public_address logger.info("postfix-relay IP: %s", relay_ip) - return {"postfix_relay_ip": relay_ip} + return {"postfix_relay_app": postfix_relay_app, "postfix_relay_ip": relay_ip} # --------------------------------------------------------------------------- @@ -488,11 +441,9 @@ def deploy_dovecot_fixture( def deploy_postfix_relay_fixture( postfix_relay_charm_file: str, self_signed_app: str, - opendkim_app: str, juju: jubilant.Juju, - pytestconfig: pytest.Config, ) -> str: - """Deploy postfix-relay and integrate with TLS provider and opendkim milter.""" + """Deploy postfix-relay and integrate with TLS provider.""" if not juju.status().apps.get(POSTFIX_RELAY_APP): charm_path = ( postfix_relay_charm_file @@ -506,86 +457,98 @@ def deploy_postfix_relay_fixture( "relay_domains": f"- {TEST_DOMAIN}", "enable_smtp_auth": "true", "smtp_auth_users": yaml.dump( - [f"{E2E_SMTP_USER}:{_sha512_dovecot_password(E2E_SMTP_PASSWORD)}"] + [f"{TEST_SMTP_USER}:{_sha512_dovecot_password(TEST_SMTP_PASSWORD)}"] ), "enable_reject_unknown_sender_domain": "false", }, ) _integrate_once(juju, f"{POSTFIX_RELAY_APP}:certificates", f"{self_signed_app}:certificates") - _integrate_once(juju, f"{POSTFIX_RELAY_APP}:milter", f"{opendkim_app}:milter") juju.wait( lambda status: ( status.apps[POSTFIX_RELAY_APP].is_active - and status.apps[opendkim_app].app_status.current in {"blocked", "active"} ), timeout=15 * 60, ) - logger.info("postfix-relay is active (opendkim is blocked or already active)") + logger.info("postfix-relay is active") return POSTFIX_RELAY_APP +@pytest.fixture(scope="session", name="postfix_relay_configurator_app") +def deploy_postfix_relay_configurator_fixture( + configurator_charm_file: str, + juju: jubilant.Juju, +) -> str: + """Deploy postfix-relay-configurator.""" + if not juju.status().apps.get(CONFIGURATOR_APP): + charm_path = ( + configurator_charm_file + if configurator_charm_file.startswith(("./", "/")) + else f"./{configurator_charm_file}" + ) + juju.deploy( + charm_path, + app=CONFIGURATOR_APP, + config={ + "sender_login_maps": yaml.dump({AUTHORIZED_SENDER: TEST_SMTP_USER}), + }, + ) + else: + juju.config(CONFIGURATOR_APP, {"sender_login_maps": yaml.dump({AUTHORIZED_SENDER: TEST_SMTP_USER})}) + logger.info("postfix-relay-configurator deployed") + return CONFIGURATOR_APP + + # --------------------------------------------------------------------------- # Deploy: postfix-relay-configurator (subordinate) # --------------------------------------------------------------------------- @pytest.fixture(scope="session", name="configurator_app") def deploy_configurator_fixture( - configurator_charm_file: str, - postfix_relay_app: str, + postfix_stack: typing.Dict[str, str], + opendkim_app: str, dovecot_app: str, juju: jubilant.Juju, ) -> str: - """Deploy the postfix-relay-configurator subordinate and configure LMTP routing. + """Deploy the postfix-relay-configurator subordinate and configure SMTP routing. The configurator's ``transport_maps`` is set to route mail for TEST_DOMAIN - to dovecot's LMTP port (24) so that postfix-relay delivers locally to dovecot. + to dovecot's Postfix on port 25, which delivers locally to dovecot via LMTP + Unix socket. """ # Resolve dovecot's IP after it is active. status = juju.status() dovecot_unit = next(iter(status.apps[dovecot_app].units.values())) dovecot_ip = dovecot_unit.public_address - logger.info("Routing %s → lmtp:inet:%s:24", TEST_DOMAIN, dovecot_ip) + logger.info("Routing %s → smtp:[%s]:25", TEST_DOMAIN, dovecot_ip) + _integrate_once(juju, f"{postfix_stack['postfix_relay_app']}:milter", f"{opendkim_app}:milter") configurator_config = { "relay_access_sources": yaml.dump({"192.0.2.0/24": "OK"}), "relay_recipient_maps": yaml.dump( - {f"noreply@{TEST_DOMAIN}": f"postmaster@{TEST_DOMAIN}"} + {f"noreply@{TEST_DOMAIN}": f"postmaster@{TEST_DOMAIN}", f"{TEST_SMTP_USER}@{TEST_DOMAIN}": "OK"} ), "restrict_recipients": yaml.dump({"blocked-recipient@example.invalid": "REJECT"}), "restrict_senders": yaml.dump({"blocked-sender@example.invalid": "REJECT"}), "sender_login_maps": yaml.dump( { + AUTHORIZED_SENDER: TEST_SMTP_USER, "auth-only@example.invalid": "nobody", - f"{E2E_SMTP_USER}@{TEST_DOMAIN}": f"{E2E_SMTP_USER}@{TEST_DOMAIN}", + f"{TEST_SMTP_USER}@{TEST_DOMAIN}": f"{TEST_SMTP_USER}@{TEST_DOMAIN}", } ), - "transport_maps": yaml.dump({TEST_DOMAIN: f"lmtp:inet:{dovecot_ip}:24"}), + "transport_maps": yaml.dump({TEST_DOMAIN: f"smtp:[{dovecot_ip}]:25"}), } - if not juju.status().apps.get(CONFIGURATOR_APP): - charm_path = ( - configurator_charm_file - if configurator_charm_file.startswith(("./", "/")) - else f"./{configurator_charm_file}" - ) - juju.deploy( - charm_path, - app=CONFIGURATOR_APP, - config=configurator_config, - ) - else: - juju.config(CONFIGURATOR_APP, configurator_config) - - _integrate_once(juju, f"{postfix_relay_app}:juju-info", f"{CONFIGURATOR_APP}:juju-info") + juju.config(CONFIGURATOR_APP, configurator_config) # Wait for both to be active. def _both_active(status: jubilant.Status) -> bool: - if not status.apps.get(postfix_relay_app): + if not status.apps.get(postfix_stack["postfix_relay_app"]): return False - if not status.apps[POSTFIX_RELAY_APP].is_active: + if not status.apps[postfix_stack["postfix_relay_app"]].is_active: return False - for unit in status.apps[POSTFIX_RELAY_APP].units.values(): + for unit in status.apps[postfix_stack["postfix_relay_app"]].units.values(): subs = unit.subordinates or {} conf_subs = {k: v for k, v in subs.items() if CONFIGURATOR_APP in k} if not conf_subs: @@ -606,7 +569,8 @@ def _both_active(status: jubilant.Status) -> bool: @pytest.fixture(scope="session", name="opendkim_configured") def configure_opendkim_fixture( opendkim_app: str, - postfix_relay_app: str, + postfix_stack: typing.Dict[str, str], + configurator_app: str, machine_ip_address: str, juju: jubilant.Juju, ) -> str: @@ -653,14 +617,14 @@ def configure_opendkim_fixture( # Inject TEST_DOMAIN → test-runner IP in /etc/hosts on the postfix-relay unit # so that Postfix can resolve the domain when it looks up the MX / transport. status = juju.status() - relay_unit = next(iter(status.apps[postfix_relay_app].units.values())) + relay_unit = next(iter(status.apps[postfix_stack["postfix_relay_app"]].units.values())) juju.exec( machine=relay_unit.machine, command=f"echo '{machine_ip_address} {TEST_DOMAIN}' | sudo tee -a /etc/hosts", ) juju.wait( - lambda status: jubilant.all_active(status, opendkim_app, postfix_relay_app), + lambda status: jubilant.all_active(status, opendkim_app, postfix_stack["postfix_relay_app"]), timeout=5 * 60, delay=5, ) @@ -675,7 +639,7 @@ def configure_opendkim_fixture( def mail_stack_fixture( juju: jubilant.Juju, dovecot_app: str, - postfix_relay_app: str, + postfix_stack: typing.Dict[str, str], opendkim_configured: str, configurator_app: str, ) -> typing.Dict[str, str]: @@ -687,22 +651,20 @@ def mail_stack_fixture( """ juju.wait( lambda status: jubilant.all_active( - status, dovecot_app, postfix_relay_app, opendkim_configured, SELF_SIGNED_APP + status, dovecot_app, postfix_stack["postfix_relay_app"], opendkim_configured, SELF_SIGNED_APP ), timeout=5 * 60, ) status = juju.status() dovecot_ip = next(iter(status.apps[dovecot_app].units.values())).public_address - relay_ip = next(iter(status.apps[postfix_relay_app].units.values())).public_address - logger.info("Mail stack ready — dovecot: %s, postfix-relay: %s", dovecot_ip, relay_ip) + logger.info("Mail stack ready — dovecot: %s, postfix-relay: %s", dovecot_ip, postfix_stack["postfix_relay_ip"]) return { "dovecot_app": dovecot_app, - "postfix_relay_app": postfix_relay_app, "opendkim_app": opendkim_configured, "configurator_app": configurator_app, "dovecot_ip": dovecot_ip, - "postfix_relay_ip": relay_ip, + **postfix_stack } diff --git a/tests/integration/test_configurator_maps.py b/tests/integration/test_configurator_maps.py index ff4681c..044714e 100644 --- a/tests/integration/test_configurator_maps.py +++ b/tests/integration/test_configurator_maps.py @@ -17,15 +17,11 @@ logger = logging.getLogger(__name__) +from conftest import AUTHORIZED_SENDER, SMTP_PORT, TEST_DOMAIN, TEST_SMTP_PASSWORD, TEST_SMTP_USER + # --------------------------------------------------------------------------- # Test-specific constants # --------------------------------------------------------------------------- -TEST_DOMAIN = "mailstack.internal" -SMTP_PORT = 587 - -AUTH_USER = "testuser" -AUTH_PASSWORD = "test-password" -AUTHORIZED_SENDER = f"authorized@{TEST_DOMAIN}" SPOOFED_SENDER = f"spoofed@{TEST_DOMAIN}" RECIPIENT = f"recipient@{TEST_DOMAIN}" @@ -49,7 +45,7 @@ def test_sender_login_map_enforcement(self, postfix_stack: typing.Dict[str, str] smtp.ehlo() smtp.starttls(context=ctx) smtp.ehlo() - smtp.login(AUTH_USER, AUTH_PASSWORD) + smtp.login(TEST_SMTP_USER, TEST_SMTP_PASSWORD) smtp.sendmail( from_addr=AUTHORIZED_SENDER, to_addrs=[RECIPIENT], @@ -76,7 +72,7 @@ def test_sender_login_map_enforcement_failure(self, postfix_stack: typing.Dict[s smtp.ehlo() smtp.starttls(context=ctx) smtp.ehlo() - smtp.login(AUTH_USER, AUTH_PASSWORD) + smtp.login(TEST_SMTP_USER, TEST_SMTP_PASSWORD) with pytest.raises(smtplib.SMTPRecipientsRefused) as exc_info: smtp.sendmail( from_addr=SPOOFED_SENDER, diff --git a/tests/integration/test_e2e.py b/tests/integration/test_e2e.py index 9b4d37b..3876e1c 100644 --- a/tests/integration/test_e2e.py +++ b/tests/integration/test_e2e.py @@ -19,10 +19,9 @@ import pytest import yaml -TEST_DOMAIN = "mailstack.internal" -TEST_USER = "e2euser" -MAILBOX_USER = f"{TEST_USER}@{TEST_DOMAIN}" -TEST_PASSWORD = token_hex(16) +from conftest import TEST_DOMAIN, TEST_SMTP_PASSWORD, TEST_SMTP_USER + +MAILBOX_USER = f"{TEST_SMTP_USER}@{TEST_DOMAIN}" SMTP_SUBMISSION_PORT = 587 IMAP_PORT = 993 @@ -37,14 +36,14 @@ def test_e2e(juju: jubilant.Juju, mail_stack: Dict[str, str]) -> None: dovecot_unit, "create-mail-user", params={ - "username": TEST_USER, - "password": TEST_PASSWORD, + "username": TEST_SMTP_USER, + "password": TEST_SMTP_PASSWORD, "mailbox-user": MAILBOX_USER, }, ) assert action_result.status == "completed" - smtp_auth_users = yaml.dump([f"{MAILBOX_USER}:{_sha512_dovecot(TEST_PASSWORD)}"]) + smtp_auth_users = yaml.dump([f"{MAILBOX_USER}:{_sha512_dovecot(TEST_SMTP_PASSWORD)}"]) juju.config( "postfix-relay", { @@ -77,10 +76,10 @@ def test_e2e(juju: jubilant.Juju, mail_stack: Dict[str, str]) -> None: server.ehlo() server.starttls(context=tls_ctx) server.ehlo() - server.login(MAILBOX_USER, TEST_PASSWORD) + server.login(MAILBOX_USER, TEST_SMTP_PASSWORD) server.sendmail(from_addr, [to_addr], message) - raw_message = _wait_for_subject(dovecot_ip, MAILBOX_USER, TEST_PASSWORD, subject) + raw_message = _wait_for_subject(dovecot_ip, MAILBOX_USER, TEST_SMTP_PASSWORD, subject) parsed = email.message_from_bytes(raw_message) assert parsed["Subject"] == subject @@ -93,6 +92,8 @@ def _sha512_dovecot(password: str, salt: bytes | None = None) -> str: salt = os.urandom(8) digest = hashlib.sha512(password.encode() + salt).digest() return "{SSHA512}" + base64.b64encode(digest + salt).decode() + + def _wait_for_subject(host: str, username: str, password: str, subject: str) -> bytes: ctx = ssl.create_default_context() ctx.check_hostname = False From 9c3a76eaa13e956250899f3a39db1cd928f440a8 Mon Sep 17 00:00:00 2001 From: Ali Ugur Date: Thu, 21 May 2026 12:11:34 +0300 Subject: [PATCH 44/46] chore: Cleanup by Pi agent --- dovecot-charm/src/charm.py | 8 +------- tests/integration/conftest.py | 7 +++---- tests/integration/test_configurator_maps.py | 7 ++----- tests/integration/test_e2e.py | 3 ++- 4 files changed, 8 insertions(+), 17 deletions(-) diff --git a/dovecot-charm/src/charm.py b/dovecot-charm/src/charm.py index 3e7e5f7..b425003 100644 --- a/dovecot-charm/src/charm.py +++ b/dovecot-charm/src/charm.py @@ -220,13 +220,7 @@ def _install(self): self.unit.status = MaintenanceStatus("Charm installation done") def _open_ports(self): - """Open mail ports. - - Exposes TLS-wrapped IMAP/POP3 listener ports (993/995) while leaving - plaintext IMAP/POP3 ports (143/110) closed. Also exposes SMTP on TCP/25 - intentionally for mail relay traffic; port 25 is standard SMTP (not - implicit TLS), and STARTTLS is expected when supported by the peer. - """ + """Open mail ports (TLS-only: plaintext 143/110 are not exposed).""" # Port 25 accepts SMTP from postfix-relay, which forwards to Dovecot via # the LMTP Unix socket for final delivery into the user mailbox. self.unit.open_port("tcp", 25) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 2e668ff..463980a 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -22,14 +22,14 @@ """ import base64 +from collections.abc import Generator import hashlib +import json import logging import pathlib from secrets import token_hex import socket import typing -from collections.abc import Generator -import json import jubilant import pytest @@ -559,7 +559,7 @@ def _both_active(status: jubilant.Status) -> bool: return True juju.wait(_both_active, error=jubilant.any_error, timeout=15 * 60) - logger.info("postfix-relay-configurator subordinate is active") + logger.info("postfix-relay + configurator active for maps tests") return CONFIGURATOR_APP @@ -667,4 +667,3 @@ def mail_stack_fixture( "dovecot_ip": dovecot_ip, **postfix_stack } - diff --git a/tests/integration/test_configurator_maps.py b/tests/integration/test_configurator_maps.py index 044714e..cb3b4b2 100644 --- a/tests/integration/test_configurator_maps.py +++ b/tests/integration/test_configurator_maps.py @@ -15,13 +15,10 @@ import pytest -logger = logging.getLogger(__name__) - from conftest import AUTHORIZED_SENDER, SMTP_PORT, TEST_DOMAIN, TEST_SMTP_PASSWORD, TEST_SMTP_USER -# --------------------------------------------------------------------------- -# Test-specific constants -# --------------------------------------------------------------------------- +logger = logging.getLogger(__name__) + SPOOFED_SENDER = f"spoofed@{TEST_DOMAIN}" RECIPIENT = f"recipient@{TEST_DOMAIN}" diff --git a/tests/integration/test_e2e.py b/tests/integration/test_e2e.py index 3876e1c..8857882 100644 --- a/tests/integration/test_e2e.py +++ b/tests/integration/test_e2e.py @@ -12,7 +12,6 @@ import smtplib import ssl import time -from secrets import token_hex from typing import Dict import jubilant @@ -80,6 +79,8 @@ def test_e2e(juju: jubilant.Juju, mail_stack: Dict[str, str]) -> None: server.sendmail(from_addr, [to_addr], message) raw_message = _wait_for_subject(dovecot_ip, MAILBOX_USER, TEST_SMTP_PASSWORD, subject) + # Dovecot accepts login with the full mailbox address (e.g. user@domain) when + # auth_username_format is unset; the system user is looked up by local part. parsed = email.message_from_bytes(raw_message) assert parsed["Subject"] == subject From 538f888423d1b3a362a83345a440f403d73a25c1 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 9 Jun 2026 06:50:20 +0000 Subject: [PATCH 45/46] fix: resolve merge conflicts with origin/main --- .github/workflows/integration_test.yaml | 9 - .github/workflows/publish_charm.yaml | 4 - .github/workflows/test.yaml | 3 - dovecot-charm/src/charm.py | 38 ----- dovecot-charm/tests/integration/conftest.py | 3 - dovecot-charm/tests/integration/helpers.py | 121 ++++++++------ dovecot-charm/tests/integration/test_ha.py | 5 - dovecot-charm/tests/integration/test_mail.py | 61 +------ pyproject.toml | 4 - tests/integration/conftest.py | 164 +------------------ tests/integration/helpers.py | 4 - tests/integration/test_configurator_maps.py | 22 --- tox.toml | 8 - uv.lock | 4 - 14 files changed, 80 insertions(+), 370 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index bee1c34..ded9ba9 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -50,10 +50,7 @@ jobs: canonical/operator-workflows/.github/workflows/integration_test.yaml@main secrets: inherit with: -<<<<<<< HEAD extra-arguments: '-x --log-format="%(asctime)s %(levelname)s %(message)s"' -======= ->>>>>>> origin/main juju-channel: 3/stable modules: | [ @@ -64,7 +61,6 @@ jobs: self-hosted-runner-label: "self-hosted-linux-amd64-noble-large" trivy-fs-enabled: false with-uv: true -<<<<<<< HEAD integration-tests-stack: uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main @@ -83,15 +79,10 @@ jobs: "test_e2e.py" ] with-uv: true -======= ->>>>>>> origin/main allure-report: if: ${{ !cancelled() && github.event_name == 'schedule' }} needs: - integration-tests-juju-3 - integration-tests-global -<<<<<<< HEAD - integration-tests-stack -======= ->>>>>>> origin/main uses: canonical/operator-workflows/.github/workflows/allure_report.yaml@main diff --git a/.github/workflows/publish_charm.yaml b/.github/workflows/publish_charm.yaml index 92de34a..8c5c25a 100644 --- a/.github/workflows/publish_charm.yaml +++ b/.github/workflows/publish_charm.yaml @@ -10,7 +10,6 @@ jobs: publish-to-edge: strategy: matrix: -<<<<<<< HEAD configuration: [ { @@ -34,12 +33,9 @@ jobs: tag-prefix: "postfix-relay-configurator", }, ] -======= - configuration: [{working-directory: "./dovecot-charm", channel: "2.3/edge", tag-prefix: "dovecot"}] permissions: actions: read contents: write ->>>>>>> origin/main uses: canonical/operator-workflows/.github/workflows/publish_charm.yaml@main secrets: inherit with: diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 4dd43b7..e4045a6 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -19,10 +19,7 @@ jobs: uses: canonical/operator-workflows/.github/workflows/test.yaml@main secrets: inherit permissions: -<<<<<<< HEAD -======= contents: read ->>>>>>> origin/main pull-requests: write with: self-hosted-runner: false diff --git a/dovecot-charm/src/charm.py b/dovecot-charm/src/charm.py index d1c8aa5..df5b38e 100644 --- a/dovecot-charm/src/charm.py +++ b/dovecot-charm/src/charm.py @@ -220,11 +220,6 @@ def _install(self): self.unit.status = MaintenanceStatus("Charm installation done") def _open_ports(self): -<<<<<<< HEAD - """Open mail ports (TLS-only: plaintext 143/110 are not exposed).""" - # Port 25 accepts SMTP from postfix-relay, which forwards to Dovecot via - # the LMTP Unix socket for final delivery into the user mailbox. -======= """Open mail ports. Exposes TLS-wrapped IMAP/POP3 listener ports (993/995) while leaving @@ -236,7 +231,6 @@ def _open_ports(self): # is not an implicit-TLS port; peers should negotiate STARTTLS when # available before Postfix forwards mail to Dovecot via the LMTP Unix # socket for final delivery into the user mailbox. ->>>>>>> origin/main self.unit.open_port("tcp", 25) self.unit.open_port("tcp", 993) self.unit.open_port("tcp", 995) @@ -248,17 +242,9 @@ def _on_create_mail_user_action(self, event): password = str(event.params.get("password", "")) mailbox_user = str(event.params.get("mailbox-user", "")).strip() -<<<<<<< HEAD - if not username: - event.fail("Parameter 'username' is required.") - return - if not password: - event.fail("Parameter 'password' is required.") -======= validation_error = self._validate_mail_user_action_params(username, password, mailbox_user) if validation_error: event.fail(validation_error) ->>>>>>> origin/main return users_to_manage = [username] @@ -274,17 +260,11 @@ def _on_create_mail_user_action(self, event): updated_users.append(user) else: self._create_system_user(user) -<<<<<<< HEAD -======= prepare_user_dir(os.path.join(MAIL_ROOT, user), user) ->>>>>>> origin/main created_users.append(user) self._ensure_user_in_mail_group(user) self._set_system_user_password(user, password) except (subprocess.CalledProcessError, KeyError, FileNotFoundError) as exc: -<<<<<<< HEAD - event.fail(f"Failed to manage users: {exc}") -======= message = f"Failed to manage users: {exc}" if isinstance(exc, subprocess.CalledProcessError): stderr = exc.stderr.strip() if isinstance(exc.stderr, str) else exc.stderr @@ -294,7 +274,6 @@ def _on_create_mail_user_action(self, event): if stdout: message += f"; stdout: {stdout}" event.fail(message) ->>>>>>> origin/main return event.set_results( @@ -315,14 +294,6 @@ def _system_user_exists(username: str) -> bool: return False @staticmethod -<<<<<<< HEAD - def _create_system_user(username: str) -> None: - """Create a local system user, allowing mailbox-style names if needed.""" - command = ["/usr/sbin/useradd", "-m", username] - if "@" in username: - command.insert(1, "--badname") - subprocess.run(command, check=True, capture_output=True, text=True) -======= def _contains_invalid_user_characters(username: str) -> bool: """Return whether username contains disallowed path/control characters.""" if username in (".", ".."): @@ -362,7 +333,6 @@ def _create_system_user(username: str) -> None: if "@" in username: command.insert(1, "--badname") subprocess.run(command, check=True, capture_output=True, text=True) # nosec B603 ->>>>>>> origin/main @staticmethod def _ensure_user_in_mail_group(username: str) -> None: @@ -372,11 +342,7 @@ def _ensure_user_in_mail_group(username: str) -> None: check=True, capture_output=True, text=True, -<<<<<<< HEAD - ) -======= ) # nosec B603 ->>>>>>> origin/main @staticmethod def _set_system_user_password(username: str, password: str) -> None: @@ -387,11 +353,7 @@ def _set_system_user_password(username: str, password: str) -> None: capture_output=True, text=True, input=f"{username}:{password}", -<<<<<<< HEAD - ) -======= ) # nosec B603 ->>>>>>> origin/main def _on_clear_queue_action(self, event): """Handle the clear-queue action.""" diff --git a/dovecot-charm/tests/integration/conftest.py b/dovecot-charm/tests/integration/conftest.py index ee0ef3d..9e6d1c4 100644 --- a/dovecot-charm/tests/integration/conftest.py +++ b/dovecot-charm/tests/integration/conftest.py @@ -16,8 +16,6 @@ # Charm mailname — must match the value passed in deploy config so tests can # construct the correct SMTP recipient addresses (@example.com). MAILNAME = "example.com" -<<<<<<< HEAD -======= # GDPR action test constants MAIL_ROOT = "/srv/mail" @@ -30,7 +28,6 @@ CREATE_MAIL_USER_TEST_USER = "cmu-testuser" CREATE_MAIL_USER_TEST_MAILBOX = "cmu-testuser@example.com" CREATE_MAIL_USER_TEST_PASSWORD = secrets.token_hex(16) ->>>>>>> origin/main @pytest.fixture(scope="session", name="juju") diff --git a/dovecot-charm/tests/integration/helpers.py b/dovecot-charm/tests/integration/helpers.py index 620e162..7c18db2 100644 --- a/dovecot-charm/tests/integration/helpers.py +++ b/dovecot-charm/tests/integration/helpers.py @@ -1,11 +1,8 @@ # Copyright 2026 Canonical Ltd. # See LICENSE file for licensing details. -<<<<<<< HEAD -======= """Shared helper functions for Dovecot integration tests.""" ->>>>>>> origin/main import contextlib import imaplib import logging @@ -15,8 +12,6 @@ from email.message import EmailMessage import jubilant -<<<<<<< HEAD -======= from tenacity import retry, retry_if_result, stop_after_attempt, wait_fixed logger = logging.getLogger(__name__) @@ -152,7 +147,6 @@ def assert_deferred_queue_empty(juju: jubilant.Juju, unit_name: str) -> None: def assert_queue_non_empty(juju: jubilant.Juju, unit_name: str) -> None: """Assert that Postfix reports a non-empty queue.""" juju.exec("postqueue -p | grep -qv 'Mail queue is empty'", unit=unit_name) ->>>>>>> origin/main def send_mail_via_smtp( @@ -177,44 +171,17 @@ def send_mail_via_smtp( smtp.send_message(msg) -<<<<<<< HEAD -======= @retry( stop=stop_after_attempt(20), wait=wait_fixed(3), retry=retry_if_result(lambda found: not found), ) ->>>>>>> origin/main def check_mail_via_imap(unit_ip: str, user: str, password: str, subject: str) -> bool: """Poll IMAP on unit_ip until the email with the given subject is found.""" context = ssl.create_default_context() context.check_hostname = False context.verify_mode = ssl.CERT_NONE -<<<<<<< HEAD - for attempt in range(20): - mail = None - try: - mail = imaplib.IMAP4_SSL(unit_ip, port=993, ssl_context=context) - mail.login(user, password) - mail.select("inbox") - _, data = mail.search(None, f'(HEADER Subject "{subject}")') - if data and data[0]: - logging.info(f"Email found via IMAP on {unit_ip}. IDs: {data[0]}") - return True - logging.info(f"Email not found yet on {unit_ip} (attempt {attempt + 1})...") - except (imaplib.IMAP4.error, OSError) as e: - logging.warning(f"IMAP attempt {attempt + 1} on {unit_ip} failed: {e}. Retrying...") - finally: - if mail is not None: - with contextlib.suppress(imaplib.IMAP4.error, OSError): - mail.close() - with contextlib.suppress(imaplib.IMAP4.error, OSError): - mail.logout() - time.sleep(3) - - return False -======= mail = None try: mail = imaplib.IMAP4_SSL(unit_ip, port=993, ssl_context=context) @@ -235,39 +202,25 @@ def check_mail_via_imap(unit_ip: str, user: str, password: str, subject: str) -> mail.close() with contextlib.suppress(imaplib.IMAP4.error, OSError): mail.logout() ->>>>>>> origin/main def setup_mail_user( juju: jubilant.Juju, primary: str, -<<<<<<< HEAD - secondary: str, - user: str, - password: str, -): - """Create a mail user on both units. -======= secondary: str | None, user: str, password: str, ): """Create a mail user on primary and optionally secondary unit. ->>>>>>> origin/main The system account and password are created on both units so PAM auth works on the secondary after sync. The Maildir is only initialised on the primary so that dsync can replicate it to the secondary without GUID conflicts. -<<<<<<< HEAD - """ - for unit in (primary, secondary): -======= Args: secondary: Secondary unit name, or None for single-unit deployments. """ for unit in (u for u in (primary, secondary) if u is not None): ->>>>>>> origin/main juju.exec( ( f"id -u {user} >/dev/null 2>&1 || " @@ -363,3 +316,77 @@ def wait_for_sync_trigger( "Timed out waiting for sync trigger on " f"{unit}; previous mtime={previous_mtime}, previous timer count={previous_timer_count}" ) + + +def get_last_sync_mtime(juju: jubilant.Juju, unit: str) -> int | None: + """Return /srv/mail/.last-dsync mtime epoch on unit, or None if missing.""" + output = juju.exec( + "stat -c %Y /srv/mail/.last-dsync 2>/dev/null || true", unit=unit + ).stdout.strip() + return int(output) if output.isdigit() else None + + +def get_sync_timer_run_count(juju: jubilant.Juju, unit: str) -> int: + """Return count of sync-to-secondary service invocations from the journal.""" + output = juju.exec( + "journalctl -u sync-to-secondary.service --no-pager -q 2>/dev/null | wc -l || true", + unit=unit, + ).stdout.strip() + return int(output) if output.isdigit() else 0 + + +def get_sync_log_content(juju: jubilant.Juju, unit: str, lines: int = 20) -> str: + """Return last N lines from the sync-to-secondary service journal for debugging.""" + output = juju.exec( + f"journalctl -u sync-to-secondary.service --no-pager -n {lines} 2>/dev/null || echo 'No journal entries for sync-to-secondary'", + unit=unit, + ).stdout + return output + + +def get_timer_status(juju: jubilant.Juju, unit: str) -> str | None: + """Return systemctl show output for the sync-to-secondary timer, or None if absent.""" + result = juju.exec( + "systemctl show sync-to-secondary.timer --property=ActiveState,LastTriggerUSec 2>/dev/null || true", + unit=unit, + ).stdout.strip() + return result if result else None + + +def wait_for_sync_trigger( + juju: jubilant.Juju, + unit: str, + previous_mtime: int | None, + previous_timer_count: int, + timeout: int = 4 * 60, + poll_interval: int = 5, +) -> int: + """Wait until /srv/mail/.last-dsync mtime advances, indicating a completed sync. + + The sync script touches .last-dsync only at the very end, so this is a + reliable end-of-sync marker. Journal timer count is checked only to log + that the timer appears to have fired while we continue waiting for + .last-dsync to be updated. + """ + deadline = time.time() + timeout + timer_fired = False + while time.time() < deadline: + current_mtime = get_last_sync_mtime(juju, unit) + if current_mtime is not None and ( + previous_mtime is None or current_mtime > previous_mtime + ): + return current_mtime + + current_timer_count = get_sync_timer_run_count(juju, unit) + if current_timer_count > previous_timer_count and not timer_fired: + logging.info( + "Timer fired (journal count increased); waiting for .last-dsync to update..." + ) + timer_fired = True + + time.sleep(poll_interval) + + raise AssertionError( + "Timed out waiting for sync trigger on " + f"{unit}; previous mtime={previous_mtime}, previous timer count={previous_timer_count}" + ) diff --git a/dovecot-charm/tests/integration/test_ha.py b/dovecot-charm/tests/integration/test_ha.py index 6589a4f..2bb4cd2 100644 --- a/dovecot-charm/tests/integration/test_ha.py +++ b/dovecot-charm/tests/integration/test_ha.py @@ -8,14 +8,9 @@ import jubilant import pytest -<<<<<<< HEAD -from conftest import MAILNAME -from helpers import ( -======= from .conftest import MAILNAME from .helpers import ( ->>>>>>> origin/main check_mail_via_imap, get_last_sync_mtime, get_sync_log_content, diff --git a/dovecot-charm/tests/integration/test_mail.py b/dovecot-charm/tests/integration/test_mail.py index 4f458df..ef20eb0 100644 --- a/dovecot-charm/tests/integration/test_mail.py +++ b/dovecot-charm/tests/integration/test_mail.py @@ -1,39 +1,27 @@ # Copyright 2026 Canonical Ltd. # See LICENSE file for licensing details. -"""Integration tests for end-to-end mail delivery via Postfix → LMTP → Dovecot.""" +"""Integration tests for end-to-end mail delivery via Postfix -> LMTP -> Dovecot.""" -<<<<<<< HEAD -import contextlib -import imaplib -======= ->>>>>>> origin/main import logging from secrets import token_hex import jubilant import pytest -from conftest import MAILNAME -from helpers import send_mail_via_smtp from .conftest import MAILNAME from .helpers import check_mail_via_imap, send_mail_via_smtp, setup_mail_user def test_mail_workflow(juju: jubilant.Juju, dovecot_charm: str): - """Test end-to-end mail delivery via Postfix LMTP → Dovecot and IMAP retrieval. + """Test end-to-end mail delivery via Postfix LMTP -> Dovecot and IMAP retrieval. Mail is submitted over SMTP on port 25. Postfix matches the recipient domain against virtual_mailbox_domains and forwards it to Dovecot via the LMTP Unix -<<<<<<< HEAD - socket (virtual_transport = lmtp:unix:private/dovecot-lmtp). The test then - verifies the message is retrievable over IMAPS. -======= socket (virtual_transport = lmtp:unix:private/dovecot-lmtp). Dovecot strips the domain from the envelope recipient (auth_username_format = %n) before the userdb lookup, so the system user 'ubuntu' is found for 'ubuntu@'. The test then verifies the message is retrievable over IMAPS. ->>>>>>> origin/main """ unit_name = f"{dovecot_charm}/0" logging.info(f"Updating primary-unit config to {unit_name}...") @@ -42,24 +30,12 @@ def test_mail_workflow(juju: jubilant.Juju, dovecot_charm: str): password = token_hex(8) logging.info("Configuring user 'ubuntu'...") -<<<<<<< HEAD - juju.exec("usermod -aG mail ubuntu", unit=unit_name) - juju.exec(f"echo 'ubuntu:{password}' | chpasswd", unit=unit_name) - # Ensure the Dovecot mail directory exists so the LMTP delivery can write - # immediately without waiting for Dovecot to auto-create it. - juju.exec( - "install -d -m 0700 -o ubuntu -g mail /srv/mail/ubuntu", - unit=unit_name, - ) -======= setup_mail_user(juju, primary=unit_name, secondary=None, user="ubuntu", password=password) - result = juju.run( unit_name, "create-mail-user", params={"username": "ubuntu", "password": password} ) assert result.status == "completed" assert result.results["status"] == "success" ->>>>>>> origin/main # Resolve the unit IP before sending so we can reuse it for the IMAP check. status = juju.status() @@ -76,38 +52,5 @@ def test_mail_workflow(juju: jubilant.Juju, dovecot_charm: str): ) logging.info(f"Verifying via IMAP at {unit_ip}:993 ...") -<<<<<<< HEAD - context = ssl.create_default_context() - context.check_hostname = False - context.verify_mode = ssl.CERT_NONE - email_found = False - for i in range(20): - mail = None - try: - mail = imaplib.IMAP4_SSL(unit_ip, port=993, ssl_context=context) - mail.login("ubuntu", password) - mail.select("inbox") - _, data = mail.search(None, f'(HEADER Subject "{subject}")') - - if data and data[0]: - logging.info(f"Email found successfully via IMAP! IDs: {data[0]}") - email_found = True - break - else: - logging.info("Email not found yet...") - except (imaplib.IMAP4.error, OSError) as e: - logging.warning(f"IMAP check attempt {i + 1} failed: {e}. Retrying...") - finally: - if mail is not None: - with contextlib.suppress(imaplib.IMAP4.error, OSError): - mail.close() - with contextlib.suppress(imaplib.IMAP4.error, OSError): - mail.logout() - - time.sleep(3) - - if not email_found: -======= if not check_mail_via_imap(unit_ip, "ubuntu", password, subject): ->>>>>>> origin/main pytest.fail("Failed to verify email delivery via IMAP.") diff --git a/pyproject.toml b/pyproject.toml index 3771c11..1cf853c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,12 +47,8 @@ static = [ integration = [ "allure-pytest>=2.8.18", "allure-pytest-collection-report @ git+https://github.com/canonical/data-platform-workflows@v24.0.0#subdirectory=python/pytest_plugins/allure_pytest_collection_report", -<<<<<<< HEAD "cryptography>=42.0", - "jubilant==1.9.0", -======= "jubilant==1.10.0", ->>>>>>> origin/main "pytest", "pyyaml>=6.0", "requests>=2.31", diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index c422a75..acae88d 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,7 +1,6 @@ # Copyright 2026 Canonical Ltd. # See LICENSE file for licensing details. -<<<<<<< HEAD """Fixtures for the full-stack mailserver integration tests. Topology @@ -31,18 +30,11 @@ from secrets import token_hex import socket import typing -======= -"""Shared fixtures and configuration for integration tests.""" - -import logging -import typing -from collections.abc import Generator ->>>>>>> origin/main import jubilant import pytest import yaml -<<<<<<< HEAD +from helpers import integrate_once as _integrate_once, select_charm_file as _select_charm_file, sha512_dovecot_password as _sha512_dovecot_password from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric import rsa @@ -75,28 +67,10 @@ # --------------------------------------------------------------------------- def pytest_addoption(parser: pytest.Parser) -> None: """Register extra CLI options consumed by the integration suite.""" -======= - -from helpers import integrate_once, select_charm_file, sha512_dovecot_password - -logger = logging.getLogger(__name__) - -POSTFIX_RELAY_APP = "postfix-relay" -CONFIGURATOR_APP = "postfix-relay-configurator" -SELF_SIGNED_APP = "self-signed-certificates" - -TEST_DOMAIN = "mailstack.internal" -SMTP_PORT = 587 - - -def pytest_addoption(parser: pytest.Parser) -> None: - """Add integration test command-line options.""" ->>>>>>> origin/main parser.addoption( "--charm-file", action="append", default=[], -<<<<<<< HEAD help=("Path to a pre-built .charm file. Pass this option multiple times (one per charm)."), ) parser.addoption( @@ -110,52 +84,17 @@ def pytest_addoption(parser: pytest.Parser) -> None: action="store", default=None, help="Use an existing Juju model by name instead of creating a temp model.", -======= - help="Path to charm file (can be used multiple times)", ->>>>>>> origin/main ) parser.addoption( "--use-existing", action="store_true", default=False, -<<<<<<< HEAD help="Attach to the current Juju model without deploying anything new.", ) -# --------------------------------------------------------------------------- -# Helper functions -# --------------------------------------------------------------------------- -def _sha512_dovecot_password(password: str) -> str: - """Generate a SSHA512 password hash compatible with dovecot.""" - salt = b"mailtest" - digest = hashlib.sha512(password.encode() + salt).digest() - return "{SSHA512}" + base64.b64encode(digest + salt).decode() - -def _integrate_once(juju: jubilant.Juju, endpoint_a: str, endpoint_b: str) -> None: - """Call ``juju integrate`` tolerating 'already related' errors.""" - try: - juju.integrate(endpoint_a, endpoint_b) - except Exception as exc: # noqa: BLE001 - msg = str(exc) - if "already exists" not in msg and "already related" not in msg: - raise - logger.debug("Relation %s ↔ %s already exists, skipping", endpoint_a, endpoint_b) - - -def _select_charm_file(pytestconfig: pytest.Config, marker: str) -> str: - """Select charm file matching marker from --charm-file options.""" - charm_files: list[str] = pytestconfig.getoption("--charm-file", default=[]) - for path in charm_files: - if marker in pathlib.Path(path).name.lower(): - return path - use_existing = pytestconfig.getoption("--use-existing", default=False) - if use_existing: - return "" - provided = ", ".join(charm_files) if charm_files else "" - raise AssertionError(f"Missing --charm-file matching '{marker}'. Provided: {provided}.") # --------------------------------------------------------------------------- @@ -165,65 +104,33 @@ def _select_charm_file(pytestconfig: pytest.Config, marker: str) -> str: def juju_fixture(request: pytest.FixtureRequest) -> Generator[jubilant.Juju, None, None]: """Session-scoped Juju client in a temporary model for integration tests.""" logging.getLogger('jubilant.wait').setLevel(logging.WARNING) -======= - help="Use existing model instead of creating a temporary one", - ) - parser.addoption( - "--model", - default=None, - help="Specific model name to use", - ) - parser.addoption( - "--keep-models", - action="store_true", - default=False, - help="Keep temporary models after tests complete", - ) - -@pytest.fixture(scope="module", name="juju") -def juju_fixture(request: pytest.FixtureRequest) -> Generator[jubilant.Juju, None, None]: - """Module-scoped Juju client in a temporary model for configurator map tests.""" - def _show_debug_log(juju: jubilant.Juju) -> None: if request.session.testsfailed: log = juju.debug_log(limit=2000) print(log, end="") ->>>>>>> origin/main use_existing = request.config.getoption("--use-existing", default=False) if use_existing: juju = jubilant.Juju() juju.model_config({"automatically-retry-hooks": True}) yield juju -<<<<<<< HEAD - return - - model = request.config.getoption("--model") -======= _show_debug_log(juju) return - model = request.config.getoption("--model", default=None) ->>>>>>> origin/main + model = request.config.getoption("--model") if model: juju = jubilant.Juju(model=model) juju.model_config({"automatically-retry-hooks": True}) yield juju -<<<<<<< HEAD - return - - keep_models = typing.cast(bool, request.config.getoption("--keep-models")) -======= _show_debug_log(juju) return - keep_models = typing.cast(bool, request.config.getoption("--keep-models", default=False)) ->>>>>>> origin/main + keep_models = typing.cast(bool, request.config.getoption("--keep-models")) with jubilant.temp_model(keep=keep_models) as juju: juju.wait_timeout = 15 * 60 juju.model_config({"automatically-retry-hooks": True}) yield juju -<<<<<<< HEAD + _show_debug_log(juju) return @@ -350,20 +257,6 @@ def generate_dkim_keypair(domain: str, selector: str) -> typing.Tuple[str, str]: @pytest.fixture(scope="session", name="self_signed_app") def deploy_self_signed_certs_fixture(juju: jubilant.Juju) -> str: """Deploy self-signed-certificates from CharmHub.""" -======= - _show_debug_log(juju) - - -@pytest.fixture(scope="module", name="postfix_stack") -def postfix_stack_fixture( - juju: jubilant.Juju, - pytestconfig: pytest.Config, -) -> typing.Dict[str, str]: - """Deploy postfix-relay + postfix-relay-configurator configured for sender_login enforcement. - - Returns a dict with ``postfix_relay_ip``. - """ ->>>>>>> origin/main if not juju.status().apps.get(SELF_SIGNED_APP): juju.deploy(SELF_SIGNED_APP, channel="latest/stable") juju.wait( @@ -371,7 +264,6 @@ def postfix_stack_fixture( error=jubilant.any_error, timeout=10 * 60, ) -<<<<<<< HEAD logger.info("self-signed-certificates is active") return SELF_SIGNED_APP @@ -390,50 +282,6 @@ def postfix_stack_fixture( juju, f"{postfix_relay_app}:juju-info", f"{postfix_relay_configurator_app}:juju-info", -======= - - # --- postfix-relay --- - auth_password = "test-password" - if not juju.status().apps.get(POSTFIX_RELAY_APP): - charm_path = select_charm_file(pytestconfig, "postfix-relay_") - if not charm_path.startswith(("./", "/")): - charm_path = f"./{charm_path}" - juju.deploy( - charm_path, - app=POSTFIX_RELAY_APP, - config={ - "relay_domains": f"- {TEST_DOMAIN}", - "enable_smtp_auth": "true", - "smtp_auth_users": yaml.dump( - [f"testuser:{sha512_dovecot_password(auth_password)}"] - ), - "enable_reject_unknown_sender_domain": "false", - }, - ) - integrate_once( - juju, - f"{POSTFIX_RELAY_APP}:certificates", - f"{SELF_SIGNED_APP}:certificates", - ) - - # --- postfix-relay-configurator --- - authorized_sender = f"authorized@{TEST_DOMAIN}" - if not juju.status().apps.get(CONFIGURATOR_APP): - charm_path = select_charm_file(pytestconfig, "postfix-relay-configurator_") - if not charm_path.startswith(("./", "/")): - charm_path = f"./{charm_path}" - juju.deploy( - charm_path, - app=CONFIGURATOR_APP, - config={ - "sender_login_maps": yaml.dump({authorized_sender: "testuser"}), - }, - ) - integrate_once( - juju, - f"{POSTFIX_RELAY_APP}:juju-info", - f"{CONFIGURATOR_APP}:juju-info", ->>>>>>> origin/main ) # Wait for both to be active. @@ -458,7 +306,6 @@ def _both_active(status: jubilant.Status) -> bool: status = juju.status() relay_ip = next(iter(status.apps[POSTFIX_RELAY_APP].units.values())).public_address logger.info("postfix-relay IP: %s", relay_ip) -<<<<<<< HEAD return {"postfix_relay_app": postfix_relay_app, "postfix_relay_ip": relay_ip} @@ -798,6 +645,3 @@ def mail_stack_fixture( "dovecot_ip": dovecot_ip, **postfix_stack } -======= - return {"postfix_relay_ip": relay_ip} ->>>>>>> origin/main diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index e8ece5a..2036994 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -13,8 +13,6 @@ logger = logging.getLogger(__name__) -<<<<<<< HEAD -======= POSTFIX_RELAY_APP = "postfix-relay" CONFIGURATOR_APP = "postfix-relay-configurator" SELF_SIGNED_APP = "self-signed-certificates" @@ -22,8 +20,6 @@ TEST_DOMAIN = "mailstack.internal" SMTP_PORT = 587 ->>>>>>> origin/main - def sha512_dovecot_password(password: str) -> str: """Generate a SSHA512 password hash compatible with dovecot.""" salt = b"mailtest" diff --git a/tests/integration/test_configurator_maps.py b/tests/integration/test_configurator_maps.py index bf155a6..cb3b4b2 100644 --- a/tests/integration/test_configurator_maps.py +++ b/tests/integration/test_configurator_maps.py @@ -15,24 +15,10 @@ import pytest -<<<<<<< HEAD from conftest import AUTHORIZED_SENDER, SMTP_PORT, TEST_DOMAIN, TEST_SMTP_PASSWORD, TEST_SMTP_USER logger = logging.getLogger(__name__) -======= -logger = logging.getLogger(__name__) - -# --------------------------------------------------------------------------- -# Test-specific constants -# --------------------------------------------------------------------------- -TEST_DOMAIN = "mailstack.internal" -SMTP_PORT = 587 - -AUTH_USER = "testuser" -AUTH_PASSWORD = "test-password" -AUTHORIZED_SENDER = f"authorized@{TEST_DOMAIN}" ->>>>>>> origin/main SPOOFED_SENDER = f"spoofed@{TEST_DOMAIN}" RECIPIENT = f"recipient@{TEST_DOMAIN}" @@ -56,11 +42,7 @@ def test_sender_login_map_enforcement(self, postfix_stack: typing.Dict[str, str] smtp.ehlo() smtp.starttls(context=ctx) smtp.ehlo() -<<<<<<< HEAD smtp.login(TEST_SMTP_USER, TEST_SMTP_PASSWORD) -======= - smtp.login(AUTH_USER, AUTH_PASSWORD) ->>>>>>> origin/main smtp.sendmail( from_addr=AUTHORIZED_SENDER, to_addrs=[RECIPIENT], @@ -87,11 +69,7 @@ def test_sender_login_map_enforcement_failure(self, postfix_stack: typing.Dict[s smtp.ehlo() smtp.starttls(context=ctx) smtp.ehlo() -<<<<<<< HEAD smtp.login(TEST_SMTP_USER, TEST_SMTP_PASSWORD) -======= - smtp.login(AUTH_USER, AUTH_PASSWORD) ->>>>>>> origin/main with pytest.raises(smtplib.SMTPRecipientsRefused) as exc_info: smtp.sendmail( from_addr=SPOOFED_SENDER, diff --git a/tox.toml b/tox.toml index a9cf76c..6730941 100644 --- a/tox.toml +++ b/tox.toml @@ -72,7 +72,6 @@ commands = [ dependency_groups = [ "static" ] [env.integration] -<<<<<<< HEAD description = "Run integration tests for all charms" allowlist_externals = [ "tox" ] commands = [ @@ -85,20 +84,13 @@ dependency_groups = [ "integration" ] [env.stack-integration] description = "Run full-stack mailserver integration tests (all charms)" -======= -description = "Run cross-charm integration tests" ->>>>>>> origin/main commands = [ [ "pytest", "-v", -<<<<<<< HEAD - "--tb", "short", -======= "--tb", "native", "tests/integration/", ->>>>>>> origin/main "--log-cli-level=INFO", "-s", "{toxinidir}/tests/integration", diff --git a/uv.lock b/uv.lock index 0dd20a2..e997c2b 100644 --- a/uv.lock +++ b/uv.lock @@ -442,12 +442,8 @@ fmt = [{ name = "ruff" }] integration = [ { name = "allure-pytest", specifier = ">=2.8.18" }, { name = "allure-pytest-collection-report", git = "https://github.com/canonical/data-platform-workflows?subdirectory=python%2Fpytest_plugins%2Fallure_pytest_collection_report&rev=v24.0.0" }, -<<<<<<< HEAD { name = "cryptography", specifier = ">=42.0" }, - { name = "jubilant", specifier = "==1.9.0" }, -======= { name = "jubilant", specifier = "==1.10.0" }, ->>>>>>> origin/main { name = "pytest" }, { name = "pyyaml", specifier = ">=6.0" }, { name = "requests", specifier = ">=2.31" }, From 31f455019379e97ef9f91ed1ebb7ff6504bd1380 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 9 Jun 2026 06:52:19 +0000 Subject: [PATCH 46/46] fix: remove duplicate test path in stack-integration tox env --- tox.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/tox.toml b/tox.toml index 6730941..5200a82 100644 --- a/tox.toml +++ b/tox.toml @@ -90,7 +90,6 @@ commands = [ "-v", "--tb", "native", - "tests/integration/", "--log-cli-level=INFO", "-s", "{toxinidir}/tests/integration",