From abb70a6b1488906b662597f0d15638c19b35aedd Mon Sep 17 00:00:00 2001 From: Treefit Date: Fri, 28 Nov 2025 09:34:44 +0100 Subject: [PATCH 01/24] Handle case where user followed the tutorial and set the CNAME reccord for mta-sts, but no TXT record for it yet. --- cmdeploy/src/cmdeploy/remote/rdns.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmdeploy/src/cmdeploy/remote/rdns.py b/cmdeploy/src/cmdeploy/remote/rdns.py index cc151e9..5b79d43 100644 --- a/cmdeploy/src/cmdeploy/remote/rdns.py +++ b/cmdeploy/src/cmdeploy/remote/rdns.py @@ -37,7 +37,10 @@ def perform_initial_checks(mail_domain, pre_command=""): return res # parse out sts-id if exists, example: "v=STSv1; id=2090123" - parts = query_dns("TXT", f"_mta-sts.{mail_domain}").split("id=") + mta_sts_txt = query_dns("TXT", f"_mta-sts.{mail_domain}") + if not mta_sts_txt: + return res + parts = mta_sts_txt.split("id=") res["sts_id"] = parts[1].rstrip('"') if len(parts) == 2 else "" return res From c98853570b4e3512ced756189e7b65b86e1c1d06 Mon Sep 17 00:00:00 2001 From: Rodrigo Camacho Date: Thu, 11 Dec 2025 09:58:09 +0100 Subject: [PATCH 02/24] updated location of the documentation for custom webpage location --- chatmaild/src/chatmaild/ini/chatmail.ini.f | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chatmaild/src/chatmaild/ini/chatmail.ini.f b/chatmaild/src/chatmaild/ini/chatmail.ini.f index f87eda9..7ec3ca8 100644 --- a/chatmaild/src/chatmaild/ini/chatmail.ini.f +++ b/chatmaild/src/chatmaild/ini/chatmail.ini.f @@ -45,7 +45,7 @@ passthrough_senders = # (space-separated, item may start with "@" to whitelist whole recipient domains) passthrough_recipients = echo@{mail_domain} -# path to www directory - documented here: https://github.com/chatmail/relay/#custom-web-pages +# path to www directory - documented here: https://chatmail.at/doc/relay/getting_started.html#custom-web-pages #www_folder = www # From 4b5e8feb96fd7c5df54985d9e7e26e15b077b284 Mon Sep 17 00:00:00 2001 From: missytake Date: Wed, 10 Dec 2025 10:37:16 +0100 Subject: [PATCH 03/24] ci: run test_status_cmd at the end to avoid flakiness --- .../src/cmdeploy/tests/online/test_1_basic.py | 42 ------------------ .../cmdeploy/tests/online/test_3_status.py | 43 +++++++++++++++++++ 2 files changed, 43 insertions(+), 42 deletions(-) create mode 100644 cmdeploy/src/cmdeploy/tests/online/test_3_status.py diff --git a/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py b/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py index 2b55f6b..e0350fb 100644 --- a/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py +++ b/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py @@ -1,5 +1,4 @@ import datetime -import os import smtplib import socket import subprocess @@ -8,7 +7,6 @@ import time import pytest from cmdeploy import remote -from cmdeploy.cmdeploy import main from cmdeploy.sshexec import SSHExec @@ -70,46 +68,6 @@ class TestSSHExecutor: assert (now - since_date).total_seconds() < 60 * 60 * 51 -def test_status_cmd(chatmail_config, capsys, request): - os.chdir(request.config.invocation_params.dir) - assert main(["status"]) == 0 - status_out = capsys.readouterr() - print(status_out.out) - - services = [ - "acmetool-redirector", - "chatmail-metadata", - "doveauth", - "dovecot", - "fcgiwrap", - "filtermail-incoming", - "filtermail", - "lastlogin", - "nginx", - "opendkim", - "postfix@-", - "systemd-journald", - "turnserver", - "unbound", - ] - not_running = [] - for service in services: - active = False - for line in status_out: - if service in line: - active = True - if not "loaded" in line: - active = False - if not "active" in line: - active = False - if not "running" in line: - active = False - break - if not active: - not_running.append(service) - assert not_running == [] - - def test_timezone_env(remote): for line in remote.iter_output("env"): print(line) diff --git a/cmdeploy/src/cmdeploy/tests/online/test_3_status.py b/cmdeploy/src/cmdeploy/tests/online/test_3_status.py new file mode 100644 index 0000000..981df23 --- /dev/null +++ b/cmdeploy/src/cmdeploy/tests/online/test_3_status.py @@ -0,0 +1,43 @@ +import os + +from cmdeploy.cmdeploy import main + + +def test_status_cmd(chatmail_config, capsys, request): + os.chdir(request.config.invocation_params.dir) + assert main(["status"]) == 0 + status_out = capsys.readouterr() + print(status_out.out) + + services = [ + "acmetool-redirector", + "chatmail-metadata", + "doveauth", + "dovecot", + "fcgiwrap", + "filtermail-incoming", + "filtermail", + "lastlogin", + "nginx", + "opendkim", + "postfix@-", + "systemd-journald", + "turnserver", + "unbound", + ] + not_running = [] + for service in services: + active = False + for line in status_out: + if service in line: + active = True + if not "loaded" in line: + active = False + if not "active" in line: + active = False + if not "running" in line: + active = False + break + if not active: + not_running.append(service) + assert not_running == [] From 96fc3d9ff6ab4085c937e92f297f64dad0e60a5f Mon Sep 17 00:00:00 2001 From: missytake Date: Fri, 12 Dec 2025 11:53:46 +0100 Subject: [PATCH 04/24] tests: don't let test_status_cmd test server state --- cmdeploy/src/cmdeploy/tests/online/test_3_status.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cmdeploy/src/cmdeploy/tests/online/test_3_status.py b/cmdeploy/src/cmdeploy/tests/online/test_3_status.py index 981df23..1783c32 100644 --- a/cmdeploy/src/cmdeploy/tests/online/test_3_status.py +++ b/cmdeploy/src/cmdeploy/tests/online/test_3_status.py @@ -9,6 +9,11 @@ def test_status_cmd(chatmail_config, capsys, request): status_out = capsys.readouterr() print(status_out.out) + assert len(status_out.out.splitlines()) > 5 + + """ + don't test actual server state: + services = [ "acmetool-redirector", "chatmail-metadata", @@ -41,3 +46,4 @@ def test_status_cmd(chatmail_config, capsys, request): if not active: not_running.append(service) assert not_running == [] + """ From c84ddf69e8e7a37d5a204fc284c736f1587662be Mon Sep 17 00:00:00 2001 From: missytake Date: Fri, 12 Dec 2025 10:39:27 +0100 Subject: [PATCH 05/24] add missing changelog entries --- CHANGELOG.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7641ed4..5253767 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,11 +2,15 @@ ## untagged -- Add imap_compress option to chatmail.ini (#760) +- Add imap_compress option to chatmail.ini + ([#760](https://github.com/chatmail/relay/pull/760)) - Remove echobot from relays ([#753](https://github.com/chatmail/relay/pull/753)) +- Fix `cmdeploy webdev` + ([#743](https://github.com/chatmail/relay/pull/743)) + - Add robots.txt to exclude all web crawlers ([#732](https://github.com/chatmail/relay/pull/732)) From e15b8ebf11c08e6b02d4e86902c46d244d3c5995 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 14 Dec 2025 10:38:13 -0800 Subject: [PATCH 06/24] docs README update There is no sphinx-build to pip install --- doc/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/README.md b/doc/README.md index 4c21f5d..b18b634 100644 --- a/doc/README.md +++ b/doc/README.md @@ -6,7 +6,7 @@ You can use the `make` command and `make html` to build web pages. You need a Python environment where the following install was excuted: - pip install sphinx-build furo sphinx-autobuild + pip install furo sphinx-autobuild To develop/change documentation, you can then do: From 1188aed0616744c34568882194790bf6f9784dc9 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 14 Dec 2025 10:14:11 -0800 Subject: [PATCH 07/24] Related: Add the Chatmail Cookbook project --- doc/source/related.rst | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/doc/source/related.rst b/doc/source/related.rst index 14f1288..0784551 100644 --- a/doc/source/related.rst +++ b/doc/source/related.rst @@ -7,7 +7,7 @@ Active development takes place in the `chatmail/relay github repository `_ and ask to get added to a non-public support chat for debugging issues. -We know of two work-in-progress alternative implementation efforts: +We know of three work-in-progress alternative implementation efforts: - `Mox `_: A Golang email server. `Work is in progress `_ to modify @@ -18,3 +18,10 @@ We know of two work-in-progress alternative implementation efforts: plugin for the `Maddy email server `_ which aims to implement the chatmail relay features and configuration options. + +- `Chatmail Cookbook `_: + A Chef Cookbook implementing a relay server. The project follows the + official relay server software and configurations converted to a Chef + Cookbook with only minor differences. The cookbook uses DNS-01 for + certificate validation and additionally supports FreeBSD. It does not + require a Chef server to use. From 9bf99cc8a9fa97358ca63fc7e23531b482e0d610 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EF=BC=93=EF=BC=97=EF=BC=93?= <151577046+ccclxxiii@users.noreply.github.com> Date: Sat, 13 Dec 2025 14:06:35 +0000 Subject: [PATCH 08/24] removes development notice --- www/src/index.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/www/src/index.md b/www/src/index.md index e167c74..aae1a0d 100644 --- a/www/src/index.md +++ b/www/src/index.md @@ -23,7 +23,3 @@ you can also **scan this QR code** with Delta Chat: 🐣 **Choose** your Avatar and Name 💬 **Start** chatting with any Delta Chat contacts using [QR invite codes](https://delta.chat/en/help#howtoe2ee) - -{% if config.mail_domain != "nine.testrun.org" %} -
Note: this is only a temporary development chatmail service
-{% endif %} From 49c66116bfee2395fe97ee2af9fa693c83969256 Mon Sep 17 00:00:00 2001 From: cliffmccarthy <16453869+cliffmccarthy@users.noreply.github.com> Date: Wed, 6 Aug 2025 07:59:23 -0500 Subject: [PATCH 09/24] feat: Remove echobot special cases --- chatmaild/src/chatmaild/lastlogin.py | 2 -- chatmaild/src/chatmaild/user.py | 10 ++++------ 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/chatmaild/src/chatmaild/lastlogin.py b/chatmaild/src/chatmaild/lastlogin.py index c9a531a..7164be8 100644 --- a/chatmaild/src/chatmaild/lastlogin.py +++ b/chatmaild/src/chatmaild/lastlogin.py @@ -13,8 +13,6 @@ class LastLoginDictProxy(DictProxy): keyname = parts[1].split("/") value = parts[2] if len(parts) > 2 else "" if keyname[0] == "shared" and keyname[1] == "last-login": - if addr.startswith("echo@"): - return True addr = keyname[2] timestamp = int(value) user = self.config.get_user(addr) diff --git a/chatmaild/src/chatmaild/user.py b/chatmaild/src/chatmaild/user.py index 934eb51..3a86958 100644 --- a/chatmaild/src/chatmaild/user.py +++ b/chatmaild/src/chatmaild/user.py @@ -19,7 +19,7 @@ class User: @property def can_track(self): - return "@" in self.addr and not self.addr.startswith("echo@") + return "@" in self.addr def get_userdb_dict(self): """Return a non-empty dovecot 'userdb' style dict @@ -55,11 +55,9 @@ class User: try: write_bytes_atomic(self.password_path, password) except PermissionError: - if not self.addr.startswith("echo@"): - logging.error(f"could not write password for: {self.addr}") - raise - if not self.addr.startswith("echo@"): - self.enforce_E2EE_path.touch() + logging.error(f"could not write password for: {self.addr}") + raise + self.enforce_E2EE_path.touch() def set_last_login_timestamp(self, timestamp): """Track login time with daily granularity From bab3de97684969d8d047788a82daa754fc31ced5 Mon Sep 17 00:00:00 2001 From: cliffmccarthy <16453869+cliffmccarthy@users.noreply.github.com> Date: Fri, 14 Nov 2025 09:36:17 -0600 Subject: [PATCH 10/24] feat: Remove echobot user from deployment --- cmdeploy/src/cmdeploy/deployers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cmdeploy/src/cmdeploy/deployers.py b/cmdeploy/src/cmdeploy/deployers.py index 4686e4c..84b8e59 100644 --- a/cmdeploy/src/cmdeploy/deployers.py +++ b/cmdeploy/src/cmdeploy/deployers.py @@ -440,7 +440,6 @@ class ChatmailVenvDeployer(Deployer): class ChatmailDeployer(Deployer): required_users = [ ("vmail", "vmail", None), - ("echobot", None, None), ("iroh", None, None), ] From d76b33def162ec05633e9e9142189e1ee32b590f Mon Sep 17 00:00:00 2001 From: cliffmccarthy <16453869+cliffmccarthy@users.noreply.github.com> Date: Mon, 8 Dec 2025 12:38:55 -0600 Subject: [PATCH 11/24] feat: Remove echo from passthrough recipients --- chatmaild/src/chatmaild/ini/chatmail.ini.f | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chatmaild/src/chatmaild/ini/chatmail.ini.f b/chatmaild/src/chatmaild/ini/chatmail.ini.f index 7ec3ca8..6e28599 100644 --- a/chatmaild/src/chatmaild/ini/chatmail.ini.f +++ b/chatmaild/src/chatmaild/ini/chatmail.ini.f @@ -43,7 +43,7 @@ passthrough_senders = # list of e-mail recipients for which to accept outbound un-encrypted mails # (space-separated, item may start with "@" to whitelist whole recipient domains) -passthrough_recipients = echo@{mail_domain} +passthrough_recipients = # path to www directory - documented here: https://chatmail.at/doc/relay/getting_started.html#custom-web-pages #www_folder = www From 40fd62c562a206b7f4fe6e2428285721f7adfdbb Mon Sep 17 00:00:00 2001 From: missytake Date: Wed, 10 Dec 2025 08:12:10 +0100 Subject: [PATCH 12/24] opendkim: report DKIM error code in SMTP response --- cmdeploy/src/cmdeploy/opendkim/final.lua | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmdeploy/src/cmdeploy/opendkim/final.lua b/cmdeploy/src/cmdeploy/opendkim/final.lua index e68a486..fef5eb9 100644 --- a/cmdeploy/src/cmdeploy/opendkim/final.lua +++ b/cmdeploy/src/cmdeploy/opendkim/final.lua @@ -10,6 +10,7 @@ if nsigs == nil then end local valid = false +local sigerrors = "" for i = 1, nsigs do sig = odkim.get_sighandle(ctx, i - 1) sigres = odkim.sig_result(sig) @@ -21,6 +22,8 @@ for i = 1, nsigs do -- means the message is acceptable. if sigres == 0 then valid = true + else + sigerrors = sigerrors .. " " .. tostring(sigres) end end @@ -31,7 +34,7 @@ if valid then odkim.del_header(ctx, "DKIM-Signature", i) end else - odkim.set_reply(ctx, "554", "5.7.1", "No valid DKIM signature found") + odkim.set_reply(ctx, "554", "5.7.1", "No valid DKIM signature found. Search https://github.com/trusteddomainproject/OpenDKIM/blob/master/libopendkim/dkim.h#L108 for " .. sigerrors) odkim.set_result(ctx, SMFIS_REJECT) end From 70da217442fd85ecb63d0a23f6ff506e5e0a599a Mon Sep 17 00:00:00 2001 From: missytake Date: Fri, 12 Dec 2025 10:25:27 +0100 Subject: [PATCH 13/24] opendkim: only display last sigerror --- cmdeploy/src/cmdeploy/opendkim/final.lua | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmdeploy/src/cmdeploy/opendkim/final.lua b/cmdeploy/src/cmdeploy/opendkim/final.lua index fef5eb9..ce909f0 100644 --- a/cmdeploy/src/cmdeploy/opendkim/final.lua +++ b/cmdeploy/src/cmdeploy/opendkim/final.lua @@ -10,7 +10,7 @@ if nsigs == nil then end local valid = false -local sigerrors = "" +local error_msg = "No valid DKIM signature found." for i = 1, nsigs do sig = odkim.get_sighandle(ctx, i - 1) sigres = odkim.sig_result(sig) @@ -23,7 +23,7 @@ for i = 1, nsigs do if sigres == 0 then valid = true else - sigerrors = sigerrors .. " " .. tostring(sigres) + error_msg = "DKIM signature is invalid, error code " .. tostring(sigres) .. ", search https://github.com/trusteddomainproject/OpenDKIM/blob/master/libopendkim/dkim.h#L108" end end @@ -34,7 +34,7 @@ if valid then odkim.del_header(ctx, "DKIM-Signature", i) end else - odkim.set_reply(ctx, "554", "5.7.1", "No valid DKIM signature found. Search https://github.com/trusteddomainproject/OpenDKIM/blob/master/libopendkim/dkim.h#L108 for " .. sigerrors) + odkim.set_reply(ctx, "554", "5.7.1", error_msg) odkim.set_result(ctx, SMFIS_REJECT) end From df21076e9bf4877567bf10bdf50ace7f6a1c390e Mon Sep 17 00:00:00 2001 From: j4n Date: Tue, 16 Dec 2025 17:56:24 +0100 Subject: [PATCH 14/24] acmetool: use a fixed name and reconcile instead of want --- cmdeploy/src/cmdeploy/acmetool/__init__.py | 17 +++++++++++++++-- cmdeploy/src/cmdeploy/acmetool/desired.yaml.j2 | 6 ++++++ 2 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 cmdeploy/src/cmdeploy/acmetool/desired.yaml.j2 diff --git a/cmdeploy/src/cmdeploy/acmetool/__init__.py b/cmdeploy/src/cmdeploy/acmetool/__init__.py index 7539806..e4e1ed8 100644 --- a/cmdeploy/src/cmdeploy/acmetool/__init__.py +++ b/cmdeploy/src/cmdeploy/acmetool/__init__.py @@ -61,6 +61,19 @@ class AcmetoolDeployer(Deployer): mode="644", ) + server.shell( + name=f"Remove old acmetool desired files for {self.domains[0]}", + commands=[f"rm -f /var/lib/acme/desired/{self.domains[0]}-*"], + ) + files.template( + src=importlib.resources.files(__package__).joinpath("desired.yaml.j2"), + dest=f"/var/lib/acme/desired/{self.domains[0]}", # 0 is mailhost TLD + user="root", + group="root", + mode="644", + domains=self.domains, + ) + service_file = files.put( src=importlib.resources.files(__package__).joinpath( "acmetool-redirector.service" @@ -123,6 +136,6 @@ class AcmetoolDeployer(Deployer): self.need_restart_reconcile_timer = False server.shell( - name=f"Request certificate for: {', '.join(self.domains)}", - commands=[f"acmetool want --xlog.severity=debug {' '.join(self.domains)}"], + name=f"Reconcile certificates for: {', '.join(self.domains)}", + commands=["acmetool --batch --xlog.severity=debug reconcile"], ) diff --git a/cmdeploy/src/cmdeploy/acmetool/desired.yaml.j2 b/cmdeploy/src/cmdeploy/acmetool/desired.yaml.j2 new file mode 100644 index 0000000..a5b340a --- /dev/null +++ b/cmdeploy/src/cmdeploy/acmetool/desired.yaml.j2 @@ -0,0 +1,6 @@ +satisfy: + names: +{%- for domain in domains %} + - {{ domain }} +{%- endfor %} + From e7bed4d2a16fbd4228ae963aa73f67fb01aff5fe Mon Sep 17 00:00:00 2001 From: missytake Date: Wed, 10 Dec 2025 09:54:42 +0100 Subject: [PATCH 15/24] dovecot: restart automatically on failure --- cmdeploy/src/cmdeploy/dovecot/deployer.py | 15 ++++++++++++--- cmdeploy/src/cmdeploy/service/10_restart.conf | 3 +++ 2 files changed, 15 insertions(+), 3 deletions(-) create mode 100644 cmdeploy/src/cmdeploy/service/10_restart.conf diff --git a/cmdeploy/src/cmdeploy/dovecot/deployer.py b/cmdeploy/src/cmdeploy/dovecot/deployer.py index dfe9de1..387dba0 100644 --- a/cmdeploy/src/cmdeploy/dovecot/deployer.py +++ b/cmdeploy/src/cmdeploy/dovecot/deployer.py @@ -27,7 +27,7 @@ class DovecotDeployer(Deployer): def configure(self): configure_remote_units(self.config.mail_domain, self.units) - self.need_restart = _configure_dovecot(self.config) + self.need_restart, self.daemon_reload = _configure_dovecot(self.config) def activate(self): activate_remote_units(self.units) @@ -42,6 +42,7 @@ class DovecotDeployer(Deployer): running=False if self.disable_mail else True, enabled=False if self.disable_mail else True, restarted=restart, + daemon_reload=self.daemon_reload, ) self.need_restart = False @@ -80,9 +81,10 @@ def _install_dovecot_package(package: str, arch: str): apt.deb(name=f"Install dovecot-{package}", src=deb_filename) -def _configure_dovecot(config: Config, debug: bool = False) -> bool: +def _configure_dovecot(config: Config, debug: bool = False) -> (bool, bool): """Configures Dovecot IMAP server.""" need_restart = False + daemon_reload = False main_config = files.template( src=get_resource("dovecot/dovecot.conf.j2"), @@ -134,4 +136,11 @@ def _configure_dovecot(config: Config, debug: bool = False) -> bool: ) need_restart |= timezone_env.changed - return need_restart + restart_conf = files.put( + name="dovecot: restart automatically on failure", + src=get_resource("service/10_restart.conf"), + dest="/etc/systemd/system/dovecot.service.d/10_restart.conf", + ) + daemon_reload |= restart_conf.changed + + return need_restart, daemon_reload diff --git a/cmdeploy/src/cmdeploy/service/10_restart.conf b/cmdeploy/src/cmdeploy/service/10_restart.conf new file mode 100644 index 0000000..37e5d3a --- /dev/null +++ b/cmdeploy/src/cmdeploy/service/10_restart.conf @@ -0,0 +1,3 @@ +[Service] +Restart=always +RestartSec=30 From 8c8c37c822927af9d7079a5ab0f370732579f57c Mon Sep 17 00:00:00 2001 From: missytake Date: Wed, 10 Dec 2025 09:55:20 +0100 Subject: [PATCH 16/24] postfix: restart automatically on failure --- cmdeploy/src/cmdeploy/postfix/deployer.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cmdeploy/src/cmdeploy/postfix/deployer.py b/cmdeploy/src/cmdeploy/postfix/deployer.py index fe35700..d0ecc0b 100644 --- a/cmdeploy/src/cmdeploy/postfix/deployer.py +++ b/cmdeploy/src/cmdeploy/postfix/deployer.py @@ -60,6 +60,13 @@ class PostfixDeployer(Deployer): mode="644", ) need_restart |= login_map.changed + + restart_conf = files.put( + name="postfix: restart automatically on failure", + src=get_resource("service/10_restart.conf"), + dest="/etc/systemd/system/dovecot.service.d/10_restart.conf", + ) + self.daemon_reload = restart_conf.changed self.need_restart = need_restart def activate(self): @@ -73,5 +80,6 @@ class PostfixDeployer(Deployer): running=False if self.disable_mail else True, enabled=False if self.disable_mail else True, restarted=restart, + daemon_reload=self.daemon_reload, ) self.need_restart = False From abe0cb5d08e97b13bff8db18d200c8b0a9315160 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Wed, 17 Dec 2025 13:07:52 +0100 Subject: [PATCH 17/24] address cliff's comments about dovecot/postfix --- cmdeploy/src/cmdeploy/dovecot/deployer.py | 2 ++ cmdeploy/src/cmdeploy/postfix/deployer.py | 1 + 2 files changed, 3 insertions(+) diff --git a/cmdeploy/src/cmdeploy/dovecot/deployer.py b/cmdeploy/src/cmdeploy/dovecot/deployer.py index 387dba0..bda9e19 100644 --- a/cmdeploy/src/cmdeploy/dovecot/deployer.py +++ b/cmdeploy/src/cmdeploy/dovecot/deployer.py @@ -13,6 +13,8 @@ from cmdeploy.basedeploy import ( class DovecotDeployer(Deployer): + daemon_reload = False + def __init__(self, config, disable_mail): self.config = config self.disable_mail = disable_mail diff --git a/cmdeploy/src/cmdeploy/postfix/deployer.py b/cmdeploy/src/cmdeploy/postfix/deployer.py index d0ecc0b..421a44e 100644 --- a/cmdeploy/src/cmdeploy/postfix/deployer.py +++ b/cmdeploy/src/cmdeploy/postfix/deployer.py @@ -5,6 +5,7 @@ from cmdeploy.basedeploy import Deployer, get_resource class PostfixDeployer(Deployer): required_users = [("postfix", None, ["opendkim"])] + daemon_reload = False def __init__(self, config, disable_mail): self.config = config From 87153667ed570e6e514ea93002fd9336bfca7a67 Mon Sep 17 00:00:00 2001 From: link2xt Date: Wed, 17 Dec 2025 21:18:02 +0000 Subject: [PATCH 18/24] chore: update the heading in the CHANGELOG.md I have checked that nobody added any entries since 1.8.0 was released. --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5253767..5139049 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog for chatmail deployment -## untagged +## 1.8.0 2025-12-12 - Add imap_compress option to chatmail.ini ([#760](https://github.com/chatmail/relay/pull/760)) From 966754a3460cd57a1135c4605d2da02fce15a7fe Mon Sep 17 00:00:00 2001 From: link2xt Date: Wed, 17 Dec 2025 21:21:03 +0000 Subject: [PATCH 19/24] chore: setup git-cliff I am running git-cliff 2.11.0. Ran `git-cliff --init` to generate `cliff.toml`. Removed emojis, replaced `doc` with `docs` to match chatmail core convention. --- cliff.toml | 94 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 cliff.toml diff --git a/cliff.toml b/cliff.toml new file mode 100644 index 0000000..b741c36 --- /dev/null +++ b/cliff.toml @@ -0,0 +1,94 @@ +# git-cliff ~ configuration file +# https://git-cliff.org/docs/configuration + + +[changelog] +# A Tera template to be rendered for each release in the changelog. +# See https://keats.github.io/tera/docs/#introduction +body = """ +{% if version %}\ + ## [{{ version | trim_start_matches(pat="v") }}] - {{ timestamp | date(format="%Y-%m-%d") }} +{% else %}\ + ## [unreleased] +{% endif %}\ +{% for group, commits in commits | group_by(attribute="group") %} + ### {{ group | striptags | trim | upper_first }} + {% for commit in commits %} + - {% if commit.scope %}*({{ commit.scope }})* {% endif %}\ + {% if commit.breaking %}[**breaking**] {% endif %}\ + {{ commit.message | upper_first }}\ + {% endfor %} +{% endfor %} +""" +# Remove leading and trailing whitespaces from the changelog's body. +trim = true +# Render body even when there are no releases to process. +render_always = true +# An array of regex based postprocessors to modify the changelog. +postprocessors = [ + # Replace the placeholder with a URL. + #{ pattern = '', replace = "https://github.com/orhun/git-cliff" }, +] +# render body even when there are no releases to process +# render_always = true +# output file path +# output = "test.md" + +[git] +# Parse commits according to the conventional commits specification. +# See https://www.conventionalcommits.org +conventional_commits = true +# Exclude commits that do not match the conventional commits specification. +filter_unconventional = true +# Require all commits to be conventional. +# Takes precedence over filter_unconventional. +require_conventional = false +# Split commits on newlines, treating each line as an individual commit. +split_commits = false +# An array of regex based parsers to modify commit messages prior to further processing. +commit_preprocessors = [ + # Replace issue numbers with link templates to be updated in `changelog.postprocessors`. + #{ pattern = '\((\w+\s)?#([0-9]+)\)', replace = "([#${2}](/issues/${2}))"}, + # Check spelling of the commit message using https://github.com/crate-ci/typos. + # If the spelling is incorrect, it will be fixed automatically. + #{ pattern = '.*', replace_command = 'typos --write-changes -' }, +] +# Prevent commits that are breaking from being excluded by commit parsers. +protect_breaking_commits = false +# An array of regex based parsers for extracting data from the commit message. +# Assigns commits to groups. +# Optionally sets the commit's scope and can decide to exclude commits from further processing. +commit_parsers = [ + { message = "^feat", group = "Features" }, + { message = "^fix", group = "Bug Fixes" }, + { message = "^docs", group = "Documentation" }, + { message = "^perf", group = "Performance" }, + { message = "^refactor", group = "Refactor" }, + { message = "^style", group = "Styling" }, + { message = "^test", group = "Testing" }, + { message = "^chore\\(release\\): prepare for", skip = true }, + { message = "^chore\\(deps.*\\)", skip = true }, + { message = "^chore\\(pr\\)", skip = true }, + { message = "^chore\\(pull\\)", skip = true }, + { message = "^chore|^ci", group = "Miscellaneous Tasks" }, + { body = ".*security", group = "Security" }, + { message = "^revert", group = "Revert" }, + { message = ".*", group = "Other" }, +] +# Exclude commits that are not matched by any commit parser. +filter_commits = false +# Fail on a commit that is not matched by any commit parser. +fail_on_unmatched_commit = false +# An array of link parsers for extracting external references, and turning them into URLs, using regex. +link_parsers = [] +# Include only the tags that belong to the current branch. +use_branch_tags = false +# Order releases topologically instead of chronologically. +topo_order = false +# Order commits topologically instead of chronologically. +topo_order_commits = true +# Order of commits in each group/release within the changelog. +# Allowed values: newest, oldest +sort_commits = "oldest" +# Process submodules commits +recurse_submodules = false From 610843a44ae1932d8c56d084e0d1284a76161268 Mon Sep 17 00:00:00 2001 From: link2xt Date: Wed, 17 Dec 2025 21:25:04 +0000 Subject: [PATCH 20/24] docs: add RELEASE.md and CONTRIBUTING.md --- CONTRIBUTING.md | 7 +++++++ RELEASE.md | 15 +++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 CONTRIBUTING.md create mode 100644 RELEASE.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..8f92ce2 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,7 @@ +# Contributing to the chatmail relay + +Commit messages follow the [Conventional Commits] notation. +We use [git-cliff] to generate the changelog from commit messages before the release. + +[Conventional Commits]: https://www.conventionalcommits.org/ +[git-cliff]: https://git-cliff.org/ diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 0000000..7d571c8 --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,15 @@ +# Releasing a new version of chatmail relay + +For example, to release version 1.9.0 of chatmail relay, do the following steps. + +1. Update the changelog: `git cliff --unreleased --tag 1.9.0 --prepend CHANGELOG.md` or `git cliff -u -t 1.9.0 -p CHANGELOG.md`. + +2. Open the changelog in the editor, edit it if required. + +3. Commit the changes to the changelog with a commit message `chore(release): prepare for 1.9.0`. + +3. Tag the release: `git tag --annotate 1.9.0`. + +4. Push the release tag: `git push origin 1.9.0`. + +5. Create a GitHub release: `gh release create 1.9.0`. From 24e3f33acda4f98c34c12a9395af16e1baf6b21c Mon Sep 17 00:00:00 2001 From: holger krekel Date: Thu, 18 Dec 2025 17:21:46 +0100 Subject: [PATCH 21/24] fix: expire messages also from DeltaChat IMAP subfolders --- chatmaild/src/chatmaild/expire.py | 21 ++++++++++++------- chatmaild/src/chatmaild/tests/test_expire.py | 22 ++++++++++++++------ 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/chatmaild/src/chatmaild/expire.py b/chatmaild/src/chatmaild/expire.py index c65d1ad..e811b4a 100644 --- a/chatmaild/src/chatmaild/expire.py +++ b/chatmaild/src/chatmaild/expire.py @@ -63,21 +63,28 @@ class MailboxStat: os.chdir(self.basedir) except FileNotFoundError: return - for name in os_listdir_if_exists("."): + try: + self.scandir(".") + finally: + os.chdir(old_cwd) + + def scandir(self, dirname): + for name in os_listdir_if_exists(dirname): + relpath = f"{dirname}/{name}" if name in ("cur", "new", "tmp"): - for msg_name in os_listdir_if_exists(name): - entry = get_file_entry(f"{name}/{msg_name}") + for msg_name in os_listdir_if_exists(relpath): + entry = get_file_entry(f"{relpath}/{msg_name}") if entry is not None: self.messages.append(entry) - + elif relpath == "./.DeltaChat": + self.scandir(name) else: - entry = get_file_entry(name) + entry = get_file_entry(relpath) if entry is not None: self.extrafiles.append(entry) if name == "password": self.last_login = entry.mtime self.extrafiles.sort(key=lambda x: -x.size) - os.chdir(old_cwd) def print_info(msg): @@ -150,7 +157,7 @@ class Expiry: self.remove_file(message.relpath, mtime=message.mtime) elif message.size > 200000 and message.mtime < cutoff_large_mails: # we only remove noticed large files (not unnoticed ones in new/) - if message.relpath.startswith("cur/"): + if "cur" in message.relpath.split("/"): self.remove_file(message.relpath, mtime=message.mtime) else: continue diff --git a/chatmaild/src/chatmaild/tests/test_expire.py b/chatmaild/src/chatmaild/tests/test_expire.py index b9ad15d..89d3a69 100644 --- a/chatmaild/src/chatmaild/tests/test_expire.py +++ b/chatmaild/src/chatmaild/tests/test_expire.py @@ -17,9 +17,7 @@ from chatmaild.expire import main as expiry_main from chatmaild.fsreport import main as report_main -def fill_mbox(basedir): - basedir1 = basedir.joinpath("mailbox1@example.org") - basedir1.mkdir() +def fill_mbox(basedir1): password = basedir1.joinpath("password") password.write_text("xxx") basedir1.joinpath("maildirsize").write_text("xxx") @@ -29,7 +27,6 @@ def fill_mbox(basedir): create_new_messages(basedir1, ["cur/msg1"], size=500) create_new_messages(basedir1, ["new/msg2"], size=600) - return basedir1 def create_new_messages(basedir, relpaths, size=1000, days=0): @@ -45,8 +42,21 @@ def create_new_messages(basedir, relpaths, size=1000, days=0): @pytest.fixture def mbox1(example_config): - basedir1 = fill_mbox(example_config.mailboxes_dir) - return MailboxStat(basedir1) + mboxdir = example_config.mailboxes_dir.joinpath("mailbox1@example.org") + mboxdir.mkdir() + fill_mbox(mboxdir) + return MailboxStat(mboxdir) + + +def test_deltachat_folder(example_config): + """Test old setups that might have a .DeltaChat folder where messages also need to get removed.""" + mboxdir = example_config.mailboxes_dir.joinpath("mailbox1@example.org") + mboxdir.mkdir() + mbox2dir = mboxdir.joinpath(".DeltaChat") + mbox2dir.mkdir() + fill_mbox(mbox2dir) + mb = MailboxStat(mboxdir) + assert len(mb.messages) == 2 def test_filentry_ordering(tmp_path): From f04a624e19a498f6f047ac9d46f0eb59305b568e Mon Sep 17 00:00:00 2001 From: holger krekel Date: Thu, 18 Dec 2025 22:51:38 +0100 Subject: [PATCH 22/24] fix: use absolute path instead of relative path, and streamline some code parts according to comments at https://github.com/chatmail/relay/pull/785 --- chatmaild/src/chatmaild/expire.py | 47 ++++++-------------- chatmaild/src/chatmaild/tests/test_expire.py | 15 ++++--- 2 files changed, 22 insertions(+), 40 deletions(-) diff --git a/chatmaild/src/chatmaild/expire.py b/chatmaild/src/chatmaild/expire.py index e811b4a..e940500 100644 --- a/chatmaild/src/chatmaild/expire.py +++ b/chatmaild/src/chatmaild/expire.py @@ -14,7 +14,7 @@ from stat import S_ISREG from chatmaild.config import read_config -FileEntry = namedtuple("FileEntry", ("relpath", "mtime", "size")) +FileEntry = namedtuple("FileEntry", ("path", "mtime", "size")) def iter_mailboxes(basedir, maxnum): @@ -51,35 +51,22 @@ class MailboxStat: def __init__(self, basedir): self.basedir = str(basedir) - # all detected messages in cur/new/tmp folders self.messages = [] - - # all detected files in mailbox top dir self.extrafiles = [] + self.scandir(self.basedir) - # scan all relevant files (without recursion) - old_cwd = os.getcwd() - try: - os.chdir(self.basedir) - except FileNotFoundError: - return - try: - self.scandir(".") - finally: - os.chdir(old_cwd) - - def scandir(self, dirname): - for name in os_listdir_if_exists(dirname): - relpath = f"{dirname}/{name}" + def scandir(self, folderdir): + for name in os_listdir_if_exists(folderdir): + path = f"{folderdir}/{name}" if name in ("cur", "new", "tmp"): - for msg_name in os_listdir_if_exists(relpath): - entry = get_file_entry(f"{relpath}/{msg_name}") + for msg_name in os_listdir_if_exists(path): + entry = get_file_entry(f"{path}/{msg_name}") if entry is not None: self.messages.append(entry) - elif relpath == "./.DeltaChat": - self.scandir(name) + elif os.path.isdir(path): + self.scandir(path) else: - entry = get_file_entry(relpath) + entry = get_file_entry(path) if entry is not None: self.extrafiles.append(entry) if name == "password": @@ -137,13 +124,6 @@ class Expiry: self.remove_mailbox(mbox.basedir) return - # all to-be-removed files are relative to the mailbox basedir - try: - os.chdir(mbox.basedir) - except FileNotFoundError: - print_info(f"mailbox not found/vanished {mbox.basedir}") - return - mboxname = os.path.basename(mbox.basedir) if self.verbose: date = datetime.fromtimestamp(mbox.last_login) if mbox.last_login else None @@ -154,11 +134,12 @@ class Expiry: self.all_files += len(mbox.messages) for message in mbox.messages: if message.mtime < cutoff_mails: - self.remove_file(message.relpath, mtime=message.mtime) + self.remove_file(message.path, mtime=message.mtime) elif message.size > 200000 and message.mtime < cutoff_large_mails: # we only remove noticed large files (not unnoticed ones in new/) - if "cur" in message.relpath.split("/"): - self.remove_file(message.relpath, mtime=message.mtime) + parts = message.path.split("/") + if len(parts) >= 2 and parts[-2] == "cur": + self.remove_file(message.path, mtime=message.mtime) else: continue changed = True diff --git a/chatmaild/src/chatmaild/tests/test_expire.py b/chatmaild/src/chatmaild/tests/test_expire.py index 89d3a69..fcdb450 100644 --- a/chatmaild/src/chatmaild/tests/test_expire.py +++ b/chatmaild/src/chatmaild/tests/test_expire.py @@ -17,16 +17,17 @@ from chatmaild.expire import main as expiry_main from chatmaild.fsreport import main as report_main -def fill_mbox(basedir1): - password = basedir1.joinpath("password") +def fill_mbox(folderdir): + password = folderdir.joinpath("password") password.write_text("xxx") - basedir1.joinpath("maildirsize").write_text("xxx") + folderdir.joinpath("maildirsize").write_text("xxx") - garbagedir = basedir1.joinpath("garbagedir") + garbagedir = folderdir.joinpath("garbagedir") garbagedir.mkdir() + garbagedir.joinpath("bimbum").write_text("hello") - create_new_messages(basedir1, ["cur/msg1"], size=500) - create_new_messages(basedir1, ["new/msg2"], size=600) + create_new_messages(folderdir, ["cur/msg1"], size=500) + create_new_messages(folderdir, ["new/msg2"], size=600) def create_new_messages(basedir, relpaths, size=1000, days=0): @@ -86,7 +87,7 @@ def test_stats_mailbox(mbox1): create_new_messages(mbox1.basedir, ["large-extra"], size=1000) create_new_messages(mbox1.basedir, ["index-something"], size=3) mbox2 = MailboxStat(mbox1.basedir) - assert len(mbox2.extrafiles) == 4 + assert len(mbox2.extrafiles) == 5 assert mbox2.extrafiles[0].size == 1000 # cope well with mailbox dirs that have no password (for whatever reason) From 1ae4c8451a2bd9c07649a80e7cfc8a7899dda4fc Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 18 Dec 2025 22:17:47 +0000 Subject: [PATCH 23/24] ci: run tests against ci-chatmail.testrun.org instead of nine.testrun.org --- .github/workflows/test-and-deploy-ipv4only.yaml | 2 +- .github/workflows/test-and-deploy.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-and-deploy-ipv4only.yaml b/.github/workflows/test-and-deploy-ipv4only.yaml index 0dc2c35..24a1177 100644 --- a/.github/workflows/test-and-deploy-ipv4only.yaml +++ b/.github/workflows/test-and-deploy-ipv4only.yaml @@ -93,7 +93,7 @@ jobs: ssh root@ns.testrun.org systemctl reload nsd - name: cmdeploy test - run: CHATMAIL_DOMAIN2=nine.testrun.org cmdeploy test --slow + run: CHATMAIL_DOMAIN2=ci-chatmail.testrun.org cmdeploy test --slow - name: cmdeploy dns run: cmdeploy dns -v diff --git a/.github/workflows/test-and-deploy.yaml b/.github/workflows/test-and-deploy.yaml index c02f455..3640acb 100644 --- a/.github/workflows/test-and-deploy.yaml +++ b/.github/workflows/test-and-deploy.yaml @@ -94,7 +94,7 @@ jobs: ssh root@ns.testrun.org systemctl reload nsd - name: cmdeploy test - run: CHATMAIL_DOMAIN2=nine.testrun.org cmdeploy test --slow + run: CHATMAIL_DOMAIN2=ci-chatmail.testrun.org cmdeploy test --slow - name: cmdeploy dns run: cmdeploy dns -v From 7191329a9f5100d220a3770733fd8d9cc66f532a Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 18 Dec 2025 23:49:38 +0000 Subject: [PATCH 24/24] chore(release): prepare for 1.9.0 --- CHANGELOG.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5139049..91ba879 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,31 @@ # Changelog for chatmail deployment +## 1.9.0 2025-12-18 + +### Documentation + +- Add RELEASE.md and CONTRIBUTING.md +- README update, mention Chatmail Cookbook project + +### Bug Fixes + +- Expire messages also from IMAP subfolders +- Use absolute path instead of relative path in message expiration script +- Restart Postfix and Dovecot automatically on failure +- acmetool: Use a fixed name and `reconcile` instead of `want` + +### Features + +- Report DKIM error code in SMTP response +- Remove development notice from the web pages + +### Miscellaneous Tasks + +- Update the heading in the CHANGELOG.md +- Setup git-cliff +- Run tests against ci-chatmail.testrun.org instead of nine.testrun.org +- Cleanup remaining echobot code, remove echobot user from deployment and passthrough recipients + ## 1.8.0 2025-12-12 - Add imap_compress option to chatmail.ini