From 70acc81bac05f25aafecd94b6d197b42a59ac3b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20B=C3=BCrki?= Date: Fri, 18 Sep 2026 16:20:52 +0200 Subject: [PATCH 01/12] roles(apache_httpd): added csv access log formats "csvio csvextensive csvsiem" and their tsv equivalents "tsvio tsvextensive tsvsiem" --- CHANGELOG.md | 1 + roles/apache_httpd/README.md | 4 +- .../etc/httpd/conf-available/logio.conf.j2 | 140 +++++++++++++++++- 3 files changed, 142 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21ed9ff3e..ac7fb63f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +* **role:apache_httpd:** Added csv access log formats `csvio csvextensive csvsiem` and their tsv equivalents `tsvio tsvextensive tsvsiem`. * **role:wordpress**: Several WordPress instances can share a host as pseudo hosts in the inventory, under different host names as well as under different paths of one host name, such as `https://example.com/blog`. * **role:fail2ban**: The `wordpress-login` filter and `z10-wordpress-login` jail ban IPs with too many failed WordPress logins, on the host whose Apache logs the visitor's address. * **role:rstudio_server, playbook:rstudio_server**: Add a role and playbook to install RStudio Server Open Source, the browser-based R development environment. Users sign in with their account on the host and have to be a member of a group to be let in at all, the PAM profile covers directory users where the vendor's covers local ones only, and the R sessions can be given a memory and process budget. diff --git a/roles/apache_httpd/README.md b/roles/apache_httpd/README.md index 81c0f0785..193002259 100644 --- a/roles/apache_httpd/README.md +++ b/roles/apache_httpd/README.md @@ -611,7 +611,7 @@ The remaining subkeys configure the contents of the vHost and are only honoured `conf_custom_log` -* The log format has to be one of: `agent`, `combined`, `combinedio`, `common`, `debug`, `fail2ban`, `linuxfabrikio`, `matomo`, `referer`, `vhost_common`. Set it to an empty string to disable the access log. See [CustomLog](https://httpd.apache.org/docs/2.4/mod/mod_log_config.html#customlog). +* The log format has to be one of: `agent`, `combined`, `combinedio`, `common`, `csvextensive`, `csvio`, `csvsiem`, `debug`, `fail2ban`, `linuxfabrikio`, `matomo`, `referer`, `tsvextensive`, `tsvio`, `tsvsiem`, `vhost_common`. Set it to an empty string to disable the access log. See [CustomLog](https://httpd.apache.org/docs/2.4/mod/mod_log_config.html#customlog). * Applies to: app, localhost, proxy, wordpress. * Type: String. * Default: `'logs/{{ conf_server_name }}-access.log linuxfabrikio'` @@ -858,7 +858,7 @@ This module is for flexible logging of client requests. Logs are written in a cu `apache_httpd__mod_log_config_custom_log` -* Global log directive that applies to requests not handled by any vHost. Each vHost defines its own log via `conf_custom_log`. One of: `agent`, `combined`, `combinedio`, `common`, `debug`, `fail2ban`, `linuxfabrikio`, `matomo`, `referer`, `vhost_common`. See [CustomLog](https://httpd.apache.org/docs/2.4/mod/mod_log_config.html#customlog). +* Global log directive that applies to requests not handled by any vHost. Each vHost defines its own log via `conf_custom_log`. One of: `agent`, `combined`, `combinedio`, `common`, `csvextensive`, `csvio`, `csvsiem`, `debug`, `fail2ban`, `linuxfabrikio`, `matomo`, `referer`, `tsvextensive`, `tsvio`, `tsvsiem`, `vhost_common`. See [CustomLog](https://httpd.apache.org/docs/2.4/mod/mod_log_config.html#customlog). * Type: String. * Default: unset diff --git a/roles/apache_httpd/templates/etc/httpd/conf-available/logio.conf.j2 b/roles/apache_httpd/templates/etc/httpd/conf-available/logio.conf.j2 index c2defdefc..318c16290 100644 --- a/roles/apache_httpd/templates/etc/httpd/conf-available/logio.conf.j2 +++ b/roles/apache_httpd/templates/etc/httpd/conf-available/logio.conf.j2 @@ -1,5 +1,5 @@ # {{ ansible_managed }} -# 2021110301 +# 2026091801 {% if item['by_role'] | d() %} # Generated by Ansible role: {{ item['by_role'] }} {% endif %} @@ -10,6 +10,144 @@ LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" c LogFormat "%h %{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" linuxfabrikio +# io: based on linuxfabrikio +# +# Field Token Name Group +# +# 1 {{ '%{%FT%T}t.%{usec_frac}t%{%z}t' }} timestamp Transaction +# 2 %v vhost Server +# 3 %a client_ip Client +# 4 %{c}a peer_ip Client +# 5 %{X-Forwarded-For}i xff Client +# 6 %u user Client +# 7 %r request_line Request +# 8 %>s status_final Response +# 9 %b bytes_resp Transfer +# 10 %I bytes_in Transfer +# 11 %O bytes_out Transfer +# 12 %{Referer}i referer Request +# 13 %{User-Agent}i useragent Request + +LogFormat "\ +\"{{ '%{%FT%T}t.%{usec_frac}t%{%z}t' }}\";\ +\"%v\";\ +\"%a\";\"%{c}a\";\"%{X-Forwarded-For}i\";\"%u\";\ +\"%r\";\ +\"%>s\";\ +\"%b\";\"%I\";\"%O\";\ +\"%{Referer}i\";\"%{User-Agent}i\"\ +" csvio + +LogFormat "\ +{{ '%{%FT%T}t.%{usec_frac}t%{%z}t' }}\t\ +%v\t\ +%a\t%{c}a\t%{X-Forwarded-For}i\t%u\t\ +%r\t\ +%>s\t\ +%b\t%I\t%O\t\ +%{Referer}i\t%{User-Agent}i\ +" tsvio + +# extensive: based on gelf logformat +# +# Field Token Name Group +# +# 1 {{ '%{%FT%T}t.%{usec_frac}t%{%z}t' }} timestamp Transaction +# 2 %v vhost Server +# 3 %a client_ip Client +# 4 %{c}a peer_ip Client +# 5 %{X-Forwarded-For}i xff Client +# 6 %u user Client +# 7 %r request_line Request +# 8 %m method Request +# 9 %U uri Request +# 10 %q query Request +# 11 %H protocol Request +# 12 %{Referer}i referer Request +# 13 %{User-Agent}i useragent Request +# 14 %s status_orig Response +# 15 %>s status_final Response +# 16 %b bytes_resp Transfer +# 17 %I bytes_in Transfer +# 18 %O bytes_out Transfer +# 19 %D duration_us Transfer + +LogFormat "\ +\"{{ '%{%FT%T}t.%{usec_frac}t%{%z}t' }}\";\ +\"%v\";\ +\"%a\";\"%{c}a\";\"%{X-Forwarded-For}i\";\"%u\";\ +\"%r\";\"%m\";\"%U\";\"%q\";\"%H\";\"%{Referer}i\";\"%{User-Agent}i\";\ +\"%s\";\"%>s\";\ +\"%O\";\"%D\";\ +" csvextensive + +LogFormat "\ +{{ '%{%FT%T}t.%{usec_frac}t%{%z}t' }}\t\ +%v\t\ +%a\t%{c}a\t%{X-Forwarded-For}i\t%u\t\ +%r\t%m\t%U\t%q\t%H\t%{Referer}i\t%{User-Agent}i\t\ +%s\t%>s\t\ +%O\t%D\t\ +" tsvextensive + +# Field Token Name Group +# +# 1 {{ '%{%FT%T}t.%{usec_frac}t%{%z}t' }} timestamp Transaction +# 2 %{UNIQUE_ID}e unique_id Transaction +# 3 %L log_id Transaction +# 4 %v vhost Server +# 5 %p server_port Server +# 6 %P pid Server +# 7 %a client_ip Client +# 8 %{c}a peer_ip Client +# 9 %{X-Forwarded-For}i xff Client +# 10 %u user Client +# 11 %r request_line Request +# 12 %m method Request +# 13 %U uri Request +# 14 %q query Request +# 15 %H protocol Request +# 16 %{Referer}i referer Request +# 17 %{User-Agent}i useragent Request +# 18 %s status_orig Response +# 19 %>s status_final Response +# 20 %{Content-Type}o content_type Response +# 21 %b bytes_resp Transfer +# 22 %I bytes_in Transfer +# 23 %O bytes_out Transfer +# 24 %D duration_us Transfer +# 25 %X conn_status Connection +# 26 %k keepalive Connection +# 27 %{SSL_PROTOCOL}x tls_protocol Connection +# 28 %{SSL_CIPHER}x tls_cipher Connection +# 29 %{SSL_CIPHER_USEKEYSIZE}x tls_cipher_usekeysize Connection +# 30 %{SSL_TLS_SNI}x tls_sni Connection +# 31 %{SSL_SESSION_RESUMED}x tls_session_resumed Connection +# 32 %{CF-RAY}i cf_ray Cloudflare +# 33 %{CF-Connecting-IP}i cf_connecting_ip Cloudflare + +LogFormat "\ +\"{{ '%{%FT%T}t.%{usec_frac}t%{%z}t' }}\";\"%{UNIQUE_ID}e\";\"%L\";\ +\"%v\";\"%p\";\"%P\";\ +\"%a\";\"%{c}a\";\"%{X-Forwarded-For}i\";\"%u\";\ +\"%r\";\"%m\";\"%U\";\"%q\";\"%H\";\"%{Referer}i\";\"%{User-Agent}i\";\ +\"%s\";\"%>s\";\"%{Content-Type}o\";\ +\"%b\";\"%I\";\"%O\";\"%D\";\ +\"%X\";\"%k\";\"%{SSL_PROTOCOL}x\";\"%{SSL_CIPHER}x\";\"%{SSL_CIPHER_USEKEYSIZE}x\";\"%{SSL_TLS_SNI}x\";\"%{SSL_SESSION_RESUMED}x\";\ +\"%{CF-RAY}i\";\"%{CF-Connecting-IP}i\"\ +" csvsiem + +LogFormat "\ +{{ '%{%FT%T}t.%{usec_frac}t%{%z}t' }}\t%{UNIQUE_ID}e\t%L\t\ +%v\t%p\t%P\t\ +%a\t%{c}a\t%{X-Forwarded-For}i\t%u\t\ +%r\t%m\t\%U\t%q\t%H\t%{Referer}i\t%{User-Agent}i\t\ +%s\t%>s\t%{Content-Type}o\t\ +%b\t%I\t%O\t%D\t\ +%X\t%k\t%{SSL_PROTOCOL}x\t%{SSL_CIPHER}x\t%{SSL_CIPHER_USEKEYSIZE}x\t%{SSL_TLS_SNI}x\t%{SSL_SESSION_RESUMED}x\t\ +%{CF-RAY}i\t%{CF-Connecting-IP}i\ +" tsvsiem + LogFormat "\n\ Bytes\n\ Total, incl. request and headers: %S B\n\ From d153d53bf69ad710c1838e61bef8efacb95884d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20B=C3=BCrki?= Date: Fri, 18 Sep 2026 16:55:44 +0200 Subject: [PATCH 02/12] fix(roles/apache_httpd): add consistent header and fix column aligmnent for csv/tsv log formats. log cloudflare headers CF-Ray CF-Connecting-IP in all csv/tsv formats --- .../etc/httpd/conf-available/logio.conf.j2 | 150 ++++++++++-------- 1 file changed, 80 insertions(+), 70 deletions(-) diff --git a/roles/apache_httpd/templates/etc/httpd/conf-available/logio.conf.j2 b/roles/apache_httpd/templates/etc/httpd/conf-available/logio.conf.j2 index 318c16290..9658971e1 100644 --- a/roles/apache_httpd/templates/etc/httpd/conf-available/logio.conf.j2 +++ b/roles/apache_httpd/templates/etc/httpd/conf-available/logio.conf.j2 @@ -10,23 +10,25 @@ LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" c LogFormat "%h %{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" linuxfabrikio -# io: based on linuxfabrikio +# csvio, tsvio: based on linuxfabrikio with additional cloudflare headers # -# Field Token Name Group +# Field Token Name Group # -# 1 {{ '%{%FT%T}t.%{usec_frac}t%{%z}t' }} timestamp Transaction -# 2 %v vhost Server -# 3 %a client_ip Client -# 4 %{c}a peer_ip Client -# 5 %{X-Forwarded-For}i xff Client -# 6 %u user Client -# 7 %r request_line Request -# 8 %>s status_final Response -# 9 %b bytes_resp Transfer -# 10 %I bytes_in Transfer -# 11 %O bytes_out Transfer -# 12 %{Referer}i referer Request -# 13 %{User-Agent}i useragent Request +# 1 {{ '%{%FT%T}t.%{usec_frac}t%{%z}t' }} timestamp Transaction +# 2 %v vhost Server +# 3 %a client_ip Client +# 4 %{c}a peer_ip Client +# 5 %{X-Forwarded-For}i xff Client +# 6 %u user Client +# 7 %r request_line Request +# 8 %>s status_final Response +# 9 %b bytes_resp Transfer +# 10 %I bytes_in Transfer +# 11 %O bytes_out Transfer +# 12 %{Referer}i referer Request +# 13 %{User-Agent}i useragent Request +# 14 %{CF-RAY}i cf_ray Cloudflare +# 15 %{CF-Connecting-IP}i cf_connecting_ip Cloudflare LogFormat "\ \"{{ '%{%FT%T}t.%{usec_frac}t%{%z}t' }}\";\ @@ -36,6 +38,7 @@ LogFormat "\ \"%>s\";\ \"%b\";\"%I\";\"%O\";\ \"%{Referer}i\";\"%{User-Agent}i\"\ +\"%{CF-RAY}i\";\"%{CF-Connecting-IP}i\"\ " csvio LogFormat "\ @@ -45,32 +48,35 @@ LogFormat "\ %r\t\ %>s\t\ %b\t%I\t%O\t\ -%{Referer}i\t%{User-Agent}i\ +%{Referer}i\t%{User-Agent}i\t\ +%{CF-RAY}i\t%{CF-Connecting-IP}i\ " tsvio -# extensive: based on gelf logformat +# csvextensive, tsvextensive: based on gelf logformat with additional cloudflare headers # -# Field Token Name Group +# Field Token Name Group # -# 1 {{ '%{%FT%T}t.%{usec_frac}t%{%z}t' }} timestamp Transaction -# 2 %v vhost Server -# 3 %a client_ip Client -# 4 %{c}a peer_ip Client -# 5 %{X-Forwarded-For}i xff Client -# 6 %u user Client -# 7 %r request_line Request -# 8 %m method Request -# 9 %U uri Request -# 10 %q query Request -# 11 %H protocol Request -# 12 %{Referer}i referer Request -# 13 %{User-Agent}i useragent Request -# 14 %s status_orig Response -# 15 %>s status_final Response -# 16 %b bytes_resp Transfer -# 17 %I bytes_in Transfer -# 18 %O bytes_out Transfer -# 19 %D duration_us Transfer +# 1 {{ '%{%FT%T}t.%{usec_frac}t%{%z}t' }} timestamp Transaction +# 2 %v vhost Server +# 3 %a client_ip Client +# 4 %{c}a peer_ip Client +# 5 %{X-Forwarded-For}i xff Client +# 6 %u user Client +# 7 %r request_line Request +# 8 %m method Request +# 9 %U uri Request +# 10 %q query Request +# 11 %H protocol Request +# 12 %{Referer}i referer Request +# 13 %{User-Agent}i useragent Request +# 14 %s status_orig Response +# 15 %>s status_final Response +# 16 %b bytes_resp Transfer +# 17 %I bytes_in Transfer +# 18 %O bytes_out Transfer +# 19 %D duration_us Transfer +# 20 %{CF-RAY}i cf_ray Cloudflare +# 21 %{CF-Connecting-IP}i cf_connecting_ip Cloudflare LogFormat "\ \"{{ '%{%FT%T}t.%{usec_frac}t%{%z}t' }}\";\ @@ -79,6 +85,7 @@ LogFormat "\ \"%r\";\"%m\";\"%U\";\"%q\";\"%H\";\"%{Referer}i\";\"%{User-Agent}i\";\ \"%s\";\"%>s\";\ \"%O\";\"%D\";\ +\"%{CF-RAY}i\";\"%{CF-Connecting-IP}i\"\ " csvextensive LogFormat "\ @@ -88,43 +95,46 @@ LogFormat "\ %r\t%m\t%U\t%q\t%H\t%{Referer}i\t%{User-Agent}i\t\ %s\t%>s\t\ %O\t%D\t\ +%{CF-RAY}i\t%{CF-Connecting-IP}i\ " tsvextensive -# Field Token Name Group +# csvsiem, tsvsiem: most comprehensive set of log fields useful for siem +# +# Field Token Name Group # # 1 {{ '%{%FT%T}t.%{usec_frac}t%{%z}t' }} timestamp Transaction -# 2 %{UNIQUE_ID}e unique_id Transaction -# 3 %L log_id Transaction -# 4 %v vhost Server -# 5 %p server_port Server -# 6 %P pid Server -# 7 %a client_ip Client -# 8 %{c}a peer_ip Client -# 9 %{X-Forwarded-For}i xff Client -# 10 %u user Client -# 11 %r request_line Request -# 12 %m method Request -# 13 %U uri Request -# 14 %q query Request -# 15 %H protocol Request -# 16 %{Referer}i referer Request -# 17 %{User-Agent}i useragent Request -# 18 %s status_orig Response -# 19 %>s status_final Response -# 20 %{Content-Type}o content_type Response -# 21 %b bytes_resp Transfer -# 22 %I bytes_in Transfer -# 23 %O bytes_out Transfer -# 24 %D duration_us Transfer -# 25 %X conn_status Connection -# 26 %k keepalive Connection -# 27 %{SSL_PROTOCOL}x tls_protocol Connection -# 28 %{SSL_CIPHER}x tls_cipher Connection -# 29 %{SSL_CIPHER_USEKEYSIZE}x tls_cipher_usekeysize Connection -# 30 %{SSL_TLS_SNI}x tls_sni Connection -# 31 %{SSL_SESSION_RESUMED}x tls_session_resumed Connection -# 32 %{CF-RAY}i cf_ray Cloudflare -# 33 %{CF-Connecting-IP}i cf_connecting_ip Cloudflare +# 2 %{UNIQUE_ID}e unique_id Transaction +# 3 %L log_id Transaction +# 4 %v vhost Server +# 5 %p server_port Server +# 6 %P pid Server +# 7 %a client_ip Client +# 8 %{c}a peer_ip Client +# 9 %{X-Forwarded-For}i xff Client +# 10 %u user Client +# 11 %r request_line Request +# 12 %m method Request +# 13 %U uri Request +# 14 %q query Request +# 15 %H protocol Request +# 16 %{Referer}i referer Request +# 17 %{User-Agent}i useragent Request +# 18 %s status_orig Response +# 19 %>s status_final Response +# 20 %{Content-Type}o content_type Response +# 21 %b bytes_resp Transfer +# 22 %I bytes_in Transfer +# 23 %O bytes_out Transfer +# 24 %D duration_us Transfer +# 25 %X conn_status Connection +# 26 %k keepalive Connection +# 27 %{SSL_PROTOCOL}x tls_protocol Connection +# 28 %{SSL_CIPHER}x tls_cipher Connection +# 29 %{SSL_CIPHER_USEKEYSIZE}x tls_cipher_usekeysize Connection +# 30 %{SSL_TLS_SNI}x tls_sni Connection +# 31 %{SSL_SESSION_RESUMED}x tls_session_resumed Connection +# 32 %{CF-RAY}i cf_ray Cloudflare +# 33 %{CF-Connecting-IP}i cf_connecting_ip Cloudflare LogFormat "\ \"{{ '%{%FT%T}t.%{usec_frac}t%{%z}t' }}\";\"%{UNIQUE_ID}e\";\"%L\";\ From 978628ffcd5ded00633fa8a38370a4ed3712ea79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20B=C3=BCrki?= Date: Fri, 18 Sep 2026 17:06:43 +0200 Subject: [PATCH 03/12] fix(roles/apache_httpd): update timestamp and descriptions --- .../templates/etc/httpd/conf-available/logio.conf.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/apache_httpd/templates/etc/httpd/conf-available/logio.conf.j2 b/roles/apache_httpd/templates/etc/httpd/conf-available/logio.conf.j2 index 9658971e1..449b59416 100644 --- a/roles/apache_httpd/templates/etc/httpd/conf-available/logio.conf.j2 +++ b/roles/apache_httpd/templates/etc/httpd/conf-available/logio.conf.j2 @@ -10,7 +10,7 @@ LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" c LogFormat "%h %{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" linuxfabrikio -# csvio, tsvio: based on linuxfabrikio with additional cloudflare headers +# csvio, tsvio: based on linuxfabrikio logformat with additional cloudflare headers # # Field Token Name Group # @@ -98,7 +98,7 @@ LogFormat "\ %{CF-RAY}i\t%{CF-Connecting-IP}i\ " tsvextensive -# csvsiem, tsvsiem: most comprehensive set of log fields useful for siem +# csvsiem, tsvsiem: most comprehensive logformat useful for siem # # Field Token Name Group # From b9674d8e6ed2215defe6e6dc3489717a9479ffda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20B=C3=BCrki?= Date: Fri, 18 Sep 2026 17:10:19 +0200 Subject: [PATCH 04/12] fix(roles/apache_httpd): update timestamp --- .../templates/etc/httpd/conf-available/logio.conf.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/apache_httpd/templates/etc/httpd/conf-available/logio.conf.j2 b/roles/apache_httpd/templates/etc/httpd/conf-available/logio.conf.j2 index 449b59416..56c635794 100644 --- a/roles/apache_httpd/templates/etc/httpd/conf-available/logio.conf.j2 +++ b/roles/apache_httpd/templates/etc/httpd/conf-available/logio.conf.j2 @@ -1,5 +1,5 @@ # {{ ansible_managed }} -# 2026091801 +# 2026091802 {% if item['by_role'] | d() %} # Generated by Ansible role: {{ item['by_role'] }} {% endif %} From eb9107da943bf3e65b8fb655d3d7713793ae5c24 Mon Sep 17 00:00:00 2001 From: Navid Sassan Date: Tue, 22 Sep 2026 15:32:21 +0200 Subject: [PATCH 05/12] fix(roles/apache_httpd): add missing separator after user agent in csvio --- .../templates/etc/httpd/conf-available/logio.conf.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/apache_httpd/templates/etc/httpd/conf-available/logio.conf.j2 b/roles/apache_httpd/templates/etc/httpd/conf-available/logio.conf.j2 index 56c635794..87b994776 100644 --- a/roles/apache_httpd/templates/etc/httpd/conf-available/logio.conf.j2 +++ b/roles/apache_httpd/templates/etc/httpd/conf-available/logio.conf.j2 @@ -1,5 +1,5 @@ # {{ ansible_managed }} -# 2026091802 +# 2026092201 {% if item['by_role'] | d() %} # Generated by Ansible role: {{ item['by_role'] }} {% endif %} @@ -37,7 +37,7 @@ LogFormat "\ \"%r\";\ \"%>s\";\ \"%b\";\"%I\";\"%O\";\ -\"%{Referer}i\";\"%{User-Agent}i\"\ +\"%{Referer}i\";\"%{User-Agent}i\";\ \"%{CF-RAY}i\";\"%{CF-Connecting-IP}i\"\ " csvio From a2087af691d836a6020541fe1e3a5b9fd00ef5a3 Mon Sep 17 00:00:00 2001 From: Navid Sassan Date: Tue, 22 Sep 2026 15:33:04 +0200 Subject: [PATCH 06/12] fix(roles/apache_httpd): log bytes_resp and bytes_in in csvextensive and tsvextensive --- .../templates/etc/httpd/conf-available/logio.conf.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/apache_httpd/templates/etc/httpd/conf-available/logio.conf.j2 b/roles/apache_httpd/templates/etc/httpd/conf-available/logio.conf.j2 index 87b994776..42d3b1c5d 100644 --- a/roles/apache_httpd/templates/etc/httpd/conf-available/logio.conf.j2 +++ b/roles/apache_httpd/templates/etc/httpd/conf-available/logio.conf.j2 @@ -84,7 +84,7 @@ LogFormat "\ \"%a\";\"%{c}a\";\"%{X-Forwarded-For}i\";\"%u\";\ \"%r\";\"%m\";\"%U\";\"%q\";\"%H\";\"%{Referer}i\";\"%{User-Agent}i\";\ \"%s\";\"%>s\";\ -\"%O\";\"%D\";\ +\"%b\";\"%I\";\"%O\";\"%D\";\ \"%{CF-RAY}i\";\"%{CF-Connecting-IP}i\"\ " csvextensive @@ -94,7 +94,7 @@ LogFormat "\ %a\t%{c}a\t%{X-Forwarded-For}i\t%u\t\ %r\t%m\t%U\t%q\t%H\t%{Referer}i\t%{User-Agent}i\t\ %s\t%>s\t\ -%O\t%D\t\ +%b\t%I\t%O\t%D\t\ %{CF-RAY}i\t%{CF-Connecting-IP}i\ " tsvextensive From 745e079f84251646da9e7e6cce0ce8a023c32a7b Mon Sep 17 00:00:00 2001 From: Navid Sassan Date: Tue, 22 Sep 2026 15:33:32 +0200 Subject: [PATCH 07/12] fix(roles/apache_httpd): remove stray backslash before the uri in tsvsiem --- .../templates/etc/httpd/conf-available/logio.conf.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/apache_httpd/templates/etc/httpd/conf-available/logio.conf.j2 b/roles/apache_httpd/templates/etc/httpd/conf-available/logio.conf.j2 index 42d3b1c5d..9cd682ad2 100644 --- a/roles/apache_httpd/templates/etc/httpd/conf-available/logio.conf.j2 +++ b/roles/apache_httpd/templates/etc/httpd/conf-available/logio.conf.j2 @@ -151,7 +151,7 @@ LogFormat "\ {{ '%{%FT%T}t.%{usec_frac}t%{%z}t' }}\t%{UNIQUE_ID}e\t%L\t\ %v\t%p\t%P\t\ %a\t%{c}a\t%{X-Forwarded-For}i\t%u\t\ -%r\t%m\t\%U\t%q\t%H\t%{Referer}i\t%{User-Agent}i\t\ +%r\t%m\t%U\t%q\t%H\t%{Referer}i\t%{User-Agent}i\t\ %s\t%>s\t%{Content-Type}o\t\ %b\t%I\t%O\t%D\t\ %X\t%k\t%{SSL_PROTOCOL}x\t%{SSL_CIPHER}x\t%{SSL_CIPHER_USEKEYSIZE}x\t%{SSL_TLS_SNI}x\t%{SSL_SESSION_RESUMED}x\t\ From 3835f25f10bf179022b68552609b15f9aceb8de5 Mon Sep 17 00:00:00 2001 From: Navid Sassan Date: Tue, 22 Sep 2026 15:33:44 +0200 Subject: [PATCH 08/12] docs(roles/apache_httpd): describe the csv and tsv access log formats --- roles/apache_httpd/README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/roles/apache_httpd/README.md b/roles/apache_httpd/README.md index 193002259..026799aa3 100644 --- a/roles/apache_httpd/README.md +++ b/roles/apache_httpd/README.md @@ -856,6 +856,18 @@ apache_httpd__mods__host_var: This module is for flexible logging of client requests. Logs are written in a customizable format, and may be written directly to a file, or to an external program. Conditional logging is provided so that individual requests may be included or excluded from the logs based on characteristics of the request. +Besides the common formats, the role provides structured formats for log shippers and SIEMs, each as a semicolon-separated CSV (`csv*`) and a tab-separated TSV (`tsv*`) variant with identical columns: + +* `csvio` / `tsvio`: the fields of `linuxfabrikio` (15 columns). +* `csvextensive` / `tsvextensive`: additionally method, URI, query string, protocol, original status and request duration (21 columns). +* `csvsiem` / `tsvsiem`: additionally unique request ID, error log ID, server port, PID, content type, connection status, keep-alive count and the TLS session details (33 columns). + +All of them start with an ISO 8601 timestamp with microseconds and end with the Cloudflare `CF-Ray` and `CF-Connecting-IP` request headers. The column list is documented in `templates/etc/httpd/conf-available/logio.conf.j2`. Things to consider: + +* Missing values are logged as `-`, for example the TLS fields on plain HTTP or the Cloudflare fields without Cloudflare in front of the server. +* In the CSV formats every field is enclosed in double quotes, and Apache escapes a double quote inside a value as `\"`, not as `""`. Configure the CSV parser with a backslash as escape character. The TSV formats need no quoting, since Apache escapes tabs in values as `\t`. +* The fail2ban filters of the `fail2ban` role do not match these formats. + `apache_httpd__mod_log_config_custom_log` * Global log directive that applies to requests not handled by any vHost. Each vHost defines its own log via `conf_custom_log`. One of: `agent`, `combined`, `combinedio`, `common`, `csvextensive`, `csvio`, `csvsiem`, `debug`, `fail2ban`, `linuxfabrikio`, `matomo`, `referer`, `tsvextensive`, `tsvio`, `tsvsiem`, `vhost_common`. See [CustomLog](https://httpd.apache.org/docs/2.4/mod/mod_log_config.html#customlog). From 4c778981dbba243a3e57ea1cc38a90bb78800822 Mon Sep 17 00:00:00 2001 From: Navid Sassan Date: Tue, 22 Sep 2026 15:34:00 +0200 Subject: [PATCH 09/12] docs(changelog): reword the apache_httpd csv/tsv log format entry --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac7fb63f8..88c452a07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,7 +29,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -* **role:apache_httpd:** Added csv access log formats `csvio csvextensive csvsiem` and their tsv equivalents `tsvio tsvextensive tsvsiem`. +* **role:apache_httpd**: CSV and TSV access log formats for log shippers and SIEMs (`csvio`, `csvextensive`, `csvsiem` and their `tsv*` equivalents), including the Cloudflare `CF-Ray` and `CF-Connecting-IP` headers. * **role:wordpress**: Several WordPress instances can share a host as pseudo hosts in the inventory, under different host names as well as under different paths of one host name, such as `https://example.com/blog`. * **role:fail2ban**: The `wordpress-login` filter and `z10-wordpress-login` jail ban IPs with too many failed WordPress logins, on the host whose Apache logs the visitor's address. * **role:rstudio_server, playbook:rstudio_server**: Add a role and playbook to install RStudio Server Open Source, the browser-based R development environment. Users sign in with their account on the host and have to be a member of a group to be let in at all, the PAM profile covers directory users where the vendor's covers local ones only, and the R sessions can be given a memory and process budget. From e9036da0181be735cc1139569aeb3945e5a7df3e Mon Sep 17 00:00:00 2001 From: Navid Sassan Date: Tue, 22 Sep 2026 15:34:37 +0200 Subject: [PATCH 10/12] style(roles/apache_httpd): tidy up the csv/tsv log format comments --- .../templates/etc/httpd/conf-available/logio.conf.j2 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/roles/apache_httpd/templates/etc/httpd/conf-available/logio.conf.j2 b/roles/apache_httpd/templates/etc/httpd/conf-available/logio.conf.j2 index 9cd682ad2..fccee739d 100644 --- a/roles/apache_httpd/templates/etc/httpd/conf-available/logio.conf.j2 +++ b/roles/apache_httpd/templates/etc/httpd/conf-available/logio.conf.j2 @@ -10,7 +10,7 @@ LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" c LogFormat "%h %{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" linuxfabrikio -# csvio, tsvio: based on linuxfabrikio logformat with additional cloudflare headers +# csvio, tsvio: based on the linuxfabrikio LogFormat, plus the Cloudflare headers # # Field Token Name Group # @@ -52,7 +52,7 @@ LogFormat "\ %{CF-RAY}i\t%{CF-Connecting-IP}i\ " tsvio -# csvextensive, tsvextensive: based on gelf logformat with additional cloudflare headers +# csvextensive, tsvextensive: based on the gelf LogFormat in log_config.conf, plus the Cloudflare headers # # Field Token Name Group # @@ -98,9 +98,9 @@ LogFormat "\ %{CF-RAY}i\t%{CF-Connecting-IP}i\ " tsvextensive -# csvsiem, tsvsiem: most comprehensive logformat useful for siem +# csvsiem, tsvsiem: the most comprehensive LogFormat, meant for a SIEM # -# Field Token Name Group +# Field Token Name Group # # 1 {{ '%{%FT%T}t.%{usec_frac}t%{%z}t' }} timestamp Transaction # 2 %{UNIQUE_ID}e unique_id Transaction From 73c0af1b8974eb584141c856714dcb099f1e4190 Mon Sep 17 00:00:00 2001 From: Navid Sassan Date: Tue, 22 Sep 2026 15:51:50 +0200 Subject: [PATCH 11/12] style(roles/apache_httpd): align the timestamp row in the rendered log format comments --- .../templates/etc/httpd/conf-available/logio.conf.j2 | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/roles/apache_httpd/templates/etc/httpd/conf-available/logio.conf.j2 b/roles/apache_httpd/templates/etc/httpd/conf-available/logio.conf.j2 index fccee739d..4b8d381df 100644 --- a/roles/apache_httpd/templates/etc/httpd/conf-available/logio.conf.j2 +++ b/roles/apache_httpd/templates/etc/httpd/conf-available/logio.conf.j2 @@ -10,11 +10,12 @@ LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" c LogFormat "%h %{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" linuxfabrikio +{# Row 1 of the field tables below looks misaligned here, but lines up in the rendered file. #} # csvio, tsvio: based on the linuxfabrikio LogFormat, plus the Cloudflare headers # # Field Token Name Group # -# 1 {{ '%{%FT%T}t.%{usec_frac}t%{%z}t' }} timestamp Transaction +# 1 {{ '%{%FT%T}t.%{usec_frac}t%{%z}t' }} timestamp Transaction # 2 %v vhost Server # 3 %a client_ip Client # 4 %{c}a peer_ip Client @@ -56,7 +57,7 @@ LogFormat "\ # # Field Token Name Group # -# 1 {{ '%{%FT%T}t.%{usec_frac}t%{%z}t' }} timestamp Transaction +# 1 {{ '%{%FT%T}t.%{usec_frac}t%{%z}t' }} timestamp Transaction # 2 %v vhost Server # 3 %a client_ip Client # 4 %{c}a peer_ip Client @@ -102,7 +103,7 @@ LogFormat "\ # # Field Token Name Group # -# 1 {{ '%{%FT%T}t.%{usec_frac}t%{%z}t' }} timestamp Transaction +# 1 {{ '%{%FT%T}t.%{usec_frac}t%{%z}t' }} timestamp Transaction # 2 %{UNIQUE_ID}e unique_id Transaction # 3 %L log_id Transaction # 4 %v vhost Server From 0eafd199c8eeb70ba750da35daf0609a19e28e68 Mon Sep 17 00:00:00 2001 From: Navid Sassan Date: Tue, 22 Sep 2026 15:53:46 +0200 Subject: [PATCH 12/12] docs(roles/apache_httpd): list the fields csvio adds to linuxfabrikio --- roles/apache_httpd/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/apache_httpd/README.md b/roles/apache_httpd/README.md index 026799aa3..9a157143c 100644 --- a/roles/apache_httpd/README.md +++ b/roles/apache_httpd/README.md @@ -858,7 +858,7 @@ This module is for flexible logging of client requests. Logs are written in a cu Besides the common formats, the role provides structured formats for log shippers and SIEMs, each as a semicolon-separated CSV (`csv*`) and a tab-separated TSV (`tsv*`) variant with identical columns: -* `csvio` / `tsvio`: the fields of `linuxfabrikio` (15 columns). +* `csvio` / `tsvio`: the fields of `linuxfabrikio` (without the identd user), plus server name and peer IP (15 columns). * `csvextensive` / `tsvextensive`: additionally method, URI, query string, protocol, original status and request duration (21 columns). * `csvsiem` / `tsvsiem`: additionally unique request ID, error log ID, server port, PID, content type, connection status, keep-alive count and the TLS session details (33 columns).