diff --git a/CHANGELOG.md b/CHANGELOG.md index 21ed9ff3..88c452a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +* **role:apache_httpd**: CSV and TSV access log formats for log shippers and SIEMs (`csvio`, `csvextensive`, `csvsiem` and their `tsv*` equivalents), including the Cloudflare `CF-Ray` and `CF-Connecting-IP` headers. * **role:wordpress**: Several WordPress instances can share a host as pseudo hosts in the inventory, under different host names as well as under different paths of one host name, such as `https://example.com/blog`. * **role:fail2ban**: The `wordpress-login` filter and `z10-wordpress-login` jail ban IPs with too many failed WordPress logins, on the host whose Apache logs the visitor's address. * **role:rstudio_server, playbook:rstudio_server**: Add a role and playbook to install RStudio Server Open Source, the browser-based R development environment. Users sign in with their account on the host and have to be a member of a group to be let in at all, the PAM profile covers directory users where the vendor's covers local ones only, and the R sessions can be given a memory and process budget. diff --git a/roles/apache_httpd/README.md b/roles/apache_httpd/README.md index 81c0f078..9a157143 100644 --- a/roles/apache_httpd/README.md +++ b/roles/apache_httpd/README.md @@ -611,7 +611,7 @@ The remaining subkeys configure the contents of the vHost and are only honoured `conf_custom_log` -* The log format has to be one of: `agent`, `combined`, `combinedio`, `common`, `debug`, `fail2ban`, `linuxfabrikio`, `matomo`, `referer`, `vhost_common`. Set it to an empty string to disable the access log. See [CustomLog](https://httpd.apache.org/docs/2.4/mod/mod_log_config.html#customlog). +* The log format has to be one of: `agent`, `combined`, `combinedio`, `common`, `csvextensive`, `csvio`, `csvsiem`, `debug`, `fail2ban`, `linuxfabrikio`, `matomo`, `referer`, `tsvextensive`, `tsvio`, `tsvsiem`, `vhost_common`. Set it to an empty string to disable the access log. See [CustomLog](https://httpd.apache.org/docs/2.4/mod/mod_log_config.html#customlog). * Applies to: app, localhost, proxy, wordpress. * Type: String. * Default: `'logs/{{ conf_server_name }}-access.log linuxfabrikio'` @@ -856,9 +856,21 @@ apache_httpd__mods__host_var: This module is for flexible logging of client requests. Logs are written in a customizable format, and may be written directly to a file, or to an external program. Conditional logging is provided so that individual requests may be included or excluded from the logs based on characteristics of the request. +Besides the common formats, the role provides structured formats for log shippers and SIEMs, each as a semicolon-separated CSV (`csv*`) and a tab-separated TSV (`tsv*`) variant with identical columns: + +* `csvio` / `tsvio`: the fields of `linuxfabrikio` (without the identd user), plus server name and peer IP (15 columns). +* `csvextensive` / `tsvextensive`: additionally method, URI, query string, protocol, original status and request duration (21 columns). +* `csvsiem` / `tsvsiem`: additionally unique request ID, error log ID, server port, PID, content type, connection status, keep-alive count and the TLS session details (33 columns). + +All of them start with an ISO 8601 timestamp with microseconds and end with the Cloudflare `CF-Ray` and `CF-Connecting-IP` request headers. The column list is documented in `templates/etc/httpd/conf-available/logio.conf.j2`. Things to consider: + +* Missing values are logged as `-`, for example the TLS fields on plain HTTP or the Cloudflare fields without Cloudflare in front of the server. +* In the CSV formats every field is enclosed in double quotes, and Apache escapes a double quote inside a value as `\"`, not as `""`. Configure the CSV parser with a backslash as escape character. The TSV formats need no quoting, since Apache escapes tabs in values as `\t`. +* The fail2ban filters of the `fail2ban` role do not match these formats. + `apache_httpd__mod_log_config_custom_log` -* Global log directive that applies to requests not handled by any vHost. Each vHost defines its own log via `conf_custom_log`. One of: `agent`, `combined`, `combinedio`, `common`, `debug`, `fail2ban`, `linuxfabrikio`, `matomo`, `referer`, `vhost_common`. See [CustomLog](https://httpd.apache.org/docs/2.4/mod/mod_log_config.html#customlog). +* Global log directive that applies to requests not handled by any vHost. Each vHost defines its own log via `conf_custom_log`. One of: `agent`, `combined`, `combinedio`, `common`, `csvextensive`, `csvio`, `csvsiem`, `debug`, `fail2ban`, `linuxfabrikio`, `matomo`, `referer`, `tsvextensive`, `tsvio`, `tsvsiem`, `vhost_common`. See [CustomLog](https://httpd.apache.org/docs/2.4/mod/mod_log_config.html#customlog). * Type: String. * Default: unset diff --git a/roles/apache_httpd/templates/etc/httpd/conf-available/logio.conf.j2 b/roles/apache_httpd/templates/etc/httpd/conf-available/logio.conf.j2 index c2defdef..4b8d381d 100644 --- a/roles/apache_httpd/templates/etc/httpd/conf-available/logio.conf.j2 +++ b/roles/apache_httpd/templates/etc/httpd/conf-available/logio.conf.j2 @@ -1,5 +1,5 @@ # {{ ansible_managed }} -# 2021110301 +# 2026092201 {% if item['by_role'] | d() %} # Generated by Ansible role: {{ item['by_role'] }} {% endif %} @@ -10,6 +10,155 @@ LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" c LogFormat "%h %{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" linuxfabrikio +{# Row 1 of the field tables below looks misaligned here, but lines up in the rendered file. #} +# csvio, tsvio: based on the linuxfabrikio LogFormat, plus the Cloudflare headers +# +# Field Token Name Group +# +# 1 {{ '%{%FT%T}t.%{usec_frac}t%{%z}t' }} timestamp Transaction +# 2 %v vhost Server +# 3 %a client_ip Client +# 4 %{c}a peer_ip Client +# 5 %{X-Forwarded-For}i xff Client +# 6 %u user Client +# 7 %r request_line Request +# 8 %>s status_final Response +# 9 %b bytes_resp Transfer +# 10 %I bytes_in Transfer +# 11 %O bytes_out Transfer +# 12 %{Referer}i referer Request +# 13 %{User-Agent}i useragent Request +# 14 %{CF-RAY}i cf_ray Cloudflare +# 15 %{CF-Connecting-IP}i cf_connecting_ip Cloudflare + +LogFormat "\ +\"{{ '%{%FT%T}t.%{usec_frac}t%{%z}t' }}\";\ +\"%v\";\ +\"%a\";\"%{c}a\";\"%{X-Forwarded-For}i\";\"%u\";\ +\"%r\";\ +\"%>s\";\ +\"%b\";\"%I\";\"%O\";\ +\"%{Referer}i\";\"%{User-Agent}i\";\ +\"%{CF-RAY}i\";\"%{CF-Connecting-IP}i\"\ +" csvio + +LogFormat "\ +{{ '%{%FT%T}t.%{usec_frac}t%{%z}t' }}\t\ +%v\t\ +%a\t%{c}a\t%{X-Forwarded-For}i\t%u\t\ +%r\t\ +%>s\t\ +%b\t%I\t%O\t\ +%{Referer}i\t%{User-Agent}i\t\ +%{CF-RAY}i\t%{CF-Connecting-IP}i\ +" tsvio + +# csvextensive, tsvextensive: based on the gelf LogFormat in log_config.conf, plus the Cloudflare headers +# +# Field Token Name Group +# +# 1 {{ '%{%FT%T}t.%{usec_frac}t%{%z}t' }} timestamp Transaction +# 2 %v vhost Server +# 3 %a client_ip Client +# 4 %{c}a peer_ip Client +# 5 %{X-Forwarded-For}i xff Client +# 6 %u user Client +# 7 %r request_line Request +# 8 %m method Request +# 9 %U uri Request +# 10 %q query Request +# 11 %H protocol Request +# 12 %{Referer}i referer Request +# 13 %{User-Agent}i useragent Request +# 14 %s status_orig Response +# 15 %>s status_final Response +# 16 %b bytes_resp Transfer +# 17 %I bytes_in Transfer +# 18 %O bytes_out Transfer +# 19 %D duration_us Transfer +# 20 %{CF-RAY}i cf_ray Cloudflare +# 21 %{CF-Connecting-IP}i cf_connecting_ip Cloudflare + +LogFormat "\ +\"{{ '%{%FT%T}t.%{usec_frac}t%{%z}t' }}\";\ +\"%v\";\ +\"%a\";\"%{c}a\";\"%{X-Forwarded-For}i\";\"%u\";\ +\"%r\";\"%m\";\"%U\";\"%q\";\"%H\";\"%{Referer}i\";\"%{User-Agent}i\";\ +\"%s\";\"%>s\";\ +\"%b\";\"%I\";\"%O\";\"%D\";\ +\"%{CF-RAY}i\";\"%{CF-Connecting-IP}i\"\ +" csvextensive + +LogFormat "\ +{{ '%{%FT%T}t.%{usec_frac}t%{%z}t' }}\t\ +%v\t\ +%a\t%{c}a\t%{X-Forwarded-For}i\t%u\t\ +%r\t%m\t%U\t%q\t%H\t%{Referer}i\t%{User-Agent}i\t\ +%s\t%>s\t\ +%b\t%I\t%O\t%D\t\ +%{CF-RAY}i\t%{CF-Connecting-IP}i\ +" tsvextensive + +# csvsiem, tsvsiem: the most comprehensive LogFormat, meant for a SIEM +# +# Field Token Name Group +# +# 1 {{ '%{%FT%T}t.%{usec_frac}t%{%z}t' }} timestamp Transaction +# 2 %{UNIQUE_ID}e unique_id Transaction +# 3 %L log_id Transaction +# 4 %v vhost Server +# 5 %p server_port Server +# 6 %P pid Server +# 7 %a client_ip Client +# 8 %{c}a peer_ip Client +# 9 %{X-Forwarded-For}i xff Client +# 10 %u user Client +# 11 %r request_line Request +# 12 %m method Request +# 13 %U uri Request +# 14 %q query Request +# 15 %H protocol Request +# 16 %{Referer}i referer Request +# 17 %{User-Agent}i useragent Request +# 18 %s status_orig Response +# 19 %>s status_final Response +# 20 %{Content-Type}o content_type Response +# 21 %b bytes_resp Transfer +# 22 %I bytes_in Transfer +# 23 %O bytes_out Transfer +# 24 %D duration_us Transfer +# 25 %X conn_status Connection +# 26 %k keepalive Connection +# 27 %{SSL_PROTOCOL}x tls_protocol Connection +# 28 %{SSL_CIPHER}x tls_cipher Connection +# 29 %{SSL_CIPHER_USEKEYSIZE}x tls_cipher_usekeysize Connection +# 30 %{SSL_TLS_SNI}x tls_sni Connection +# 31 %{SSL_SESSION_RESUMED}x tls_session_resumed Connection +# 32 %{CF-RAY}i cf_ray Cloudflare +# 33 %{CF-Connecting-IP}i cf_connecting_ip Cloudflare + +LogFormat "\ +\"{{ '%{%FT%T}t.%{usec_frac}t%{%z}t' }}\";\"%{UNIQUE_ID}e\";\"%L\";\ +\"%v\";\"%p\";\"%P\";\ +\"%a\";\"%{c}a\";\"%{X-Forwarded-For}i\";\"%u\";\ +\"%r\";\"%m\";\"%U\";\"%q\";\"%H\";\"%{Referer}i\";\"%{User-Agent}i\";\ +\"%s\";\"%>s\";\"%{Content-Type}o\";\ +\"%b\";\"%I\";\"%O\";\"%D\";\ +\"%X\";\"%k\";\"%{SSL_PROTOCOL}x\";\"%{SSL_CIPHER}x\";\"%{SSL_CIPHER_USEKEYSIZE}x\";\"%{SSL_TLS_SNI}x\";\"%{SSL_SESSION_RESUMED}x\";\ +\"%{CF-RAY}i\";\"%{CF-Connecting-IP}i\"\ +" csvsiem + +LogFormat "\ +{{ '%{%FT%T}t.%{usec_frac}t%{%z}t' }}\t%{UNIQUE_ID}e\t%L\t\ +%v\t%p\t%P\t\ +%a\t%{c}a\t%{X-Forwarded-For}i\t%u\t\ +%r\t%m\t%U\t%q\t%H\t%{Referer}i\t%{User-Agent}i\t\ +%s\t%>s\t%{Content-Type}o\t\ +%b\t%I\t%O\t%D\t\ +%X\t%k\t%{SSL_PROTOCOL}x\t%{SSL_CIPHER}x\t%{SSL_CIPHER_USEKEYSIZE}x\t%{SSL_TLS_SNI}x\t%{SSL_SESSION_RESUMED}x\t\ +%{CF-RAY}i\t%{CF-Connecting-IP}i\ +" tsvsiem + LogFormat "\n\ Bytes\n\ Total, incl. request and headers: %S B\n\