From 3df212085300142b6c0210f0698d2f8dbd2567c3 Mon Sep 17 00:00:00 2001 From: I504010 Date: Fri, 12 Jun 2026 08:31:32 +0200 Subject: [PATCH 1/2] chore(logs): rename ingress-nginx GROK fields to OTel semantic conventions Signed-off-by: I504010 --- logs/charts/Chart.yaml | 2 +- logs/charts/templates/_containerd-config.tpl | 2 +- logs/plugindefinition.yaml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/logs/charts/Chart.yaml b/logs/charts/Chart.yaml index a2be53368..55ae16ba0 100644 --- a/logs/charts/Chart.yaml +++ b/logs/charts/Chart.yaml @@ -4,7 +4,7 @@ apiVersion: v2 appVersion: 0.152.0 name: logs -version: 0.2.8 +version: 0.2.9 description: OpenTelemetry Operator Helm chart for Kubernetes icon: https://raw.githubusercontent.com/cncf/artwork/a718fa97fffec1b9fd14147682e9e3ac0c8817cb/projects/opentelemetry/icon/color/opentelemetry-icon-color.png type: application diff --git a/logs/charts/templates/_containerd-config.tpl b/logs/charts/templates/_containerd-config.tpl index d9380bcde..6b04fb53d 100644 --- a/logs/charts/templates/_containerd-config.tpl +++ b/logs/charts/templates/_containerd-config.tpl @@ -46,7 +46,7 @@ transform/ingress: conditions: - resource.attributes["app.label.name"] == "ingress-nginx" statements: - - merge_maps(log.attributes, ExtractGrokPatterns(log.body, "%{IP:client.address} %{NOTSPACE:client.ident} %{NOTSPACE:client.auth} \\[%{HTTPDATE:httpdate}\\] \"%{WORD:request.method} %{NOTSPACE:request.path} %{WORD:network.protocol.name}/%{NOTSPACE:network.protocol.version}\" %{NUMBER:response} %{NUMBER:content_length:int} %{QUOTEDSTRING} \"%{GREEDYDATA:user_agent}\" %{NUMBER:request.length:int} %{BASE10NUM:request.time:float}( \\[%{NOTSPACE:service}\\])? ?(\\[\\])? %{IP:server.address}\\:%{NUMBER:server.port} %{NUMBER:upstream_response_length:int} %{BASE10NUM:upstream_response_time:float} %{NOTSPACE:upstream_status} %{NOTSPACE:request.id}", true),"upsert") + - merge_maps(log.attributes, ExtractGrokPatterns(log.body, "%{IP:client.address} %{NOTSPACE:client.ident} %{NOTSPACE:client.auth} \\[%{HTTPDATE:httpdate}\\] \"%{WORD:http.request.method} %{NOTSPACE:url.path} %{WORD:network.protocol.name}/%{NOTSPACE:network.protocol.version}\" %{NUMBER:http.response.status_code} %{NUMBER:http.response.body.size:int} %{QUOTEDSTRING} \"%{GREEDYDATA:user_agent.original}\" %{NUMBER:http.request.body.size:int} %{BASE10NUM:request.time:float}( \\[%{NOTSPACE:service}\\])? ?(\\[\\])? %{IP:server.address}\\:%{NUMBER:server.port} %{NUMBER:upstream_response_length:int} %{BASE10NUM:upstream_response_time:float} %{NOTSPACE:upstream_status} %{NOTSPACE:request.id}", true),"upsert") - set(log.attributes["network.protocol.name"], ConvertCase(log.attributes["network.protocol.name"], "lower")) where log.attributes["network.protocol.name"] != nil - set(log.attributes["config.parsed"], "ingress-nginx") where log.attributes["client.address"] != nil diff --git a/logs/plugindefinition.yaml b/logs/plugindefinition.yaml index 10d78e8f0..2c4e9cc9a 100644 --- a/logs/plugindefinition.yaml +++ b/logs/plugindefinition.yaml @@ -6,14 +6,14 @@ kind: PluginDefinition metadata: name: logs spec: - version: 0.13.8 + version: 0.13.9 displayName: Logs description: Observability framework for instrumenting, generating, collecting, and exporting logs. icon: https://raw.githubusercontent.com/cloudoperators/greenhouse-extensions/main/logs/logo.png helmChart: name: logs repository: oci://ghcr.io/cloudoperators/greenhouse-extensions/charts - version: 0.2.8 + version: 0.2.9 options: - default: true description: Set to true to enable the installation of the OpenTelemetry Operator. From ef591e8151910ff4af26e482bde804fdca17858d Mon Sep 17 00:00:00 2001 From: I504010 Date: Fri, 12 Jun 2026 09:01:22 +0200 Subject: [PATCH 2/2] chore(logs): rename GROK fields to OTel semantic conventions in openstack pipeline Align all parsers in _openstack-config.tpl with OTel semantic conventions, consistent with the containerd pipeline changes: - request.method -> http.request.method - request.path -> url.path - response -> http.response.status_code - content_length -> http.response.body.size - user_agent -> user_agent.original - request.length -> http.request.body.size Affected transforms: ingress, neutron_agent, neutron_errors, openstack_api, non_openstack, elektra, keystone_api, swift_proxy Signed-off-by: I504010 --- logs/charts/templates/_openstack-config.tpl | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/logs/charts/templates/_openstack-config.tpl b/logs/charts/templates/_openstack-config.tpl index 67c8ff323..abf2bee59 100644 --- a/logs/charts/templates/_openstack-config.tpl +++ b/logs/charts/templates/_openstack-config.tpl @@ -24,7 +24,7 @@ transform/ingress: conditions: - resource.attributes["app.label.name"] == "ingress-nginx" statements: - - merge_maps(log.attributes, ExtractGrokPatterns(log.body, "%{IP:client.address} %{NOTSPACE:client.ident} %{NOTSPACE:client.auth} \\[%{HTTPDATE:httpdate}\\] \"%{WORD:request.method} %{NOTSPACE:request.path} %{WORD:network.protocol.name}/%{NOTSPACE:network.protocol.version}\" %{NUMBER:response} %{NUMBER:content_length:int} %{QUOTEDSTRING} \"%{GREEDYDATA:user_agent}\" %{NUMBER:request.length:int} %{BASE10NUM:request.time:float}( \\[%{NOTSPACE:service}\\])? ?(\\[\\])? %{IP:server.address}\\:%{NUMBER:server.port} %{NUMBER:upstream.response.length:int} %{BASE10NUM:upstream.response.time:float} %{NOTSPACE:upstream.status} %{NOTSPACE:request.id}", true),"upsert") + - merge_maps(log.attributes, ExtractGrokPatterns(log.body, "%{IP:client.address} %{NOTSPACE:client.ident} %{NOTSPACE:client.auth} \\[%{HTTPDATE:httpdate}\\] \"%{WORD:http.request.method} %{NOTSPACE:url.path} %{WORD:network.protocol.name}/%{NOTSPACE:network.protocol.version}\" %{NUMBER:http.response.status_code} %{NUMBER:http.response.body.size:int} %{QUOTEDSTRING} \"%{GREEDYDATA:user_agent.original}\" %{NUMBER:http.request.body.size:int} %{BASE10NUM:request.time:float}( \\[%{NOTSPACE:service}\\])? ?(\\[\\])? %{IP:server.address}\\:%{NUMBER:server.port} %{NUMBER:upstream.response.length:int} %{BASE10NUM:upstream.response.time:float} %{NOTSPACE:upstream.status} %{NOTSPACE:request.id}", true),"upsert") - set(log.attributes["network.protocol.name"], ConvertCase(log.attributes["network.protocol.name"], "lower")) where log.attributes["network.protocol.name"] != nil - set(log.attributes["config.parsed"], "ingress-nginx") where log.attributes["client.address"] != nil @@ -35,7 +35,7 @@ transform/neutron_agent: conditions: - resource.attributes["k8s.container.name"] == "neutron-metadata-agent" statements: - - merge_maps(log.attributes, ExtractGrokPatterns(log.body, "(%{TIMESTAMP_ISO8601}|)( )?%{TIMESTAMP_ISO8601}.%{NOTSPACE}? %{NUMBER:pid} %{WORD:log.level} %{NOTSPACE:logger} \\[-\\] %{GREEDYDATA} %{NOTSPACE} %{WORD:request.method} %{NOTSPACE} %{QUOTEDSTRING:request.path} %{NOTSPACE} %{NUMBER:request.status} %{NOTSPACE} %{IPV4:client.address} %{NOTSPACE} %{NOTSPACE:project.id} %{NOTSPACE} %{NOTSPACE:openstack.network.id} %{NOTSPACE} %{NOTSPACE:openstack.router.id} %{NOTSPACE} %{NOTSPACE:openstack.instance.id} %{NOTSPACE} %{BASE10NUM:request.duration} %{NOTSPACE} %{QUOTEDSTRING:user_agent}", true),"upsert") + - merge_maps(log.attributes, ExtractGrokPatterns(log.body, "(%{TIMESTAMP_ISO8601}|)( )?%{TIMESTAMP_ISO8601}.%{NOTSPACE}? %{NUMBER:pid} %{WORD:log.level} %{NOTSPACE:logger} \\[-\\] %{GREEDYDATA} %{NOTSPACE} %{WORD:http.request.method} %{NOTSPACE} %{QUOTEDSTRING:url.path} %{NOTSPACE} %{NUMBER:request.status} %{NOTSPACE} %{IPV4:client.address} %{NOTSPACE} %{NOTSPACE:project.id} %{NOTSPACE} %{NOTSPACE:openstack.network.id} %{NOTSPACE} %{NOTSPACE:openstack.router.id} %{NOTSPACE} %{NOTSPACE:openstack.instance.id} %{NOTSPACE} %{BASE10NUM:request.duration} %{NOTSPACE} %{QUOTEDSTRING:user_agent.original}", true),"upsert") transform/neutron_errors: error_mode: ignore @@ -45,7 +45,7 @@ transform/neutron_errors: - resource.attributes["k8s.container.name"] == "neutron-server" - resource.attributes["k8s.container.name"] == "neutron-asr1k" statements: - - merge_maps(log.attributes, ExtractGrokPatterns(log.body, "%{TIMESTAMP_ISO8601} %{NOTSPACE} %{NOTSPACE:log.level} %{NOTSPACE:logger} (\\[)?(req-)%{NOTSPACE:request.id} ?(g%{NOTSPACE:global.request.id}) ?%{NOTSPACE:user.id} ?%{NOTSPACE:project.id} ?%{NOTSPACE:domain.id} ?%{NOTSPACE:user.domain.id} ?%{NOTSPACE:project.domain.id}\\] %{IPV4:client.address} \"%{WORD:request.method} %{NOTSPACE:request.path} HTTP/%{NOTSPACE}\" %{NOTSPACE} %{NUMBER:response}?( ).*%{NOTSPACE} %{NUMBER:content_length:int} %{NOTSPACE} %{BASE10NUM:request.time:float} %{NOTSPACE} %{NOTSPACE:agent}", true),"upsert") + - merge_maps(log.attributes, ExtractGrokPatterns(log.body, "%{TIMESTAMP_ISO8601} %{NOTSPACE} %{NOTSPACE:log.level} %{NOTSPACE:logger} (\\[)?(req-)%{NOTSPACE:request.id} ?(g%{NOTSPACE:global.request.id}) ?%{NOTSPACE:user.id} ?%{NOTSPACE:project.id} ?%{NOTSPACE:domain.id} ?%{NOTSPACE:user.domain.id} ?%{NOTSPACE:project.domain.id}\\] %{IPV4:client.address} \"%{WORD:http.request.method} %{NOTSPACE:url.path} HTTP/%{NOTSPACE}\" %{NOTSPACE} %{NUMBER:http.response.status_code}?( ).*%{NOTSPACE} %{NUMBER:http.response.body.size:int} %{NOTSPACE} %{BASE10NUM:request.time:float} %{NOTSPACE} %{NOTSPACE:agent}", true),"upsert") - merge_maps(log.attributes, ExtractGrokPatterns(log.body, "%{TIMESTAMP_ISO8601} %{NOTSPACE} %{NOTSPACE:log.level} %{NOTSPACE:logger} (\\[)?(req-)%{NOTSPACE:request.id} ?(g%{NOTSPACE:global.request.id}) ?%{NOTSPACE:user.id} ?%{NOTSPACE:project.id} ?%{NOTSPACE:domain.id} ?%{NOTSPACE:user.domain.id} ?%{NOTSPACE:project.domain.id}\\]", true), "upsert") - merge_maps(log.attributes, ExtractGrokPatterns(log.body, "Encoutered a requeable lock exception executing %{WORD:neutronTask:string} for model %{WORD:neutronModel:string} on device %{IP:neutronIp:string}", true),"upsert") - merge_maps(log.attributes, ExtractGrokPatterns(log.body, "asr1k_exceptions.InconsistentModelException?(:) %{WORD:neutronTask} for model %{WORD:neutronModel} cannot be executed on %{IP:neutronIp}", true),"upsert") @@ -64,7 +64,7 @@ transform/openstack_api: - resource.attributes["k8s.deployment.name"] == "manila-api" - resource.attributes["k8s.deployment.name"] == "ironic-api" statements: - - merge_maps(log.attributes, ExtractGrokPatterns(log.body, "%{TIMESTAMP_ISO8601} %{NOTSPACE} %{NOTSPACE:log.level} %{NOTSPACE:logger} (\\[)?(req-)%{NOTSPACE:request.id} ?(g%{NOTSPACE:global.request.id}) ?%{NOTSPACE:user.id} ?%{NOTSPACE:project.id} ?%{NOTSPACE:domain.id} ?%{NOTSPACE:user.domain.id} ?%{NOTSPACE:project.domain.id}\\] %{IPV4:ip.client}(,%{IPV4:ip.target})? \"%{WORD:request.method} %{NOTSPACE:request.path} HTTP/%{NOTSPACE}\" %{NOTSPACE} %{NUMBER:response}?( ).*%{NOTSPACE} %{NUMBER:content_length:int} %{NOTSPACE} %{BASE10NUM:request.time:float} ?%{NOTSPACE} ?%{NOTSPACE:agent}", true), "upsert") + - merge_maps(log.attributes, ExtractGrokPatterns(log.body, "%{TIMESTAMP_ISO8601} %{NOTSPACE} %{NOTSPACE:log.level} %{NOTSPACE:logger} (\\[)?(req-)%{NOTSPACE:request.id} ?(g%{NOTSPACE:global.request.id}) ?%{NOTSPACE:user.id} ?%{NOTSPACE:project.id} ?%{NOTSPACE:domain.id} ?%{NOTSPACE:user.domain.id} ?%{NOTSPACE:project.domain.id}\\] %{IPV4:ip.client}(,%{IPV4:ip.target})? \"%{WORD:http.request.method} %{NOTSPACE:url.path} HTTP/%{NOTSPACE}\" %{NOTSPACE} %{NUMBER:http.response.status_code}?( ).*%{NOTSPACE} %{NUMBER:http.response.body.size:int} %{NOTSPACE} %{BASE10NUM:request.time:float} ?%{NOTSPACE} ?%{NOTSPACE:agent}", true), "upsert") - merge_maps(log.attributes, ExtractGrokPatterns(log.body, "%{TIMESTAMP_ISO8601} %{NOTSPACE} %{NOTSPACE:log.level} %{NOTSPACE:logger} (\\[)?(req-)%{NOTSPACE:request.id} ?(g%{NOTSPACE:global.request.id}) ?%{NOTSPACE:user.id} ?%{NOTSPACE:project.id} ?%{NOTSPACE:domain.id} ?%{NOTSPACE:user.domain.id} ?%{NOTSPACE:project.domain.id}\\]", true), "upsert") transform/non_openstack: @@ -75,7 +75,7 @@ transform/non_openstack: - resource.attributes["k8s.container.name"] == "sentry" - resource.attributes["k8s.deployment.name"] == "arc-api" statements: - - merge_maps(log.attributes, ExtractGrokPatterns(log.body, "%{IP:ip.target} %{NOTSPACE:ident} %{NOTSPACE:auth} \\[%{HAPROXYDATE:proxy_date}\\] \"%{WORD:request.method} %{NOTSPACE:request.path} %{NOTSPACE:httpversion}\" %{NUMBER:response} %{NUMBER:content_length:int} %{QUOTEDSTRING:url} \"%{GREEDYDATA:user_agent}\"?( )?(%{BASE10NUM:request.time:float})", true, ["HAPROXYDATE=%{MONTHDAY}/%{MONTH}/%{YEAR}:%{HAPROXYTIME}.%{INT}", "HAPROXYTIME=%{HOUR}:%{MINUTE}(?::%{SECOND})"]),"upsert") + - merge_maps(log.attributes, ExtractGrokPatterns(log.body, "%{IP:ip.target} %{NOTSPACE:ident} %{NOTSPACE:auth} \\[%{HAPROXYDATE:proxy_date}\\] \"%{WORD:http.request.method} %{NOTSPACE:url.path} %{NOTSPACE:httpversion}\" %{NUMBER:http.response.status_code} %{NUMBER:http.response.body.size:int} %{QUOTEDSTRING:url} \"%{GREEDYDATA:user_agent.original}\"?( )?(%{BASE10NUM:request.time:float})", true, ["HAPROXYDATE=%{MONTHDAY}/%{MONTH}/%{YEAR}:%{HAPROXYTIME}.%{INT}", "HAPROXYTIME=%{HOUR}:%{MINUTE}(?::%{SECOND})"]),"upsert") transform/network_generic_ssh_exporter: error_mode: ignore @@ -147,8 +147,8 @@ transform/elektra: conditions: - resource.attributes["k8s.deployment.name"] == "elektra" statements: - - merge_maps(log.attributes, ExtractGrokPatterns(log.body, "\\[%{NOTSPACE:request.id}\\] %{WORD} %{WORD:request.method} \"%{NOTSPACE:url} %{WORD} %{IP:ip.client} %{WORD} %{TIMESTAMP_ISO8601}", true), "upsert") - - merge_maps(log.attributes, ExtractGrokPatterns(log.body, "\\[%{NOTSPACE:request.id}\\] %{WORD} %{NUMBER:response}", true), "upsert") + - merge_maps(log.attributes, ExtractGrokPatterns(log.body, "\\[%{NOTSPACE:request.id}\\] %{WORD} %{WORD:http.request.method} \"%{NOTSPACE:url} %{WORD} %{IP:ip.client} %{WORD} %{TIMESTAMP_ISO8601}", true), "upsert") + - merge_maps(log.attributes, ExtractGrokPatterns(log.body, "\\[%{NOTSPACE:request.id}\\] %{WORD} %{NUMBER:http.response.status_code}", true), "upsert") - merge_maps(log.attributes, ExtractGrokPatterns(log.body, "\\[%{NOTSPACE:request.id}\\]", true), "upsert") transform/keystone_api: @@ -158,7 +158,7 @@ transform/keystone_api: conditions: - resource.attributes["k8s.container.name"] == "keystone-api" statements: - - merge_maps(log.attributes, ExtractGrokPatterns(log.body, "%{DATE_EU} %{TIME} %{NUMBER} %{NOTSPACE:log.level} %{NOTSPACE:component} \\[%{NOTSPACE:request.id} %{NOTSPACE:global.request.id} usr %{NOTSPACE:user.id} prj %{NOTSPACE:project.id} dom %{NOTSPACE:domain.id} usr-dom %{NOTSPACE:user.domain.id} prj-dom %{NOTSPACE:project.domain.id}\\] %{GREEDYDATA}'%{WORD:request.method} %{URIPATH:uri}' %{WORD:action}", true), "upsert") + - merge_maps(log.attributes, ExtractGrokPatterns(log.body, "%{DATE_EU} %{TIME} %{NUMBER} %{NOTSPACE:log.level} %{NOTSPACE:component} \\[%{NOTSPACE:request.id} %{NOTSPACE:global.request.id} usr %{NOTSPACE:user.id} prj %{NOTSPACE:project.id} dom %{NOTSPACE:domain.id} usr-dom %{NOTSPACE:user.domain.id} prj-dom %{NOTSPACE:project.domain.id}\\] %{GREEDYDATA}'%{WORD:http.request.method} %{URIPATH:uri}' %{WORD:action}", true), "upsert") filter/hermes_logstash: error_mode: ignore @@ -174,7 +174,7 @@ transform/swift_proxy: - resource.attributes["k8s.daemonset.name"] == "swift-proxy-cluster-3" - resource.attributes["app.label.component"] == "swift-servers" statements: - - merge_maps(log.attributes, ExtractGrokPatterns(log.body, "%{SYSLOGTIMESTAMP:date} %{HOSTNAME:host} %{WORD}.%{LOGLEVEL} %{SYSLOGPROG}%{NOTSPACE} %{HOSTNAME:client.address} %{HOSTNAME:server.ip} %{NOTSPACE:datetime} %{WORD:request.method} %{NOTSPACE:request.path}?( )?(%{NOTSPACE:request.param}) ?(%{NOTSPACE:protocol})?( )%{NUMBER:response} %{NOTSPACE} %{NOTSPACE:user_agent} %{NOTSPACE:auth_token} %{NOTSPACE:bytes_recvd} %{NOTSPACE:bytes_sent} %{NOTSPACE:client.etag} %{NOTSPACE:transaction_id} %{NOTSPACE:headers} %{BASE10NUM:request.time:float} %{NOTSPACE:source} %{NOTSPACE:log_info} %{BASE10NUM:request.start.time} %{BASE10NUM:request.end.time} %{NOTSPACE:policy_index}", true), "upsert") + - merge_maps(log.attributes, ExtractGrokPatterns(log.body, "%{SYSLOGTIMESTAMP:date} %{HOSTNAME:host} %{WORD}.%{LOGLEVEL} %{SYSLOGPROG}%{NOTSPACE} %{HOSTNAME:client.address} %{HOSTNAME:server.ip} %{NOTSPACE:datetime} %{WORD:http.request.method} %{NOTSPACE:url.path}?( )?(%{NOTSPACE:request.param}) ?(%{NOTSPACE:protocol})?( )%{NUMBER:http.response.status_code} %{NOTSPACE} %{NOTSPACE:user_agent.original} %{NOTSPACE:auth_token} %{NOTSPACE:bytes_recvd} %{NOTSPACE:bytes_sent} %{NOTSPACE:client.etag} %{NOTSPACE:transaction_id} %{NOTSPACE:headers} %{BASE10NUM:request.time:float} %{NOTSPACE:source} %{NOTSPACE:log_info} %{BASE10NUM:request.start.time} %{BASE10NUM:request.end.time} %{NOTSPACE:policy_index}", true), "upsert") - set(log.attributes["bytes_recvd"], 0) where log.attributes["bytes_recvd"] == "-" - set(log.attributes["bytes_sent"], 0) where log.attributes["bytes_sent"] == "-"