From 2bc017e7c34ae300fdf60c1a7e908680c0817b19 Mon Sep 17 00:00:00 2001 From: shimoncohen Date: Sun, 3 Aug 2025 13:41:04 +0300 Subject: [PATCH 1/5] fix: log format doesn't uphold opentelemetry field naming --- helm/config/log_format.conf | 73 +++++++++++++++++++++---------------- helm/config/nginx.conf | 10 +++++ 2 files changed, 51 insertions(+), 32 deletions(-) diff --git a/helm/config/log_format.conf b/helm/config/log_format.conf index 52b66aa..ec81ac2 100644 --- a/helm/config/log_format.conf +++ b/helm/config/log_format.conf @@ -6,42 +6,51 @@ log_format main escape=json '{' '"Timestamp":$nanosec,' '"Attributes":{' - '"mapcolonies.time_local":"$time_local",' + '"mapcolonies.time_local": "$time_local",' {{ if .Values.authorization.enabled }} - '"mapcolonies.http.auth.token.client_name":"$jwt_payload_sub",' + '"mapcolonies.http.auth.token.client_name": "$jwt_payload_sub",' {{ end }} - '"http.request.method":"$request_method",' - '"http.request.header.referer":"$http_referer",' - '"http.request.body.size":"$content_length",' - '"http.response.body.size":"$body_bytes_sent",' - '"http.response.header.x_forwarded_for":"$proxy_add_x_forwarded_for",' - '"http.response.status_code":"$status",' - '"user_agent.original":"$http_user_agent",' - '"network.protocol":"$server_protocol",' - '"mapcolonies.request_time":"$request_time",' - '"mapcolonies.http.upstream_connect_time":"$upstream_connect_time",' - '"mapcolonies.http.upstream_response_time":"$upstream_response_time",' - '"mapcolonies.http.upstream_addr":"$upstream_addr",' - '"mapcolonies.http.upstream_status_code":"$upstream_status",' - '"mapcolonies.http.upstream_cache_status":"$upstream_cache_status",' - '"mapcolonies.server":"$hostname",' - '"server.address":"$host",' - '"server.port":"$server_port",' - '"client.address":"$remote_addr",' - '"client.port":"$remote_port",' - '"url.scheme":"$scheme",' - '"url.path":"$uri",' - '"url.full":"$request_uri"' + '"http.request.method": "$request_method",' + '"http.request.header.host": "$host",' + '"http.request.header.referer": "$http_referer",' + '"http.request.header.content-type": "$content_type",' + '"http.request.header.content-length": "$content_length",' + '"http.request.body.size": $request_length,' + '"http.request.size": $bytes_sent,' + '"http.response.body.size": $body_bytes_sent,' + '"http.response.header.x_forwarded_for": "$proxy_add_x_forwarded_for",' + '"http.response.status_code": "$status",' + '"user_agent.original": "$http_user_agent",' + '"network.protocol.name": "$otel_network_protocol_name",' + '"network.protocol.version": "$otel_network_protocol_version",' + '"mapcolonies.request_time": "$request_time",' + '"mapcolonies.http.upstream_addr": "$upstream_addr",' + '"mapcolonies.http.upstream_status_code": "$upstream_status",' + '"mapcolonies.http.upstream_connect_time": "$upstream_connect_time",' + '"mapcolonies.http.upstream_response_time": "$upstream_response_time",' + '"mapcolonies.http.upstream_response_length": $upstream_response_length,' + '"mapcolonies.http.upstream_bytes_sent": $upstream_bytes_sent,' + '"mapcolonies.http.upstream_bytes_received": $upstream_bytes_received,' + '"mapcolonies.http.upstream_cache_status": "$upstream_cache_status",' + '"host.name": "$hostname",' + '"server.address": "$server_addr",' + '"server.port": "$server_port",' + '"client.address": "$remote_addr",' + '"client.port": "$remote_port",' + '"url.scheme": "$scheme",' + '"url.path": "$uri",' + '"url.full": "$request_uri",' + '"url.query": "$query_string"' '},' '"Resource":{' # Additional important log details should be added here - '"service.name":"{{ .Values.image.repository }}",' - '"service.version":"{{ .Values.image.tag }}"' + '"service.name": "{{ .Values.image.repository }}",' + '"service.version": "{{ .Values.image.tag }}"' '},' - '"TraceId":"$opentelemetry_trace_id",' ## this is a byte sequence (hex-encoded in JSON) - '"SpanId":"$opentelemetry_span_id",' - '"SeverityText":"INFO",' - '"SeverityNumber":9,' - '"InstrumentationScope":"access.log",' - '"Body":"$request"' + '"TraceId": "$opentelemetry_trace_id",' ## this is a byte sequence (hex-encoded in JSON) + '"SpanId": "$opentelemetry_span_id",' + '"SeverityText": "INFO",' + '"SeverityNumber": 9,' + '"InstrumentationScope": "access.log",' + '"Body": "$request"' '}'; diff --git a/helm/config/nginx.conf b/helm/config/nginx.conf index e050b99..28ebe6a 100644 --- a/helm/config/nginx.conf +++ b/helm/config/nginx.conf @@ -18,6 +18,16 @@ http { endpoint {{ .Values.opentelemetry.exporterHost }}:{{ .Values.opentelemetry.exporterPort }}; } + map $server_protocol $otel_network_protocol_name { + ~^(.+)/.*$ $1; + default "http"; + } + + map $server_protocol $otel_network_protocol_version { + ~^.+/(.+)$ $1; + default "1.1"; + } + include /etc/nginx/mime.types; default_type application/octet-stream; From a46320f02e62b0a9b8b3ee62acb0626980ea15b2 Mon Sep 17 00:00:00 2001 From: shimoncohen Date: Sun, 3 Aug 2025 15:31:15 +0300 Subject: [PATCH 2/5] chore: simplify nginx.conf network_protocol regex --- helm/config/nginx.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm/config/nginx.conf b/helm/config/nginx.conf index 28ebe6a..478d43a 100644 --- a/helm/config/nginx.conf +++ b/helm/config/nginx.conf @@ -19,12 +19,12 @@ http { } map $server_protocol $otel_network_protocol_name { - ~^(.+)/.*$ $1; + (.+)/.+$ $1; default "http"; } map $server_protocol $otel_network_protocol_version { - ~^.+/(.+)$ $1; + .+/(.+)$ $1; default "1.1"; } From 0b756e5e875255add8d0f952db8229794101b71c Mon Sep 17 00:00:00 2001 From: shimoncohen Date: Sun, 3 Aug 2025 17:10:47 +0300 Subject: [PATCH 3/5] chore: quote upstream_response_length in log_format.conf --- helm/config/log_format.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/config/log_format.conf b/helm/config/log_format.conf index ec81ac2..2c77051 100644 --- a/helm/config/log_format.conf +++ b/helm/config/log_format.conf @@ -28,7 +28,7 @@ log_format main escape=json '"mapcolonies.http.upstream_status_code": "$upstream_status",' '"mapcolonies.http.upstream_connect_time": "$upstream_connect_time",' '"mapcolonies.http.upstream_response_time": "$upstream_response_time",' - '"mapcolonies.http.upstream_response_length": $upstream_response_length,' + '"mapcolonies.http.upstream_response_length": "$upstream_response_length",' '"mapcolonies.http.upstream_bytes_sent": $upstream_bytes_sent,' '"mapcolonies.http.upstream_bytes_received": $upstream_bytes_received,' '"mapcolonies.http.upstream_cache_status": "$upstream_cache_status",' From 538358b3780d9b5c47d40cedc3819b82cb30e190 Mon Sep 17 00:00:00 2001 From: Shimon Cohen <33935191+shimoncohen@users.noreply.github.com> Date: Mon, 8 Sep 2025 20:00:09 +0300 Subject: [PATCH 4/5] Update helm/config/log_format.conf Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- helm/config/log_format.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/config/log_format.conf b/helm/config/log_format.conf index 2c77051..ee20b0e 100644 --- a/helm/config/log_format.conf +++ b/helm/config/log_format.conf @@ -16,7 +16,7 @@ log_format main escape=json '"http.request.header.content-type": "$content_type",' '"http.request.header.content-length": "$content_length",' '"http.request.body.size": $request_length,' - '"http.request.size": $bytes_sent,' + '"http.request.size": $request_length,' '"http.response.body.size": $body_bytes_sent,' '"http.response.header.x_forwarded_for": "$proxy_add_x_forwarded_for",' '"http.response.status_code": "$status",' From 0aaf0b3221d94cb6f90eb2d6337632fb8fbfc9b5 Mon Sep 17 00:00:00 2001 From: Shimon Cohen <33935191+shimoncohen@users.noreply.github.com> Date: Mon, 8 Sep 2025 20:01:01 +0300 Subject: [PATCH 5/5] Update helm/config/log_format.conf --- helm/config/log_format.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/config/log_format.conf b/helm/config/log_format.conf index ee20b0e..76a9f28 100644 --- a/helm/config/log_format.conf +++ b/helm/config/log_format.conf @@ -15,7 +15,7 @@ log_format main escape=json '"http.request.header.referer": "$http_referer",' '"http.request.header.content-type": "$content_type",' '"http.request.header.content-length": "$content_length",' - '"http.request.body.size": $request_length,' + '"http.request.body.size": $content_length,' '"http.request.size": $request_length,' '"http.response.body.size": $body_bytes_sent,' '"http.response.header.x_forwarded_for": "$proxy_add_x_forwarded_for",'