diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 20b31270d..d684fbd1a 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -27,6 +27,13 @@ updates: open-pull-requests-limit: 0 + - package-ecosystem: "bundler" + directory: "/docker-image/v1.19/arm64/debian-datadog" + schedule: + interval: "daily" + open-pull-requests-limit: 0 + + - package-ecosystem: "bundler" directory: "/docker-image/v1.19/arm64/debian-elasticsearch7" schedule: @@ -146,6 +153,13 @@ updates: open-pull-requests-limit: 0 + - package-ecosystem: "bundler" + directory: "/docker-image/v1.19/debian-datadog" + schedule: + interval: "daily" + open-pull-requests-limit: 0 + + - package-ecosystem: "bundler" directory: "/docker-image/v1.19/debian-elasticsearch7" schedule: diff --git a/.github/workflows/check-build.yml b/.github/workflows/check-build.yml index 490b6944c..a65330336 100644 --- a/.github/workflows/check-build.yml +++ b/.github/workflows/check-build.yml @@ -13,6 +13,7 @@ jobs: [ "debian-azureblob", "debian-cloudwatch", + "debian-datadog", "debian-elasticsearch7", "debian-elasticsearch8", "debian-forward", diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 54a2acad8..83acdc068 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -15,6 +15,7 @@ jobs: [ "debian-azureblob", "debian-cloudwatch", + "debian-datadog", "debian-elasticsearch7", "debian-elasticsearch8", "debian-forward", diff --git a/Makefile b/Makefile index 889f62255..125b4efc8 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,8 @@ X86_IMAGES := \ v1.19/debian-logzio:v1.19.0-debian-logzio-amd64-1.0,v1.19-debian-logzio-amd64-1 \ v1.19/debian-kafka:v1.19.0-debian-kafka-amd64-1.0,v1.19-debian-kafka-amd64-1 \ v1.19/debian-kafka2:v1.19.0-debian-kafka2-amd64-1.0,v1.19-debian-kafka2-amd64-1 \ - v1.19/debian-kinesis:v1.19.0-debian-kinesis-amd64-1.0,v1.19-debian-kinesis-amd64-1 + v1.19/debian-kinesis:v1.19.0-debian-kinesis-amd64-1.0,v1.19-debian-kinesis-amd64-1 \ + v1.19/debian-datadog:v1.19.0-debian-datadog-amd64-1.0,v1.19-debian-datadog-amd64-1 # :,,,... @@ -45,7 +46,8 @@ ARM64_IMAGES := \ v1.19/arm64/debian-logzio:v1.19.0-debian-logzio-arm64-1.0,v1.19-debian-logzio-arm64-1 \ v1.19/arm64/debian-kafka:v1.19.0-debian-kafka-arm64-1.0,v1.19-debian-kafka-arm64-1 \ v1.19/arm64/debian-kafka2:v1.19.0-debian-kafka2-arm64-1.0,v1.19-debian-kafka2-arm64-1 \ - v1.19/arm64/debian-kinesis:v1.19.0-debian-kinesis-arm64-1.0,v1.19-debian-kinesis-arm64-1 + v1.19/arm64/debian-kinesis:v1.19.0-debian-kinesis-arm64-1.0,v1.19-debian-kinesis-arm64-1 \ + v1.19/arm64/debian-datadog:v1.19.0-debian-datadog-arm64-1.0,v1.19-debian-datadog-arm64-1 ALL_IMAGES := $(X86_IMAGES) $(ARM64_IMAGES) diff --git a/README.md b/README.md index 22a28cd8e..4e322ada9 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,9 @@ See also dockerhub tags page: https://hub.docker.com/r/fluent/fluentd-kubernetes - `Kinesis` - `docker pull fluent/fluentd-kubernetes-daemonset:v1.19.0-debian-kinesis-1.0` - `docker pull fluent/fluentd-kubernetes-daemonset:v1.19-debian-kinesis-1` +- `Datadog` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.19.0-debian-datadog-1.0` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.19-debian-datadog-1` ##### x86_64 images - `Azureblob` [Dockerfile](docker-image/v1.19/debian-azureblob/Dockerfile) @@ -133,6 +136,9 @@ See also dockerhub tags page: https://hub.docker.com/r/fluent/fluentd-kubernetes - `Kinesis` [Dockerfile](docker-image/v1.19/debian-kinesis/Dockerfile) - `docker pull fluent/fluentd-kubernetes-daemonset:v1.19.0-debian-kinesis-amd64-1.0` - `docker pull fluent/fluentd-kubernetes-daemonset:v1.19-debian-kinesis-amd64-1` +- `Datadog` [Dockerfile](docker-image/v1.19/debian-datadog/Dockerfile) + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.19.0-debian-datadog-amd64-1.0` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.19-debian-datadog-amd64-1` ##### arm64 images - `Azureblob` [Dockerfile](docker-image/v1.19/arm64/debian-azureblob/Dockerfile) @@ -187,6 +193,9 @@ See also dockerhub tags page: https://hub.docker.com/r/fluent/fluentd-kubernetes - `Kinesis` [Dockerfile](docker-image/v1.19/arm64/debian-kinesis/Dockerfile) - `docker pull fluent/fluentd-kubernetes-daemonset:v1.19.0-debian-kinesis-arm64-1.0` - `docker pull fluent/fluentd-kubernetes-daemonset:v1.19-debian-kinesis-arm64-1` +- `Datadog` [Dockerfile](docker-image/v1.19/arm64/debian-datadog/Dockerfile) + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.19.0-debian-datadog-arm64-1.0` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.19-debian-datadog-arm64-1` You can also use `v1-debian-PLUGIN` tag to refer latest v1 image, e.g. `v1-debian-elasticsearch`. On production, strict tag is better to avoid unexpected update. @@ -373,6 +382,7 @@ Some images are contributed by users. If you have a problem/question for followi - Amazon Kinesis : @shiftky - logz.io : @SaMnCo / @jamielennox - splunkhec: @FutureSharks +- Datadog : @nbifrye Currently, we don't accept new destination request without contribution. See https://github.com/fluent/fluentd-kubernetes-daemonset/issues/293 diff --git a/docker-image/v1.19/arm64/debian-datadog/.dockerignore b/docker-image/v1.19/arm64/debian-datadog/.dockerignore new file mode 100644 index 000000000..921ed3fe9 --- /dev/null +++ b/docker-image/v1.19/arm64/debian-datadog/.dockerignore @@ -0,0 +1 @@ +**/*.gitkeep \ No newline at end of file diff --git a/docker-image/v1.19/arm64/debian-datadog/Dockerfile b/docker-image/v1.19/arm64/debian-datadog/Dockerfile new file mode 100644 index 000000000..0eea1b560 --- /dev/null +++ b/docker-image/v1.19/arm64/debian-datadog/Dockerfile @@ -0,0 +1,56 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Dockerfile.erb + +# For multiarch build on Docker hub automated build. +FROM fluent/fluentd:v1.19.0-debian-arm64 + +LABEL maintainer="Eduardo Silva " +USER root +WORKDIR /home/fluent +ENV PATH /fluentd/vendor/bundle/ruby/3.4.0/bin:$PATH +ENV GEM_PATH /fluentd/vendor/bundle/ruby/3.4.0 +ENV GEM_HOME /fluentd/vendor/bundle/ruby/3.4.0 +# skip runtime bundler installation +ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1 + +COPY Gemfile* /fluentd/ +RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev" \ + runtimeDeps="" \ + && apt-get update \ + && apt-get upgrade -y \ + && apt-get install \ + -y --no-install-recommends \ + $buildDeps $runtimeDeps net-tools \ + && gem install bundler --version 2.4.17 \ + && bundle config silence_root_warning true \ + && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \ + && SUDO_FORCE_REMOVE=yes \ + apt-get purge -y --auto-remove \ + -o APT::AutoRemove::RecommendsImportant=false \ + $buildDeps \ + '*-dev' \ + && rm -rf /var/lib/apt/lists/* \ + && gem sources --clear-all \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem +# Copy configuration files +COPY ./conf/fluent.conf /fluentd/etc/ +COPY ./conf/systemd.conf /fluentd/etc/ +COPY ./conf/kubernetes.conf /fluentd/etc/ +COPY ./conf/prometheus.conf /fluentd/etc/ +COPY ./conf/tail_container_parse.conf /fluentd/etc/ +COPY ./conf/kubernetes/*.conf /fluentd/etc/kubernetes/ +RUN touch /fluentd/etc/disable.conf + +# Copy plugins +COPY plugins /fluentd/plugins/ +COPY entrypoint.sh /fluentd/entrypoint.sh + +# Environment variables +ENV FLUENTD_OPT="" +ENV FLUENTD_CONF="fluent.conf" +# Override LD_PRELOAD in fluentd docker image +# Set "/usr/lib/libjemalloc.so.2" if you want to enable jemalloc explicitly +ENV LD_PRELOAD="" + +# Overwrite ENTRYPOINT to run fluentd as root for /var/log / /var/lib +ENTRYPOINT ["tini", "--", "/fluentd/entrypoint.sh"] diff --git a/docker-image/v1.19/arm64/debian-datadog/Gemfile b/docker-image/v1.19/arm64/debian-datadog/Gemfile new file mode 100644 index 000000000..96e460a6d --- /dev/null +++ b/docker-image/v1.19/arm64/debian-datadog/Gemfile @@ -0,0 +1,21 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb + +source "https://rubygems.org" + +gem "fluentd", "1.19.0" +gem "oj", "~> 3.16.11" +gem "rexml", "~> 3.4.1" +gem "fluent-plugin-multi-format-parser", "~> 1.0.0" +gem "fluent-plugin-concat", "~> 2.6.0" +gem "fluent-plugin-grok-parser", "~> 2.6.2" +gem "fluent-plugin-prometheus", "~> 2.2.1" +gem 'fluent-plugin-json-in-json-2', ">= 1.0.2" +gem "fluent-plugin-record-modifier", "~> 2.2.0" +gem "fluent-plugin-detect-exceptions", "~> 0.0.16" +gem "fluent-plugin-rewrite-tag-filter", "~> 2.4.0" +gem "fluent-plugin-parser-cri", "~> 0.1.1" +gem "fluent-plugin-datadog", "~> 0.14.4" +gem "fluent-plugin-kubernetes_metadata_filter", "~> 3.8.0" +gem "ffi" +gem "fluent-plugin-systemd", "~> 1.1.1" diff --git a/docker-image/v1.19/arm64/debian-datadog/Gemfile.lock b/docker-image/v1.19/arm64/debian-datadog/Gemfile.lock new file mode 100644 index 000000000..a69daaf0e --- /dev/null +++ b/docker-image/v1.19/arm64/debian-datadog/Gemfile.lock @@ -0,0 +1,217 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) + async (2.24.0) + console (~> 1.29) + fiber-annotation + io-event (~> 1.9) + metrics (~> 0.12) + traces (~> 0.15) + async-http (0.91.0) + async (>= 2.10.2) + async-pool (~> 0.11) + io-endpoint (~> 0.14) + io-stream (~> 0.6) + metrics (~> 0.12) + protocol-http (~> 0.49) + protocol-http1 (~> 0.30) + protocol-http2 (~> 0.22) + traces (~> 0.10) + async-pool (0.11.0) + async (>= 2.0) + base64 (0.3.0) + bigdecimal (3.2.3) + concurrent-ruby (1.3.5) + connection_pool (2.5.4) + console (1.34.0) + fiber-annotation + fiber-local (~> 1.1) + json + cool.io (1.9.0) + csv (3.3.5) + domain_name (0.6.20240107) + drb (2.2.3) + ffi (1.17.2) + ffi (1.17.2-aarch64-linux-gnu) + ffi (1.17.2-aarch64-linux-musl) + ffi (1.17.2-arm-linux-gnu) + ffi (1.17.2-arm-linux-musl) + ffi (1.17.2-arm64-darwin) + ffi (1.17.2-x86-linux-gnu) + ffi (1.17.2-x86-linux-musl) + ffi (1.17.2-x86_64-darwin) + ffi (1.17.2-x86_64-linux-gnu) + ffi (1.17.2-x86_64-linux-musl) + ffi-compiler (1.3.2) + ffi (>= 1.15.5) + rake + fiber-annotation (0.2.0) + fiber-local (1.1.0) + fiber-storage + fiber-storage (1.0.1) + fluent-config-regexp-type (1.0.0) + fluentd (> 1.0.0, < 2) + fluent-plugin-concat (2.6.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-datadog (0.14.4) + fluentd (>= 1, < 2) + net-http-persistent (~> 4.0.1) + fluent-plugin-detect-exceptions (0.0.16) + fluentd (>= 1.14.2) + fluent-plugin-grok-parser (2.6.2) + fluentd (>= 0.14.6, < 2) + fluent-plugin-json-in-json-2 (1.0.2) + fluentd (>= 0.14.0, < 2) + yajl-ruby (~> 1.0) + fluent-plugin-kubernetes_metadata_filter (3.8.0) + fluentd (>= 0.14.0, < 1.20) + kubeclient (>= 4.0.0, < 5.0.0) + sin_lru_redux + fluent-plugin-multi-format-parser (1.0.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-parser-cri (0.1.1) + fluentd (>= 1) + fluent-plugin-prometheus (2.2.1) + fluentd (>= 1.9.1, < 2) + prometheus-client (>= 2.1.0) + fluent-plugin-record-modifier (2.2.1) + fluentd (>= 1.1, < 2) + fluent-plugin-rewrite-tag-filter (2.4.0) + fluent-config-regexp-type + fluentd (>= 0.14.2, < 2) + fluent-plugin-systemd (1.1.1) + fluentd (>= 0.14.11, < 2) + systemd-journal (~> 2.1.0) + fluentd (1.19.0) + async-http (~> 0.86) + base64 (~> 0.2) + bundler + cool.io (>= 1.4.5, < 2.0.0) + csv (~> 3.2) + drb (~> 2.2) + http_parser.rb (>= 0.5.1, < 0.9.0) + io-event (< 1.11.0) + io-stream (< 0.8.0) + logger (~> 1.6) + msgpack (>= 1.3.1, < 2.0.0) + serverengine (>= 2.3.2, < 3.0.0) + sigdump (~> 0.2.5) + strptime (>= 0.2.4, < 1.0.0) + tzinfo (>= 1.0, < 3.0) + tzinfo-data (~> 1.0) + uri (~> 1.0) + webrick (~> 1.4) + yajl-ruby (~> 1.0) + zstd-ruby (~> 1.5) + http (5.3.1) + addressable (~> 2.8) + http-cookie (~> 1.0) + http-form_data (~> 2.2) + llhttp-ffi (~> 0.5.0) + http-accept (1.7.0) + http-cookie (1.0.8) + domain_name (~> 0.5) + http-form_data (2.3.0) + http_parser.rb (0.8.0) + io-endpoint (0.15.2) + io-event (1.10.2) + io-stream (0.7.0) + json (2.13.2) + jsonpath (1.1.5) + multi_json + kubeclient (4.12.0) + http (>= 3.0, < 6.0) + jsonpath (~> 1.0) + recursive-open-struct (~> 1.1, >= 1.1.1) + rest-client (~> 2.0) + llhttp-ffi (0.5.1) + ffi-compiler (~> 1.0) + rake (~> 13.0) + logger (1.7.0) + metrics (0.14.0) + mime-types (3.7.0) + logger + mime-types-data (~> 3.2025, >= 3.2025.0507) + mime-types-data (3.2025.0902) + msgpack (1.8.0) + multi_json (1.17.0) + net-http-persistent (4.0.6) + connection_pool (~> 2.2, >= 2.2.4) + netrc (0.11.0) + oj (3.16.11) + bigdecimal (>= 3.0) + ostruct (>= 0.2) + ostruct (0.6.3) + prometheus-client (4.2.5) + base64 + protocol-hpack (1.5.1) + protocol-http (0.53.0) + protocol-http1 (0.35.1) + protocol-http (~> 0.22) + protocol-http2 (0.22.1) + protocol-hpack (~> 1.4) + protocol-http (~> 0.47) + public_suffix (6.0.2) + rake (13.3.0) + recursive-open-struct (1.3.1) + ostruct + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + rexml (3.4.3) + serverengine (2.4.0) + base64 (~> 0.1) + logger (~> 1.4) + sigdump (~> 0.2.2) + sigdump (0.2.5) + sin_lru_redux (2.5.2) + strptime (0.2.5) + systemd-journal (2.1.1) + ffi (~> 1.9) + traces (0.18.1) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + tzinfo-data (1.2025.2) + tzinfo (>= 1.0.0) + uri (1.0.3) + webrick (1.9.1) + yajl-ruby (1.4.3) + zstd-ruby (1.5.7.0) + +PLATFORMS + aarch64-linux-gnu + aarch64-linux-musl + arm-linux-gnu + arm-linux-musl + arm64-darwin + ruby + x86-linux-gnu + x86-linux-musl + x86_64-darwin + x86_64-linux-gnu + x86_64-linux-musl + +DEPENDENCIES + ffi + fluent-plugin-concat (~> 2.6.0) + fluent-plugin-datadog (~> 0.14.4) + fluent-plugin-detect-exceptions (~> 0.0.16) + fluent-plugin-grok-parser (~> 2.6.2) + fluent-plugin-json-in-json-2 (>= 1.0.2) + fluent-plugin-kubernetes_metadata_filter (~> 3.8.0) + fluent-plugin-multi-format-parser (~> 1.0.0) + fluent-plugin-parser-cri (~> 0.1.1) + fluent-plugin-prometheus (~> 2.2.1) + fluent-plugin-record-modifier (~> 2.2.0) + fluent-plugin-rewrite-tag-filter (~> 2.4.0) + fluent-plugin-systemd (~> 1.1.1) + fluentd (= 1.19.0) + oj (~> 3.16.11) + rexml (~> 3.4.1) + +BUNDLED WITH + 2.6.9 diff --git a/docker-image/v1.19/arm64/debian-datadog/conf/fluent.conf b/docker-image/v1.19/arm64/debian-datadog/conf/fluent.conf new file mode 100644 index 000000000..1c04a3dc6 --- /dev/null +++ b/docker-image/v1.19/arm64/debian-datadog/conf/fluent.conf @@ -0,0 +1,15 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/fluent.conf.erb + +@include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf" +@include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf" +@include kubernetes.conf +@include conf.d/*.conf + + + @type datadog + @id out_datadog + api_key "#{ENV['DD_API_KEY']}" + + diff --git a/docker-image/v1.19/arm64/debian-datadog/conf/kubernetes.conf b/docker-image/v1.19/arm64/debian-datadog/conf/kubernetes.conf new file mode 100644 index 000000000..bbdd453d6 --- /dev/null +++ b/docker-image/v1.19/arm64/debian-datadog/conf/kubernetes.conf @@ -0,0 +1,39 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/kubernetes.conf.erb + + + +@include kubernetes/cluster-autoscaler.conf +@include kubernetes/containers.conf +@include kubernetes/docker.conf +@include kubernetes/etcd.conf +@include kubernetes/glbc.conf +@include kubernetes/kube-apiserver-audit.conf +@include kubernetes/kube-apiserver.conf +@include kubernetes/kube-controller-manager.conf +@include kubernetes/kube-proxy.conf +@include kubernetes/kube-scheduler.conf +@include kubernetes/kubelet.conf +@include kubernetes/rescheduler.conf +@include kubernetes/salt.conf +@include kubernetes/startupscript.conf + + + + @type kubernetes_metadata + @id filter_kube_metadata + kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || (host = ENV.fetch('KUBERNETES_SERVICE_HOST'); port = ENV.fetch('KUBERNETES_SERVICE_PORT'); host = (IPAddr.new(host).ipv6? ? '[' + host + ']' : host rescue host); 'https://' + host + ':' + port + '/api')}" + verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}" + ca_file "#{ENV['KUBERNETES_CA_FILE']}" + skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}" + skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}" + skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}" + skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}" + watch "#{ENV['FLUENT_KUBERNETES_WATCH'] || 'true'}" + + diff --git a/docker-image/v1.19/arm64/debian-datadog/conf/kubernetes/cluster-autoscaler.conf b/docker-image/v1.19/arm64/debian-datadog/conf/kubernetes/cluster-autoscaler.conf new file mode 100644 index 000000000..5e045b017 --- /dev/null +++ b/docker-image/v1.19/arm64/debian-datadog/conf/kubernetes/cluster-autoscaler.conf @@ -0,0 +1,11 @@ + + @type tail + @id in_tail_cluster_autoscaler + multiline_flush_interval 5s + path /var/log/cluster-autoscaler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-cluster-autoscaler.log.pos')}" + tag cluster-autoscaler + + @type kubernetes + + diff --git a/docker-image/v1.19/arm64/debian-datadog/conf/kubernetes/containers.conf b/docker-image/v1.19/arm64/debian-datadog/conf/kubernetes/containers.conf new file mode 100644 index 000000000..1ea4d1ea4 --- /dev/null +++ b/docker-image/v1.19/arm64/debian-datadog/conf/kubernetes/containers.conf @@ -0,0 +1,10 @@ + + @type tail + @id in_tail_container_logs + path "#{ENV['FLUENT_CONTAINER_TAIL_PATH'] || '/var/log/containers/*.log'}" + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-containers.log.pos')}" + tag "#{ENV['FLUENT_CONTAINER_TAIL_TAG'] || 'kubernetes.*'}" + exclude_path "#{ENV['FLUENT_CONTAINER_TAIL_EXCLUDE_PATH'] || use_default}" + read_from_head true + @include ../tail_container_parse.conf + diff --git a/docker-image/v1.19/arm64/debian-datadog/conf/kubernetes/docker.conf b/docker-image/v1.19/arm64/debian-datadog/conf/kubernetes/docker.conf new file mode 100644 index 000000000..f79e010d4 --- /dev/null +++ b/docker-image/v1.19/arm64/debian-datadog/conf/kubernetes/docker.conf @@ -0,0 +1,11 @@ + + @type tail + @id in_tail_docker + path /var/log/docker.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-docker.log.pos')}" + tag docker + + @type regexp + expression /^time="(? + diff --git a/docker-image/v1.19/arm64/debian-datadog/conf/kubernetes/etcd.conf b/docker-image/v1.19/arm64/debian-datadog/conf/kubernetes/etcd.conf new file mode 100644 index 000000000..be565daef --- /dev/null +++ b/docker-image/v1.19/arm64/debian-datadog/conf/kubernetes/etcd.conf @@ -0,0 +1,10 @@ + + @type tail + @id in_tail_etcd + path /var/log/etcd.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-etcd.log.pos')}" + tag etcd + + @type none + + diff --git a/docker-image/v1.19/arm64/debian-datadog/conf/kubernetes/glbc.conf b/docker-image/v1.19/arm64/debian-datadog/conf/kubernetes/glbc.conf new file mode 100644 index 000000000..4010f4f1d --- /dev/null +++ b/docker-image/v1.19/arm64/debian-datadog/conf/kubernetes/glbc.conf @@ -0,0 +1,11 @@ + + @type tail + @id in_tail_glbc + multiline_flush_interval 5s + path /var/log/glbc.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-glbc.log.pos')}" + tag glbc + + @type kubernetes + + diff --git a/docker-image/v1.19/arm64/debian-datadog/conf/kubernetes/kube-apiserver-audit.conf b/docker-image/v1.19/arm64/debian-datadog/conf/kubernetes/kube-apiserver-audit.conf new file mode 100644 index 000000000..b48743bd7 --- /dev/null +++ b/docker-image/v1.19/arm64/debian-datadog/conf/kubernetes/kube-apiserver-audit.conf @@ -0,0 +1,23 @@ +# Example: +# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="" asgroups="" namespace="default" uri="/api/v1/namespaces/default/pods" +# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200" + + @type tail + @id in_tail_kube_apiserver_audit + multiline_flush_interval 5s + path /var/log/kubernetes/kube-apiserver-audit.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'kube-apiserver-audit.log.pos')}" + tag kube-apiserver-audit + + @type multiline + format_firstline /^\S+\s+AUDIT:/ + # Fields must be explicitly captured by name to be parsed into the record. + # Fields may not always be present, and order may change, so this just looks + # for a list of key="\"quoted\" value" pairs separated by spaces. + # Unknown fields are ignored. + # Note: We can't separate query/response lines as format1/format2 because + # they don't always come one after the other for a given query. + format1 /^(? + diff --git a/docker-image/v1.19/arm64/debian-datadog/conf/kubernetes/kube-apiserver.conf b/docker-image/v1.19/arm64/debian-datadog/conf/kubernetes/kube-apiserver.conf new file mode 100644 index 000000000..4e4bfad9e --- /dev/null +++ b/docker-image/v1.19/arm64/debian-datadog/conf/kubernetes/kube-apiserver.conf @@ -0,0 +1,11 @@ + + @type tail + @id in_tail_kube_apiserver + multiline_flush_interval 5s + path /var/log/kube-apiserver.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-apiserver.log.pos')}" + tag kube-apiserver + + @type kubernetes + + diff --git a/docker-image/v1.19/arm64/debian-datadog/conf/kubernetes/kube-controller-manager.conf b/docker-image/v1.19/arm64/debian-datadog/conf/kubernetes/kube-controller-manager.conf new file mode 100644 index 000000000..c35f7edd7 --- /dev/null +++ b/docker-image/v1.19/arm64/debian-datadog/conf/kubernetes/kube-controller-manager.conf @@ -0,0 +1,11 @@ + + @type tail + @id in_tail_kube_controller_manager + multiline_flush_interval 5s + path /var/log/kube-controller-manager.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-controller-manager.log.pos')}" + tag kube-controller-manager + + @type kubernetes + + diff --git a/docker-image/v1.19/arm64/debian-datadog/conf/kubernetes/kube-proxy.conf b/docker-image/v1.19/arm64/debian-datadog/conf/kubernetes/kube-proxy.conf new file mode 100644 index 000000000..2ae19a287 --- /dev/null +++ b/docker-image/v1.19/arm64/debian-datadog/conf/kubernetes/kube-proxy.conf @@ -0,0 +1,11 @@ + + @type tail + @id in_tail_kube_proxy + multiline_flush_interval 5s + path /var/log/kube-proxy.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-proxy.log.pos')}" + tag kube-proxy + + @type kubernetes + + diff --git a/docker-image/v1.19/arm64/debian-datadog/conf/kubernetes/kube-scheduler.conf b/docker-image/v1.19/arm64/debian-datadog/conf/kubernetes/kube-scheduler.conf new file mode 100644 index 000000000..53ccdda55 --- /dev/null +++ b/docker-image/v1.19/arm64/debian-datadog/conf/kubernetes/kube-scheduler.conf @@ -0,0 +1,11 @@ + + @type tail + @id in_tail_kube_scheduler + multiline_flush_interval 5s + path /var/log/kube-scheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-scheduler.log.pos')}" + tag kube-scheduler + + @type kubernetes + + diff --git a/docker-image/v1.19/arm64/debian-datadog/conf/kubernetes/kubelet.conf b/docker-image/v1.19/arm64/debian-datadog/conf/kubernetes/kubelet.conf new file mode 100644 index 000000000..5a1e9eef5 --- /dev/null +++ b/docker-image/v1.19/arm64/debian-datadog/conf/kubernetes/kubelet.conf @@ -0,0 +1,11 @@ + + @type tail + @id in_tail_kubelet + multiline_flush_interval 5s + path /var/log/kubelet.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kubelet.log.pos')}" + tag kubelet + + @type kubernetes + + diff --git a/docker-image/v1.19/arm64/debian-datadog/conf/kubernetes/rescheduler.conf b/docker-image/v1.19/arm64/debian-datadog/conf/kubernetes/rescheduler.conf new file mode 100644 index 000000000..69d72a237 --- /dev/null +++ b/docker-image/v1.19/arm64/debian-datadog/conf/kubernetes/rescheduler.conf @@ -0,0 +1,11 @@ + + @type tail + @id in_tail_rescheduler + multiline_flush_interval 5s + path /var/log/rescheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-rescheduler.log.pos')}" + tag rescheduler + + @type kubernetes + + diff --git a/docker-image/v1.19/arm64/debian-datadog/conf/kubernetes/salt.conf b/docker-image/v1.19/arm64/debian-datadog/conf/kubernetes/salt.conf new file mode 100644 index 000000000..60b6b2b16 --- /dev/null +++ b/docker-image/v1.19/arm64/debian-datadog/conf/kubernetes/salt.conf @@ -0,0 +1,13 @@ + + + @type tail + @id in_tail_minion + path /var/log/salt/minion + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-salt.pos')}" + tag salt + + @type regexp + expression /^(? + diff --git a/docker-image/v1.19/arm64/debian-datadog/conf/kubernetes/startupscript.conf b/docker-image/v1.19/arm64/debian-datadog/conf/kubernetes/startupscript.conf new file mode 100644 index 000000000..e44bb75a2 --- /dev/null +++ b/docker-image/v1.19/arm64/debian-datadog/conf/kubernetes/startupscript.conf @@ -0,0 +1,10 @@ + + @type tail + @id in_tail_startupscript + path /var/log/startupscript.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_DIR', ''), 'fluentd-startupscript.log.pos')}" + tag startupscript + + @type syslog + + diff --git a/docker-image/v1.19/arm64/debian-datadog/conf/prometheus.conf b/docker-image/v1.19/arm64/debian-datadog/conf/prometheus.conf new file mode 100644 index 000000000..37599dd8a --- /dev/null +++ b/docker-image/v1.19/arm64/debian-datadog/conf/prometheus.conf @@ -0,0 +1,16 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/prometheus.conf.erb + +# Prometheus metric exposed on 0.0.0.0:24231/metrics + + @type prometheus + @id in_prometheus + bind "#{ENV['FLUENTD_PROMETHEUS_BIND'] || '0.0.0.0'}" + port "#{ENV['FLUENTD_PROMETHEUS_PORT'] || '24231'}" + metrics_path "#{ENV['FLUENTD_PROMETHEUS_PATH'] || '/metrics'}" + + + + @type prometheus_output_monitor + @id in_prometheus_output_monitor + diff --git a/docker-image/v1.19/arm64/debian-datadog/conf/systemd.conf b/docker-image/v1.19/arm64/debian-datadog/conf/systemd.conf new file mode 100644 index 000000000..cfc73eba8 --- /dev/null +++ b/docker-image/v1.19/arm64/debian-datadog/conf/systemd.conf @@ -0,0 +1,46 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/systemd.conf.erb + +# Logs from systemd-journal for interesting services. + + @type systemd + @id in_systemd_kubelet + matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] + + @type local + persistent true + path /var/log/fluentd-journald-kubelet-cursor.json + + read_from_head true + tag kubelet + + +# Logs from docker-systemd + + @type systemd + @id in_systemd_docker + matches [{ "_SYSTEMD_UNIT": "docker.service" }] + + @type local + persistent true + path /var/log/fluentd-journald-docker-cursor.json + + read_from_head true + tag docker.systemd + + +# Logs from systemd-journal for interesting services. + + @type systemd + @id in_systemd_bootkube + matches [{ "_SYSTEMD_UNIT": "bootkube.service" }] + + @type local + persistent true + path /var/log/fluentd-journald-bootkube-cursor.json + + read_from_head true + tag bootkube + + diff --git a/docker-image/v1.19/arm64/debian-datadog/conf/tail_container_parse.conf b/docker-image/v1.19/arm64/debian-datadog/conf/tail_container_parse.conf new file mode 100644 index 000000000..873c6e5dc --- /dev/null +++ b/docker-image/v1.19/arm64/debian-datadog/conf/tail_container_parse.conf @@ -0,0 +1,4 @@ + + @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}" + time_format "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TIME_FORMAT'] || '%Y-%m-%dT%H:%M:%S.%NZ'}" + diff --git a/docker-image/v1.19/arm64/debian-datadog/entrypoint.sh b/docker-image/v1.19/arm64/debian-datadog/entrypoint.sh new file mode 100755 index 000000000..a09f4369f --- /dev/null +++ b/docker-image/v1.19/arm64/debian-datadog/entrypoint.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/entrypoint.sh.erb + + +exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT} diff --git a/docker-image/v1.19/arm64/debian-datadog/plugins/.gitkeep b/docker-image/v1.19/arm64/debian-datadog/plugins/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/docker-image/v1.19/arm64/debian-datadog/plugins/parser_kubernetes.rb b/docker-image/v1.19/arm64/debian-datadog/plugins/parser_kubernetes.rb new file mode 100644 index 000000000..0f593caaf --- /dev/null +++ b/docker-image/v1.19/arm64/debian-datadog/plugins/parser_kubernetes.rb @@ -0,0 +1,68 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# +# @type kubernetes +# +# +# +# ---- EOF --- + +require 'fluent/plugin/parser_regexp' + +module Fluent + module Plugin + class KubernetesParser < RegexpParser + Fluent::Plugin.register_parser("kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?\w)(?