Skip to content

Add minute-level log retention for log groomer | issue #61814 #61859

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
MiteshSahu wants to merge 2 commits into from
Closed

Add minute-level log retention for log groomer | issue #61814 #61859

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7d9d4e3
Add minute-level log retention for log groomer
Feb 13, 2026
3097595
Add minute-level log retention for log groomer
Feb 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 15 additions & 6 deletions Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1639,7 +1639,8 @@ COPY <<"EOF" /clean-logs.sh
set -euo pipefail

readonly DIRECTORY="${AIRFLOW_HOME:-/usr/local/airflow}"
readonly RETENTION="${AIRFLOW__LOG_RETENTION_DAYS:-15}"
readonly RETENTION_DAYS="${AIRFLOW__LOG_RETENTION_DAYS:-15}"
readonly RETENTION_MINUTES="${AIRFLOW__LOG_RETENTION_MINUTES:-}"
readonly FREQUENCY="${AIRFLOW__LOG_CLEANUP_FREQUENCY_MINUTES:-15}"

trap "exit" INT TERM
Expand All @@ -1649,11 +1650,19 @@ readonly EVERY=$((FREQUENCY*60))
echo "Cleaning logs every $EVERY seconds"

while true; do
echo "Trimming airflow logs to ${RETENTION} days."
find "${DIRECTORY}"/logs \
-type d -name 'lost+found' -prune -o \
-type f -mtime +"${RETENTION}" -name '*.log' -print0 | \
xargs -0 rm -f || true
if [[ -n "${RETENTION_MINUTES}" && "${RETENTION_MINUTES}" -ge 0 ]]; then
echo "Trimming airflow logs older than ${RETENTION_MINUTES} minutes."
find "${DIRECTORY}"/logs \
-type d -name 'lost+found' -prune -o \
-type f -mmin +"${RETENTION_MINUTES}" -name '*.log' -print0 | \
xargs -0 rm -f || true
else
echo "Trimming airflow logs to ${RETENTION_DAYS} days."
find "${DIRECTORY}"/logs \
-type d -name 'lost+found' -prune -o \
-type f -mtime +"${RETENTION_DAYS}" -name '*.log' -print0 | \
xargs -0 rm -f || true
fi

find "${DIRECTORY}"/logs -type d -empty -delete || true

Expand Down
1 change: 1 addition & 0 deletions airflow-core/newsfragments/61814.feature.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* Add ``AIRFLOW__LOG_RETENTION_MINUTES`` and Helm chart ``logGroomerSidecar.retentionMinutes`` for minute-level log retention (previously days only). Minutes take precedence over days.
1 change: 1 addition & 0 deletions airflow-core/newsfragments/99999.feature.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* Add ``AIRFLOW__LOG_RETENTION_MINUTES`` and Helm chart ``logGroomerSidecar.retentionMinutes`` for minute-level log retention (previously days only). Minutes take precedence over days.
28 changes: 28 additions & 0 deletions airflow-core/tests/unit/charts/log_groomer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,34 @@ def test_log_groomer_retention_days_overrides(self, retention_days, retention_re
else:
assert len(jmespath.search("spec.template.spec.containers[1].env", docs[0])) == 2

@pytest.mark.parametrize(("retention_minutes", "retention_result"), [(None, None), (60, "60")])
def test_log_groomer_retention_minutes_overrides(self, retention_minutes, retention_result):
if self.obj_name == "dag-processor":
values = {
"dagProcessor": {
"enabled": True,
"logGroomerSidecar": {"retentionMinutes": retention_minutes},
}
}
else:
values = {f"{self.folder}": {"logGroomerSidecar": {"retentionMinutes": retention_minutes}}}

docs = render_chart(
values=values,
show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"],
)

if retention_result:
assert (
jmespath.search(
"spec.template.spec.containers[1].env[?name=='AIRFLOW__LOG_RETENTION_MINUTES'].value | [0]",
docs[0],
)
== retention_result
)
else:
assert len(jmespath.search("spec.template.spec.containers[1].env", docs[0])) == 2

@pytest.mark.parametrize(("frequency_minutes", "frequency_result"), [(None, None), (20, "20")])
def test_log_groomer_frequency_minutes_overrides(self, frequency_minutes, frequency_result):
if self.obj_name == "dag-processor":
Expand Down
7 changes: 6 additions & 1 deletion chart/templates/dag-processor/dag-processor-deployment.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -211,9 +211,14 @@ spec:
args: {{- tpl (toYaml .Values.dagProcessor.logGroomerSidecar.args) . | nindent 12 }}
{{- end }}
env:
{{- if .Values.dagProcessor.logGroomerSidecar.retentionDays }}
{{- if or .Values.dagProcessor.logGroomerSidecar.retentionDays .Values.dagProcessor.logGroomerSidecar.retentionMinutes }}
{{- if .Values.dagProcessor.logGroomerSidecar.retentionMinutes }}
- name: AIRFLOW__LOG_RETENTION_MINUTES
value: "{{ .Values.dagProcessor.logGroomerSidecar.retentionMinutes }}"
{{- else }}
- name: AIRFLOW__LOG_RETENTION_DAYS
value: "{{ .Values.dagProcessor.logGroomerSidecar.retentionDays }}"
{{- end }}
{{- end }}
{{- if .Values.dagProcessor.logGroomerSidecar.frequencyMinutes }}
- name: AIRFLOW__LOG_CLEANUP_FREQUENCY_MINUTES
Expand Down
7 changes: 6 additions & 1 deletion chart/templates/scheduler/scheduler-deployment.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -286,9 +286,14 @@ spec:
args: {{- tpl (toYaml .Values.scheduler.logGroomerSidecar.args) . | nindent 12 }}
{{- end }}
env:
{{- if .Values.scheduler.logGroomerSidecar.retentionDays }}
{{- if or .Values.scheduler.logGroomerSidecar.retentionDays .Values.scheduler.logGroomerSidecar.retentionMinutes }}
{{- if .Values.scheduler.logGroomerSidecar.retentionMinutes }}
- name: AIRFLOW__LOG_RETENTION_MINUTES
value: "{{ .Values.scheduler.logGroomerSidecar.retentionMinutes }}"
{{- else }}
- name: AIRFLOW__LOG_RETENTION_DAYS
value: "{{ .Values.scheduler.logGroomerSidecar.retentionDays }}"
{{- end }}
{{- end }}
{{- if .Values.scheduler.logGroomerSidecar.frequencyMinutes }}
- name: AIRFLOW__LOG_CLEANUP_FREQUENCY_MINUTES
Expand Down
7 changes: 6 additions & 1 deletion chart/templates/triggerer/triggerer-deployment.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -246,9 +246,14 @@ spec:
args: {{- tpl (toYaml .Values.triggerer.logGroomerSidecar.args) . | nindent 12 }}
{{- end }}
env:
{{- if .Values.triggerer.logGroomerSidecar.retentionDays }}
{{- if or .Values.triggerer.logGroomerSidecar.retentionDays .Values.triggerer.logGroomerSidecar.retentionMinutes }}
{{- if .Values.triggerer.logGroomerSidecar.retentionMinutes }}
- name: AIRFLOW__LOG_RETENTION_MINUTES
value: "{{ .Values.triggerer.logGroomerSidecar.retentionMinutes }}"
{{- else }}
- name: AIRFLOW__LOG_RETENTION_DAYS
value: "{{ .Values.triggerer.logGroomerSidecar.retentionDays }}"
{{- end }}
{{- end }}
{{- if .Values.triggerer.logGroomerSidecar.frequencyMinutes }}
- name: AIRFLOW__LOG_CLEANUP_FREQUENCY_MINUTES
Expand Down
7 changes: 6 additions & 1 deletion chart/templates/workers/worker-deployment.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -357,9 +357,14 @@ spec:
args: {{ tpl (toYaml .Values.workers.logGroomerSidecar.args) . | nindent 12 }}
{{- end }}
env:
{{- if .Values.workers.logGroomerSidecar.retentionDays }}
{{- if or .Values.workers.logGroomerSidecar.retentionDays .Values.workers.logGroomerSidecar.retentionMinutes }}
{{- if .Values.workers.logGroomerSidecar.retentionMinutes }}
- name: AIRFLOW__LOG_RETENTION_MINUTES
value: "{{ .Values.workers.logGroomerSidecar.retentionMinutes }}"
{{- else }}
- name: AIRFLOW__LOG_RETENTION_DAYS
value: "{{ .Values.workers.logGroomerSidecar.retentionDays }}"
{{- end }}
{{- end }}
{{- if .Values.workers.logGroomerSidecar.frequencyMinutes }}
- name: AIRFLOW__LOG_CLEANUP_FREQUENCY_MINUTES
Expand Down
5 changes: 5 additions & 0 deletions chart/values.schema.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13727,6 +13727,11 @@
"type": "integer",
"default": 15
},
"retentionMinutes": {
"description": "Number of minutes to retain the logs when running the Airflow log groomer sidecar. Overrides retentionDays if set.",
"type": "integer",
"default": 0
},
"frequencyMinutes": {
"description": "Number of minutes between attempts to groom the Airflow logs in log groomer sidecar.",
"type": "integer",
Expand Down
3 changes: 3 additions & 0 deletions chart/values.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -981,6 +981,9 @@ workers:
# Number of days to retain logs
retentionDays: 15

# Number of minutes to retain logs. Overrides retentionDays if set.
retentionMinutes: 0

# Frequency to attempt to groom logs (in minutes)
frequencyMinutes: 15

Expand Down
28 changes: 28 additions & 0 deletions helm-tests/tests/chart_utils/log_groomer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,34 @@ def test_log_groomer_retention_days_overrides(self, retention_days, retention_re
else:
assert len(jmespath.search("spec.template.spec.containers[1].env", docs[0])) == 2

@pytest.mark.parametrize(("retention_minutes", "retention_result"), [(None, None), (60, "60")])
def test_log_groomer_retention_minutes_overrides(self, retention_minutes, retention_result):
if self.obj_name == "dag-processor":
values = {
"dagProcessor": {
"enabled": True,
"logGroomerSidecar": {"retentionMinutes": retention_minutes},
}
}
else:
values = {f"{self.folder}": {"logGroomerSidecar": {"retentionMinutes": retention_minutes}}}

docs = render_chart(
values=values,
show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"],
)

if retention_result:
assert (
jmespath.search(
"spec.template.spec.containers[1].env[?name=='AIRFLOW__LOG_RETENTION_MINUTES'].value | [0]",
docs[0],
)
== retention_result
)
else:
assert len(jmespath.search("spec.template.spec.containers[1].env", docs[0])) == 2

@pytest.mark.parametrize(("frequency_minutes", "frequency_result"), [(None, None), (20, "20")])
def test_log_groomer_frequency_minutes_overrides(self, frequency_minutes, frequency_result):
if self.obj_name == "dag-processor":
Expand Down
23 changes: 17 additions & 6 deletions scripts/docker/clean-logs.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,10 @@
set -euo pipefail

readonly DIRECTORY="${AIRFLOW_HOME:-/usr/local/airflow}"
readonly RETENTION="${AIRFLOW__LOG_RETENTION_DAYS:-15}"
# Allow retention to be specified in either days (legacy) or minutes.
# Minutes take precedence when set to a positive integer.
readonly RETENTION_DAYS="${AIRFLOW__LOG_RETENTION_DAYS:-15}"
readonly RETENTION_MINUTES="${AIRFLOW__LOG_RETENTION_MINUTES:-}"
readonly FREQUENCY="${AIRFLOW__LOG_CLEANUP_FREQUENCY_MINUTES:-15}"

trap "exit" INT TERM
Expand All @@ -30,11 +33,19 @@ readonly EVERY=$((FREQUENCY*60))
echo "Cleaning logs every $EVERY seconds"

while true; do
echo "Trimming airflow logs to ${RETENTION} days."
find "${DIRECTORY}"/logs \
-type d -name 'lost+found' -prune -o \
-type f -mtime +"${RETENTION}" -name '*.log' -print0 | \
xargs -0 rm -f || true
if [[ -n "${RETENTION_MINUTES}" && "${RETENTION_MINUTES}" -ge 0 ]]; then
echo "Trimming airflow logs older than ${RETENTION_MINUTES} minutes."
find "${DIRECTORY}"/logs \
-type d -name 'lost+found' -prune -o \
-type f -mmin +"${RETENTION_MINUTES}" -name '*.log' -print0 | \
xargs -0 rm -f || true
else
echo "Trimming airflow logs to ${RETENTION_DAYS} days."
find "${DIRECTORY}"/logs \
-type d -name 'lost+found' -prune -o \
-type f -mtime +"${RETENTION_DAYS}" -name '*.log' -print0 | \
xargs -0 rm -f || true
fi

find "${DIRECTORY}"/logs -type d -empty -delete || true

Expand Down
Loading