feat: remove kube-rbac-proxy

Tiltfile

@@ -139,12 +139,13 @@ k8s_resource(
         'promoter-gitcommitstatus-admin-role:clusterrole',
         'promoter-gitcommitstatus-editor-role:clusterrole',
         'promoter-gitcommitstatus-viewer-role:clusterrole',
-        'promoter-metrics-reader:clusterrole',
-        'promoter-proxy-role:clusterrole',
         'promoter-timedcommitstatus-admin-role:clusterrole',
         'promoter-timedcommitstatus-editor-role:clusterrole',
         'promoter-timedcommitstatus-viewer-role:clusterrole',
-        'promoter-proxy-rolebinding:clusterrolebinding',
+        'promoter-metrics-auth-role:clusterrole',
+        'promoter-metrics-auth-rolebinding:clusterrolebinding',
+        'promoter-metrics-reader:clusterrole',
+        'promoter-metrics-reader-rolebinding:clusterrolebinding',
         'promoter-controller-configuration:controllerconfiguration',
     ]
 )

cmd/main.go

@@ -26,13 +26,13 @@ import (
 	"runtime/debug"
 	"syscall"
 
-	"go.uber.org/zap/zapcore"
-	"sigs.k8s.io/controller-runtime/pkg/cluster"
-
 	"github.com/argoproj-labs/gitops-promoter/cmd/demo"
 	"github.com/argoproj-labs/gitops-promoter/internal/controller"
 	"github.com/argoproj-labs/gitops-promoter/internal/utils"
 	"github.com/argoproj-labs/gitops-promoter/internal/webserver"
+	"go.uber.org/zap/zapcore"
+	"sigs.k8s.io/controller-runtime/pkg/cluster"
+	"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
 
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 
@@ -161,14 +161,17 @@ func runController(
 
 	// Create the provider first, then the manager with the provider
 	provider := kubeconfigprovider.New(providerOpts)
-
+	metricsOpts := metricsserver.Options{
+		BindAddress:   metricsAddr,
+		SecureServing: secureMetrics,
+		TLSOpts:       tlsOpts,
+	}
+	if secureMetrics {
+		metricsOpts.FilterProvider = filters.WithAuthenticationAndAuthorization
+	}
 	mcMgr, err := mcmanager.New(ctrl.GetConfigOrDie(), provider, ctrl.Options{
-		Scheme: scheme,
-		Metrics: metricsserver.Options{
-			BindAddress:   metricsAddr,
-			SecureServing: secureMetrics,
-			TLSOpts:       tlsOpts,
-		},
+		Scheme:                 scheme,
+		Metrics:                metricsOpts,
 		WebhookServer:          webhookServer,
 		HealthProbeBindAddress: probeAddr,
 		PprofBindAddress:       pprofAddr,

config/default/manager_auth_proxy_patch.yaml

@@ -1,5 +1,4 @@
-# This patch inject a sidecar container which is a HTTP proxy for the
-# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews.
+# This patch configures the controller manager with proper bind addresses
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -13,28 +12,5 @@ spec:
         image: quay.io/argoprojlabs/gitops-promoter:latest
         args:
           - "--health-probe-bind-address=:8081"
-          - "--metrics-bind-address=127.0.0.1:8080"
+          - "--metrics-bind-address=:9080"
           - "--leader-elect"
-      - name: kube-rbac-proxy
-        securityContext:
-          allowPrivilegeEscalation: false
-          capabilities:
-            drop:
-              - "ALL"
-        image: quay.io/brancz/kube-rbac-proxy:v0.20.2
-        args:
-          - "--secure-listen-address=0.0.0.0:8443"
-          - "--upstream=http://127.0.0.1:8080/"
-          - "--logtostderr=true"
-          - "--v=0"
-        ports:
-          - containerPort: 8443
-            protocol: TCP
-            name: https
-        resources:
-          limits:
-            cpu: 500m
-            memory: 128Mi
-          requests:
-            cpu: 5m
-            memory: 64Mi

config/manager/deployment.yaml

@@ -37,6 +37,16 @@ spec:
             - --leader-elect
           image: quay.io/argoprojlabs/gitops-promoter:latest
           name: manager
+          ports:
+            - containerPort: 8443
+              name: https
+              protocol: TCP
+            - containerPort: 9080
+              name: http
+              protocol: TCP
+            - containerPort: 8081
+              name: health
+              protocol: TCP
           securityContext:
             allowPrivilegeEscalation: false
             capabilities:

config/manager/kustomization.yaml

@@ -5,3 +5,4 @@ resources:
   - namespace.yaml
   - deployment.yaml
   - webhook_service.yaml
+  - metrics_service.yaml

config/rbac/auth_proxy_service.yaml → config/manager/metrics_service.yaml

@@ -1,21 +1,24 @@
 apiVersion: v1
 kind: Service
 metadata:
+  name: controller-manager-metrics-service
   labels:
     control-plane: controller-manager
     app.kubernetes.io/name: service
     app.kubernetes.io/instance: controller-manager-metrics-service
-    app.kubernetes.io/component: kube-rbac-proxy
+    app.kubernetes.io/component: metrics
     app.kubernetes.io/created-by: promoter
     app.kubernetes.io/part-of: promoter
     app.kubernetes.io/managed-by: kustomize
-  name: controller-manager-metrics-service
-  namespace: system
 spec:
-  ports:
-  - name: https
-    port: 8443
-    protocol: TCP
-    targetPort: https
   selector:
     control-plane: controller-manager
+  ports:
+    - name: https
+      port: 8443
+      protocol: TCP
+      targetPort: https
+    - name: http
+      port: 9080
+      protocol: TCP
+      targetPort: http

config/rbac/kustomization.yaml

@@ -9,13 +9,10 @@ resources:
 - role_binding.yaml
 - leader_election_role.yaml
 - leader_election_role_binding.yaml
-# Comment the following 4 lines if you want to disable
-# the auth proxy (https://github.com/brancz/kube-rbac-proxy)
-# which protects your /metrics endpoint.
-- auth_proxy_service.yaml
-- auth_proxy_role.yaml
-- auth_proxy_role_binding.yaml
-- auth_proxy_client_clusterrole.yaml
+- metrics_auth_role.yaml
+- metrics_auth_role_binding.yaml
+- metrics_reader_role.yaml
+- metrics_reader_role_binding.yaml
 # For each CRD, "Editor" and "Viewer" roles are scaffolded by
 # default, aiding admins in cluster management. Those roles are
 # not used by the Project itself. You can comment the following lines

config/rbac/metrics_auth_role.yaml

@@ -0,0 +1,17 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: metrics-auth-role
+rules:
+  - apiGroups:
+      - authentication.k8s.io
+    resources:
+      - tokenreviews
+    verbs:
+      - create
+  - apiGroups:
+      - authorization.k8s.io
+    resources:
+      - subjectaccessreviews
+    verbs:
+      - create

config/rbac/metrics_auth_role_binding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: metrics-auth-rolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: metrics-auth-role
+subjects:
+  - kind: ServiceAccount
+    name: controller-manager
+    namespace: system

config/rbac/metrics_reader_role.yaml

@@ -0,0 +1,9 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: metrics-reader
+rules:
+  - nonResourceURLs:
+      - "/metrics"
+    verbs:
+      - get

config/rbac/metrics_reader_role_binding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: metrics-reader-rolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: metrics-reader
+subjects:
+  - kind: ServiceAccount
+    name: controller-manager
+    namespace: system