Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -650,7 +650,7 @@ func TestAdminResizeControlPlane(t *testing.T) {
Return(healthyKubeAPIServerJSON(), nil).
AnyTimes()
k.EXPECT().
KubeList(gomock.Any(), "Pod", "openshift-kube-apiserver").
KubeList(gomock.Any(), "Pod", "openshift-kube-apiserver", "app=openshift-kube-apiserver").
Return(healthyKubeAPIServerPodsJSON(), nil).
AnyTimes()
k.EXPECT().
Expand Down
28 changes: 12 additions & 16 deletions pkg/frontend/admin_openshiftcluster_vmresize_pre_validation.go
Original file line number Diff line number Diff line change
Expand Up @@ -289,11 +289,11 @@ func validateAPIServerHealth(ctx context.Context, k adminactions.KubeActions) er

func validateAPIServerPods(ctx context.Context, k adminactions.KubeActions) error {
const (
kubeAPIServerNamespace = "openshift-kube-apiserver"
kubeAPIServerAppLabel = "openshift-kube-apiserver"
kubeAPIServerNamespace = "openshift-kube-apiserver"
kubeAPIServerLabelSelector = "app=openshift-kube-apiserver"
)

rawPods, err := k.KubeList(ctx, "Pod", kubeAPIServerNamespace)
rawPods, err := k.KubeList(ctx, "Pod", kubeAPIServerNamespace, kubeAPIServerLabelSelector)
if err != nil {
return api.NewCloudError(
http.StatusInternalServerError,
Expand All @@ -309,19 +309,15 @@ func validateAPIServerPods(ctx context.Context, k adminactions.KubeActions) erro
fmt.Sprintf("Failed to parse pod list: %v", err))
}

var apiServerPodCount int
var unhealthyPods []string
for _, pod := range podList.Items {
if pod.Labels["app"] != kubeAPIServerAppLabel {
continue
}

apiServerPodCount++
if healthy, reason := isPodHealthy(&pod); !healthy {
unhealthyPods = append(unhealthyPods, fmt.Sprintf("%s (%s)", pod.Name, reason))
if err := validatePodHealth(&pod); err != nil {
unhealthyPods = append(unhealthyPods, fmt.Sprintf("%s (%s)", pod.Name, err.Error()))
}
}

apiServerPodCount := len(podList.Items)

if apiServerPodCount != api.ControlPlaneNodeCount {
return api.NewCloudError(
http.StatusConflict,
Expand All @@ -341,21 +337,21 @@ func validateAPIServerPods(ctx context.Context, k adminactions.KubeActions) erro
return nil
}

func isPodHealthy(pod *corev1.Pod) (bool, string) {
func validatePodHealth(pod *corev1.Pod) error {
if pod.Status.Phase != corev1.PodRunning {
return false, fmt.Sprintf("phase: %s", pod.Status.Phase)
return fmt.Errorf("phase: %s", pod.Status.Phase)
}

for _, cond := range pod.Status.Conditions {
if cond.Type == corev1.PodReady {
if cond.Status != corev1.ConditionTrue {
return false, "not ready"
return fmt.Errorf("not ready")
}
return true, ""
return nil
}
}

return false, "Ready condition not found"
return fmt.Errorf("ready condition not found")
}

// validateEtcdHealth verifies that the etcd ClusterOperator is healthy.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ func allKubeChecksHealthyMock(k *mock_adminactions.MockKubeActions) {
Return(healthyKubeAPIServerJSON(), nil).
AnyTimes()
k.EXPECT().
KubeList(gomock.Any(), "Pod", "openshift-kube-apiserver").
KubeList(gomock.Any(), "Pod", "openshift-kube-apiserver", "app=openshift-kube-apiserver").
Return(healthyKubeAPIServerPodsJSON(), nil).
AnyTimes()
k.EXPECT().
Expand Down Expand Up @@ -904,15 +904,15 @@ func TestValidateAPIServerPods(t *testing.T) {
name: "all kube-apiserver pods healthy",
mocks: func(k *mock_adminactions.MockKubeActions) {
k.EXPECT().
KubeList(gomock.Any(), "Pod", "openshift-kube-apiserver").
KubeList(gomock.Any(), "Pod", "openshift-kube-apiserver", "app=openshift-kube-apiserver").
Return(healthyKubeAPIServerPodsJSON(), nil)
},
},
{
name: "kube-apiserver pod count mismatch",
mocks: func(k *mock_adminactions.MockKubeActions) {
k.EXPECT().
KubeList(gomock.Any(), "Pod", "openshift-kube-apiserver").
KubeList(gomock.Any(), "Pod", "openshift-kube-apiserver", "app=openshift-kube-apiserver").
Return(fakePodListJSON(
fakeKubeAPIServerPod("kube-apiserver-master-0", corev1.PodRunning, true),
fakeKubeAPIServerPod("kube-apiserver-master-1", corev1.PodRunning, true),
Expand All @@ -924,7 +924,7 @@ func TestValidateAPIServerPods(t *testing.T) {
name: "kube-apiserver pod unhealthy phase",
mocks: func(k *mock_adminactions.MockKubeActions) {
k.EXPECT().
KubeList(gomock.Any(), "Pod", "openshift-kube-apiserver").
KubeList(gomock.Any(), "Pod", "openshift-kube-apiserver", "app=openshift-kube-apiserver").
Return(fakePodListJSON(
fakeKubeAPIServerPod("kube-apiserver-master-0", corev1.PodRunning, true),
fakeKubeAPIServerPod("kube-apiserver-master-1", corev1.PodPending, false),
Expand All @@ -937,7 +937,7 @@ func TestValidateAPIServerPods(t *testing.T) {
name: "kube-apiserver pod not ready",
mocks: func(k *mock_adminactions.MockKubeActions) {
k.EXPECT().
KubeList(gomock.Any(), "Pod", "openshift-kube-apiserver").
KubeList(gomock.Any(), "Pod", "openshift-kube-apiserver", "app=openshift-kube-apiserver").
Return(fakePodListJSON(
fakeKubeAPIServerPod("kube-apiserver-master-0", corev1.PodRunning, true),
fakeKubeAPIServerPod("kube-apiserver-master-1", corev1.PodRunning, false),
Expand All @@ -947,31 +947,22 @@ func TestValidateAPIServerPods(t *testing.T) {
wantErr: "409: RequestNotAllowed: kube-apiserver-pods: Unhealthy kube-apiserver pods: [kube-apiserver-master-1 (not ready)]. Resize is not safe without full API server redundancy.",
},
{
name: "non-apiserver pods are ignored",
name: "uses server-side label selector filtering",
mocks: func(k *mock_adminactions.MockKubeActions) {
k.EXPECT().
KubeList(gomock.Any(), "Pod", "openshift-kube-apiserver").
KubeList(gomock.Any(), "Pod", "openshift-kube-apiserver", "app=openshift-kube-apiserver").
Return(fakePodListJSON(
fakeKubeAPIServerPod("kube-apiserver-master-0", corev1.PodRunning, true),
fakeKubeAPIServerPod("kube-apiserver-master-1", corev1.PodRunning, true),
fakeKubeAPIServerPod("kube-apiserver-master-2", corev1.PodRunning, true),
corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "unrelated-pod",
Labels: map[string]string{"app": "something-else"},
},
Status: corev1.PodStatus{
Phase: corev1.PodFailed,
},
},
), nil)
},
},
{
name: "KubeList returns error",
mocks: func(k *mock_adminactions.MockKubeActions) {
k.EXPECT().
KubeList(gomock.Any(), "Pod", "openshift-kube-apiserver").
KubeList(gomock.Any(), "Pod", "openshift-kube-apiserver", "app=openshift-kube-apiserver").
Return(nil, fmt.Errorf("connection refused"))
},
wantErr: "500: InternalServerError: kube-apiserver-pods: Failed to list pods in openshift-kube-apiserver namespace: connection refused",
Expand Down
13 changes: 10 additions & 3 deletions pkg/frontend/adminactions/kubeactions.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"context"
"fmt"
"net/http"
"strings"

"github.com/sirupsen/logrus"

Expand All @@ -32,7 +33,7 @@ import (
// KubeActions are those that involve k8s objects, and thus depend upon k8s clients being createable
type KubeActions interface {
KubeGet(ctx context.Context, groupKind, namespace, name string) ([]byte, error)
KubeList(ctx context.Context, groupKind, namespace string) ([]byte, error)
KubeList(ctx context.Context, groupKind, namespace string, labelSelector ...string) ([]byte, error)
KubeCreateOrUpdate(ctx context.Context, obj *unstructured.Unstructured) error
KubeDelete(ctx context.Context, groupKind, namespace, name string, force bool, propagationPolicy *metav1.DeletionPropagation) error
ResolveGVR(groupKind string, optionalVersion string) (schema.GroupVersionResource, error)
Expand Down Expand Up @@ -117,14 +118,20 @@ func (k *kubeActions) KubeGet(ctx context.Context, groupKind, namespace, name st
return un.MarshalJSON()
}

func (k *kubeActions) KubeList(ctx context.Context, groupKind, namespace string) ([]byte, error) {
// KubeList lists resources. Pass optional label selectors to filter results (e.g., "app=foo", "env=prod").
func (k *kubeActions) KubeList(ctx context.Context, groupKind, namespace string, labelSelector ...string) ([]byte, error) {
gvr, err := k.ResolveGVR(groupKind, "")
if err != nil {
return nil, err
}

selector := strings.Join(labelSelector, ",")

// protect RP memory by not reading in more than 1000 items
ul, err := k.dyn.Resource(gvr).Namespace(namespace).List(ctx, metav1.ListOptions{Limit: 1000})
ul, err := k.dyn.Resource(gvr).Namespace(namespace).List(ctx, metav1.ListOptions{
Limit: 1000,
LabelSelector: selector,
})
if err != nil {
return nil, err
}
Expand Down
13 changes: 9 additions & 4 deletions pkg/util/mocks/adminactions/kubeactions.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading