diff --git a/pkg/controller/node/node.go b/pkg/controller/node/node.go index e49c6196..2c9cba00 100644 --- a/pkg/controller/node/node.go +++ b/pkg/controller/node/node.go @@ -426,10 +426,6 @@ func (r *ReconcileNode) handleEFLO(ctx context.Context, k8sNode *corev1.Node, no NodeID: &node.Spec.NodeMetadata.InstanceID, } - if types.NodeExclusiveENIMode(node.Labels) != types.ExclusiveENIOnly { - return reconcile.Result{}, fmt.Errorf("exclusive ENI mode must be enabled for EFLO nodes") - } - resp, err := r.aliyun.GetEFLOController().DescribeNode(ctx, describeNodeReq) if err != nil { return reconcile.Result{}, err @@ -448,6 +444,11 @@ func (r *ReconcileNode) handleEFLO(ctx context.Context, k8sNode *corev1.Node, no if (node.Spec.NodeCap.Adapters <= 1 && limit.HighDenseQuantity > 0) || k8sNode.Annotations[types.ENOApi] == types.APIEcsHDeni { // check k8s config + + if types.NodeExclusiveENIMode(node.Labels) != types.ExclusiveENIOnly { + return reconcile.Result{}, fmt.Errorf("exclusive ENI mode must be enabled for EFLO nodes") + } + node.Spec.NodeCap.Adapters = limit.HighDenseQuantity node.Spec.NodeCap.TotalAdapters = limit.HighDenseQuantity diff --git a/pkg/controller/node/node_controller_test.go b/pkg/controller/node/node_controller_test.go index 959194dd..37ecb263 100644 --- a/pkg/controller/node/node_controller_test.go +++ b/pkg/controller/node/node_controller_test.go @@ -501,33 +501,5 @@ var _ = Describe("Node Controller", func() { verifyNetworkCardsCount(ctx, nodeName, 0) }) }) - - Context("Error Cases", func() { - It("should reject EFLO node without exclusive ENI mode", func() { - nodeName := "test-eflo-node-error" - defer cleanupNode(ctx, nodeName) - - k8sNode := testutil.NewK8sNodeBuilder(nodeName). - WithEFLO(). - WithInstanceType("eflo.instance"). - WithProviderID("instanceID-error"). - Build() - Expect(k8sClient.Create(ctx, k8sNode)).To(Succeed()) - - openAPI.On("DescribeNetworkInterfaceV2", mock.Anything, mock.Anything).Return([]*aliyunClient.NetworkInterface{ - { - NetworkInterfaceID: "eni-test", - Type: aliyunClient.ENITypePrimary, - }, - }, nil).Maybe() - - reconciler := createReconciler(true, true) - _, err := reconciler.Reconcile(ctx, reconcile.Request{ - NamespacedName: types.NamespacedName{Name: nodeName}, - }) - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("exclusive ENI mode must be enabled for EFLO nodes")) - }) - }) }) }) diff --git a/tests/connective_test.go b/tests/connective_test.go index 6c765f4e..fd0baf29 100644 --- a/tests/connective_test.go +++ b/tests/connective_test.go @@ -39,7 +39,7 @@ type PodConfig struct { // generatePodConfigs generates pod configurations with proper node affinity to avoid exclusive ENI nodes func generatePodConfigs(testName string) []PodConfig { // Get node affinity exclude labels to avoid scheduling on exclusive ENI nodes - nodeAffinityExclude := GetNodeAffinityExcludeForType(NodeTypeNormal) + nodeAffinityExclude := GetNodeAffinityExcludeForType(NodeTypeECSSharedENI) var mutateConfig []PodConfig if affinityLabel == "" { diff --git a/tests/connectivity_scenarios_test.go b/tests/connectivity_scenarios_test.go index 0771496b..537be01d 100644 --- a/tests/connectivity_scenarios_test.go +++ b/tests/connectivity_scenarios_test.go @@ -14,22 +14,26 @@ import ( "sigs.k8s.io/e2e-framework/pkg/features" ) -// TestConnectivity_AllNodeTypes tests basic connectivity across all available node types -// Note: Trunk and Exclusive ENI mode tests are in trunk_test.go and exclusive_eni_test.go respectively +// TestConnectivity_AllNodeTypes tests basic connectivity across all available node types and ENI modes +// Tests all combinations: ECS/Lingjun nodes × Shared/Exclusive ENI modes func TestConnectivity_AllNodeTypes(t *testing.T) { var feats []features.Feature - // Test normal nodes - normalFeature := createConnectivityTest("Connectivity/NormalNode", NodeTypeNormal, "normal") - feats = append(feats, normalFeature) + // Test ECS nodes with shared ENI mode + ecsSharedFeature := createConnectivityTest("Connectivity/ECS-SharedENI", NodeTypeECSSharedENI, "ecs-shared") + feats = append(feats, ecsSharedFeature) - // Test exclusive ENI nodes (basic connectivity only, not exclusive ENI mode) - exclusiveENIFeature := createConnectivityTest("Connectivity/ExclusiveENINode", NodeTypeExclusiveENI, "exclusive-eni") - feats = append(feats, exclusiveENIFeature) + // Test ECS nodes with exclusive ENI mode + ecsExclusiveFeature := createConnectivityTest("Connectivity/ECS-ExclusiveENI", NodeTypeECSExclusiveENI, "ecs-exclusive") + feats = append(feats, ecsExclusiveFeature) - // Test Lingjun nodes - lingjunFeature := createConnectivityTest("Connectivity/LingjunNode", NodeTypeLingjun, "lingjun") - feats = append(feats, lingjunFeature) + // Test Lingjun nodes with shared ENI mode + lingjunSharedFeature := createConnectivityTest("Connectivity/Lingjun-SharedENI", NodeTypeLingjunSharedENI, "lingjun-shared") + feats = append(feats, lingjunSharedFeature) + + // Test Lingjun nodes with exclusive ENI mode + lingjunExclusiveFeature := createConnectivityTest("Connectivity/Lingjun-ExclusiveENI", NodeTypeLingjunExclusiveENI, "lingjun-exclusive") + feats = append(feats, lingjunExclusiveFeature) testenv.Test(t, feats...) } @@ -69,6 +73,17 @@ func createConnectivityTest(testName string, nodeType NodeType, label string) fe server = server.WithNodeAffinityExclude(nodeAffinityExclude) } + // Add tolerations for Lingjun nodes (both shared and exclusive ENI modes) + if nodeType == NodeTypeLingjunSharedENI || nodeType == NodeTypeLingjunExclusiveENI { + server = server.WithTolerations([]corev1.Toleration{ + { + Key: "node-role.alibabacloud.com/lingjun", + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoSchedule, + }, + }) + } + err = config.Client().Resources().Create(ctx, server.Pod) if err != nil { t.Fatalf("create server pod failed, %v", err) @@ -88,6 +103,17 @@ func createConnectivityTest(testName string, nodeType NodeType, label string) fe client = client.WithNodeAffinityExclude(nodeAffinityExclude) } + // Add tolerations for Lingjun nodes (both shared and exclusive ENI modes) + if nodeType == NodeTypeLingjunSharedENI || nodeType == NodeTypeLingjunExclusiveENI { + client = client.WithTolerations([]corev1.Toleration{ + { + Key: "node-role.alibabacloud.com/lingjun", + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoSchedule, + }, + }) + } + err = config.Client().Resources().Create(ctx, client.Pod) if err != nil { t.Fatalf("create client pod failed, %v", err) @@ -159,7 +185,7 @@ func createConnectivityTest(testName string, nodeType NodeType, label string) fe func TestConnectivity_CrossNode(t *testing.T) { var feats []features.Feature - for _, nodeType := range []NodeType{NodeTypeNormal, NodeTypeExclusiveENI, NodeTypeLingjun} { + for _, nodeType := range []NodeType{NodeTypeECSSharedENI, NodeTypeECSExclusiveENI, NodeTypeLingjunSharedENI, NodeTypeLingjunExclusiveENI} { feat := createCrossNodeTest(nodeType) feats = append(feats, feat) } @@ -199,6 +225,17 @@ func createCrossNodeTest(nodeType NodeType) features.Feature { server = server.WithNodeAffinityExclude(nodeAffinityExclude) } + // Add tolerations for Lingjun nodes (both shared and exclusive ENI modes) + if nodeType == NodeTypeLingjunSharedENI || nodeType == NodeTypeLingjunExclusiveENI { + server = server.WithTolerations([]corev1.Toleration{ + { + Key: "node-role.alibabacloud.com/lingjun", + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoSchedule, + }, + }) + } + err = config.Client().Resources().Create(ctx, server.Pod) if err != nil { t.Fatalf("create server pod failed, %v", err) @@ -217,6 +254,17 @@ func createCrossNodeTest(nodeType NodeType) features.Feature { client = client.WithNodeAffinityExclude(nodeAffinityExclude) } + // Add tolerations for Lingjun nodes (both shared and exclusive ENI modes) + if nodeType == NodeTypeLingjunSharedENI || nodeType == NodeTypeLingjunExclusiveENI { + client = client.WithTolerations([]corev1.Toleration{ + { + Key: "node-role.alibabacloud.com/lingjun", + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoSchedule, + }, + }) + } + err = config.Client().Resources().Create(ctx, client.Pod) if err != nil { t.Fatalf("create client pod failed, %v", err) @@ -274,7 +322,7 @@ func createCrossNodeTest(nodeType NodeType) features.Feature { func TestConnectivity_CrossZone(t *testing.T) { var feats []features.Feature - for _, nodeType := range []NodeType{NodeTypeNormal, NodeTypeExclusiveENI, NodeTypeLingjun} { + for _, nodeType := range []NodeType{NodeTypeECSSharedENI, NodeTypeECSExclusiveENI, NodeTypeLingjunSharedENI, NodeTypeLingjunExclusiveENI} { feat := createCrossZoneTest(nodeType) feats = append(feats, feat) } @@ -338,6 +386,17 @@ func createCrossZoneTest(nodeType NodeType) features.Feature { server = server.WithNodeAffinityExclude(nodeAffinityExclude) } + // Add tolerations for Lingjun nodes (both shared and exclusive ENI modes) + if nodeType == NodeTypeLingjunSharedENI || nodeType == NodeTypeLingjunExclusiveENI { + server = server.WithTolerations([]corev1.Toleration{ + { + Key: "node-role.alibabacloud.com/lingjun", + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoSchedule, + }, + }) + } + err = config.Client().Resources().Create(ctx, server.Pod) if err != nil { t.Fatalf("create server pod failed, %v", err) @@ -356,6 +415,17 @@ func createCrossZoneTest(nodeType NodeType) features.Feature { client = client.WithNodeAffinityExclude(nodeAffinityExclude) } + // Add tolerations for Lingjun nodes (both shared and exclusive ENI modes) + if nodeType == NodeTypeLingjunSharedENI || nodeType == NodeTypeLingjunExclusiveENI { + client = client.WithTolerations([]corev1.Toleration{ + { + Key: "node-role.alibabacloud.com/lingjun", + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoSchedule, + }, + }) + } + err = config.Client().Resources().Create(ctx, client.Pod) if err != nil { t.Fatalf("create client pod failed, %v", err) diff --git a/tests/multi_network_test.go b/tests/multi_network_test.go index b8df4af1..561d0293 100644 --- a/tests/multi_network_test.go +++ b/tests/multi_network_test.go @@ -44,7 +44,7 @@ type MultiNetworkConfig struct { // NewMultiNetworkConfig creates a default multi-network test configuration func NewMultiNetworkConfig() *MultiNetworkConfig { config := &MultiNetworkConfig{ - NodeTypes: []NodeType{NodeTypeNormal, NodeTypeExclusiveENI}, + NodeTypes: []NodeType{NodeTypeECSSharedENI, NodeTypeECSExclusiveENI, NodeTypeLingjunSharedENI, NodeTypeLingjunExclusiveENI}, EnableDefaultMode: true, // Always test default mode } @@ -133,7 +133,7 @@ func createMultiNetworkTestFeature(testName string, nodeType NodeType, mode stri // Create primary PodNetworking (always use default config) pn1 := NewPodNetworking(pnPrimary) - if nodeType == NodeTypeExclusiveENI { + if nodeType == NodeTypeECSExclusiveENI || nodeType == NodeTypeLingjunExclusiveENI { pn1 = pn1.WithENIAttachType(networkv1beta1.ENIOptionTypeENI) } err = CreatePodNetworkingAndWaitReady(ctx, config.Client(), pn1.PodNetworking) @@ -145,7 +145,7 @@ func createMultiNetworkTestFeature(testName string, nodeType NodeType, mode stri // Create secondary PodNetworking based on mode pn2 := NewPodNetworking(pnSecondary) - if nodeType == NodeTypeExclusiveENI { + if nodeType == NodeTypeECSExclusiveENI || nodeType == NodeTypeLingjunExclusiveENI { pn2 = pn2.WithENIAttachType(networkv1beta1.ENIOptionTypeENI) } if mode == "custom" { diff --git a/tests/node_utils_test.go b/tests/node_utils_test.go index 9208483c..df5ce2e3 100644 --- a/tests/node_utils_test.go +++ b/tests/node_utils_test.go @@ -17,45 +17,48 @@ import ( type NodeType string const ( - NodeTypeNormal NodeType = "normal" - NodeTypeExclusiveENI NodeType = "exclusive-eni" - NodeTypeLingjun NodeType = "lingjun" + NodeTypeECSSharedENI NodeType = "ecs-shared-eni" + NodeTypeECSExclusiveENI NodeType = "ecs-exclusive-eni" + NodeTypeLingjunSharedENI NodeType = "lingjun-shared-eni" + NodeTypeLingjunExclusiveENI NodeType = "lingjun-exclusive-eni" ) // NodeTypeInfo contains information about available node types in the cluster type NodeTypeInfo struct { - NormalNodes []corev1.Node - ExclusiveENINodes []corev1.Node - LingjunNodes []corev1.Node - AllNodes []corev1.Node + ECSSharedENINodes []corev1.Node + ECSExclusiveENINodes []corev1.Node + LingjunSharedENINodes []corev1.Node + LingjunExclusiveENINodes []corev1.Node + AllNodes []corev1.Node } // HasTrunkNodes checks if the cluster has nodes that support trunk mode func (n *NodeTypeInfo) HasTrunkNodes() bool { - // Normal nodes and exclusive ENI nodes support trunk mode - // Lingjun nodes do NOT support trunk mode - return len(n.NormalNodes) > 0 || len(n.ExclusiveENINodes) > 0 + // ECS nodes support trunk mode, Lingjun nodes do NOT support trunk mode + return len(n.ECSSharedENINodes) > 0 || len(n.ECSExclusiveENINodes) > 0 } // HasExclusiveENINodes checks if the cluster has exclusive ENI nodes func (n *NodeTypeInfo) HasExclusiveENINodes() bool { - return len(n.ExclusiveENINodes) > 0 + return len(n.ECSExclusiveENINodes) > 0 || len(n.LingjunExclusiveENINodes) > 0 } // HasLingjunNodes checks if the cluster has Lingjun nodes func (n *NodeTypeInfo) HasLingjunNodes() bool { - return len(n.LingjunNodes) > 0 + return len(n.LingjunSharedENINodes) > 0 || len(n.LingjunExclusiveENINodes) > 0 } // GetNodesByType returns nodes of a specific type func (n *NodeTypeInfo) GetNodesByType(nodeType NodeType) []corev1.Node { switch nodeType { - case NodeTypeNormal: - return n.NormalNodes - case NodeTypeExclusiveENI: - return n.ExclusiveENINodes - case NodeTypeLingjun: - return n.LingjunNodes + case NodeTypeECSSharedENI: + return n.ECSSharedENINodes + case NodeTypeECSExclusiveENI: + return n.ECSExclusiveENINodes + case NodeTypeLingjunSharedENI: + return n.LingjunSharedENINodes + case NodeTypeLingjunExclusiveENI: + return n.LingjunExclusiveENINodes default: return []corev1.Node{} } @@ -76,12 +79,14 @@ func DiscoverNodeTypes(ctx context.Context, client klient.Client) (*NodeTypeInfo for _, node := range nodes.Items { nodeType := classifyNode(node) switch nodeType { - case NodeTypeNormal: - info.NormalNodes = append(info.NormalNodes, node) - case NodeTypeExclusiveENI: - info.ExclusiveENINodes = append(info.ExclusiveENINodes, node) - case NodeTypeLingjun: - info.LingjunNodes = append(info.LingjunNodes, node) + case NodeTypeECSSharedENI: + info.ECSSharedENINodes = append(info.ECSSharedENINodes, node) + case NodeTypeECSExclusiveENI: + info.ECSExclusiveENINodes = append(info.ECSExclusiveENINodes, node) + case NodeTypeLingjunSharedENI: + info.LingjunSharedENINodes = append(info.LingjunSharedENINodes, node) + case NodeTypeLingjunExclusiveENI: + info.LingjunExclusiveENINodes = append(info.LingjunExclusiveENINodes, node) } } @@ -90,18 +95,23 @@ func DiscoverNodeTypes(ctx context.Context, client klient.Client) (*NodeTypeInfo // classifyNode determines the type of a node based on its labels and resources func classifyNode(node corev1.Node) NodeType { - // Check if it's a Lingjun node first + // Check if it's a Lingjun exclusive ENI node first (both Lingjun and exclusive ENI) + if isLingjunNode(node) && isExclusiveENINode(node) { + return NodeTypeLingjunExclusiveENI + } + + // Check if it's a Lingjun shared ENI node if isLingjunNode(node) { - return NodeTypeLingjun + return NodeTypeLingjunSharedENI } - // Check if it's an exclusive ENI node + // Check if it's an ECS exclusive ENI node if isExclusiveENINode(node) { - return NodeTypeExclusiveENI + return NodeTypeECSExclusiveENI } - // Default to normal node - return NodeTypeNormal + // Default to ECS shared ENI node + return NodeTypeECSSharedENI } // isLingjunNode checks if a node is a Lingjun node @@ -119,12 +129,19 @@ func isExclusiveENINode(node corev1.Node) bool { // GetNodeAffinityForType returns node affinity labels for scheduling pods to specific node types func GetNodeAffinityForType(nodeType NodeType) map[string]string { switch nodeType { - case NodeTypeLingjun: + case NodeTypeECSSharedENI: + return map[string]string{} + case NodeTypeECSExclusiveENI: + return map[string]string{ + "k8s.aliyun.com/exclusive-mode-eni-type": "eniOnly", + } + case NodeTypeLingjunSharedENI: return map[string]string{ "alibabacloud.com/lingjun-worker": "true", } - case NodeTypeExclusiveENI: + case NodeTypeLingjunExclusiveENI: return map[string]string{ + "alibabacloud.com/lingjun-worker": "true", "k8s.aliyun.com/exclusive-mode-eni-type": "eniOnly", } default: @@ -135,12 +152,25 @@ func GetNodeAffinityForType(nodeType NodeType) map[string]string { // GetNodeAffinityExcludeForType returns node affinity exclusion labels for specific node types func GetNodeAffinityExcludeForType(nodeType NodeType) map[string]string { switch nodeType { - case NodeTypeNormal: - // For normal nodes, exclude Lingjun and exclusive ENI nodes + case NodeTypeECSSharedENI: + // For ECS shared ENI nodes, exclude all Lingjun nodes and ECS exclusive ENI nodes return map[string]string{ "alibabacloud.com/lingjun-worker": "true", "k8s.aliyun.com/exclusive-mode-eni-type": "eniOnly", } + case NodeTypeECSExclusiveENI: + // For ECS exclusive ENI nodes, exclude all Lingjun nodes + return map[string]string{ + "alibabacloud.com/lingjun-worker": "true", + } + case NodeTypeLingjunSharedENI: + // For Lingjun shared ENI nodes, exclude exclusive ENI nodes + return map[string]string{ + "k8s.aliyun.com/exclusive-mode-eni-type": "eniOnly", + } + case NodeTypeLingjunExclusiveENI: + // For Lingjun exclusive ENI nodes, no exclusions needed (already precisely matched) + return map[string]string{} default: return map[string]string{} } @@ -208,11 +238,11 @@ func ValidateNodeTypeRequirements(nodeInfo *NodeTypeInfo, requiredTypes []NodeTy func (n *NodeTypeInfo) GetSupportedNodeTypesForTrunk() []NodeType { var supported []NodeType - if len(n.NormalNodes) > 0 { - supported = append(supported, NodeTypeNormal) + if len(n.ECSSharedENINodes) > 0 { + supported = append(supported, NodeTypeECSSharedENI) } - if len(n.ExclusiveENINodes) > 0 { - supported = append(supported, NodeTypeExclusiveENI) + if len(n.ECSExclusiveENINodes) > 0 { + supported = append(supported, NodeTypeECSExclusiveENI) } // Lingjun nodes do NOT support trunk mode @@ -223,22 +253,25 @@ func (n *NodeTypeInfo) GetSupportedNodeTypesForTrunk() []NodeType { func (n *NodeTypeInfo) GetSupportedNodeTypesForExclusiveENI() []NodeType { var supported []NodeType - // Check normal nodes for aliyun/eni resource - for _, node := range n.NormalNodes { + // Check ECS shared ENI nodes for aliyun/eni resource + for _, node := range n.ECSSharedENINodes { if CheckNodeSupportsExclusiveENI(node) { - supported = append(supported, NodeTypeNormal) + supported = append(supported, NodeTypeECSSharedENI) break } } - // Check exclusive ENI nodes - if len(n.ExclusiveENINodes) > 0 { - supported = append(supported, NodeTypeExclusiveENI) + // Check ECS exclusive ENI nodes + if len(n.ECSExclusiveENINodes) > 0 { + supported = append(supported, NodeTypeECSExclusiveENI) } // Lingjun nodes support exclusive ENI but don't expose aliyun/eni resource - if len(n.LingjunNodes) > 0 { - supported = append(supported, NodeTypeLingjun) + if len(n.LingjunSharedENINodes) > 0 { + supported = append(supported, NodeTypeLingjunSharedENI) + } + if len(n.LingjunExclusiveENINodes) > 0 { + supported = append(supported, NodeTypeLingjunExclusiveENI) } return supported @@ -249,11 +282,11 @@ func (n *NodeTypeInfo) GetSupportedNodeTypesForMultiNetwork() []NodeType { var supported []NodeType // Multi-network does not support Lingjun nodes - if len(n.NormalNodes) > 0 { - supported = append(supported, NodeTypeNormal) + if len(n.ECSSharedENINodes) > 0 { + supported = append(supported, NodeTypeECSSharedENI) } - if len(n.ExclusiveENINodes) > 0 { - supported = append(supported, NodeTypeExclusiveENI) + if len(n.ECSExclusiveENINodes) > 0 { + supported = append(supported, NodeTypeECSExclusiveENI) } return supported diff --git a/tests/utils_test.go b/tests/utils_test.go index 9aaa6f4d..578e70df 100644 --- a/tests/utils_test.go +++ b/tests/utils_test.go @@ -275,6 +275,11 @@ func (p *Pod) WithResourceLimits(resources corev1.ResourceList) *Pod { return p } +func (p *Pod) WithTolerations(tolerations []corev1.Toleration) *Pod { + p.Spec.Tolerations = append(p.Spec.Tolerations, tolerations...) + return p +} + type Service struct { *corev1.Service }