-
Notifications
You must be signed in to change notification settings - Fork 226
NE-2421: Support dual-stack IngressController on AWS #1376
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: master
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -11,6 +11,7 @@ import ( | |
| iov1 "github.com/openshift/api/operatoringress/v1" | ||
| "github.com/openshift/cluster-ingress-operator/pkg/dns" | ||
| logf "github.com/openshift/cluster-ingress-operator/pkg/log" | ||
| awsutil "github.com/openshift/cluster-ingress-operator/pkg/util/aws" | ||
|
|
||
| "github.com/aws/aws-sdk-go/aws" | ||
| "github.com/aws/aws-sdk-go/aws/arn" | ||
|
|
@@ -109,6 +110,11 @@ type Config struct { | |
| // Client is a Kubernetes client, which the provider uses to annotate | ||
| // DNSRecord CRs. | ||
| Client client.Client | ||
|
|
||
| // IPFamily is the cluster's IP family configuration from the | ||
| // Infrastructure CR. When dual-stack, the provider creates both | ||
| // Alias A and Alias AAAA Route53 records. | ||
| IPFamily configv1.IPFamilyType | ||
| } | ||
|
|
||
| // ServiceEndpoint stores the configuration of a custom url to | ||
|
|
@@ -619,22 +625,34 @@ func (m *Provider) updateRecord(domain, zoneID, target, targetHostedZoneID, acti | |
| }, | ||
| } | ||
| } else { | ||
| input.ChangeBatch = &route53.ChangeBatch{ | ||
| Changes: []*route53.Change{ | ||
| { | ||
| Action: aws.String(action), | ||
| ResourceRecordSet: &route53.ResourceRecordSet{ | ||
| Name: aws.String(domain), | ||
| Type: aws.String(route53.RRTypeA), | ||
| AliasTarget: &route53.AliasTarget{ | ||
| HostedZoneId: aws.String(targetHostedZoneID), | ||
| DNSName: aws.String(target), | ||
| EvaluateTargetHealth: aws.Bool(false), | ||
| }, | ||
| }, | ||
| aliasTarget := &route53.AliasTarget{ | ||
| HostedZoneId: aws.String(targetHostedZoneID), | ||
| DNSName: aws.String(target), | ||
| EvaluateTargetHealth: aws.Bool(false), | ||
| } | ||
| changes := []*route53.Change{ | ||
| { | ||
| Action: aws.String(action), | ||
| ResourceRecordSet: &route53.ResourceRecordSet{ | ||
| Name: aws.String(domain), | ||
| Type: aws.String(route53.RRTypeA), | ||
| AliasTarget: aliasTarget, | ||
| }, | ||
| }, | ||
| } | ||
| if awsutil.IsDualStack(m.config.IPFamily) { | ||
| changes = append(changes, &route53.Change{ | ||
| Action: aws.String(action), | ||
| ResourceRecordSet: &route53.ResourceRecordSet{ | ||
| Name: aws.String(domain), | ||
| Type: aws.String(route53.RRTypeAaaa), | ||
| AliasTarget: aliasTarget, | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Both
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yeah, something like a C++ pointer to a constant would avoid this confusion, but Golang is way simpler. I think it's fine to use a pointer to the same object. AWS SDK should not mess with the input parameters especially from the same change batch. If it does (unlikely), we may have problems with other variables too, like |
||
| }, | ||
| }) | ||
| } | ||
| input.ChangeBatch = &route53.ChangeBatch{ | ||
| Changes: changes, | ||
| } | ||
| } | ||
| resp, err := m.route53.ChangeResourceRecordSets(&input) | ||
| if err != nil { | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -15,6 +15,7 @@ import ( | |
|
|
||
| "github.com/openshift/cluster-ingress-operator/pkg/manifests" | ||
| "github.com/openshift/cluster-ingress-operator/pkg/operator/controller" | ||
| awsutil "github.com/openshift/cluster-ingress-operator/pkg/util/aws" | ||
| corev1 "k8s.io/api/core/v1" | ||
|
|
||
| configv1 "github.com/openshift/api/config/v1" | ||
|
|
@@ -496,6 +497,35 @@ func desiredLoadBalancerService(ci *operatorv1.IngressController, deploymentRef | |
| } | ||
| } | ||
|
|
||
| // Set ipFamilies and ipFamilyPolicy for dual-stack clusters. | ||
| if platform.AWS != nil && awsutil.IsDualStack(platform.AWS.IPFamily) { | ||
| if lbStatus != nil && lbStatus.ProviderParameters != nil && lbStatus.ProviderParameters.AWS != nil { | ||
| switch lbStatus.ProviderParameters.AWS.Type { | ||
| case operatorv1.AWSNetworkLoadBalancer: | ||
| // NLB supports dual-stack natively. | ||
| ipFamilyPolicy := corev1.IPFamilyPolicyRequireDualStack | ||
| service.Spec.IPFamilyPolicy = &ipFamilyPolicy | ||
| if platform.AWS.IPFamily == configv1.DualStackIPv4Primary { | ||
| service.Spec.IPFamilies = []corev1.IPFamily{corev1.IPv4Protocol, corev1.IPv6Protocol} | ||
| } else { | ||
| service.Spec.IPFamilies = []corev1.IPFamily{corev1.IPv6Protocol, corev1.IPv4Protocol} | ||
| } | ||
| case operatorv1.AWSClassicLoadBalancer: | ||
| // CLB does not support dual-stack and only forwards | ||
| // IPv4 traffic. On DualStackIPv4Primary clusters, | ||
| // the service defaults to SingleStack/IPv4, which | ||
| // is correct. However on DualStackIPv6Primary, the | ||
| // default would be SingleStack/IPv6, causing OVN to | ||
| // refuse IPv4 traffic on the service's NodePort. | ||
| // Explicitly set SingleStack/IPv4 to ensure CLB | ||
| // traffic is always accepted. | ||
| ipFamilyPolicy := corev1.IPFamilyPolicySingleStack | ||
| service.Spec.IPFamilyPolicy = &ipFamilyPolicy | ||
| service.Spec.IPFamilies = []corev1.IPFamily{corev1.IPv4Protocol} | ||
| } | ||
| } | ||
| } | ||
|
|
||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. There is no else statement so if
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We are fine with this, it's consistent with the rest of |
||
| // Set the load balancer for AWS to be as aggressive as Azure (2 fail @ 5s interval, 2 healthy) | ||
| service.Annotations[awsLBHealthCheckTimeoutAnnotation] = awsLBHealthCheckTimeoutDefault | ||
| service.Annotations[awsLBHealthCheckUnhealthyThresholdAnnotation] = awsLBHealthCheckUnhealthyThresholdDefault | ||
|
|
@@ -715,6 +745,27 @@ func loadBalancerServiceChanged(current, expected *corev1.Service) (bool, *corev | |
| } | ||
| } | ||
|
|
||
| // Reconcile ipFamilies and ipFamilyPolicy when the desired service | ||
| // specifies them (i.e. the cluster is dual-stack). | ||
| if len(expected.Spec.IPFamilies) != 0 { | ||
| if !reflect.DeepEqual(current.Spec.IPFamilies, expected.Spec.IPFamilies) { | ||
| if !changed { | ||
| changed = true | ||
| updated = current.DeepCopy() | ||
| } | ||
| updated.Spec.IPFamilies = expected.Spec.IPFamilies | ||
| } | ||
| } | ||
| if expected.Spec.IPFamilyPolicy != nil { | ||
| if current.Spec.IPFamilyPolicy == nil || *current.Spec.IPFamilyPolicy != *expected.Spec.IPFamilyPolicy { | ||
| if !changed { | ||
| changed = true | ||
| updated = current.DeepCopy() | ||
| } | ||
| updated.Spec.IPFamilyPolicy = expected.Spec.IPFamilyPolicy | ||
| } | ||
| } | ||
alebedev87 marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
|
||
| return changed, updated | ||
| } | ||
|
|
||
|
|
@@ -858,6 +909,10 @@ func loadBalancerServiceIsProgressing(ic *operatorv1.IngressController, service | |
| if wantLBType != haveLBType { | ||
| patchSpec := fmt.Sprintf(`{"spec":{"endpointPublishingStrategy":{"type":"LoadBalancerService","loadBalancer":{"providerParameters":{"type":"AWS","aws":{"type":"%s"}}}}}}`, haveLBType) | ||
| err := createEffectuationMessage("loadBalancer.providerParameters.aws.type", string(haveLBType), string(wantLBType), patchSpec, ic, service) | ||
| // Add a note to the effectuation message about CLB not supporting dual-stack. | ||
| if wantLBType == operatorv1.AWSClassicLoadBalancer && platform.AWS != nil && awsutil.IsDualStack(platform.AWS.IPFamily) { | ||
| err = fmt.Errorf("%w Classic Load Balancers do not support dual-stack. The IngressController %q will use IPv4-only despite that the cluster is configured as %q. Use an NLB type to support dual-stack networking.", err, ic.Name, platform.AWS.IPFamily) | ||
| } | ||
| errs = append(errs, err) | ||
| } | ||
|
|
||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -893,6 +893,176 @@ func TestDesiredLoadBalancerServiceAWSIdleTimeout(t *testing.T) { | |
| } | ||
| } | ||
|
|
||
| // TestDesiredLoadBalancerServiceDualStack verifies that | ||
| // desiredLoadBalancerService sets the expected ipFamilies and ipFamilyPolicy | ||
| // based on the infrastructure platform status ipFamily field. | ||
| func TestDesiredLoadBalancerServiceDualStack(t *testing.T) { | ||
| testCases := []struct { | ||
| name string | ||
| strategyStatus *operatorv1.EndpointPublishingStrategy | ||
| platformStatus *configv1.PlatformStatus | ||
| expectedIPFamilies []corev1.IPFamily | ||
| expectedIPFamilyPolicy *corev1.IPFamilyPolicyType | ||
| }{ | ||
| { | ||
| name: "NLB with DualStackIPv4Primary sets IPv4-primary dual-stack", | ||
| strategyStatus: nlbStrategy(operatorv1.ExternalLoadBalancer), | ||
| platformStatus: &configv1.PlatformStatus{ | ||
| Type: configv1.AWSPlatformType, | ||
| AWS: &configv1.AWSPlatformStatus{ | ||
| IPFamily: configv1.DualStackIPv4Primary, | ||
| }, | ||
| }, | ||
| expectedIPFamilies: []corev1.IPFamily{corev1.IPv4Protocol, corev1.IPv6Protocol}, | ||
| expectedIPFamilyPolicy: ipFamilyPolicyPtr(corev1.IPFamilyPolicyRequireDualStack), | ||
| }, | ||
| { | ||
| name: "NLB with DualStackIPv6Primary sets IPv6-primary dual-stack", | ||
| strategyStatus: nlbStrategy(operatorv1.ExternalLoadBalancer), | ||
| platformStatus: &configv1.PlatformStatus{ | ||
| Type: configv1.AWSPlatformType, | ||
| AWS: &configv1.AWSPlatformStatus{ | ||
| IPFamily: configv1.DualStackIPv6Primary, | ||
| }, | ||
| }, | ||
| expectedIPFamilies: []corev1.IPFamily{corev1.IPv6Protocol, corev1.IPv4Protocol}, | ||
| expectedIPFamilyPolicy: ipFamilyPolicyPtr(corev1.IPFamilyPolicyRequireDualStack), | ||
| }, | ||
| { | ||
| name: "NLB with IPv4 does not set ipFamilies", | ||
| strategyStatus: nlbStrategy(operatorv1.ExternalLoadBalancer), | ||
| platformStatus: &configv1.PlatformStatus{ | ||
| Type: configv1.AWSPlatformType, | ||
| AWS: &configv1.AWSPlatformStatus{ | ||
| IPFamily: configv1.IPv4, | ||
| }, | ||
| }, | ||
| expectedIPFamilies: nil, | ||
| expectedIPFamilyPolicy: nil, | ||
| }, | ||
| { | ||
| name: "NLB with empty ipFamily does not set ipFamilies", | ||
| strategyStatus: nlbStrategy(operatorv1.ExternalLoadBalancer), | ||
| platformStatus: &configv1.PlatformStatus{ | ||
| Type: configv1.AWSPlatformType, | ||
| AWS: &configv1.AWSPlatformStatus{}, | ||
| }, | ||
| expectedIPFamilies: nil, | ||
| expectedIPFamilyPolicy: nil, | ||
| }, | ||
| { | ||
| name: "NLB without AWS platform status does not set ipFamilies", | ||
| strategyStatus: nlbStrategy(operatorv1.ExternalLoadBalancer), | ||
| platformStatus: &configv1.PlatformStatus{ | ||
| Type: configv1.AWSPlatformType, | ||
| }, | ||
| expectedIPFamilies: nil, | ||
| expectedIPFamilyPolicy: nil, | ||
| }, | ||
| { | ||
| name: "CLB with DualStackIPv4Primary sets SingleStack/IPv4", | ||
| strategyStatus: clbStrategy(operatorv1.ExternalLoadBalancer), | ||
| platformStatus: &configv1.PlatformStatus{ | ||
| Type: configv1.AWSPlatformType, | ||
| AWS: &configv1.AWSPlatformStatus{ | ||
| IPFamily: configv1.DualStackIPv4Primary, | ||
| }, | ||
| }, | ||
| expectedIPFamilies: []corev1.IPFamily{corev1.IPv4Protocol}, | ||
| expectedIPFamilyPolicy: ipFamilyPolicyPtr(corev1.IPFamilyPolicySingleStack), | ||
| }, | ||
| { | ||
| name: "CLB with DualStackIPv6Primary sets SingleStack/IPv4", | ||
| strategyStatus: clbStrategy(operatorv1.ExternalLoadBalancer), | ||
| platformStatus: &configv1.PlatformStatus{ | ||
| Type: configv1.AWSPlatformType, | ||
| AWS: &configv1.AWSPlatformStatus{ | ||
| IPFamily: configv1.DualStackIPv6Primary, | ||
| }, | ||
| }, | ||
| expectedIPFamilies: []corev1.IPFamily{corev1.IPv4Protocol}, | ||
| expectedIPFamilyPolicy: ipFamilyPolicyPtr(corev1.IPFamilyPolicySingleStack), | ||
| }, | ||
| { | ||
| name: "NLB with DualStackIPv4Primary on non-AWS platform does not set ipFamilies", | ||
| strategyStatus: nlbStrategy(operatorv1.ExternalLoadBalancer), | ||
| platformStatus: &configv1.PlatformStatus{ | ||
| Type: configv1.GCPPlatformType, | ||
| AWS: &configv1.AWSPlatformStatus{ | ||
| IPFamily: configv1.DualStackIPv6Primary, | ||
| }, | ||
| }, | ||
alebedev87 marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| expectedIPFamilies: nil, | ||
| expectedIPFamilyPolicy: nil, | ||
| }, | ||
| } | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Another useful tests are configuring aws dual stack without a platform type, another one using a platform type other than aws. Both shouldn't configure IP families.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Added a new test case for the platform different than AWS.
This should not happen, the platform type is required in Infrastructure API and should be set by the installer. |
||
|
|
||
| for _, tc := range testCases { | ||
| t.Run(tc.name, func(t *testing.T) { | ||
| ic := &operatorv1.IngressController{ | ||
| ObjectMeta: metav1.ObjectMeta{Name: "default"}, | ||
| Status: operatorv1.IngressControllerStatus{ | ||
| EndpointPublishingStrategy: tc.strategyStatus, | ||
| }, | ||
| } | ||
| trueVar := true | ||
| deploymentRef := metav1.OwnerReference{ | ||
| APIVersion: "apps/v1", | ||
| Kind: "Deployment", | ||
| Name: "router-default", | ||
| UID: "1", | ||
| Controller: &trueVar, | ||
| } | ||
| haveSvc, svc, err := desiredLoadBalancerService(ic, deploymentRef, tc.platformStatus, false) | ||
| if err != nil { | ||
| t.Fatal(err) | ||
| } | ||
| if !haveSvc { | ||
| t.Fatal("desiredLoadBalancerService didn't return a service") | ||
| } | ||
| assert.Equal(t, tc.expectedIPFamilies, svc.Spec.IPFamilies) | ||
| assert.Equal(t, tc.expectedIPFamilyPolicy, svc.Spec.IPFamilyPolicy) | ||
| }) | ||
| } | ||
| } | ||
|
|
||
| // nlbStrategy returns an EndpointPublishingStrategy for an AWS NLB. | ||
| func nlbStrategy(scope operatorv1.LoadBalancerScope) *operatorv1.EndpointPublishingStrategy { | ||
| return &operatorv1.EndpointPublishingStrategy{ | ||
| Type: operatorv1.LoadBalancerServiceStrategyType, | ||
| LoadBalancer: &operatorv1.LoadBalancerStrategy{ | ||
| Scope: scope, | ||
| ProviderParameters: &operatorv1.ProviderLoadBalancerParameters{ | ||
| Type: operatorv1.AWSLoadBalancerProvider, | ||
| AWS: &operatorv1.AWSLoadBalancerParameters{ | ||
| Type: operatorv1.AWSNetworkLoadBalancer, | ||
| }, | ||
| }, | ||
| }, | ||
| } | ||
| } | ||
|
|
||
| // clbStrategy returns an EndpointPublishingStrategy for an AWS CLB. | ||
| func clbStrategy(scope operatorv1.LoadBalancerScope) *operatorv1.EndpointPublishingStrategy { | ||
| return &operatorv1.EndpointPublishingStrategy{ | ||
| Type: operatorv1.LoadBalancerServiceStrategyType, | ||
| LoadBalancer: &operatorv1.LoadBalancerStrategy{ | ||
| Scope: scope, | ||
| ProviderParameters: &operatorv1.ProviderLoadBalancerParameters{ | ||
| Type: operatorv1.AWSLoadBalancerProvider, | ||
| AWS: &operatorv1.AWSLoadBalancerParameters{ | ||
| Type: operatorv1.AWSClassicLoadBalancer, | ||
| }, | ||
| }, | ||
| }, | ||
| } | ||
| } | ||
|
|
||
| // ipFamilyPolicyPtr returns a pointer to the given IPFamilyPolicyType. | ||
| func ipFamilyPolicyPtr(p corev1.IPFamilyPolicyType) *corev1.IPFamilyPolicyType { | ||
| return &p | ||
| } | ||
|
|
||
| // Test_shouldUseLocalWithFallback verifies that shouldUseLocalWithFallback | ||
| // behaves as expected. | ||
| func Test_shouldUseLocalWithFallback(t *testing.T) { | ||
|
|
@@ -1148,6 +1318,21 @@ func Test_loadBalancerServiceChanged(t *testing.T) { | |
| }, | ||
| expect: false, | ||
| }, | ||
| { | ||
| description: "if .spec.ipFamilies is added", | ||
| mutate: func(svc *corev1.Service) { | ||
| svc.Spec.IPFamilies = []corev1.IPFamily{corev1.IPv4Protocol, corev1.IPv6Protocol} | ||
| }, | ||
| expect: true, | ||
| }, | ||
| { | ||
| description: "if .spec.ipFamilyPolicy is added", | ||
| mutate: func(svc *corev1.Service) { | ||
| policy := corev1.IPFamilyPolicyRequireDualStack | ||
| svc.Spec.IPFamilyPolicy = &policy | ||
| }, | ||
| expect: true, | ||
| }, | ||
| } | ||
|
|
||
| for _, tc := range testCases { | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| package aws | ||
|
|
||
| import configv1 "github.com/openshift/api/config/v1" | ||
|
|
||
| // IsDualStack returns true if the given IPFamilyType indicates a dual-stack | ||
| // configuration. | ||
| func IsDualStack(ipFamily configv1.IPFamilyType) bool { | ||
| return ipFamily == configv1.DualStackIPv4Primary || ipFamily == configv1.DualStackIPv6Primary | ||
| } |
Uh oh!
There was an error while loading. Please reload this page.