diff --git a/cmd/terway-cli/node.go b/cmd/terway-cli/node.go index 57b87198..aa7d403b 100644 --- a/cmd/terway-cli/node.go +++ b/cmd/terway-cli/node.go @@ -7,6 +7,7 @@ import ( "os" "time" + "github.com/Jeffail/gabs/v2" "github.com/spf13/cobra" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -62,6 +63,10 @@ var tasks = []Task{ Name: "kpr", Func: enableKPR, }, + { + Name: "hostport", + Func: setupHostPort, + }, } var nodeconfigCmd = &cobra.Command{ @@ -207,3 +212,73 @@ func enableKPR(cmd *cobra.Command, args []string) error { return store.Save() } + +func setupHostPort(cmd *cobra.Command, args []string) error { + cni, err := os.ReadFile(cniFilePath) + if err != nil { + return err + } + cniJSON, err := gabs.ParseJSON(cni) + if err != nil { + return err + } + + // has portmap plugin ? + plugins := cniJSON.Path("plugins").Children() + + foundPortmap := false + for _, plugin := range plugins { + if plugin.Exists("type") && plugin.S("type").String() == `"portmap"` { + foundPortmap = true + break + } + } + + if !foundPortmap { + return nil + } + fmt.Printf("found portMap plugin\n") + + enableSymmetricRouting := false + for _, plugin := range plugins { + if plugin.Exists("type") && plugin.S("type").String() == `"terway"` { + if plugin.Exists("symmetric_routing") { + if v, ok := plugin.S("symmetric_routing").Data().(bool); ok && v { + enableSymmetricRouting = true + } + } + break + } + } + if !enableSymmetricRouting { + fmt.Printf("symmetric routing disabled, ignore setup for portMap plugin\n") + return nil + } + + // ignores if ipv6 only + if eniCfg.IPStack == "ipv6" { + fmt.Printf("ipv6 only, ignore setup for portMap plugin\n") + return nil + } + + // ignore if KPR enabled + store := nodecap.NewFileNodeCapabilities(nodeCapabilitiesFile) + + err = store.Load() + if err != nil { + return err + } + + prev := store.Get(nodecap.NodeCapabilityKubeProxyReplacement) + if prev == True { + return nil + } + + // setup ip rule ( will not clean up ,if disabled ) + err = configureNetworkRules() + if err != nil { + return err + } + + return nil +} diff --git a/cmd/terway-cli/portmap.go b/cmd/terway-cli/portmap.go new file mode 100644 index 00000000..4dd55597 --- /dev/null +++ b/cmd/terway-cli/portmap.go @@ -0,0 +1,135 @@ +package main + +import ( + "context" + "fmt" + "net" + + "github.com/AliyunContainerService/terway/plugin/driver/utils" + "github.com/coreos/go-iptables/iptables" + "github.com/vishvananda/netlink" +) + +// Configure iptables and ip rule +func configureNetworkRules() error { + ctx := context.Background() + + var ( + mark = 0x10 + mask = 0x10 + tableID = 100 + defaultInterface = "eth0" + comment = "terway-portmap" // Rule comment + rulePrio = 600 + ) + + markHexStr := fmt.Sprintf("0x%X", mark) + maskHexStr := fmt.Sprintf("0x%X", mask) + + eth0, err := netlink.LinkByName(defaultInterface) + if err != nil { + return fmt.Errorf("failed to get interface %s: %v", defaultInterface, err) + } + + routes, err := netlink.RouteList(eth0, netlink.FAMILY_V4) + if err != nil { + return fmt.Errorf("failed to get routes for interface %s: %v", defaultInterface, err) + } + + var defaultGw net.IP + for _, route := range routes { + if route.Dst != nil { + continue + } + defaultGw = route.Gw + break + } + + if defaultGw == nil { + return fmt.Errorf("no default gateway found") + } + + ipt, err := iptables.New() + if err != nil { + return err + } + + nfRules := []ConnmarkRule{ + { + Table: "mangle", + Chain: "PREROUTING", + Args: []string{ + "-i", defaultInterface, + "-j", "CONNMARK", "--set-xmark", fmt.Sprintf("%s/%s", markHexStr, maskHexStr), "-m", "comment", "--comment", comment}, + }, + { + Table: "mangle", + Chain: "PREROUTING", + Args: []string{ + "-j", "CONNMARK", "--restore-mark", "--mask", maskHexStr, "-m", "comment", "--comment", comment}, + }, + } + + for _, rule := range nfRules { + err = ensureNFRules(ipt, &rule) + if err != nil { + return fmt.Errorf("failed to add nf rule: %v", err) + } + } + // should only needed on ipv4 case + prio := rulePrio + fwMark := mark + fwMask := mask + family := netlink.FAMILY_V4 + + rule := netlink.NewRule() + rule.Priority = prio + rule.Family = family + rule.Table = tableID + rule.Mark = fwMark + rule.Mask = fwMask + + _, err = utils.EnsureIPRule(ctx, rule) + if err != nil { + return fmt.Errorf("failed to add ip rule: %v", err) + } + route := &netlink.Route{ + LinkIndex: eth0.Attrs().Index, + Dst: &net.IPNet{IP: net.ParseIP("0.0.0.0"), Mask: net.CIDRMask(0, 32)}, + Gw: defaultGw, + Table: tableID, + Scope: netlink.SCOPE_UNIVERSE, + Flags: int(netlink.FLAG_ONLINK), + } + _, err = utils.EnsureRoute(ctx, route) + if err != nil { + return fmt.Errorf("failed to add route: %v", err) + } + return nil +} + +type ConnmarkRule struct { + Table string + Chain string + + Args []string +} + +func ensureNFRules(ipt *iptables.IPTables, rule *ConnmarkRule) error { + exists, err := ipt.Exists(rule.Table, rule.Chain, rule.Args...) + if err != nil { + return fmt.Errorf("failed to check if rule exists: %w", err) + } + + if exists { + fmt.Printf("Rule already exists in %s table %s chain %v\n", rule.Table, rule.Chain, rule.Args) + return nil + } + + err = ipt.Append(rule.Table, rule.Chain, rule.Args...) + if err != nil { + return fmt.Errorf("failed to add rule: %w", err) + } + fmt.Printf("Added rule to %s table %s chain %v\n", rule.Table, rule.Chain, rule.Args) + return nil +} diff --git a/cmd/terway-cli/portmap_test.go b/cmd/terway-cli/portmap_test.go new file mode 100644 index 00000000..2a712150 --- /dev/null +++ b/cmd/terway-cli/portmap_test.go @@ -0,0 +1,209 @@ +package main + +import ( + "fmt" + "net" + "os" + "runtime" + "testing" + + "github.com/containernetworking/plugins/pkg/ns" + "github.com/containernetworking/plugins/pkg/testutils" + "github.com/coreos/go-iptables/iptables" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" +) + +// TestConfigureNetworkRules tests network rules configuration functionality +func TestConfigureNetworkRules(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("This test requires root privileges") + } + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + containerNS, err := testutils.NewNS() + require.NoError(t, err) + defer func() { + err := containerNS.Close() + require.NoError(t, err) + + err = testutils.UnmountNS(containerNS) + require.NoError(t, err) + }() + + _ = containerNS.Do(func(ns ns.NetNS) error { + dummy := netlink.Dummy{} + dummy.Name = "eth0" + err := netlink.LinkAdd(&dummy) + require.NoError(t, err) + + dummyLink, err := netlink.LinkByName("eth0") + require.NoError(t, err) + + err = netlink.AddrAdd(dummyLink, &netlink.Addr{ + IPNet: &net.IPNet{ + IP: net.ParseIP("192.168.1.2"), + Mask: net.IPv4Mask(255, 255, 255, 255), + }, + }) + require.NoError(t, err) + + err = netlink.LinkSetUp(dummyLink) + require.NoError(t, err) + + defaultRoute := netlink.Route{ + LinkIndex: dummyLink.Attrs().Index, + Dst: &net.IPNet{IP: net.ParseIP("0.0.0.0"), Mask: net.CIDRMask(0, 32)}, + Flags: int(netlink.FLAG_ONLINK), + Gw: net.ParseIP("192.168.1.1"), + Table: unix.RT_TABLE_MAIN, + } + err = netlink.RouteAdd(&defaultRoute) + require.NoError(t, err) + + err = configureNetworkRules() + require.NoError(t, err) + + // Verify iptables rules + ipt, err := iptables.New() + require.NoError(t, err, "Failed to create iptables instance") + + // Check CONNMARK setting rule + exists, err := ipt.Exists("mangle", "PREROUTING", "-i", "eth0", "-j", "CONNMARK", "--set-xmark", "0x10/0x10", "-m", "comment", "--comment", "terway-portmap") + assert.NoError(t, err, "Failed to check CONNMARK set rule") + assert.True(t, exists, "CONNMARK set rule should exist") + + return nil + }) + +} + +// TestEnsureNFRules tests iptables rule management functionality +func TestEnsureNFRules(t *testing.T) { + // Check if running as root + if os.Geteuid() != 0 { + t.Skip("This test requires root privileges") + } + + ipt, err := iptables.New() + require.NoError(t, err, "Failed to create iptables instance") + + // Test rule + testRule := &ConnmarkRule{ + Table: "mangle", + Chain: "PREROUTING", + Args: []string{ + "-i", "lo", + "-j", "CONNMARK", + "--set-xmark", "0x20/0x20", + "-m", "comment", + "--comment", `"test-rule"`, + }, + } + + // Clean up test rule + defer func() { + ipt.Delete("mangle", "PREROUTING", testRule.Args...) + }() + + // Test adding rule + err = ensureNFRules(ipt, testRule) + assert.NoError(t, err, "ensureNFRules should succeed") + + // Verify rule exists + exists, err := ipt.Exists(testRule.Table, testRule.Chain, testRule.Args...) + assert.NoError(t, err, "Failed to check if rule exists") + assert.True(t, exists, "Rule should exist after adding") + + // Test duplicate addition (should not error) + err = ensureNFRules(ipt, testRule) + assert.NoError(t, err, "ensureNFRules should not error when rule already exists") + + // clean up + _ = ipt.Delete("mangle", "PREROUTING", testRule.Args...) +} + +// TestNetworkInterfaceDetection tests network interface detection functionality +func TestNetworkInterfaceDetection(t *testing.T) { + // Check if running as root + if os.Geteuid() != 0 { + t.Skip("This test requires root privileges") + } + + // Test getting eth0 interface + eth0, err := netlink.LinkByName("eth0") + if err != nil { + t.Skip("eth0 interface not found, skipping test") + } + + assert.NotNil(t, eth0, "eth0 interface should be found") + assert.Equal(t, "eth0", eth0.Attrs().Name, "Interface name should be eth0") + + // Test getting routes + routes, err := netlink.RouteList(eth0, netlink.FAMILY_V4) + assert.NoError(t, err, "Failed to get routes for eth0") + assert.NotEmpty(t, routes, "Should have at least one route") + + // Find default gateway + var defaultGw net.IP + for _, route := range routes { + if route.Dst == nil { + defaultGw = route.Gw + break + } + } + + if defaultGw != nil { + assert.True(t, defaultGw.To4() != nil, "Default gateway should be IPv4") + t.Logf("Found default gateway: %s", defaultGw.String()) + } else { + t.Log("No default gateway found") + } +} + +// TestMarkAndMaskValues tests correctness of mark and mask values +func TestMarkAndMaskValues(t *testing.T) { + mark := 0x10 + mask := 0x10 + + markHexStr := fmt.Sprintf("0x%X", mark) + maskHexStr := fmt.Sprintf("0x%X", mask) + + assert.Equal(t, "0x10", markHexStr, "Mark hex string should be 0x10") + assert.Equal(t, "0x10", maskHexStr, "Mask hex string should be 0x10") + + // Test mark/mask combination + markMaskStr := fmt.Sprintf("%s/%s", markHexStr, maskHexStr) + assert.Equal(t, "0x10/0x10", markMaskStr, "Mark/mask combination should be 0x10/0x10") +} + +// TestErrorHandling tests error handling +func TestErrorHandling(t *testing.T) { + // Test non-existent interface + _, err := netlink.LinkByName("eth2") + assert.Error(t, err, "Should error when interface doesn't exist") + + // Test invalid iptables rule + ipt, err := iptables.New() + if err != nil { + t.Skip("Cannot create iptables instance, skipping error handling test") + } + + invalidRule := &ConnmarkRule{ + Table: "mangle", + Chain: "PREROUTING", + Args: []string{ + "-i", "eth2", + "-j", "INVALID_TARGET", + }, + } + + err = ensureNFRules(ipt, invalidRule) + // This test might fail due to varying behavior of iptables across systems + if err != nil { + t.Logf("Expected error for invalid rule: %v", err) + } +} diff --git a/go.mod b/go.mod index 0b1a2c65..c7bdccf7 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/boltdb/bolt v1.3.1 github.com/containernetworking/cni v1.1.2 github.com/containernetworking/plugins v1.3.0 + github.com/coreos/go-iptables v0.6.0 github.com/evanphx/json-patch v5.6.0+incompatible github.com/go-logr/logr v1.4.2 github.com/go-playground/mold/v4 v4.2.0 @@ -82,7 +83,6 @@ require ( github.com/clbanning/mxj/v2 v2.7.0 // indirect github.com/containerd/cgroups v1.1.0 // indirect github.com/containerd/console v1.0.3 // indirect - github.com/coreos/go-iptables v0.6.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch/v5 v5.9.11 // indirect diff --git a/plugin/datapath/policy_router_linux.go b/plugin/datapath/policy_router_linux.go index 6decb0dc..f3ae2ede 100644 --- a/plugin/datapath/policy_router_linux.go +++ b/plugin/datapath/policy_router_linux.go @@ -456,21 +456,33 @@ func (d *PolicyRoute) Teardown(ctx context.Context, cfg *types.TeardownCfg, netN return nil } if extender.IPv4 != nil { - err := exec(&netlink.Rule{Priority: fromContainerPriority, Src: extender.IPv4}) + rule := netlink.NewRule() + rule.Priority = fromContainerPriority + rule.Src = extender.IPv4 + err := exec(rule) if err != nil { return err } - err = exec(&netlink.Rule{Priority: toContainerPriority, Dst: extender.IPv4}) + rule = netlink.NewRule() + rule.Priority = toContainerPriority + rule.Dst = extender.IPv4 + err = exec(rule) if err != nil { return err } } if extender.IPv6 != nil { - err := exec(&netlink.Rule{Priority: fromContainerPriority, Src: extender.IPv6}) + rule := netlink.NewRule() + rule.Priority = fromContainerPriority + rule.Src = extender.IPv6 + err := exec(rule) if err != nil { return err } - err = exec(&netlink.Rule{Priority: toContainerPriority, Dst: extender.IPv6}) + rule = netlink.NewRule() + rule.Priority = toContainerPriority + rule.Dst = extender.IPv6 + err = exec(rule) if err != nil { return err } diff --git a/plugin/datapath/policy_router_linux_test.go b/plugin/datapath/policy_router_linux_test.go index 2806b9f0..887fafc4 100644 --- a/plugin/datapath/policy_router_linux_test.go +++ b/plugin/datapath/policy_router_linux_test.go @@ -139,27 +139,28 @@ func TestDataPathPolicyRoute(t *testing.T) { assert.Equal(t, 1, len(routes)) // 512 from all to 169.10.0.10 look up main - rules, err := netlink.RuleListFiltered(netlink.FAMILY_V4, &netlink.Rule{ - Priority: toContainerPriority, - Table: unix.RT_TABLE_MAIN, - Dst: &net.IPNet{ - IP: cfg.ContainerIPNet.IPv4.IP, - Mask: net.CIDRMask(32, 32), - }, - }, netlink.RT_FILTER_TABLE|netlink.RT_FILTER_DST|netlink.RT_FILTER_PRIORITY) + rule := netlink.NewRule() + rule.Priority = toContainerPriority + rule.Table = unix.RT_TABLE_MAIN + rule.Dst = &net.IPNet{ + IP: cfg.ContainerIPNet.IPv4.IP, + Mask: net.CIDRMask(32, 32), + } + + rules, err := netlink.RuleListFiltered(netlink.FAMILY_V4, rule, netlink.RT_FILTER_TABLE|netlink.RT_FILTER_DST|netlink.RT_FILTER_PRIORITY) assert.NoError(t, err) assert.Equal(t, len(rules), 1) assert.Nil(t, rules[0].Src) // 2048 from 169.10.0.10 lookup table - rules, err = netlink.RuleListFiltered(netlink.FAMILY_V4, &netlink.Rule{ - Priority: fromContainerPriority, - Table: utils.GetRouteTableID(eni.Attrs().Index), - Src: &net.IPNet{ - IP: cfg.ContainerIPNet.IPv4.IP, - Mask: net.CIDRMask(32, 32), - }, - }, netlink.RT_FILTER_TABLE|netlink.RT_FILTER_SRC|netlink.RT_FILTER_PRIORITY|netlink.RT_FILTER_IIF) + rule = netlink.NewRule() + rule.Priority = fromContainerPriority + rule.Table = utils.GetRouteTableID(eni.Attrs().Index) + rule.Src = &net.IPNet{ + IP: cfg.ContainerIPNet.IPv4.IP, + Mask: net.CIDRMask(32, 32), + } + rules, err = netlink.RuleListFiltered(netlink.FAMILY_V4, rule, netlink.RT_FILTER_TABLE|netlink.RT_FILTER_SRC|netlink.RT_FILTER_PRIORITY|netlink.RT_FILTER_IIF) assert.NoError(t, err) assert.Equal(t, len(rules), 1) diff --git a/plugin/driver/utils/utils_linux.go b/plugin/driver/utils/utils_linux.go index 7db62eb2..be68fc0c 100644 --- a/plugin/driver/utils/utils_linux.go +++ b/plugin/driver/utils/utils_linux.go @@ -215,8 +215,8 @@ func FindIPRule(rule *netlink.Rule) ([]netlink.Rule, error) { var filterMask uint64 family := netlink.FAMILY_V4 - if rule.Src == nil && rule.Dst == nil && rule.OifName == "" { - return nil, errors.New("both src and dst is nil") + if rule.Src == nil && rule.Dst == nil && rule.OifName == "" && rule.Mask == 0 && rule.Mark == 0 { + return nil, errors.New("rule is empty") } if rule.Src != nil { @@ -235,6 +235,14 @@ func FindIPRule(rule *netlink.Rule) ([]netlink.Rule, error) { if rule.Priority >= 0 { filterMask = filterMask | netlink.RT_FILTER_PRIORITY } + + if rule.Mask >= 0 { + filterMask = filterMask | netlink.RT_FILTER_MASK + } + + if rule.Mark >= 0 { + filterMask = filterMask | netlink.RT_FILTER_MARK + } return netlink.RuleListFiltered(family, rule, filterMask) }