incus: make instance ips host reachable

daemon/process/vmnet/deps.go

@@ -1,12 +1,10 @@
 package vmnet
 
 import (
-	"bytes"
 	"fmt"
 	"os"
 	"path/filepath"
 	"runtime"
-	"strings"
 
 	"github.com/abiosoft/colima/daemon/process"
 	"github.com/abiosoft/colima/embedded"
@@ -18,41 +16,11 @@ var _ process.Dependency = sudoerFile{}
 type sudoerFile struct{}
 
 // Installed implements Dependency
-func (s sudoerFile) Installed() bool {
-	if _, err := os.Stat(s.path()); err != nil {
-		return false
-	}
-	b, err := os.ReadFile(s.path())
-	if err != nil {
-		return false
-	}
-	txt, err := embedded.Read(s.embeddedPath())
-	if err != nil {
-		return false
-	}
-	return bytes.Contains(b, txt)
-}
+func (s sudoerFile) Installed() bool { return embedded.SudoersInstalled() }
 
-func (s sudoerFile) path() string         { return "/etc/sudoers.d/colima" }
-func (s sudoerFile) embeddedPath() string { return "network/sudo.txt" }
+// Install implements Dependency
 func (s sudoerFile) Install(host environment.HostActions) error {
-	// read embedded file contents
-	txt, err := embedded.ReadString("network/sudo.txt")
-	if err != nil {
-		return fmt.Errorf("error retrieving embedded sudo file: %w", err)
-	}
-	// ensure parent directory exists
-	dir := filepath.Dir(s.path())
-	if err := host.RunInteractive("sudo", "mkdir", "-p", dir); err != nil {
-		return fmt.Errorf("error preparing sudoers directory: %w", err)
-	}
-	// persist file to desired location
-	stdin := strings.NewReader(txt)
-	stdout := &bytes.Buffer{}
-	if err := host.RunWith(stdin, stdout, "sudo", "sh", "-c", "cat > "+s.path()); err != nil {
-		return fmt.Errorf("error writing sudoers file, stderr: %s, err: %w", stdout.String(), err)
-	}
-	return nil
+	return embedded.InstallSudoers(host)
 }
 
 var _ process.Dependency = vmnetFile{}

embedded/network/sudo.txt

@@ -5,3 +5,7 @@
 %staff ALL=(root:wheel) NOPASSWD:NOSETENV: /usr/bin/pkill -F /opt/colima/run/*.pid
 # validating vmnet daemon
 %staff ALL=(root:wheel) NOPASSWD:NOSETENV: /usr/bin/pkill -0 -F /opt/colima/run/*.pid
+# adding route to Incus container network
+%staff ALL=(root:wheel) NOPASSWD:NOSETENV: /sbin/route add -net 192.168.100.0/24 *
+# removing route to Incus container network
+%staff ALL=(root:wheel) NOPASSWD:NOSETENV: /sbin/route delete -net 192.168.100.0/24

embedded/sudoers.go

@@ -0,0 +1,63 @@
+package embedded
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	log "github.com/sirupsen/logrus"
+)
+
+const sudoersPath = "/etc/sudoers.d/colima"
+const sudoersEmbeddedPath = "network/sudo.txt"
+
+// SudoersInstaller provides the ability to run commands on the host
+// for installing the sudoers file.
+type SudoersInstaller interface {
+	RunInteractive(args ...string) error
+	RunWith(stdin io.Reader, stdout io.Writer, args ...string) error
+}
+
+// SudoersInstalled checks if the sudoers file contains the expected embedded content.
+func SudoersInstalled() bool {
+	txt, err := Read(sudoersEmbeddedPath)
+	if err != nil {
+		return false
+	}
+	b, err := os.ReadFile(sudoersPath)
+	if err != nil {
+		return false
+	}
+	return bytes.Contains(b, txt)
+}
+
+// InstallSudoers installs the embedded sudoers file if it is not already
+// installed with the expected content. This may prompt for a sudo password.
+func InstallSudoers(host SudoersInstaller) error {
+	if SudoersInstalled() {
+		return nil
+	}
+
+	txt, err := ReadString(sudoersEmbeddedPath)
+	if err != nil {
+		return fmt.Errorf("error reading embedded sudoers file: %w", err)
+	}
+
+	log.Println("setting up network permissions, sudo password may be required")
+
+	dir := filepath.Dir(sudoersPath)
+	if err := host.RunInteractive("sudo", "mkdir", "-p", dir); err != nil {
+		return fmt.Errorf("error preparing sudoers directory: %w", err)
+	}
+
+	stdin := strings.NewReader(txt)
+	stdout := &bytes.Buffer{}
+	if err := host.RunWith(stdin, stdout, "sudo", "sh", "-c", "cat > "+sudoersPath); err != nil {
+		return fmt.Errorf("error writing sudoers file: %w", err)
+	}
+
+	return nil
+}

environment/container/incus/config.yaml

@@ -1,6 +1,6 @@
 networks:
   - config:
-      ipv4.address: 192.168.100.1/24
+      ipv4.address: {{.BridgeGateway}}
       ipv4.nat: "true"
       ipv6.address: auto
     description: ""

environment/container/incus/incus.go

@@ -81,12 +81,14 @@ func (c *incusRuntime) Provision(ctx context.Context) error {
 	}
 
 	var value struct {
-		Disk       int
-		Interface  string
-		SetStorage bool
+		Disk          int
+		Interface     string
+		BridgeGateway string
+		SetStorage    bool
 	}
 	value.Disk = conf.Disk
 	value.Interface = incusBridgeInterface
+	value.BridgeGateway = bridgeGateway
 	value.SetStorage = emptyDisk // set only when the disk is empty
 
 	buf, err := util.ParseTemplate(configYaml, value)
@@ -195,13 +197,25 @@ func (c *incusRuntime) Start(ctx context.Context) error {
 		return nil
 	})
 
+	a.Add(func() error {
+		if err := c.addContainerRoute(); err != nil {
+			return cli.ErrNonFatal(err)
+		}
+		return nil
+	})
+
 	return a.Exec()
 }
 
 // Stop implements environment.Container.
 func (c *incusRuntime) Stop(ctx context.Context) error {
 	a := c.Init(ctx)
 
+	a.Add(func() error {
+		_ = c.removeContainerRoute()
+		return nil
+	})
+
 	a.Add(func() error {
 		return c.guest.RunQuiet("sudo", "incus", "admin", "shutdown")
 	})
@@ -215,6 +229,11 @@ func (c *incusRuntime) Stop(ctx context.Context) error {
 func (c *incusRuntime) Teardown(ctx context.Context) error {
 	a := c.Init(ctx)
 
+	a.Add(func() error {
+		_ = c.removeContainerRoute()
+		return nil
+	})
+
 	a.Add(c.unsetRemote)
 
 	return a.Exec()

environment/container/incus/route.go

@@ -0,0 +1,59 @@
+package incus
+
+import (
+	"fmt"
+
+	"github.com/abiosoft/colima/config"
+	"github.com/abiosoft/colima/embedded"
+	"github.com/abiosoft/colima/environment/vm/lima/limautil"
+	"github.com/abiosoft/colima/util"
+	log "github.com/sirupsen/logrus"
+)
+
+const BridgeSubnet = "192.168.100.0/24"
+const bridgeGateway = "192.168.100.1/24"
+
+// addContainerRoute adds a macOS route for the Incus container subnet
+// via the VM's col0 IP address, making containers directly reachable from the host.
+func (c *incusRuntime) addContainerRoute() error {
+	if !util.MacOS() {
+		return nil
+	}
+
+	vmIP := limautil.IPAddress(config.CurrentProfile().ID)
+	if vmIP == "127.0.0.1" || vmIP == "" {
+		return nil
+	}
+
+	if !util.SubnetAvailable(BridgeSubnet) {
+		log.Warnf("subnet %s conflicts with host network, skipping route setup", BridgeSubnet)
+		return nil
+	}
+
+	if err := embedded.InstallSudoers(c.host); err != nil {
+		return fmt.Errorf("error setting up sudoers for route: %w", err)
+	}
+
+	// delete any stale route first (ignore errors)
+	_ = c.removeContainerRoute()
+
+	if err := c.host.RunQuiet("sudo", "/sbin/route", "add", "-net", BridgeSubnet, vmIP); err != nil {
+		return fmt.Errorf("error adding route for %s via %s: %w", BridgeSubnet, vmIP, err)
+	}
+
+	return nil
+}
+
+// removeContainerRoute removes the macOS route for the Incus container subnet.
+// This is best-effort and errors are silently ignored.
+func (c *incusRuntime) removeContainerRoute() error {
+	if !util.MacOS() {
+		return nil
+	}
+
+	if !util.RouteExists(BridgeSubnet) {
+		return nil
+	}
+
+	return c.host.RunQuiet("sudo", "/sbin/route", "delete", "-net", BridgeSubnet)
+}

environment/vm/lima/daemon.go

@@ -9,14 +9,13 @@ import (
 	"github.com/abiosoft/colima/daemon"
 	"github.com/abiosoft/colima/daemon/process/inotify"
 	"github.com/abiosoft/colima/daemon/process/vmnet"
-	"github.com/abiosoft/colima/environment/container/incus"
 	"github.com/abiosoft/colima/environment/vm/lima/limaconfig"
 	"github.com/abiosoft/colima/util"
 )
 
 func (l *limaVM) startDaemon(ctx context.Context, conf config.Config) (context.Context, error) {
-	// vmnet is used by QEMU and always used by incus (even with VZ)
-	useVmnet := conf.VMType == limaconfig.QEMU || conf.Runtime == incus.Name || conf.Network.Mode == "bridged"
+	// vmnet is used by QEMU or bridged mode
+	useVmnet := conf.VMType == limaconfig.QEMU || conf.Network.Mode == "bridged"
 
 	// network daemon is only needed for vmnet
 	conf.Network.Address = conf.Network.Address && useVmnet

environment/vm/lima/lima.go

@@ -211,6 +211,8 @@ func (l limaVM) Stop(ctx context.Context, force bool) error {
 
 	a.Add(func() error { l.removeHostAddresses(); return nil })
 
+	a.Add(func() error { l.removeIncusContainerRoute(); return nil })
+
 	a.Add(func() error {
 		if force {
 			return l.host.Run(limactl, "stop", "--force", config.CurrentProfile().ID)

environment/vm/lima/network.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/abiosoft/colima/config"
 	"github.com/abiosoft/colima/config/configmanager"
+	"github.com/abiosoft/colima/environment/container/incus"
 	"github.com/abiosoft/colima/environment/vm/lima/limautil"
 	"github.com/abiosoft/colima/util"
 	"github.com/sirupsen/logrus"
@@ -76,3 +77,21 @@ func (l *limaVM) removeHostAddresses() {
 		}
 	}
 }
+
+// removeIncusContainerRoute is a safety net for force-stop,
+// where the Incus container Stop() is skipped.
+func (l *limaVM) removeIncusContainerRoute() {
+	if !util.MacOS() {
+		return
+	}
+
+	if l.conf.Runtime != incus.Name {
+		return
+	}
+
+	if !util.RouteExists(incus.BridgeSubnet) {
+		return
+	}
+
+	_ = l.host.RunQuiet("sudo", "/sbin/route", "delete", "-net", incus.BridgeSubnet)
+}

environment/vm/lima/yaml.go

@@ -142,8 +142,8 @@ func newConf(ctx context.Context, conf config.Config) (l limaconfig.Config, err
 			if conf.Network.PreferredRoute {
 				metric = limautil.NetMetricPreferred
 			}
-			// vmnet is always used for incus runtime or bridged mode
-			if l.VMType == limaconfig.VZ && conf.Runtime != incus.Name && conf.Network.Mode != "bridged" {
+			// vmnet is used for bridged mode, otherwise VZ uses VZNAT
+			if l.VMType == limaconfig.VZ && conf.Network.Mode != "bridged" {
 				l.Networks = append(l.Networks, limaconfig.Network{
 					VZNAT:     true,
 					Interface: limautil.NetInterface,

util/util.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"net"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"strings"
 
@@ -54,6 +55,65 @@ func HostIPAddresses() []net.IP {
 	return addresses
 }
 
+// SubnetAvailable checks if a subnet (in CIDR notation) does not conflict
+// with any existing host network interface addresses.
+func SubnetAvailable(subnet string) bool {
+	_, cidr, err := net.ParseCIDR(subnet)
+	if err != nil {
+		return false
+	}
+
+	addrs, err := net.InterfaceAddrs()
+	if err != nil {
+		return false
+	}
+
+	for _, addr := range addrs {
+		ip, _, err := net.ParseCIDR(addr.String())
+		if err != nil {
+			continue
+		}
+		if ip = ip.To4(); ip == nil {
+			continue
+		}
+		if cidr.Contains(ip) {
+			return false
+		}
+	}
+
+	return true
+}
+
+// RouteExists checks if a route exists for the given subnet on macOS.
+func RouteExists(subnet string) bool {
+	if !MacOS() {
+		return false
+	}
+
+	ip, _, err := net.ParseCIDR(subnet)
+	if err != nil {
+		return false
+	}
+
+	out, err := exec.Command("netstat", "-rn", "-f", "inet").Output()
+	if err != nil {
+		return false
+	}
+
+	// macOS netstat shows /24 subnets without trailing .0
+	// e.g. "192.168.100" instead of "192.168.100.0"
+	networkAddr := strings.TrimSuffix(ip.String(), ".0")
+
+	for _, line := range strings.Split(string(out), "\n") {
+		fields := strings.Fields(line)
+		if len(fields) > 0 && (fields[0] == networkAddr || fields[0] == subnet) {
+			return true
+		}
+	}
+
+	return false
+}
+
 // ShellSplit splits cmd into arguments using.
 func ShellSplit(cmd string) []string {
 	split, err := shlex.Split(cmd)