ebpf: new way of compiling the modules

- Don't rename libbpf's bpf_map_def struct, and distribute the needed bpf
  headers.
  The bpf_map_def struct has been deprecated for quite some time now,
  and it was been removed on >= 6.2 anyway.
  We still need it, because we use gobpf.
- Improved compilation behaviour:
  - We don't require the kernel sources anymore. We can just use the
    kernel headers from the distribution.
  - There's no need to copy the sources to the kernel tree, the modules
    can be compiled from the ebpf_prog/ dir.
- Compiling against kernels 6.x seems to solve the problem we had with
  VPNs, where connections were not intercepted with modules compiled
  against 5.8, on kernels >= 5.19.

The modules has been tested on kernels 4.17, 5.4, 5.10, 5.15, 6.1 and
6.2 (kernel connections included).

Closes: #939

Author: gustavo-iniguez-goya

.github/workflows/build_ebpf_modules.yml

@@ -24,11 +24,11 @@ jobs:
   # The matrix configuration will execute the steps, once per dimension defined:
   # kernel 5.8 + tag 1.5.0
   # kernel 5.8 + tag master
-  # kernel 5.19 + tag 1.5.0, etc
+  # kernel 6.0 + tag 1.5.0, etc
   build:  
     strategy:
         matrix:
-          kernel: ["5.8", "5.19"]
+          kernel: ["5.8", "6.0"]
           tag: ["1.5.0", "master"]
 
     runs-on: ubuntu-20.04

ebpf_prog/Makefile

@@ -1,159 +1,57 @@
-#taken from /samples/bpf/Makefile and removed all targets 
-
-# SPDX-License-Identifier: GPL-2.0
-
-BPF_SAMPLES_PATH ?= $(abspath $(srctree)/$(src))
-TOOLS_PATH := $(BPF_SAMPLES_PATH)/../../tools
-
-# Libbpf dependencies
-LIBBPF = $(TOOLS_PATH)/lib/bpf/libbpf.a
-
-CGROUP_HELPERS := ../../tools/testing/selftests/bpf/cgroup_helpers.o
-TRACE_HELPERS := ../../tools/testing/selftests/bpf/trace_helpers.o
-
-always-y += opensnitch.o opensnitch-dns.o opensnitch-procs.o
-
-ifeq ($(ARCH), arm)
-# Strip all except -D__LINUX_ARM_ARCH__ option needed to handle linux
-# headers when arm instruction set identification is requested.
-ARM_ARCH_SELECTOR := $(filter -D__LINUX_ARM_ARCH__%, $(KBUILD_CFLAGS))
-BPF_EXTRA_CFLAGS := $(ARM_ARCH_SELECTOR)
-TPROGS_CFLAGS += $(ARM_ARCH_SELECTOR)
-endif
-
-TPROGS_CFLAGS += -Wall -O2
-TPROGS_CFLAGS += -Wmissing-prototypes
-TPROGS_CFLAGS += -Wstrict-prototypes
-
-TPROGS_CFLAGS += -I$(objtree)/usr/include
-TPROGS_CFLAGS += -I$(srctree)/tools/testing/selftests/bpf/
-TPROGS_CFLAGS += -I$(srctree)/tools/lib/
-TPROGS_CFLAGS += -I$(srctree)/tools/include
-TPROGS_CFLAGS += -I$(srctree)/tools/perf
-TPROGS_CFLAGS += -DHAVE_ATTR_TEST=0
-
-ifdef SYSROOT
-TPROGS_CFLAGS += --sysroot=$(SYSROOT)
-TPROGS_LDFLAGS := -L$(SYSROOT)/usr/lib
-endif
-
-TPROGCFLAGS_bpf_load.o += -Wno-unused-variable
-
-TPROGS_LDLIBS			+= $(LIBBPF) -lelf -lz
-TPROGLDLIBS_tracex4		+= -lrt
-TPROGLDLIBS_trace_output	+= -lrt
-TPROGLDLIBS_map_perf_test	+= -lrt
-TPROGLDLIBS_test_overhead	+= -lrt
-TPROGLDLIBS_xdpsock		+= -pthread
-
-# Allows pointing LLC/CLANG to a LLVM backend with bpf support, redefine on cmdline:
-#  make M=samples/bpf/ LLC=~/git/llvm/build/bin/llc CLANG=~/git/llvm/build/bin/clang
-LLC ?= llc
+# OpenSnitch - 2023
+#
+# On Debian based distros we need the following 2 directories.
+# Otherwise, just use the kernel headers from the kernel sources.
+#
+KERNEL_DIR ?= /lib/modules/$(shell uname -r)/source
+KERNEL_HEADERS ?= /usr/src/linux-headers-$(shell uname -r)/
 CLANG ?= clang
-LLVM_OBJCOPY ?= llvm-objcopy
-BTF_PAHOLE ?= pahole
-
-# Detect that we're cross compiling and use the cross compiler
-ifdef CROSS_COMPILE
-CLANG_ARCH_ARGS = --target=$(notdir $(CROSS_COMPILE:%-=%))
-endif
-
-# Don't evaluate probes and warnings if we need to run make recursively
-ifneq ($(src),)
-HDR_PROBE := $(shell printf "\#include <linux/types.h>\n struct list_head { int a; }; int main() { return 0; }" | \
-	$(CC) $(TPROGS_CFLAGS) $(TPROGS_LDFLAGS) -x c - \
-	-o /dev/null 2>/dev/null && echo okay)
-
-ifeq ($(HDR_PROBE),)
-$(warning WARNING: Detected possible issues with include path.)
-$(warning WARNING: Please install kernel headers locally (make headers_install).)
-endif
-
-BTF_LLC_PROBE := $(shell $(LLC) -march=bpf -mattr=help 2>&1 | grep dwarfris)
-BTF_PAHOLE_PROBE := $(shell $(BTF_PAHOLE) --help 2>&1 | grep BTF)
-BTF_OBJCOPY_PROBE := $(shell $(LLVM_OBJCOPY) --help 2>&1 | grep -i 'usage.*llvm')
-BTF_LLVM_PROBE := $(shell echo "int main() { return 0; }" | \
-			  $(CLANG) -target bpf -O2 -g -c -x c - -o ./llvm_btf_verify.o; \
-			  readelf -S ./llvm_btf_verify.o | grep BTF; \
-			  /bin/rm -f ./llvm_btf_verify.o)
-
-BPF_EXTRA_CFLAGS += -fno-stack-protector
-ifneq ($(BTF_LLVM_PROBE),)
-	BPF_EXTRA_CFLAGS += -g
-else
-ifneq ($(and $(BTF_LLC_PROBE),$(BTF_PAHOLE_PROBE),$(BTF_OBJCOPY_PROBE)),)
-	BPF_EXTRA_CFLAGS += -g
-	LLC_FLAGS += -mattr=dwarfris
-	DWARF2BTF = y
-endif
-endif
-endif
-
-# Trick to allow make to be run from this directory
-all:
-	$(MAKE) -C ../../ M=$(CURDIR) BPF_SAMPLES_PATH=$(CURDIR)
-
+LLC ?= llc
+LLVM_STRIP ?= llvm-strip -g
+ARCH ?= $(shell arch)
+
+# as in /usr/src/linux-headers-*/arch/
+# TODO: extract correctly the archs, and add more if needed.
+ifeq ($(ARCH),x86_64)
+	ARCH := x86
+else ifeq ($(ARCH),i686)
+	ARCH := x86
+else ifeq ($(ARCH),armv7l)
+	ARCH := arm
+else ifeq ($(ARCH),aarch64)
+	ARCH := arm64
+endif
+
+ifeq ($(ARCH),arm)
+	# on previous archs, it fails with "SMP not supported on pre-ARMv6"
+	EXTRA_FLAGS = "-D__LINUX_ARM_ARCH__=7"
+endif
+
+BIN := opensnitch.o opensnitch-procs.o opensnitch-dns.o
+CLANG_FLAGS = -I. \
+	-I$(KERNEL_HEADERS)/arch/x86/include/generated/ \
+	-I$(KERNEL_HEADERS)/include \
+	-include $(KERNEL_DIR)/include/linux/kconfig.h \
+	-I$(KERNEL_DIR)/include \
+	-I$(KERNEL_DIR)/include/uapi \
+	-I$(KERNEL_DIR)/include/generated/uapi \
+	-I$(KERNEL_DIR)/arch/$(ARCH)/include \
+	-I$(KERNEL_DIR)/arch/$(ARCH)/include/generated \
+	-I$(KERNEL_DIR)/arch/$(ARCH)/include/uapi \
+	-I$(KERNEL_DIR)/arch/$(ARCH)/include/generated/uapi \
+	-I$(KERNEL_DIR)/tools/testing/selftests/bpf/ \
+	-D__KERNEL__ -D__BPF_TRACING__ -Wno-unused-value -Wno-pointer-sign \
+	-D__TARGET_ARCH_$(ARCH) -Wno-compare-distinct-pointer-types \
+	$(EXTRA_FLAGS) \
+	-Wno-gnu-variable-sized-type-not-at-end \
+	-Wno-address-of-packed-member -Wno-tautological-compare \
+	-Wno-unknown-warning-option  \
+	-g -O2 -emit-llvm
+
+all: $(BIN)
+
+%.o: %.c
+	$(CLANG) $(CLANG_FLAGS) -c $< -o - | \
+		$(LLC) -march=bpf -mcpu=$(CPU) -filetype=obj -o $@
 clean:
-	$(MAKE) -C ../../ M=$(CURDIR) clean
-	@find $(CURDIR) -type f -name '*~' -delete
-
-$(LIBBPF): FORCE
-# Fix up variables inherited from Kbuild that tools/ build system won't like
-	$(MAKE) -C $(dir $@) RM='rm -rf' EXTRA_CFLAGS="$(TPROGS_CFLAGS)" \
-		LDFLAGS=$(TPROGS_LDFLAGS) srctree=$(BPF_SAMPLES_PATH)/../../ O=
-
-$(obj)/syscall_nrs.h:	$(obj)/syscall_nrs.s FORCE
-	$(call filechk,offsets,__SYSCALL_NRS_H__)
-
-targets += syscall_nrs.s
-clean-files += syscall_nrs.h
-
-FORCE:
-
-
-# Verify LLVM compiler tools are available and bpf target is supported by llc
-.PHONY: verify_cmds verify_target_bpf $(CLANG) $(LLC)
-
-verify_cmds: $(CLANG) $(LLC)
-	@for TOOL in $^ ; do \
-		if ! (which -- "$${TOOL}" > /dev/null 2>&1); then \
-			echo "*** ERROR: Cannot find LLVM tool $${TOOL}" ;\
-			exit 1; \
-		else true; fi; \
-	done
-
-verify_target_bpf: verify_cmds
-	@if ! (${LLC} -march=bpf -mattr=help > /dev/null 2>&1); then \
-		echo "*** ERROR: LLVM (${LLC}) does not support 'bpf' target" ;\
-		echo "   NOTICE: LLVM version >= 3.7.1 required" ;\
-		exit 2; \
-	else true; fi
-
-$(BPF_SAMPLES_PATH)/*.c: verify_target_bpf $(LIBBPF)
-$(src)/*.c: verify_target_bpf $(LIBBPF)
-
-$(obj)/tracex5_kern.o: $(obj)/syscall_nrs.h
-$(obj)/hbm_out_kern.o: $(src)/hbm.h $(src)/hbm_kern.h
-$(obj)/hbm.o: $(src)/hbm.h
-$(obj)/hbm_edt_kern.o: $(src)/hbm.h $(src)/hbm_kern.h
-
--include $(BPF_SAMPLES_PATH)/Makefile.target
-
-# asm/sysreg.h - inline assembly used by it is incompatible with llvm.
-# But, there is no easy way to fix it, so just exclude it since it is
-# useless for BPF samples.
-$(obj)/%.o: $(src)/%.c
-	@echo "  CLANG-bpf " $@
-	$(Q)$(CLANG) $(NOSTDINC_FLAGS) $(LINUXINCLUDE) $(BPF_EXTRA_CFLAGS) \
-		-I$(obj) -I$(srctree)/tools/testing/selftests/bpf/ \
-		-I$(srctree)/tools/lib/ \
-		-D__KERNEL__ -D__BPF_TRACING__ -Wno-unused-value -Wno-pointer-sign \
-		-D__TARGET_ARCH_$(SRCARCH) -Wno-compare-distinct-pointer-types \
-		-Wno-gnu-variable-sized-type-not-at-end \
-		-Wno-address-of-packed-member -Wno-tautological-compare \
-		-Wno-unknown-warning-option $(CLANG_ARCH_ARGS) \
-		-I$(srctree)/samples/bpf/ -include asm_goto_workaround.h \
-		-O2 -emit-llvm -c $< -o -| $(LLC) -march=bpf $(LLC_FLAGS) -filetype=obj -o $@
-ifeq ($(DWARF2BTF),y)
-	$(BTF_PAHOLE) -J $@
-endif
+	rm -f *.o

ebpf_prog/README

@@ -1,4 +1,4 @@
-Compilation requires getting kernel sources.
+Compilation requires getting kernel sources for now.
 
 There's a helper script to automate this process:
  https://github.com/evilsocket/opensnitch/blob/master/utils/packaging/build_modules.sh
@@ -9,24 +9,36 @@ The basic steps to compile the modules are:
   cd opensnitch
   wget https://github.com/torvalds/linux/archive/v5.8.tar.gz
   tar -xf v5.8.tar.gz
-  patch linux-5.8/tools/lib/bpf/bpf_helpers.h < ebpf_prog/file.patch
-  cp ebpf_prog/opensnitch*.c ebpf_prog/common.h ebpf_prog/Makefile linux-5.8/samples/bpf
+  cp ebpf_prog/opensnitch*.c ebpf_prog/common* ebpf_prog/Makefile linux-5.8/samples/bpf/
+  cp -r ebpf_prog/bpf_headers/ linux-5.8/samples/bpf/
   cd linux-5.8 && yes "" | make oldconfig && make prepare && make headers_install # (1 min)
-  cd samples/bpf && make
-  objdump -h opensnitch.o #you should see many section, number 1 should be called kprobe/tcp_v4_connect
-  llvm-strip -g opensnitch.o #remove debug info
-  sudo cp opensnitch*.o /etc/opensnitchd/
+  cd samples/bpf && make KERNEL_DIR=../../linux-5.8/
+  objdump -h opensnitch.o # you should see many sections, number 1 should be called kprobe/tcp_v4_connect
+  llvm-strip -g opensnitch*.o # remove debug info
+  sudo cp opensnitch*.o /usr/lib/opensnitchd/ebpf/ # or /etc/opensnitchd for < v1.6.x
   cd ../../../daemon
 
-opensnitchd expects to find opensnitch.o in:
+Since v1.6.0, opensnitchd expects to find the opensnitch*.o modules under:
  /usr/local/lib/opensnitchd/ebpf/
  /usr/lib/opensnitchd/ebpf/
- /etc/opensnitchd/ # deprecated
+ /etc/opensnitchd/ # deprecated, only on < v1.5.x
 
 start opensnitchd with:
 
   opensnitchd -rules-path /etc/opensnitchd/rules -process-monitor-method ebpf
 
+---
+
+### Compiling for Fedora (and others rpm based systems)
+
+You need to install the kernel-devel, clang and llvm packages.
+
+Then: `cd ebpf_prog/ ; make KERNEL_DIR=/usr/src/kernels/$(uname -r)/`
+
+(or just pass the kernel version you want)
+
+### Notes
+
 The kernel where you intend to run it must have some options activated:
 
  $ grep BPF /boot/config-$(uname -r)
@@ -42,11 +54,19 @@ For the opensnitch-procs.o module to work, this option must be enabled:
  $ grep FTRACE_SYSCALLS /boot/config-$(uname -r)
   CONFIG_FTRACE_SYSCALLS=y
 
-Also, in some distributions debugfs is not mounted automatically, so you need
-to do it manually:
+(https://github.com/iovisor/bcc/blob/master/docs/kernel_config.md)
+
+Also, in some distributions debugfs is not mounted automatically.
+Since v1.6.0 we try to mount it automatically. If you're running
+a lower version so you'll need to mount it manually:
 
  $ sudo mount -t debugfs none /sys/kernel/debug
 
 In order to make it permanent add it to /etc/fstab:
 
 debugfs    /sys/kernel/debug      debugfs  defaults  0 0
+
+
+opensnitch-procs.o and opensnitch-dns.o are only compatible with kernels >= 5.5,
+bpf_probe_read_user*() were added on that kernel on:
+https://github.com/iovisor/bcc/blob/master/docs/kernel-versions.md#helpers