diff options
Diffstat (limited to 'samples/bpf')
83 files changed, 620 insertions, 175 deletions
diff --git a/samples/bpf/Makefile b/samples/bpf/Makefile index 1fc42ad8ff49..b0e8adf7eb01 100644 --- a/samples/bpf/Makefile +++ b/samples/bpf/Makefile @@ -38,6 +38,8 @@ tprogs-y += tc_l2_redirect tprogs-y += lwt_len_hist tprogs-y += xdp_tx_iptunnel tprogs-y += test_map_in_map +tprogs-y += per_socket_stats_example +tprogs-y += xdp_redirect tprogs-y += xdp_redirect_map tprogs-y += xdp_redirect_cpu tprogs-y += xdp_monitor @@ -182,7 +184,6 @@ TPROGS_CFLAGS += -Wmissing-prototypes TPROGS_CFLAGS += -Wstrict-prototypes TPROGS_CFLAGS += -I$(objtree)/usr/include -TPROGS_CFLAGS += -I$(srctree)/tools/lib/bpf/ TPROGS_CFLAGS += -I$(srctree)/tools/testing/selftests/bpf/ TPROGS_CFLAGS += -I$(srctree)/tools/lib/ TPROGS_CFLAGS += -I$(srctree)/tools/include @@ -196,7 +197,7 @@ endif TPROGCFLAGS_bpf_load.o += -Wno-unused-variable -TPROGS_LDLIBS += $(LIBBPF) -lelf +TPROGS_LDLIBS += $(LIBBPF) -lelf -lz TPROGLDLIBS_tracex4 += -lrt TPROGLDLIBS_trace_output += -lrt TPROGLDLIBS_map_perf_test += -lrt @@ -234,6 +235,7 @@ BTF_LLVM_PROBE := $(shell echo "int main() { return 0; }" | \ readelf -S ./llvm_btf_verify.o | grep BTF; \ /bin/rm -f ./llvm_btf_verify.o) +BPF_EXTRA_CFLAGS += -fno-stack-protector ifneq ($(BTF_LLVM_PROBE),) BPF_EXTRA_CFLAGS += -g else @@ -251,7 +253,7 @@ all: clean: $(MAKE) -C ../../ M=$(CURDIR) clean - @rm -f *~ + @find $(CURDIR) -type f -name '*~' -delete $(LIBBPF): FORCE # Fix up variables inherited from Kbuild that tools/ build system won't like @@ -302,7 +304,7 @@ $(obj)/%.o: $(src)/%.c @echo " CLANG-bpf " $@ $(Q)$(CLANG) $(NOSTDINC_FLAGS) $(LINUXINCLUDE) $(BPF_EXTRA_CFLAGS) \ -I$(obj) -I$(srctree)/tools/testing/selftests/bpf/ \ - -I$(srctree)/tools/lib/bpf/ \ + -I$(srctree)/tools/lib/ \ -D__KERNEL__ -D__BPF_TRACING__ -Wno-unused-value -Wno-pointer-sign \ -D__TARGET_ARCH_$(SRCARCH) -Wno-compare-distinct-pointer-types \ -Wno-gnu-variable-sized-type-not-at-end \ diff --git a/samples/bpf/cpustat_kern.c b/samples/bpf/cpustat_kern.c index 68c84da065b1..a86a19d5f033 100644 --- a/samples/bpf/cpustat_kern.c +++ b/samples/bpf/cpustat_kern.c @@ -3,7 +3,7 @@ #include <linux/version.h> #include <linux/ptrace.h> #include <uapi/linux/bpf.h> -#include "bpf_helpers.h" +#include <bpf/bpf_helpers.h> /* * The CPU number, cstate number and pstate number are based diff --git a/samples/bpf/fds_example.c b/samples/bpf/fds_example.c index 2d4b717726b6..d5992f787232 100644 --- a/samples/bpf/fds_example.c +++ b/samples/bpf/fds_example.c @@ -14,7 +14,7 @@ #include <bpf/bpf.h> -#include "libbpf.h" +#include <bpf/libbpf.h> #include "bpf_insn.h" #include "sock_example.h" diff --git a/samples/bpf/hbm.c b/samples/bpf/hbm.c index 829b68d87687..7d7153777678 100644 --- a/samples/bpf/hbm.c +++ b/samples/bpf/hbm.c @@ -50,8 +50,8 @@ #include "cgroup_helpers.h" #include "hbm.h" #include "bpf_util.h" -#include "bpf.h" -#include "libbpf.h" +#include <bpf/bpf.h> +#include <bpf/libbpf.h> bool outFlag = true; int minRate = 1000; /* cgroup rate limit in Mbps */ diff --git a/samples/bpf/hbm_kern.h b/samples/bpf/hbm_kern.h index 4edaf47876ca..e00f26f6afba 100644 --- a/samples/bpf/hbm_kern.h +++ b/samples/bpf/hbm_kern.h @@ -22,8 +22,8 @@ #include <uapi/linux/pkt_cls.h> #include <net/ipv6.h> #include <net/inet_ecn.h> -#include "bpf_endian.h" -#include "bpf_helpers.h" +#include <bpf/bpf_endian.h> +#include <bpf/bpf_helpers.h> #include "hbm.h" #define DROP_PKT 0 diff --git a/samples/bpf/ibumad_kern.c b/samples/bpf/ibumad_kern.c index f281df7e0089..3a91b4c1989a 100644 --- a/samples/bpf/ibumad_kern.c +++ b/samples/bpf/ibumad_kern.c @@ -13,7 +13,7 @@ #define KBUILD_MODNAME "ibumad_count_pkts_by_class" #include <uapi/linux/bpf.h> -#include "bpf_helpers.h" +#include <bpf/bpf_helpers.h> struct bpf_map_def SEC("maps") read_count = { diff --git a/samples/bpf/ibumad_user.c b/samples/bpf/ibumad_user.c index cb5a8f994849..fa06eef31a84 100644 --- a/samples/bpf/ibumad_user.c +++ b/samples/bpf/ibumad_user.c @@ -25,7 +25,7 @@ #include "bpf_load.h" #include "bpf_util.h" -#include "libbpf.h" +#include <bpf/libbpf.h> static void dump_counts(int fd) { diff --git a/samples/bpf/lathist_kern.c b/samples/bpf/lathist_kern.c index 18fa088473cd..ca9c2e4e69aa 100644 --- a/samples/bpf/lathist_kern.c +++ b/samples/bpf/lathist_kern.c @@ -8,7 +8,7 @@ #include <linux/version.h> #include <linux/ptrace.h> #include <uapi/linux/bpf.h> -#include "bpf_helpers.h" +#include <bpf/bpf_helpers.h> #define MAX_ENTRIES 20 #define MAX_CPU 4 diff --git a/samples/bpf/lwt_len_hist_kern.c b/samples/bpf/lwt_len_hist_kern.c index df75383280f9..9ed63e10e170 100644 --- a/samples/bpf/lwt_len_hist_kern.c +++ b/samples/bpf/lwt_len_hist_kern.c @@ -14,7 +14,7 @@ #include <uapi/linux/if_ether.h> #include <uapi/linux/ip.h> #include <uapi/linux/in.h> -#include "bpf_helpers.h" +#include <bpf/bpf_helpers.h> # define printk(fmt, ...) \ ({ \ diff --git a/samples/bpf/map_perf_test_kern.c b/samples/bpf/map_perf_test_kern.c index 281bcdaee58e..12e91ae64d4d 100644 --- a/samples/bpf/map_perf_test_kern.c +++ b/samples/bpf/map_perf_test_kern.c @@ -8,9 +8,9 @@ #include <linux/netdevice.h> #include <linux/version.h> #include <uapi/linux/bpf.h> -#include "bpf_helpers.h" +#include <bpf/bpf_helpers.h> #include "bpf_legacy.h" -#include "bpf_tracing.h" +#include <bpf/bpf_tracing.h> #define MAX_ENTRIES 1000 #define MAX_NR_CPUS 1024 diff --git a/samples/bpf/offwaketime_kern.c b/samples/bpf/offwaketime_kern.c index 9cb5207a692f..c4ec10dbfc3b 100644 --- a/samples/bpf/offwaketime_kern.c +++ b/samples/bpf/offwaketime_kern.c @@ -5,8 +5,8 @@ * License as published by the Free Software Foundation. */ #include <uapi/linux/bpf.h> -#include "bpf_helpers.h" -#include "bpf_tracing.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> #include <uapi/linux/ptrace.h> #include <uapi/linux/perf_event.h> #include <linux/version.h> diff --git a/samples/bpf/offwaketime_user.c b/samples/bpf/offwaketime_user.c index fc8767d001f6..51c7da5341cc 100644 --- a/samples/bpf/offwaketime_user.c +++ b/samples/bpf/offwaketime_user.c @@ -12,7 +12,7 @@ #include <assert.h> #include <stdbool.h> #include <sys/resource.h> -#include "libbpf.h" +#include <bpf/libbpf.h> #include "bpf_load.h" #include "trace_helpers.h" diff --git a/samples/bpf/parse_ldabs.c b/samples/bpf/parse_ldabs.c index ef5892377beb..c6f65f90a097 100644 --- a/samples/bpf/parse_ldabs.c +++ b/samples/bpf/parse_ldabs.c @@ -11,7 +11,7 @@ #include <linux/tcp.h> #include <linux/udp.h> #include <uapi/linux/bpf.h> -#include "bpf_helpers.h" +#include <bpf/bpf_helpers.h> #include "bpf_legacy.h" #define DEFAULT_PKTGEN_UDP_PORT 9 diff --git a/samples/bpf/parse_simple.c b/samples/bpf/parse_simple.c index 10af53d33cc2..4a486cb1e0df 100644 --- a/samples/bpf/parse_simple.c +++ b/samples/bpf/parse_simple.c @@ -12,7 +12,7 @@ #include <linux/udp.h> #include <uapi/linux/bpf.h> #include <net/ip.h> -#include "bpf_helpers.h" +#include <bpf/bpf_helpers.h> #define DEFAULT_PKTGEN_UDP_PORT 9 diff --git a/samples/bpf/parse_varlen.c b/samples/bpf/parse_varlen.c index 0b6f22feb2c9..d8623846e810 100644 --- a/samples/bpf/parse_varlen.c +++ b/samples/bpf/parse_varlen.c @@ -14,7 +14,7 @@ #include <linux/udp.h> #include <uapi/linux/bpf.h> #include <net/ip.h> -#include "bpf_helpers.h" +#include <bpf/bpf_helpers.h> #define DEFAULT_PKTGEN_UDP_PORT 9 #define DEBUG 0 diff --git a/samples/bpf/sampleip_kern.c b/samples/bpf/sampleip_kern.c index 4a190893894f..e504dc308371 100644 --- a/samples/bpf/sampleip_kern.c +++ b/samples/bpf/sampleip_kern.c @@ -8,8 +8,8 @@ #include <linux/ptrace.h> #include <uapi/linux/bpf.h> #include <uapi/linux/bpf_perf_event.h> -#include "bpf_helpers.h" -#include "bpf_tracing.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> #define MAX_IPS 8192 diff --git a/samples/bpf/sampleip_user.c b/samples/bpf/sampleip_user.c index 6b5dc26d9701..b0f115f938bc 100644 --- a/samples/bpf/sampleip_user.c +++ b/samples/bpf/sampleip_user.c @@ -15,7 +15,7 @@ #include <linux/ptrace.h> #include <linux/bpf.h> #include <sys/ioctl.h> -#include "libbpf.h" +#include <bpf/libbpf.h> #include "bpf_load.h" #include "perf-sys.h" #include "trace_helpers.h" diff --git a/samples/bpf/sock_flags_kern.c b/samples/bpf/sock_flags_kern.c index 05dcdf8a4baa..6d0ac7569d6f 100644 --- a/samples/bpf/sock_flags_kern.c +++ b/samples/bpf/sock_flags_kern.c @@ -3,7 +3,7 @@ #include <linux/net.h> #include <uapi/linux/in.h> #include <uapi/linux/in6.h> -#include "bpf_helpers.h" +#include <bpf/bpf_helpers.h> SEC("cgroup/sock1") int bpf_prog1(struct bpf_sock *sk) diff --git a/samples/bpf/sockex1_kern.c b/samples/bpf/sockex1_kern.c index 2408dbfb7a21..431c956460ad 100644 --- a/samples/bpf/sockex1_kern.c +++ b/samples/bpf/sockex1_kern.c @@ -2,7 +2,7 @@ #include <uapi/linux/if_ether.h> #include <uapi/linux/if_packet.h> #include <uapi/linux/ip.h> -#include "bpf_helpers.h" +#include <bpf/bpf_helpers.h> #include "bpf_legacy.h" struct { diff --git a/samples/bpf/sockex1_user.c b/samples/bpf/sockex1_user.c index a219442afbee..3c83722877dc 100644 --- a/samples/bpf/sockex1_user.c +++ b/samples/bpf/sockex1_user.c @@ -3,7 +3,7 @@ #include <assert.h> #include <linux/bpf.h> #include <bpf/bpf.h> -#include "libbpf.h" +#include <bpf/libbpf.h> #include "sock_example.h" #include <unistd.h> #include <arpa/inet.h> diff --git a/samples/bpf/sockex2_kern.c b/samples/bpf/sockex2_kern.c index a7bcd03bf529..a41dd520bc53 100644 --- a/samples/bpf/sockex2_kern.c +++ b/samples/bpf/sockex2_kern.c @@ -1,5 +1,5 @@ #include <uapi/linux/bpf.h> -#include "bpf_helpers.h" +#include <bpf/bpf_helpers.h> #include "bpf_legacy.h" #include <uapi/linux/in.h> #include <uapi/linux/if.h> diff --git a/samples/bpf/sockex2_user.c b/samples/bpf/sockex2_user.c index 6de383ddd08b..af925a5afd1d 100644 --- a/samples/bpf/sockex2_user.c +++ b/samples/bpf/sockex2_user.c @@ -3,7 +3,7 @@ #include <assert.h> #include <linux/bpf.h> #include <bpf/bpf.h> -#include "libbpf.h" +#include <bpf/libbpf.h> #include "sock_example.h" #include <unistd.h> #include <arpa/inet.h> diff --git a/samples/bpf/sockex3_kern.c b/samples/bpf/sockex3_kern.c index 151dd842ecc0..36d4dac23549 100644 --- a/samples/bpf/sockex3_kern.c +++ b/samples/bpf/sockex3_kern.c @@ -5,7 +5,7 @@ * License as published by the Free Software Foundation. */ #include <uapi/linux/bpf.h> -#include "bpf_helpers.h" +#include <bpf/bpf_helpers.h> #include "bpf_legacy.h" #include <uapi/linux/in.h> #include <uapi/linux/if.h> diff --git a/samples/bpf/spintest_kern.c b/samples/bpf/spintest_kern.c index 6e9478aa2938..f508af357251 100644 --- a/samples/bpf/spintest_kern.c +++ b/samples/bpf/spintest_kern.c @@ -9,8 +9,8 @@ #include <linux/version.h> #include <uapi/linux/bpf.h> #include <uapi/linux/perf_event.h> -#include "bpf_helpers.h" -#include "bpf_tracing.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> struct bpf_map_def SEC("maps") my_map = { .type = BPF_MAP_TYPE_HASH, diff --git a/samples/bpf/spintest_user.c b/samples/bpf/spintest_user.c index 2556af2d9b3e..fb430ea2ef51 100644 --- a/samples/bpf/spintest_user.c +++ b/samples/bpf/spintest_user.c @@ -5,7 +5,7 @@ #include <string.h> #include <assert.h> #include <sys/resource.h> -#include "libbpf.h" +#include <bpf/libbpf.h> #include "bpf_load.h" #include "trace_helpers.h" diff --git a/samples/bpf/syscall_tp_kern.c b/samples/bpf/syscall_tp_kern.c index 630ce8c4d5a2..5a62b03b1f88 100644 --- a/samples/bpf/syscall_tp_kern.c +++ b/samples/bpf/syscall_tp_kern.c @@ -2,7 +2,7 @@ /* Copyright (c) 2017 Facebook */ #include <uapi/linux/bpf.h> -#include "bpf_helpers.h" +#include <bpf/bpf_helpers.h> struct syscalls_enter_open_args { unsigned long long unused; diff --git a/samples/bpf/task_fd_query_kern.c b/samples/bpf/task_fd_query_kern.c index fb56fc2a3e5d..278ade5427c8 100644 --- a/samples/bpf/task_fd_query_kern.c +++ b/samples/bpf/task_fd_query_kern.c @@ -2,7 +2,7 @@ #include <linux/version.h> #include <linux/ptrace.h> #include <uapi/linux/bpf.h> -#include "bpf_helpers.h" +#include <bpf/bpf_helpers.h> SEC("kprobe/blk_mq_start_request") int bpf_prog1(struct pt_regs *ctx) diff --git a/samples/bpf/task_fd_query_user.c b/samples/bpf/task_fd_query_user.c index 4c31b305e6ef..ff2e9c1c7266 100644 --- a/samples/bpf/task_fd_query_user.c +++ b/samples/bpf/task_fd_query_user.c @@ -15,7 +15,7 @@ #include <sys/stat.h> #include <linux/perf_event.h> -#include "libbpf.h" +#include <bpf/libbpf.h> #include "bpf_load.h" #include "bpf_util.h" #include "perf-sys.h" diff --git a/samples/bpf/tc_l2_redirect_kern.c b/samples/bpf/tc_l2_redirect_kern.c index 7ef2a12b25b2..fd2fa0004330 100644 --- a/samples/bpf/tc_l2_redirect_kern.c +++ b/samples/bpf/tc_l2_redirect_kern.c @@ -15,7 +15,7 @@ #include <uapi/linux/filter.h> #include <uapi/linux/pkt_cls.h> #include <net/ipv6.h> -#include "bpf_helpers.h" +#include <bpf/bpf_helpers.h> #define _htonl __builtin_bswap32 diff --git a/samples/bpf/tcbpf1_kern.c b/samples/bpf/tcbpf1_kern.c index ff43341bdfce..e9356130f84e 100644 --- a/samples/bpf/tcbpf1_kern.c +++ b/samples/bpf/tcbpf1_kern.c @@ -7,7 +7,7 @@ #include <uapi/linux/tcp.h> #include <uapi/linux/filter.h> #include <uapi/linux/pkt_cls.h> -#include "bpf_helpers.h" +#include <bpf/bpf_helpers.h> #include "bpf_legacy.h" /* compiler workaround */ diff --git a/samples/bpf/tcp_basertt_kern.c b/samples/bpf/tcp_basertt_kern.c index 9dba48c2b920..8dfe09a92fec 100644 --- a/samples/bpf/tcp_basertt_kern.c +++ b/samples/bpf/tcp_basertt_kern.c @@ -16,8 +16,8 @@ #include <uapi/linux/if_packet.h> #include <uapi/linux/ip.h> #include <linux/socket.h> -#include "bpf_helpers.h" -#include "bpf_endian.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_endian.h> #define DEBUG 1 diff --git a/samples/bpf/tcp_bufs_kern.c b/samples/bpf/tcp_bufs_kern.c index af8486f33771..6a80d08952ad 100644 --- a/samples/bpf/tcp_bufs_kern.c +++ b/samples/bpf/tcp_bufs_kern.c @@ -17,8 +17,8 @@ #include <uapi/linux/if_packet.h> #include <uapi/linux/ip.h> #include <linux/socket.h> -#include "bpf_helpers.h" -#include "bpf_endian.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_endian.h> #define DEBUG 1 diff --git a/samples/bpf/tcp_clamp_kern.c b/samples/bpf/tcp_clamp_kern.c index 26c0fd091f3c..e88bd9ab0695 100644 --- a/samples/bpf/tcp_clamp_kern.c +++ b/samples/bpf/tcp_clamp_kern.c @@ -17,8 +17,8 @@ #include <uapi/linux/if_packet.h> #include <uapi/linux/ip.h> #include <linux/socket.h> -#include "bpf_helpers.h" -#include "bpf_endian.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_endian.h> #define DEBUG 1 diff --git a/samples/bpf/tcp_cong_kern.c b/samples/bpf/tcp_cong_kern.c index 6d4dc4c7dd1e..2311fc9dde85 100644 --- a/samples/bpf/tcp_cong_kern.c +++ b/samples/bpf/tcp_cong_kern.c @@ -16,8 +16,8 @@ #include <uapi/linux/if_packet.h> #include <uapi/linux/ip.h> #include <linux/socket.h> -#include "bpf_helpers.h" -#include "bpf_endian.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_endian.h> #define DEBUG 1 diff --git a/samples/bpf/tcp_dumpstats_kern.c b/samples/bpf/tcp_dumpstats_kern.c index 8557913106a0..e80d3afd24bd 100644 --- a/samples/bpf/tcp_dumpstats_kern.c +++ b/samples/bpf/tcp_dumpstats_kern.c @@ -4,8 +4,8 @@ */ #include <linux/bpf.h> -#include "bpf_helpers.h" -#include "bpf_endian.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_endian.h> #define INTERVAL 1000000000ULL diff --git a/samples/bpf/tcp_iw_kern.c b/samples/bpf/tcp_iw_kern.c index da61d53378b3..d1444557358e 100644 --- a/samples/bpf/tcp_iw_kern.c +++ b/samples/bpf/tcp_iw_kern.c @@ -17,8 +17,8 @@ #include <uapi/linux/if_packet.h> #include <uapi/linux/ip.h> #include <linux/socket.h> -#include "bpf_helpers.h" -#include "bpf_endian.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_endian.h> #define DEBUG 1 diff --git a/samples/bpf/tcp_rwnd_kern.c b/samples/bpf/tcp_rwnd_kern.c index d011e38b80d2..223d9c23b10c 100644 --- a/samples/bpf/tcp_rwnd_kern.c +++ b/samples/bpf/tcp_rwnd_kern.c @@ -16,8 +16,8 @@ #include <uapi/linux/if_packet.h> #include <uapi/linux/ip.h> #include <linux/socket.h> -#include "bpf_helpers.h" -#include "bpf_endian.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_endian.h> #define DEBUG 1 diff --git a/samples/bpf/tcp_synrto_kern.c b/samples/bpf/tcp_synrto_kern.c index 720d1950322d..d58004eef124 100644 --- a/samples/bpf/tcp_synrto_kern.c +++ b/samples/bpf/tcp_synrto_kern.c @@ -16,8 +16,8 @@ #include <uapi/linux/if_packet.h> #include <uapi/linux/ip.h> #include <linux/socket.h> -#include "bpf_helpers.h" -#include "bpf_endian.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_endian.h> #define DEBUG 1 diff --git a/samples/bpf/tcp_tos_reflect_kern.c b/samples/bpf/tcp_tos_reflect_kern.c index 369faca70a15..953fedc79ce1 100644 --- a/samples/bpf/tcp_tos_reflect_kern.c +++ b/samples/bpf/tcp_tos_reflect_kern.c @@ -15,8 +15,8 @@ #include <uapi/linux/ipv6.h> #include <uapi/linux/in.h> #include <linux/socket.h> -#include "bpf_helpers.h" -#include "bpf_endian.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_endian.h> #define DEBUG 1 diff --git a/samples/bpf/test_cgrp2_tc_kern.c b/samples/bpf/test_cgrp2_tc_kern.c index 1547b36a7b7b..4dd532a312b9 100644 --- a/samples/bpf/test_cgrp2_tc_kern.c +++ b/samples/bpf/test_cgrp2_tc_kern.c @@ -10,7 +10,7 @@ #include <uapi/linux/ipv6.h> #include <uapi/linux/pkt_cls.h> #include <uapi/linux/bpf.h> -#include "bpf_helpers.h" +#include <bpf/bpf_helpers.h> /* copy of 'struct ethhdr' without __packed */ struct eth_hdr { diff --git a/samples/bpf/test_current_task_under_cgroup_kern.c b/samples/bpf/test_current_task_under_cgroup_kern.c index 86b28d7d6c99..6dc4f41bb6cb 100644 --- a/samples/bpf/test_current_task_under_cgroup_kern.c +++ b/samples/bpf/test_current_task_under_cgroup_kern.c @@ -8,7 +8,7 @@ #include <linux/ptrace.h> #include <uapi/linux/bpf.h> #include <linux/version.h> -#include "bpf_helpers.h" +#include <bpf/bpf_helpers.h> #include <uapi/linux/utsname.h> struct bpf_map_def SEC("maps") cgroup_map = { diff --git a/samples/bpf/test_lwt_bpf.c b/samples/bpf/test_lwt_bpf.c index bacc8013436b..1b568575ad11 100644 --- a/samples/bpf/test_lwt_bpf.c +++ b/samples/bpf/test_lwt_bpf.c @@ -20,7 +20,7 @@ #include <linux/udp.h> #include <linux/icmpv6.h> #include <linux/if_ether.h> -#include "bpf_helpers.h" +#include <bpf/bpf_helpers.h> #include <string.h> # define printk(fmt, ...) \ diff --git a/samples/bpf/test_map_in_map_kern.c b/samples/bpf/test_map_in_map_kern.c index 32ee752f19df..6cee61e8ce9b 100644 --- a/samples/bpf/test_map_in_map_kern.c +++ b/samples/bpf/test_map_in_map_kern.c @@ -10,9 +10,9 @@ #include <linux/version.h> #include <uapi/linux/bpf.h> #include <uapi/linux/in6.h> -#include "bpf_helpers.h" +#include <bpf/bpf_helpers.h> #include "bpf_legacy.h" -#include "bpf_tracing.h" +#include <bpf/bpf_tracing.h> #define MAX_NR_PORTS 65536 diff --git a/samples/bpf/test_overhead_kprobe_kern.c b/samples/bpf/test_overhead_kprobe_kern.c index 8d2518e68db9..8b811c29dc79 100644 --- a/samples/bpf/test_overhead_kprobe_kern.c +++ b/samples/bpf/test_overhead_kprobe_kern.c @@ -7,8 +7,8 @@ #include <linux/version.h> #include <linux/ptrace.h> #include <uapi/linux/bpf.h> -#include "bpf_helpers.h" -#include "bpf_tracing.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> #define _(P) ({typeof(P) val = 0; bpf_probe_read(&val, sizeof(val), &P); val;}) diff --git a/samples/bpf/test_overhead_raw_tp_kern.c b/samples/bpf/test_overhead_raw_tp_kern.c index d2af8bc1c805..8763181a32f3 100644 --- a/samples/bpf/test_overhead_raw_tp_kern.c +++ b/samples/bpf/test_overhead_raw_tp_kern.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2018 Facebook */ #include <uapi/linux/bpf.h> -#include "bpf_helpers.h" +#include <bpf/bpf_helpers.h> SEC("raw_tracepoint/task_rename") int prog(struct bpf_raw_tracepoint_args *ctx) diff --git a/samples/bpf/test_overhead_tp_kern.c b/samples/bpf/test_overhead_tp_kern.c index 38f5c0b9da9f..eaa32693f8fc 100644 --- a/samples/bpf/test_overhead_tp_kern.c +++ b/samples/bpf/test_overhead_tp_kern.c @@ -5,7 +5,7 @@ * License as published by the Free Software Foundation. */ #include <uapi/linux/bpf.h> -#include "bpf_helpers.h" +#include <bpf/bpf_helpers.h> /* from /sys/kernel/debug/tracing/events/task/task_rename/format */ struct task_rename { diff --git a/samples/bpf/test_probe_write_user_kern.c b/samples/bpf/test_probe_write_user_kern.c index b7c48f37132c..f033f36a13a3 100644 --- a/samples/bpf/test_probe_write_user_kern.c +++ b/samples/bpf/test_probe_write_user_kern.c @@ -8,8 +8,8 @@ #include <linux/netdevice.h> #include <uapi/linux/bpf.h> #include <linux/version.h> -#include "bpf_helpers.h" -#include "bpf_tracing.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> struct bpf_map_def SEC("maps") dnat_map = { .type = BPF_MAP_TYPE_HASH, diff --git a/samples/bpf/trace_event_kern.c b/samples/bpf/trace_event_kern.c index 8dc18d233a27..da1d69e20645 100644 --- a/samples/bpf/trace_event_kern.c +++ b/samples/bpf/trace_event_kern.c @@ -9,8 +9,8 @@ #include <uapi/linux/bpf.h> #include <uapi/linux/bpf_perf_event.h> #include <uapi/linux/perf_event.h> -#include "bpf_helpers.h" -#include "bpf_tracing.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> struct key_t { char comm[TASK_COMM_LEN]; diff --git a/samples/bpf/trace_event_user.c b/samples/bpf/trace_event_user.c index 749a50f2f9f3..356171bc392b 100644 --- a/samples/bpf/trace_event_user.c +++ b/samples/bpf/trace_event_user.c @@ -15,7 +15,7 @@ #include <assert.h> #include <errno.h> #include <sys/resource.h> -#include "libbpf.h" +#include <bpf/libbpf.h> #include "bpf_load.h" #include "perf-sys.h" #include "trace_helpers.h" diff --git a/samples/bpf/trace_output_kern.c b/samples/bpf/trace_output_kern.c index 9b96f4fb8cea..1d7d422cae6f 100644 --- a/samples/bpf/trace_output_kern.c +++ b/samples/bpf/trace_output_kern.c @@ -1,7 +1,7 @@ #include <linux/ptrace.h> #include <linux/version.h> #include <uapi/linux/bpf.h> -#include "bpf_helpers.h" +#include <bpf/bpf_helpers.h> struct bpf_map_def SEC("maps") my_map = { .type = BPF_MAP_TYPE_PERF_EVENT_ARRAY, diff --git a/samples/bpf/trace_output_user.c b/samples/bpf/trace_output_user.c index 8ee47699a870..60a17dd05345 100644 --- a/samples/bpf/trace_output_user.c +++ b/samples/bpf/trace_output_user.c @@ -15,7 +15,7 @@ #include <sys/mman.h> #include <time.h> #include <signal.h> -#include <libbpf.h> +#include <bpf/libbpf.h> #include "bpf_load.h" #include "perf-sys.h" diff --git a/samples/bpf/tracex1_kern.c b/samples/bpf/tracex1_kern.c index 1a15f6605129..8e2610e14475 100644 --- a/samples/bpf/tracex1_kern.c +++ b/samples/bpf/tracex1_kern.c @@ -8,8 +8,8 @@ #include <linux/netdevice.h> #include <uapi/linux/bpf.h> #include <linux/version.h> -#include "bpf_helpers.h" -#include "bpf_tracing.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> #define _(P) ({typeof(P) val = 0; bpf_probe_read(&val, sizeof(val), &P); val;}) diff --git a/samples/bpf/tracex2_kern.c b/samples/bpf/tracex2_kern.c index d70b3ea79ea7..d865bb309bcb 100644 --- a/samples/bpf/tracex2_kern.c +++ b/samples/bpf/tracex2_kern.c @@ -8,8 +8,8 @@ #include <linux/netdevice.h> #include <linux/version.h> #include <uapi/linux/bpf.h> -#include "bpf_helpers.h" -#include "bpf_tracing.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> struct bpf_map_def SEC("maps") my_map = { .type = BPF_MAP_TYPE_HASH, diff --git a/samples/bpf/tracex3_kern.c b/samples/bpf/tracex3_kern.c index 9af546bebfa9..fe21c14feb8d 100644 --- a/samples/bpf/tracex3_kern.c +++ b/samples/bpf/tracex3_kern.c @@ -8,8 +8,8 @@ #include <linux/netdevice.h> #include <linux/version.h> #include <uapi/linux/bpf.h> -#include "bpf_helpers.h" -#include "bpf_tracing.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> struct bpf_map_def SEC("maps") my_map = { .type = BPF_MAP_TYPE_HASH, diff --git a/samples/bpf/tracex4_kern.c b/samples/bpf/tracex4_kern.c index 2a02cbe9d9a1..b1bb9df88f8e 100644 --- a/samples/bpf/tracex4_kern.c +++ b/samples/bpf/tracex4_kern.c @@ -7,8 +7,8 @@ #include <linux/ptrace.h> #include <linux/version.h> #include <uapi/linux/bpf.h> -#include "bpf_helpers.h" -#include "bpf_tracing.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> struct pair { u64 val; diff --git a/samples/bpf/tracex5_kern.c b/samples/bpf/tracex5_kern.c index b3557b21a8fe..481790fde864 100644 --- a/samples/bpf/tracex5_kern.c +++ b/samples/bpf/tracex5_kern.c @@ -10,8 +10,8 @@ #include <uapi/linux/seccomp.h> #include <uapi/linux/unistd.h> #include "syscall_nrs.h" -#include "bpf_helpers.h" -#include "bpf_tracing.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> #define PROG(F) SEC("kprobe/"__stringify(F)) int bpf_func_##F diff --git a/samples/bpf/tracex6_kern.c b/samples/bpf/tracex6_kern.c index 46c557afac73..96c234efa852 100644 --- a/samples/bpf/tracex6_kern.c +++ b/samples/bpf/tracex6_kern.c @@ -1,7 +1,7 @@ #include <linux/ptrace.h> #include <linux/version.h> #include <uapi/linux/bpf.h> -#include "bpf_helpers.h" +#include <bpf/bpf_helpers.h> struct bpf_map_def SEC("maps") counters = { .type = BPF_MAP_TYPE_PERF_EVENT_ARRAY, diff --git a/samples/bpf/tracex7_kern.c b/samples/bpf/tracex7_kern.c index 1ab308a43e0f..c5a92df8ac31 100644 --- a/samples/bpf/tracex7_kern.c +++ b/samples/bpf/tracex7_kern.c @@ -1,7 +1,7 @@ #include <uapi/linux/ptrace.h> #include <uapi/linux/bpf.h> #include <linux/version.h> -#include "bpf_helpers.h" +#include <bpf/bpf_helpers.h> SEC("kprobe/open_ctree") int bpf_prog1(struct pt_regs *ctx) diff --git a/samples/bpf/xdp1_kern.c b/samples/bpf/xdp1_kern.c index db6870aee42c..34b64394ed9c 100644 --- a/samples/bpf/xdp1_kern.c +++ b/samples/bpf/xdp1_kern.c @@ -12,7 +12,7 @@ #include <linux/if_vlan.h> #include <linux/ip.h> #include <linux/ipv6.h> -#include "bpf_helpers.h" +#include <bpf/bpf_helpers.h> struct { __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); diff --git a/samples/bpf/xdp1_user.c b/samples/bpf/xdp1_user.c index 3e553eed95a7..c447ad9e3a1d 100644 --- a/samples/bpf/xdp1_user.c +++ b/samples/bpf/xdp1_user.c @@ -15,8 +15,8 @@ #include <net/if.h> #include "bpf_util.h" -#include "bpf.h" -#include "libbpf.h" +#include <bpf/bpf.h> +#include <bpf/libbpf.h> static int ifindex; static __u32 xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST; @@ -98,7 +98,7 @@ int main(int argc, char **argv) xdp_flags |= XDP_FLAGS_SKB_MODE; break; case 'N': - xdp_flags |= XDP_FLAGS_DRV_MODE; + /* default, set below */ break; case 'F': xdp_flags &= ~XDP_FLAGS_UPDATE_IF_NOEXIST; @@ -109,6 +109,9 @@ int main(int argc, char **argv) } } + if (!(xdp_flags & XDP_FLAGS_SKB_MODE)) + xdp_flags |= XDP_FLAGS_DRV_MODE; + if (optind == argc) { usage(basename(argv[0])); return 1; diff --git a/samples/bpf/xdp2_kern.c b/samples/bpf/xdp2_kern.c index c74b52c6d945..c787f4b49646 100644 --- a/samples/bpf/xdp2_kern.c +++ b/samples/bpf/xdp2_kern.c @@ -12,7 +12,7 @@ #include <linux/if_vlan.h> #include <linux/ip.h> #include <linux/ipv6.h> -#include "bpf_helpers.h" +#include <bpf/bpf_helpers.h> struct { __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); diff --git a/samples/bpf/xdp2skb_meta_kern.c b/samples/bpf/xdp2skb_meta_kern.c index 0c12048ac79f..9b783316e860 100644 --- a/samples/bpf/xdp2skb_meta_kern.c +++ b/samples/bpf/xdp2skb_meta_kern.c @@ -12,7 +12,7 @@ #include <uapi/linux/bpf.h> #include <uapi/linux/pkt_cls.h> -#include "bpf_helpers.h" +#include <bpf/bpf_helpers.h> /* * This struct is stored in the XDP 'data_meta' area, which is located diff --git a/samples/bpf/xdp_adjust_tail_kern.c b/samples/bpf/xdp_adjust_tail_kern.c index 0f707e0fb375..ffdd548627f0 100644 --- a/samples/bpf/xdp_adjust_tail_kern.c +++ b/samples/bpf/xdp_adjust_tail_kern.c @@ -18,7 +18,7 @@ #include <linux/if_vlan.h> #include <linux/ip.h> #include <linux/icmp.h> -#include "bpf_helpers.h" +#include <bpf/bpf_helpers.h> #define DEFAULT_TTL 64 #define MAX_PCKT_SIZE 600 diff --git a/samples/bpf/xdp_adjust_tail_user.c b/samples/bpf/xdp_adjust_tail_user.c index d86e9ad0356b..ba482dc3da33 100644 --- a/samples/bpf/xdp_adjust_tail_user.c +++ b/samples/bpf/xdp_adjust_tail_user.c @@ -19,8 +19,8 @@ #include <netinet/ether.h> #include <unistd.h> #include <time.h> -#include "bpf.h" -#include "libbpf.h" +#include <bpf/bpf.h> +#include <bpf/libbpf.h> #define STATS_INTERVAL_S 2U #define MAX_PCKT_SIZE 600 @@ -120,7 +120,7 @@ int main(int argc, char **argv) xdp_flags |= XDP_FLAGS_SKB_MODE; break; case 'N': - xdp_flags |= XDP_FLAGS_DRV_MODE; + /* default, set below */ break; case 'F': xdp_flags &= ~XDP_FLAGS_UPDATE_IF_NOEXIST; @@ -132,6 +132,9 @@ int main(int argc, char **argv) opt_flags[opt] = 0; } + if (!(xdp_flags & XDP_FLAGS_SKB_MODE)) + xdp_flags |= XDP_FLAGS_DRV_MODE; + for (i = 0; i < strlen(optstr); i++) { if (opt_flags[(unsigned int)optstr[i]]) { fprintf(stderr, "Missing argument -%c\n", optstr[i]); diff --git a/samples/bpf/xdp_fwd_kern.c b/samples/bpf/xdp_fwd_kern.c index d013029aeaa2..54c099cbd639 100644 --- a/samples/bpf/xdp_fwd_kern.c +++ b/samples/bpf/xdp_fwd_kern.c @@ -19,7 +19,7 @@ #include <linux/ip.h> #include <linux/ipv6.h> -#include "bpf_helpers.h" +#include <bpf/bpf_helpers.h> #define IPV6_FLOWINFO_MASK cpu_to_be32(0x0FFFFFFF) diff --git a/samples/bpf/xdp_fwd_user.c b/samples/bpf/xdp_fwd_user.c index 97ff1dad7669..74a4583d0d86 100644 --- a/samples/bpf/xdp_fwd_user.c +++ b/samples/bpf/xdp_fwd_user.c @@ -24,14 +24,16 @@ #include <fcntl.h> #include <libgen.h> -#include "libbpf.h" +#include <bpf/libbpf.h> #include <bpf/bpf.h> +static __u32 xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST; + static int do_attach(int idx, int prog_fd, int map_fd, const char *name) { int err; - err = bpf_set_link_xdp_fd(idx, prog_fd, 0); + err = bpf_set_link_xdp_fd(idx, prog_fd, xdp_flags); if (err < 0) { printf("ERROR: failed to attach program to %s\n", name); return err; @@ -49,7 +51,7 @@ static int do_detach(int idx, const char *name) { int err; - err = bpf_set_link_xdp_fd(idx, -1, 0); + err = bpf_set_link_xdp_fd(idx, -1, xdp_flags); if (err < 0) printf("ERROR: failed to detach program from %s\n", name); @@ -83,11 +85,17 @@ int main(int argc, char **argv) int attach = 1; int ret = 0; - while ((opt = getopt(argc, argv, ":dD")) != -1) { + while ((opt = getopt(argc, argv, ":dDSF")) != -1) { switch (opt) { case 'd': attach = 0; break; + case 'S': + xdp_flags |= XDP_FLAGS_SKB_MODE; + break; + case 'F': + xdp_flags &= ~XDP_FLAGS_UPDATE_IF_NOEXIST; + break; case 'D': prog_name = "xdp_fwd_direct"; break; @@ -97,6 +105,9 @@ int main(int argc, char **argv) } } + if (!(xdp_flags & XDP_FLAGS_SKB_MODE)) + xdp_flags |= XDP_FLAGS_DRV_MODE; + if (optind == argc) { usage(basename(argv[0])); return 1; diff --git a/samples/bpf/xdp_monitor_kern.c b/samples/bpf/xdp_monitor_kern.c index ad10fe700d7d..3d33cca2d48a 100644 --- a/samples/bpf/xdp_monitor_kern.c +++ b/samples/bpf/xdp_monitor_kern.c @@ -4,7 +4,7 @@ * XDP monitor tool, based on tracepoints */ #include <uapi/linux/bpf.h> -#include "bpf_helpers.h" +#include <bpf/bpf_helpers.h> struct bpf_map_def SEC("maps") redirect_err_cnt = { .type = BPF_MAP_TYPE_PERCPU_ARRAY, @@ -222,14 +222,12 @@ struct bpf_map_def SEC("maps") devmap_xmit_cnt = { */ struct devmap_xmit_ctx { u64 __pad; // First 8 bytes are not accessible by bpf code - int map_id; // offset:8; size:4; signed:1; + int from_ifindex; // offset:8; size:4; signed:1; u32 act; // offset:12; size:4; signed:0; - u32 map_index; // offset:16; size:4; signed:0; + int to_ifindex; // offset:16; size:4; signed:1; int drops; // offset:20; size:4; signed:1; int sent; // offset:24; size:4; signed:1; - int from_ifindex; // offset:28; size:4; signed:1; - int to_ifindex; // offset:32; size:4; signed:1; - int err; // offset:36; size:4; signed:1; + int err; // offset:28; size:4; signed:1; }; SEC("tracepoint/xdp/xdp_devmap_xmit") diff --git a/samples/bpf/xdp_redirect_cpu_kern.c b/samples/bpf/xdp_redirect_cpu_kern.c index cfcc31e51197..313a8fe6d125 100644 --- a/samples/bpf/xdp_redirect_cpu_kern.c +++ b/samples/bpf/xdp_redirect_cpu_kern.c @@ -12,7 +12,7 @@ #include <uapi/linux/udp.h> #include <uapi/linux/bpf.h> -#include "bpf_helpers.h" +#include <bpf/bpf_helpers.h> #include "hash_func01.h" #define MAX_CPUS 64 /* WARNING - sync with _user.c */ diff --git a/samples/bpf/xdp_redirect_cpu_user.c b/samples/bpf/xdp_redirect_cpu_user.c index 0da6e9e7132e..15bdf047a222 100644 --- a/samples/bpf/xdp_redirect_cpu_user.c +++ b/samples/bpf/xdp_redirect_cpu_user.c @@ -16,6 +16,10 @@ static const char *__doc__ = #include <getopt.h> #include <net/if.h> #include <time.h> +#include <linux/limits.h> + +#define __must_check +#include <linux/err.h> #include <arpa/inet.h> #include <linux/if_link.h> @@ -26,7 +30,7 @@ static const char *__doc__ = #define MAX_PROG 6 #include <bpf/bpf.h> -#include "libbpf.h" +#include <bpf/libbpf.h> #include "bpf_util.h" @@ -46,6 +50,10 @@ static int cpus_count_map_fd; static int cpus_iterator_map_fd; static int exception_cnt_map_fd; +#define NUM_TP 5 +struct bpf_link *tp_links[NUM_TP] = { 0 }; +static int tp_cnt = 0; + /* Exit return codes */ #define EXIT_OK 0 #define EXIT_FAIL 1 @@ -88,6 +96,10 @@ static void int_exit(int sig) printf("program on interface changed, not removing\n"); } } + /* Detach tracepoints */ + while (tp_cnt) + bpf_link__destroy(tp_links[--tp_cnt]); + exit(EXIT_OK); } @@ -588,23 +600,61 @@ static void stats_poll(int interval, bool use_separators, char *prog_name, free_stats_record(prev); } +static struct bpf_link * attach_tp(struct bpf_object *obj, + const char *tp_category, + const char* tp_name) +{ + struct bpf_program *prog; + struct bpf_link *link; + char sec_name[PATH_MAX]; + int len; + + len = snprintf(sec_name, PATH_MAX, "tracepoint/%s/%s", + tp_category, tp_name); + if (len < 0) + exit(EXIT_FAIL); + + prog = bpf_object__find_program_by_title(obj, sec_name); + if (!prog) { + fprintf(stderr, "ERR: finding progsec: %s\n", sec_name); + exit(EXIT_FAIL_BPF); + } + + link = bpf_program__attach_tracepoint(prog, tp_category, tp_name); + if (IS_ERR(link)) + exit(EXIT_FAIL_BPF); + + return link; +} + +static void init_tracepoints(struct bpf_object *obj) { + tp_links[tp_cnt++] = attach_tp(obj, "xdp", "xdp_redirect_err"); + tp_links[tp_cnt++] = attach_tp(obj, "xdp", "xdp_redirect_map_err"); + tp_links[tp_cnt++] = attach_tp(obj, "xdp", "xdp_exception"); + tp_links[tp_cnt++] = attach_tp(obj, "xdp", "xdp_cpumap_enqueue"); + tp_links[tp_cnt++] = attach_tp(obj, "xdp", "xdp_cpumap_kthread"); +} + static int init_map_fds(struct bpf_object *obj) { - cpu_map_fd = bpf_object__find_map_fd_by_name(obj, "cpu_map"); - rx_cnt_map_fd = bpf_object__find_map_fd_by_name(obj, "rx_cnt"); + /* Maps updated by tracepoints */ redirect_err_cnt_map_fd = bpf_object__find_map_fd_by_name(obj, "redirect_err_cnt"); + exception_cnt_map_fd = + bpf_object__find_map_fd_by_name(obj, "exception_cnt"); cpumap_enqueue_cnt_map_fd = bpf_object__find_map_fd_by_name(obj, "cpumap_enqueue_cnt"); cpumap_kthread_cnt_map_fd = bpf_object__find_map_fd_by_name(obj, "cpumap_kthread_cnt"); + + /* Maps used by XDP */ + rx_cnt_map_fd = bpf_object__find_map_fd_by_name(obj, "rx_cnt"); + cpu_map_fd = bpf_object__find_map_fd_by_name(obj, "cpu_map"); cpus_available_map_fd = bpf_object__find_map_fd_by_name(obj, "cpus_available"); cpus_count_map_fd = bpf_object__find_map_fd_by_name(obj, "cpus_count"); cpus_iterator_map_fd = bpf_object__find_map_fd_by_name(obj, "cpus_iterator"); - exception_cnt_map_fd = - bpf_object__find_map_fd_by_name(obj, "exception_cnt"); if (cpu_map_fd < 0 || rx_cnt_map_fd < 0 || redirect_err_cnt_map_fd < 0 || cpumap_enqueue_cnt_map_fd < 0 || @@ -662,6 +712,7 @@ int main(int argc, char **argv) strerror(errno)); return EXIT_FAIL; } + init_tracepoints(obj); if (init_map_fds(obj) < 0) { fprintf(stderr, "bpf_object__find_map_fd_by_name failed\n"); return EXIT_FAIL; @@ -728,6 +779,10 @@ int main(int argc, char **argv) return EXIT_FAIL_OPTION; } } + + if (!(xdp_flags & XDP_FLAGS_SKB_MODE)) + xdp_flags |= XDP_FLAGS_DRV_MODE; + /* Required option */ if (ifindex == -1) { fprintf(stderr, "ERR: required option --dev missing\n"); diff --git a/samples/bpf/xdp_redirect_kern.c b/samples/bpf/xdp_redirect_kern.c index 1f0b7d05abb2..d26ec3aa215e 100644 --- a/samples/bpf/xdp_redirect_kern.c +++ b/samples/bpf/xdp_redirect_kern.c @@ -17,7 +17,7 @@ #include <linux/if_vlan.h> #include <linux/ip.h> #include <linux/ipv6.h> -#include "bpf_helpers.h" +#include <bpf/bpf_helpers.h> struct { __uint(type, BPF_MAP_TYPE_ARRAY); diff --git a/samples/bpf/xdp_redirect_map_kern.c b/samples/bpf/xdp_redirect_map_kern.c index 4631b484c432..6489352ab7a4 100644 --- a/samples/bpf/xdp_redirect_map_kern.c +++ b/samples/bpf/xdp_redirect_map_kern.c @@ -17,7 +17,7 @@ #include <linux/if_vlan.h> #include <linux/ip.h> #include <linux/ipv6.h> -#include "bpf_helpers.h" +#include <bpf/bpf_helpers.h> struct { __uint(type, BPF_MAP_TYPE_DEVMAP); diff --git a/samples/bpf/xdp_redirect_map_user.c b/samples/bpf/xdp_redirect_map_user.c index f70ee33907fd..35e16dee613e 100644 --- a/samples/bpf/xdp_redirect_map_user.c +++ b/samples/bpf/xdp_redirect_map_user.c @@ -17,7 +17,7 @@ #include "bpf_util.h" #include <bpf/bpf.h> -#include "libbpf.h" +#include <bpf/libbpf.h> static int ifindex_in; static int ifindex_out; @@ -116,7 +116,7 @@ int main(int argc, char **argv) xdp_flags |= XDP_FLAGS_SKB_MODE; break; case 'N': - xdp_flags |= XDP_FLAGS_DRV_MODE; + /* default, set below */ break; case 'F': xdp_flags &= ~XDP_FLAGS_UPDATE_IF_NOEXIST; @@ -127,6 +127,9 @@ int main(int argc, char **argv) } } + if (!(xdp_flags & XDP_FLAGS_SKB_MODE)) + xdp_flags |= XDP_FLAGS_DRV_MODE; + if (optind == argc) { printf("usage: %s <IFNAME|IFINDEX>_IN <IFNAME|IFINDEX>_OUT\n", argv[0]); return 1; diff --git a/samples/bpf/xdp_redirect_user.c b/samples/bpf/xdp_redirect_user.c index 5440cd620607..9ca2bf457cda 100644 --- a/samples/bpf/xdp_redirect_user.c +++ b/samples/bpf/xdp_redirect_user.c @@ -17,7 +17,7 @@ #include "bpf_util.h" #include <bpf/bpf.h> -#include "libbpf.h" +#include <bpf/libbpf.h> static int ifindex_in; static int ifindex_out; @@ -117,7 +117,7 @@ int main(int argc, char **argv) xdp_flags |= XDP_FLAGS_SKB_MODE; break; case 'N': - xdp_flags |= XDP_FLAGS_DRV_MODE; + /* default, set below */ break; case 'F': xdp_flags &= ~XDP_FLAGS_UPDATE_IF_NOEXIST; @@ -128,6 +128,9 @@ int main(int argc, char **argv) } } + if (!(xdp_flags & XDP_FLAGS_SKB_MODE)) + xdp_flags |= XDP_FLAGS_DRV_MODE; + if (optind == argc) { printf("usage: %s <IFNAME|IFINDEX>_IN <IFNAME|IFINDEX>_OUT\n", argv[0]); return 1; diff --git a/samples/bpf/xdp_router_ipv4_kern.c b/samples/bpf/xdp_router_ipv4_kern.c index bf11efc8e949..b37ca2b13063 100644 --- a/samples/bpf/xdp_router_ipv4_kern.c +++ b/samples/bpf/xdp_router_ipv4_kern.c @@ -12,7 +12,7 @@ #include <linux/if_vlan.h> #include <linux/ip.h> #include <linux/ipv6.h> -#include "bpf_helpers.h" +#include <bpf/bpf_helpers.h> #include <linux/slab.h> #include <net/ip_fib.h> diff --git a/samples/bpf/xdp_router_ipv4_user.c b/samples/bpf/xdp_router_ipv4_user.c index 1469b66ebad1..c2da1b51ff95 100644 --- a/samples/bpf/xdp_router_ipv4_user.c +++ b/samples/bpf/xdp_router_ipv4_user.c @@ -21,7 +21,7 @@ #include <sys/ioctl.h> #include <sys/syscall.h> #include "bpf_util.h" -#include "libbpf.h" +#include <bpf/libbpf.h> #include <sys/resource.h> #include <libgen.h> @@ -662,6 +662,9 @@ int main(int ac, char **argv) } } + if (!(flags & XDP_FLAGS_SKB_MODE)) + flags |= XDP_FLAGS_DRV_MODE; + if (optind == ac) { usage(basename(argv[0])); return 1; diff --git a/samples/bpf/xdp_rxq_info_kern.c b/samples/bpf/xdp_rxq_info_kern.c index 272d0f82a6b5..5e7459f9bf3e 100644 --- a/samples/bpf/xdp_rxq_info_kern.c +++ b/samples/bpf/xdp_rxq_info_kern.c @@ -6,7 +6,7 @@ #include <uapi/linux/bpf.h> #include <uapi/linux/if_ether.h> #include <uapi/linux/in.h> -#include "bpf_helpers.h" +#include <bpf/bpf_helpers.h> /* Config setup from with userspace * diff --git a/samples/bpf/xdp_rxq_info_user.c b/samples/bpf/xdp_rxq_info_user.c index 8fc3ad01de72..4fe47502ebed 100644 --- a/samples/bpf/xdp_rxq_info_user.c +++ b/samples/bpf/xdp_rxq_info_user.c @@ -22,8 +22,8 @@ static const char *__doc__ = " XDP RX-queue info extract example\n\n" #include <arpa/inet.h> #include <linux/if_link.h> -#include "bpf.h" -#include "libbpf.h" +#include <bpf/bpf.h> +#include <bpf/libbpf.h> #include "bpf_util.h" static int ifindex = -1; @@ -551,6 +551,10 @@ int main(int argc, char **argv) return EXIT_FAIL_OPTION; } } + + if (!(xdp_flags & XDP_FLAGS_SKB_MODE)) + xdp_flags |= XDP_FLAGS_DRV_MODE; + /* Required option */ if (ifindex == -1) { fprintf(stderr, "ERR: required option --dev missing\n"); diff --git a/samples/bpf/xdp_sample_pkts_kern.c b/samples/bpf/xdp_sample_pkts_kern.c index 6c7c7e0aaeda..33377289e2a8 100644 --- a/samples/bpf/xdp_sample_pkts_kern.c +++ b/samples/bpf/xdp_sample_pkts_kern.c @@ -2,7 +2,7 @@ #include <linux/ptrace.h> #include <linux/version.h> #include <uapi/linux/bpf.h> -#include "bpf_helpers.h" +#include <bpf/bpf_helpers.h> #define SAMPLE_SIZE 64ul #define MAX_CPUS 128 diff --git a/samples/bpf/xdp_sample_pkts_user.c b/samples/bpf/xdp_sample_pkts_user.c index a5760e8bf2c4..991ef6f0880b 100644 --- a/samples/bpf/xdp_sample_pkts_user.c +++ b/samples/bpf/xdp_sample_pkts_user.c @@ -10,7 +10,7 @@ #include <sys/sysinfo.h> #include <sys/ioctl.h> #include <signal.h> -#include <libbpf.h> +#include <bpf/libbpf.h> #include <bpf/bpf.h> #include <sys/resource.h> #include <libgen.h> @@ -52,13 +52,13 @@ static int do_detach(int idx, const char *name) __u32 curr_prog_id = 0; int err = 0; - err = bpf_get_link_xdp_id(idx, &curr_prog_id, 0); + err = bpf_get_link_xdp_id(idx, &curr_prog_id, xdp_flags); if (err) { printf("bpf_get_link_xdp_id failed\n"); return err; } if (prog_id == curr_prog_id) { - err = bpf_set_link_xdp_fd(idx, -1, 0); + err = bpf_set_link_xdp_fd(idx, -1, xdp_flags); if (err < 0) printf("ERROR: failed to detach prog from %s\n", name); } else if (!curr_prog_id) { @@ -115,7 +115,7 @@ int main(int argc, char **argv) .prog_type = BPF_PROG_TYPE_XDP, }; struct perf_buffer_opts pb_opts = {}; - const char *optstr = "F"; + const char *optstr = "FS"; int prog_fd, map_fd, opt; struct bpf_object *obj; struct bpf_map *map; @@ -127,12 +127,18 @@ int main(int argc, char **argv) case 'F': xdp_flags &= ~XDP_FLAGS_UPDATE_IF_NOEXIST; break; + case 'S': + xdp_flags |= XDP_FLAGS_SKB_MODE; + break; default: usage(basename(argv[0])); return 1; } } + if (!(xdp_flags & XDP_FLAGS_SKB_MODE)) + xdp_flags |= XDP_FLAGS_DRV_MODE; + if (optind == argc) { usage(basename(argv[0])); return 1; diff --git a/samples/bpf/xdp_tx_iptunnel_kern.c b/samples/bpf/xdp_tx_iptunnel_kern.c index 6db450a5c1ca..575d57e4b8d6 100644 --- a/samples/bpf/xdp_tx_iptunnel_kern.c +++ b/samples/bpf/xdp_tx_iptunnel_kern.c @@ -16,7 +16,7 @@ #include <linux/if_vlan.h> #include <linux/ip.h> #include <linux/ipv6.h> -#include "bpf_helpers.h" +#include <bpf/bpf_helpers.h> #include "xdp_tx_iptunnel_common.h" struct { diff --git a/samples/bpf/xdp_tx_iptunnel_user.c b/samples/bpf/xdp_tx_iptunnel_user.c index 2fe4c7f5ffe5..a419bee151a8 100644 --- a/samples/bpf/xdp_tx_iptunnel_user.c +++ b/samples/bpf/xdp_tx_iptunnel_user.c @@ -15,7 +15,7 @@ #include <netinet/ether.h> #include <unistd.h> #include <time.h> -#include "libbpf.h" +#include <bpf/libbpf.h> #include <bpf/bpf.h> #include "bpf_util.h" #include "xdp_tx_iptunnel_common.h" @@ -231,7 +231,7 @@ int main(int argc, char **argv) xdp_flags |= XDP_FLAGS_SKB_MODE; break; case 'N': - xdp_flags |= XDP_FLAGS_DRV_MODE; + /* default, set below */ break; case 'F': xdp_flags &= ~XDP_FLAGS_UPDATE_IF_NOEXIST; @@ -243,6 +243,9 @@ int main(int argc, char **argv) opt_flags[opt] = 0; } + if (!(xdp_flags & XDP_FLAGS_SKB_MODE)) + xdp_flags |= XDP_FLAGS_DRV_MODE; + for (i = 0; i < strlen(optstr); i++) { if (opt_flags[(unsigned int)optstr[i]]) { fprintf(stderr, "Missing argument -%c\n", optstr[i]); diff --git a/samples/bpf/xdpsock_kern.c b/samples/bpf/xdpsock_kern.c index a06177c262cd..05430484375c 100644 --- a/samples/bpf/xdpsock_kern.c +++ b/samples/bpf/xdpsock_kern.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 #include <linux/bpf.h> -#include "bpf_helpers.h" +#include <bpf/bpf_helpers.h> #include "xdpsock.h" /* This XDP program is only needed for the XDP_SHARED_UMEM mode. diff --git a/samples/bpf/xdpsock_user.c b/samples/bpf/xdpsock_user.c index a15480010828..0b5acd722306 100644 --- a/samples/bpf/xdpsock_user.c +++ b/samples/bpf/xdpsock_user.c @@ -10,6 +10,9 @@ #include <linux/if_link.h> #include <linux/if_xdp.h> #include <linux/if_ether.h> +#include <linux/ip.h> +#include <linux/udp.h> +#include <arpa/inet.h> #include <locale.h> #include <net/ethernet.h> #include <net/if.h> @@ -27,10 +30,10 @@ #include <time.h> #include <unistd.h> -#include "libbpf.h" -#include "xsk.h" -#include "xdpsock.h" +#include <bpf/libbpf.h> +#include <bpf/xsk.h> #include <bpf/bpf.h> +#include "xdpsock.h" #ifndef SOL_XDP #define SOL_XDP 283 @@ -45,12 +48,14 @@ #endif #define NUM_FRAMES (4 * 1024) -#define BATCH_SIZE 64 +#define MIN_PKT_SIZE 64 #define DEBUG_HEXDUMP 0 typedef __u64 u64; typedef __u32 u32; +typedef __u16 u16; +typedef __u8 u8; static unsigned long prev_time; @@ -65,6 +70,13 @@ static u32 opt_xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST; static const char *opt_if = ""; static int opt_ifindex; static int opt_queue; +static unsigned long opt_duration; +static unsigned long start_time; +static bool benchmark_done; +static u32 opt_batch_size = 64; +static int opt_pkt_count; +static u16 opt_pkt_size = MIN_PKT_SIZE; +static u32 opt_pkt_fill_pattern = 0x12345678; static int opt_poll; static int opt_interval = 1; static u32 opt_xdp_bind_flags = XDP_USE_NEED_WAKEUP; @@ -167,10 +179,21 @@ static void dump_stats(void) } } +static bool is_benchmark_done(void) +{ + if (opt_duration > 0) { + unsigned long dt = (get_nsecs() - start_time); + + if (dt >= opt_duration) + benchmark_done = true; + } + return benchmark_done; +} + static void *poller(void *arg) { (void)arg; - for (;;) { + while (!is_benchmark_done()) { sleep(opt_interval); dump_stats(); } @@ -196,6 +219,11 @@ static void remove_xdp_program(void) static void int_exit(int sig) { + benchmark_done = true; +} + +static void xdpsock_cleanup(void) +{ struct xsk_umem *umem = xsks[0]->umem->umem; int i; @@ -204,8 +232,6 @@ static void int_exit(int sig) xsk_socket__delete(xsks[i]->xsk); (void)xsk_umem__delete(umem); remove_xdp_program(); - - exit(EXIT_SUCCESS); } static void __exit_with_error(int error, const char *file, const char *func, @@ -220,13 +246,6 @@ static void __exit_with_error(int error, const char *file, const char *func, #define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, \ __LINE__) - -static const char pkt_data[] = - "\x3c\xfd\xfe\x9e\x7f\x71\xec\xb1\xd7\x98\x3a\xc0\x08\x00\x45\x00" - "\x00\x2e\x00\x00\x00\x00\x40\x11\x88\x97\x05\x08\x07\x08\xc8\x14" - "\x1e\x04\x10\x92\x10\x92\x00\x1a\x6d\xa3\x34\x33\x1f\x69\x40\x6b" - "\x54\x59\xb6\x14\x2d\x11\x44\xbf\xaf\xd9\xbe\xaa"; - static void swap_mac_addresses(void *data) { struct ether_header *eth = (struct ether_header *)data; @@ -274,11 +293,243 @@ static void hex_dump(void *pkt, size_t length, u64 addr) printf("\n"); } -static size_t gen_eth_frame(struct xsk_umem_info *umem, u64 addr) +static void *memset32_htonl(void *dest, u32 val, u32 size) +{ + u32 *ptr = (u32 *)dest; + int i; + + val = htonl(val); + + for (i = 0; i < (size & (~0x3)); i += 4) + ptr[i >> 2] = val; + + for (; i < size; i++) + ((char *)dest)[i] = ((char *)&val)[i & 3]; + + return dest; +} + +/* + * This function code has been taken from + * Linux kernel lib/checksum.c + */ +static inline unsigned short from32to16(unsigned int x) +{ + /* add up 16-bit and 16-bit for 16+c bit */ + x = (x & 0xffff) + (x >> 16); + /* add up carry.. */ + x = (x & 0xffff) + (x >> 16); + return x; +} + +/* + * This function code has been taken from + * Linux kernel lib/checksum.c + */ +static unsigned int do_csum(const unsigned char *buff, int len) +{ + unsigned int result = 0; + int odd; + + if (len <= 0) + goto out; + odd = 1 & (unsigned long)buff; + if (odd) { +#ifdef __LITTLE_ENDIAN + result += (*buff << 8); +#else + result = *buff; +#endif + len--; + buff++; + } + if (len >= 2) { + if (2 & (unsigned long)buff) { + result += *(unsigned short *)buff; + len -= 2; + buff += 2; + } + if (len >= 4) { + const unsigned char *end = buff + + ((unsigned int)len & ~3); + unsigned int carry = 0; + + do { + unsigned int w = *(unsigned int *)buff; + + buff += 4; + result += carry; + result += w; + carry = (w > result); + } while (buff < end); + result += carry; + result = (result & 0xffff) + (result >> 16); + } + if (len & 2) { + result += *(unsigned short *)buff; + buff += 2; + } + } + if (len & 1) +#ifdef __LITTLE_ENDIAN + result += *buff; +#else + result += (*buff << 8); +#endif + result = from32to16(result); + if (odd) + result = ((result >> 8) & 0xff) | ((result & 0xff) << 8); +out: + return result; +} + +__sum16 ip_fast_csum(const void *iph, unsigned int ihl); + +/* + * This is a version of ip_compute_csum() optimized for IP headers, + * which always checksum on 4 octet boundaries. + * This function code has been taken from + * Linux kernel lib/checksum.c + */ +__sum16 ip_fast_csum(const void *iph, unsigned int ihl) +{ + return (__force __sum16)~do_csum(iph, ihl * 4); +} + +/* + * Fold a partial checksum + * This function code has been taken from + * Linux kernel include/asm-generic/checksum.h + */ +static inline __sum16 csum_fold(__wsum csum) +{ + u32 sum = (__force u32)csum; + + sum = (sum & 0xffff) + (sum >> 16); + sum = (sum & 0xffff) + (sum >> 16); + return (__force __sum16)~sum; +} + +/* + * This function code has been taken from + * Linux kernel lib/checksum.c + */ +static inline u32 from64to32(u64 x) +{ + /* add up 32-bit and 32-bit for 32+c bit */ + x = (x & 0xffffffff) + (x >> 32); + /* add up carry.. */ + x = (x & 0xffffffff) + (x >> 32); + return (u32)x; +} + +__wsum csum_tcpudp_nofold(__be32 saddr, __be32 daddr, + __u32 len, __u8 proto, __wsum sum); + +/* + * This function code has been taken from + * Linux kernel lib/checksum.c + */ +__wsum csum_tcpudp_nofold(__be32 saddr, __be32 daddr, + __u32 len, __u8 proto, __wsum sum) +{ + unsigned long long s = (__force u32)sum; + + s += (__force u32)saddr; + s += (__force u32)daddr; +#ifdef __BIG_ENDIAN__ + s += proto + len; +#else + s += (proto + len) << 8; +#endif + return (__force __wsum)from64to32(s); +} + +/* + * This function has been taken from + * Linux kernel include/asm-generic/checksum.h + */ +static inline __sum16 +csum_tcpudp_magic(__be32 saddr, __be32 daddr, __u32 len, + __u8 proto, __wsum sum) +{ + return csum_fold(csum_tcpudp_nofold(saddr, daddr, len, proto, sum)); +} + +static inline u16 udp_csum(u32 saddr, u32 daddr, u32 len, + u8 proto, u16 *udp_pkt) +{ + u32 csum = 0; + u32 cnt = 0; + + /* udp hdr and data */ + for (; cnt < len; cnt += 2) + csum += udp_pkt[cnt >> 1]; + + return csum_tcpudp_magic(saddr, daddr, len, proto, csum); +} + +#define ETH_FCS_SIZE 4 + +#define PKT_HDR_SIZE (sizeof(struct ethhdr) + sizeof(struct iphdr) + \ + sizeof(struct udphdr)) + +#define PKT_SIZE (opt_pkt_size - ETH_FCS_SIZE) +#define IP_PKT_SIZE (PKT_SIZE - sizeof(struct ethhdr)) +#define UDP_PKT_SIZE (IP_PKT_SIZE - sizeof(struct iphdr)) +#define UDP_PKT_DATA_SIZE (UDP_PKT_SIZE - sizeof(struct udphdr)) + +static u8 pkt_data[XSK_UMEM__DEFAULT_FRAME_SIZE]; + +static void gen_eth_hdr_data(void) +{ + struct udphdr *udp_hdr = (struct udphdr *)(pkt_data + + sizeof(struct ethhdr) + + sizeof(struct iphdr)); + struct iphdr *ip_hdr = (struct iphdr *)(pkt_data + + sizeof(struct ethhdr)); + struct ethhdr *eth_hdr = (struct ethhdr *)pkt_data; + + /* ethernet header */ + memcpy(eth_hdr->h_dest, "\x3c\xfd\xfe\x9e\x7f\x71", ETH_ALEN); + memcpy(eth_hdr->h_source, "\xec\xb1\xd7\x98\x3a\xc0", ETH_ALEN); + eth_hdr->h_proto = htons(ETH_P_IP); + + /* IP header */ + ip_hdr->version = IPVERSION; + ip_hdr->ihl = 0x5; /* 20 byte header */ + ip_hdr->tos = 0x0; + ip_hdr->tot_len = htons(IP_PKT_SIZE); + ip_hdr->id = 0; + ip_hdr->frag_off = 0; + ip_hdr->ttl = IPDEFTTL; + ip_hdr->protocol = IPPROTO_UDP; + ip_hdr->saddr = htonl(0x0a0a0a10); + ip_hdr->daddr = htonl(0x0a0a0a20); + + /* IP header checksum */ + ip_hdr->check = 0; + ip_hdr->check = ip_fast_csum((const void *)ip_hdr, ip_hdr->ihl); + + /* UDP header */ + udp_hdr->source = htons(0x1000); + udp_hdr->dest = htons(0x1000); + udp_hdr->len = htons(UDP_PKT_SIZE); + + /* UDP data */ + memset32_htonl(pkt_data + PKT_HDR_SIZE, opt_pkt_fill_pattern, + UDP_PKT_DATA_SIZE); + + /* UDP header checksum */ + udp_hdr->check = 0; + udp_hdr->check = udp_csum(ip_hdr->saddr, ip_hdr->daddr, UDP_PKT_SIZE, + IPPROTO_UDP, (u16 *)udp_hdr); +} + +static void gen_eth_frame(struct xsk_umem_info *umem, u64 addr) { memcpy(xsk_umem__get_data(umem->buffer, addr), pkt_data, - sizeof(pkt_data) - 1); - return sizeof(pkt_data) - 1; + PKT_SIZE); } static struct xsk_umem_info *xsk_configure_umem(void *buffer, u64 size) @@ -375,6 +626,11 @@ static struct option long_options[] = { {"unaligned", no_argument, 0, 'u'}, {"shared-umem", no_argument, 0, 'M'}, {"force", no_argument, 0, 'F'}, + {"duration", required_argument, 0, 'd'}, + {"batch-size", required_argument, 0, 'b'}, + {"tx-pkt-count", required_argument, 0, 'C'}, + {"tx-pkt-size", required_argument, 0, 's'}, + {"tx-pkt-pattern", required_argument, 0, 'P'}, {0, 0, 0, 0} }; @@ -399,8 +655,21 @@ static void usage(const char *prog) " -u, --unaligned Enable unaligned chunk placement\n" " -M, --shared-umem Enable XDP_SHARED_UMEM\n" " -F, --force Force loading the XDP prog\n" + " -d, --duration=n Duration in secs to run command.\n" + " Default: forever.\n" + " -b, --batch-size=n Batch size for sending or receiving\n" + " packets. Default: %d\n" + " -C, --tx-pkt-count=n Number of packets to send.\n" + " Default: Continuous packets.\n" + " -s, --tx-pkt-size=n Transmit packet size.\n" + " (Default: %d bytes)\n" + " Min size: %d, Max size %d.\n" + " -P, --tx-pkt-pattern=nPacket fill pattern. Default: 0x%x\n" "\n"; - fprintf(stderr, str, prog, XSK_UMEM__DEFAULT_FRAME_SIZE); + fprintf(stderr, str, prog, XSK_UMEM__DEFAULT_FRAME_SIZE, + opt_batch_size, MIN_PKT_SIZE, MIN_PKT_SIZE, + XSK_UMEM__DEFAULT_FRAME_SIZE, opt_pkt_fill_pattern); + exit(EXIT_FAILURE); } @@ -411,7 +680,7 @@ static void parse_command_line(int argc, char **argv) opterr = 0; for (;;) { - c = getopt_long(argc, argv, "Frtli:q:psSNn:czf:muM", + c = getopt_long(argc, argv, "Frtli:q:pSNn:czf:muMd:b:C:s:P:", long_options, &option_index); if (c == -1) break; @@ -440,7 +709,7 @@ static void parse_command_line(int argc, char **argv) opt_xdp_bind_flags |= XDP_COPY; break; case 'N': - opt_xdp_flags |= XDP_FLAGS_DRV_MODE; + /* default, set below */ break; case 'n': opt_interval = atoi(optarg); @@ -469,11 +738,37 @@ static void parse_command_line(int argc, char **argv) case 'M': opt_num_xsks = MAX_SOCKS; break; + case 'd': + opt_duration = atoi(optarg); + opt_duration *= 1000000000; + break; + case 'b': + opt_batch_size = atoi(optarg); + break; + case 'C': + opt_pkt_count = atoi(optarg); + break; + case 's': + opt_pkt_size = atoi(optarg); + if (opt_pkt_size > (XSK_UMEM__DEFAULT_FRAME_SIZE) || + opt_pkt_size < MIN_PKT_SIZE) { + fprintf(stderr, + "ERROR: Invalid frame size %d\n", + opt_pkt_size); + usage(basename(argv[0])); + } + break; + case 'P': + opt_pkt_fill_pattern = strtol(optarg, NULL, 16); + break; default: usage(basename(argv[0])); } } + if (!(opt_xdp_flags & XDP_FLAGS_SKB_MODE)) + opt_xdp_flags |= XDP_FLAGS_DRV_MODE; + opt_ifindex = if_nametoindex(opt_if); if (!opt_ifindex) { fprintf(stderr, "ERROR: interface \"%s\" does not exist\n", @@ -513,7 +808,7 @@ static inline void complete_tx_l2fwd(struct xsk_socket_info *xsk, if (!opt_need_wakeup || xsk_ring_prod__needs_wakeup(&xsk->tx)) kick_tx(xsk); - ndescs = (xsk->outstanding_tx > BATCH_SIZE) ? BATCH_SIZE : + ndescs = (xsk->outstanding_tx > opt_batch_size) ? opt_batch_size : xsk->outstanding_tx; /* re-add completed Tx buffers */ @@ -542,7 +837,8 @@ static inline void complete_tx_l2fwd(struct xsk_socket_info *xsk, } } -static inline void complete_tx_only(struct xsk_socket_info *xsk) +static inline void complete_tx_only(struct xsk_socket_info *xsk, + int batch_size) { unsigned int rcvd; u32 idx; @@ -553,7 +849,7 @@ static inline void complete_tx_only(struct xsk_socket_info *xsk) if (!opt_need_wakeup || xsk_ring_prod__needs_wakeup(&xsk->tx)) kick_tx(xsk); - rcvd = xsk_ring_cons__peek(&xsk->umem->cq, BATCH_SIZE, &idx); + rcvd = xsk_ring_cons__peek(&xsk->umem->cq, batch_size, &idx); if (rcvd > 0) { xsk_ring_cons__release(&xsk->umem->cq, rcvd); xsk->outstanding_tx -= rcvd; @@ -567,7 +863,7 @@ static void rx_drop(struct xsk_socket_info *xsk, struct pollfd *fds) u32 idx_rx = 0, idx_fq = 0; int ret; - rcvd = xsk_ring_cons__peek(&xsk->rx, BATCH_SIZE, &idx_rx); + rcvd = xsk_ring_cons__peek(&xsk->rx, opt_batch_size, &idx_rx); if (!rcvd) { if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq)) ret = poll(fds, num_socks, opt_timeout); @@ -619,36 +915,68 @@ static void rx_drop_all(void) for (i = 0; i < num_socks; i++) rx_drop(xsks[i], fds); + + if (benchmark_done) + break; } } -static void tx_only(struct xsk_socket_info *xsk, u32 frame_nb) +static void tx_only(struct xsk_socket_info *xsk, u32 frame_nb, int batch_size) { u32 idx; + unsigned int i; - if (xsk_ring_prod__reserve(&xsk->tx, BATCH_SIZE, &idx) == BATCH_SIZE) { - unsigned int i; - - for (i = 0; i < BATCH_SIZE; i++) { - xsk_ring_prod__tx_desc(&xsk->tx, idx + i)->addr = - (frame_nb + i) << XSK_UMEM__DEFAULT_FRAME_SHIFT; - xsk_ring_prod__tx_desc(&xsk->tx, idx + i)->len = - sizeof(pkt_data) - 1; - } + while (xsk_ring_prod__reserve(&xsk->tx, batch_size, &idx) < + batch_size) { + complete_tx_only(xsk, batch_size); + } - xsk_ring_prod__submit(&xsk->tx, BATCH_SIZE); - xsk->outstanding_tx += BATCH_SIZE; - frame_nb += BATCH_SIZE; - frame_nb %= NUM_FRAMES; + for (i = 0; i < batch_size; i++) { + struct xdp_desc *tx_desc = xsk_ring_prod__tx_desc(&xsk->tx, + idx + i); + tx_desc->addr = (frame_nb + i) << XSK_UMEM__DEFAULT_FRAME_SHIFT; + tx_desc->len = PKT_SIZE; } - complete_tx_only(xsk); + xsk_ring_prod__submit(&xsk->tx, batch_size); + xsk->outstanding_tx += batch_size; + frame_nb += batch_size; + frame_nb %= NUM_FRAMES; + complete_tx_only(xsk, batch_size); +} + +static inline int get_batch_size(int pkt_cnt) +{ + if (!opt_pkt_count) + return opt_batch_size; + + if (pkt_cnt + opt_batch_size <= opt_pkt_count) + return opt_batch_size; + + return opt_pkt_count - pkt_cnt; +} + +static void complete_tx_only_all(void) +{ + bool pending; + int i; + + do { + pending = false; + for (i = 0; i < num_socks; i++) { + if (xsks[i]->outstanding_tx) { + complete_tx_only(xsks[i], opt_batch_size); + pending = !!xsks[i]->outstanding_tx; + } + } + } while (pending); } static void tx_only_all(void) { struct pollfd fds[MAX_SOCKS] = {}; u32 frame_nb[MAX_SOCKS] = {}; + int pkt_cnt = 0; int i, ret; for (i = 0; i < num_socks; i++) { @@ -656,7 +984,9 @@ static void tx_only_all(void) fds[0].events = POLLOUT; } - for (;;) { + while ((opt_pkt_count && pkt_cnt < opt_pkt_count) || !opt_pkt_count) { + int batch_size = get_batch_size(pkt_cnt); + if (opt_poll) { ret = poll(fds, num_socks, opt_timeout); if (ret <= 0) @@ -667,8 +997,16 @@ static void tx_only_all(void) } for (i = 0; i < num_socks; i++) - tx_only(xsks[i], frame_nb[i]); + tx_only(xsks[i], frame_nb[i], batch_size); + + pkt_cnt += batch_size; + + if (benchmark_done) + break; } + + if (opt_pkt_count) + complete_tx_only_all(); } static void l2fwd(struct xsk_socket_info *xsk, struct pollfd *fds) @@ -679,7 +1017,7 @@ static void l2fwd(struct xsk_socket_info *xsk, struct pollfd *fds) complete_tx_l2fwd(xsk, fds); - rcvd = xsk_ring_cons__peek(&xsk->rx, BATCH_SIZE, &idx_rx); + rcvd = xsk_ring_cons__peek(&xsk->rx, opt_batch_size, &idx_rx); if (!rcvd) { if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq)) ret = poll(fds, num_socks, opt_timeout); @@ -736,6 +1074,9 @@ static void l2fwd_all(void) for (i = 0; i < num_socks; i++) l2fwd(xsks[i], fds); + + if (benchmark_done) + break; } } @@ -831,9 +1172,12 @@ int main(int argc, char **argv) for (i = 0; i < opt_num_xsks; i++) xsks[num_socks++] = xsk_configure_socket(umem, rx, tx); - if (opt_bench == BENCH_TXONLY) + if (opt_bench == BENCH_TXONLY) { + gen_eth_hdr_data(); + for (i = 0; i < NUM_FRAMES; i++) gen_eth_frame(umem, i * opt_xsk_frame_size); + } if (opt_num_xsks > 1 && opt_bench != BENCH_TXONLY) enter_xsks_into_map(obj); @@ -849,6 +1193,7 @@ int main(int argc, char **argv) exit_with_error(ret); prev_time = get_nsecs(); + start_time = prev_time; if (opt_bench == BENCH_RXDROP) rx_drop_all(); @@ -857,5 +1202,11 @@ int main(int argc, char **argv) else l2fwd_all(); + benchmark_done = true; + + pthread_join(pt, NULL); + + xdpsock_cleanup(); + return 0; } |