From cb658e9724e3c34973eee913b1ff0cb9c50b8e53 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Mon, 4 Mar 2019 20:56:19 -0800 Subject: [PATCH] AF_XDP: add xsk.{c,h} to Makefile and fix build This patch makes sure we build AF_XDP-related code as part of libbpf. This also required copying few uapi/linux headers and adding few used definitions in include headers. Signed-off-by: Andrii Nakryiko --- include/asm/barrier.h | 7 ++++ include/linux/compiler.h | 52 +++++++++++++++++++++++++ include/linux/filter.h | 66 +++++++++++++++++++++++++++++++ include/linux/kernel.h | 1 + include/linux/ring_buffer.h | 35 +---------------- include/uapi/linux/if_xdp.h | 78 +++++++++++++++++++++++++++++++++++++ src/.gitignore | 2 + src/Makefile | 2 +- 8 files changed, 208 insertions(+), 35 deletions(-) create mode 100644 include/asm/barrier.h create mode 100644 include/linux/compiler.h create mode 100644 include/uapi/linux/if_xdp.h create mode 100644 src/.gitignore diff --git a/include/asm/barrier.h b/include/asm/barrier.h new file mode 100644 index 0000000..1fc6aee --- /dev/null +++ b/include/asm/barrier.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ +#ifndef __ASM_BARRIER_H +#define __ASM_BARRIER_H + +#include + +#endif diff --git a/include/linux/compiler.h b/include/linux/compiler.h new file mode 100644 index 0000000..ba96a2e --- /dev/null +++ b/include/linux/compiler.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ + +#ifndef __LINUX_COMPILER_H +#define __LINUX_COMPILER_H + +#define likely(x) __builtin_expect(!!(x), 1) +#define unlikely(x) __builtin_expect(!!(x), 0) + +#define READ_ONCE(x) (*(volatile typeof(x) *)&x) +#define WRITE_ONCE(x, v) (*(volatile typeof(x) *)&x) = (v) + +#define barrier() asm volatile("" ::: "memory") + +#if defined(__x86_64__) + +# define smp_rmb() asm volatile("lfence" ::: "memory") +# define smp_wmb() asm volatile("sfence" ::: "memory") + +# define smp_store_release(p, v) \ +do { \ + barrier(); \ + WRITE_ONCE(*p, v); \ +} while (0) + +# define smp_load_acquire(p) \ +({ \ + typeof(*p) ___p = READ_ONCE(*p); \ + barrier(); \ + ___p; \ +}) + +#else + +# define smp_mb() __sync_synchronize() +# define smp_rmb() smp_mb() +# define smp_wmb() smp_mb() + +# define smp_store_release(p, v) \ +do { \ + smp_mb(); \ + WRITE_ONCE(*p, v); \ +} while (0) + +# define smp_load_acquire(p) \ +({ \ + typeof(*p) ___p = READ_ONCE(*p); \ + smp_mb(); \ + ___p; \ +}) + +#endif /* defined(__x86_64__) */ +#endif diff --git a/include/linux/filter.h b/include/linux/filter.h index d9a7504..521bfd5 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -5,6 +5,14 @@ #include +#define BPF_ALU64_IMM(OP, DST, IMM) \ + ((struct bpf_insn) { \ + .code = BPF_ALU64 | BPF_OP(OP) | BPF_K, \ + .dst_reg = DST, \ + .src_reg = 0, \ + .off = 0, \ + .imm = IMM }) + #define BPF_MOV64_IMM(DST, IMM) \ ((struct bpf_insn) { \ .code = BPF_ALU64 | BPF_MOV | BPF_K, \ @@ -29,4 +37,62 @@ .off = 0, \ .imm = ((FUNC) - BPF_FUNC_unspec) }) +#define BPF_LDX_MEM(SIZE, DST, SRC, OFF) \ + ((struct bpf_insn) { \ + .code = BPF_LDX | BPF_SIZE(SIZE) | BPF_MEM, \ + .dst_reg = DST, \ + .src_reg = SRC, \ + .off = OFF, \ + .imm = 0 }) + +#define BPF_STX_MEM(SIZE, DST, SRC, OFF) \ + ((struct bpf_insn) { \ + .code = BPF_STX | BPF_SIZE(SIZE) | BPF_MEM, \ + .dst_reg = DST, \ + .src_reg = SRC, \ + .off = OFF, \ + .imm = 0 }) + +#define BPF_MOV64_REG(DST, SRC) \ + ((struct bpf_insn) { \ + .code = BPF_ALU64 | BPF_MOV | BPF_X, \ + .dst_reg = DST, \ + .src_reg = SRC, \ + .off = 0, \ + .imm = 0 }) + +#define BPF_MOV32_IMM(DST, IMM) \ + ((struct bpf_insn) { \ + .code = BPF_ALU | BPF_MOV | BPF_K, \ + .dst_reg = DST, \ + .src_reg = 0, \ + .off = 0, \ + .imm = IMM }) + +#define BPF_LD_IMM64_RAW(DST, SRC, IMM) \ + ((struct bpf_insn) { \ + .code = BPF_LD | BPF_DW | BPF_IMM, \ + .dst_reg = DST, \ + .src_reg = SRC, \ + .off = 0, \ + .imm = (__u32) (IMM) }), \ + ((struct bpf_insn) { \ + .code = 0, \ + .dst_reg = 0, \ + .src_reg = 0, \ + .off = 0, \ + .imm = ((__u64) (IMM)) >> 32 }) + +#define BPF_LD_MAP_FD(DST, MAP_FD) \ + BPF_LD_IMM64_RAW(DST, BPF_PSEUDO_MAP_FD, MAP_FD) + +#define BPF_JMP_IMM(OP, DST, IMM, OFF) \ + ((struct bpf_insn) { \ + .code = BPF_JMP | BPF_OP(OP) | BPF_K, \ + .dst_reg = DST, \ + .src_reg = 0, \ + .off = OFF, \ + .imm = IMM }) + + #endif diff --git a/include/linux/kernel.h b/include/linux/kernel.h index cf897a6..a4a7a9d 100644 --- a/include/linux/kernel.h +++ b/include/linux/kernel.h @@ -39,5 +39,6 @@ #endif #define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0])) +#define __KERNEL_DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d)) #endif diff --git a/include/linux/ring_buffer.h b/include/linux/ring_buffer.h index a671bbb..fc4677b 100644 --- a/include/linux/ring_buffer.h +++ b/include/linux/ring_buffer.h @@ -2,40 +2,7 @@ #ifndef _TOOLS_LINUX_RING_BUFFER_H_ #define _TOOLS_LINUX_RING_BUFFER_H_ -#define READ_ONCE(x) (*(volatile typeof(x) *)&x) -#define WRITE_ONCE(x, v) (*(volatile typeof(x) *)&x) = (v) - -#define barrier() asm volatile("" ::: "memory") - -#if defined(__x86_64__) -# define smp_store_release(p, v) \ -do { \ - barrier(); \ - WRITE_ONCE(*p, v); \ -} while (0) - -# define smp_load_acquire(p) \ -({ \ - typeof(*p) ___p = READ_ONCE(*p); \ - barrier(); \ - ___p; \ -}) -#else -# define smp_mb() __sync_synchronize() - -# define smp_store_release(p, v) \ -do { \ - smp_mb(); \ - WRITE_ONCE(*p, v); \ -} while (0) - -# define smp_load_acquire(p) \ -({ \ - typeof(*p) ___p = READ_ONCE(*p); \ - smp_mb(); \ - ___p; \ -}) -#endif /* defined(__x86_64__) */ +#include static inline __u64 ring_buffer_read_head(struct perf_event_mmap_page *base) { diff --git a/include/uapi/linux/if_xdp.h b/include/uapi/linux/if_xdp.h new file mode 100644 index 0000000..caed8b1 --- /dev/null +++ b/include/uapi/linux/if_xdp.h @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * if_xdp: XDP socket user-space interface + * Copyright(c) 2018 Intel Corporation. + * + * Author(s): Björn Töpel + * Magnus Karlsson + */ + +#ifndef _LINUX_IF_XDP_H +#define _LINUX_IF_XDP_H + +#include + +/* Options for the sxdp_flags field */ +#define XDP_SHARED_UMEM (1 << 0) +#define XDP_COPY (1 << 1) /* Force copy-mode */ +#define XDP_ZEROCOPY (1 << 2) /* Force zero-copy mode */ + +struct sockaddr_xdp { + __u16 sxdp_family; + __u16 sxdp_flags; + __u32 sxdp_ifindex; + __u32 sxdp_queue_id; + __u32 sxdp_shared_umem_fd; +}; + +struct xdp_ring_offset { + __u64 producer; + __u64 consumer; + __u64 desc; +}; + +struct xdp_mmap_offsets { + struct xdp_ring_offset rx; + struct xdp_ring_offset tx; + struct xdp_ring_offset fr; /* Fill */ + struct xdp_ring_offset cr; /* Completion */ +}; + +/* XDP socket options */ +#define XDP_MMAP_OFFSETS 1 +#define XDP_RX_RING 2 +#define XDP_TX_RING 3 +#define XDP_UMEM_REG 4 +#define XDP_UMEM_FILL_RING 5 +#define XDP_UMEM_COMPLETION_RING 6 +#define XDP_STATISTICS 7 + +struct xdp_umem_reg { + __u64 addr; /* Start of packet data area */ + __u64 len; /* Length of packet data area */ + __u32 chunk_size; + __u32 headroom; +}; + +struct xdp_statistics { + __u64 rx_dropped; /* Dropped for reasons other than invalid desc */ + __u64 rx_invalid_descs; /* Dropped due to invalid descriptor */ + __u64 tx_invalid_descs; /* Dropped due to invalid descriptor */ +}; + +/* Pgoff for mmaping the rings */ +#define XDP_PGOFF_RX_RING 0 +#define XDP_PGOFF_TX_RING 0x80000000 +#define XDP_UMEM_PGOFF_FILL_RING 0x100000000ULL +#define XDP_UMEM_PGOFF_COMPLETION_RING 0x180000000ULL + +/* Rx/Tx descriptor */ +struct xdp_desc { + __u64 addr; + __u32 len; + __u32 options; +}; + +/* UMEM descriptor is __u64 */ + +#endif /* _LINUX_IF_XDP_H */ diff --git a/src/.gitignore b/src/.gitignore new file mode 100644 index 0000000..e0292b1 --- /dev/null +++ b/src/.gitignore @@ -0,0 +1,2 @@ +*.o +*.a diff --git a/src/Makefile b/src/Makefile index cdd7c24..7ea1ff1 100644 --- a/src/Makefile +++ b/src/Makefile @@ -20,7 +20,7 @@ ALL_CFLAGS += $(CFLAGS) OBJDIR ?= . OBJS := $(addprefix $(OBJDIR)/,bpf.o btf.o libbpf.o libbpf_errno.o netlink.o \ - nlattr.o str_error.o libbpf_probes.o) + nlattr.o str_error.o libbpf_probes.o bpf_prog_linfo.o xsk.o) LIBS := $(OBJDIR)/libbpf.a ifdef BUILD_SHARED