From 3cb70413041fdf028fa1ba3986fd0c6aec9e3dcb Mon Sep 17 00:00:00 2001 From: Pu Lehui Date: Mon, 30 May 2022 17:28:11 +0800 Subject: bpf, riscv: Support riscv jit to provide bpf_line_info Add support for riscv jit to provide bpf_line_info. We need to consider the prologue offset in ctx->offset, but unlike x86 and arm64, ctx->offset of riscv does not provide an extra slot for the prologue, so here we just calculate the len of prologue and add it to ctx->offset at the end. Both RV64 and RV32 have been tested. Signed-off-by: Pu Lehui Signed-off-by: Daniel Borkmann Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20220530092815.1112406-3-pulehui@huawei.com --- arch/riscv/net/bpf_jit.h | 1 + arch/riscv/net/bpf_jit_core.c | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) (limited to 'arch') diff --git a/arch/riscv/net/bpf_jit.h b/arch/riscv/net/bpf_jit.h index 2a3715bf29fe..d926e0f7ef57 100644 --- a/arch/riscv/net/bpf_jit.h +++ b/arch/riscv/net/bpf_jit.h @@ -69,6 +69,7 @@ struct rv_jit_context { struct bpf_prog *prog; u16 *insns; /* RV insns */ int ninsns; + int body_len; int epilogue_offset; int *offset; /* BPF to RV */ int nexentries; diff --git a/arch/riscv/net/bpf_jit_core.c b/arch/riscv/net/bpf_jit_core.c index be743d700aa7..737baf8715da 100644 --- a/arch/riscv/net/bpf_jit_core.c +++ b/arch/riscv/net/bpf_jit_core.c @@ -44,7 +44,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) unsigned int prog_size = 0, extable_size = 0; bool tmp_blinded = false, extra_pass = false; struct bpf_prog *tmp, *orig_prog = prog; - int pass = 0, prev_ninsns = 0, i; + int pass = 0, prev_ninsns = 0, prologue_len, i; struct rv_jit_data *jit_data; struct rv_jit_context *ctx; @@ -95,6 +95,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) prog = orig_prog; goto out_offset; } + ctx->body_len = ctx->ninsns; bpf_jit_build_prologue(ctx); ctx->epilogue_offset = ctx->ninsns; bpf_jit_build_epilogue(ctx); @@ -161,6 +162,11 @@ skip_init_ctx: if (!prog->is_func || extra_pass) { bpf_jit_binary_lock_ro(jit_data->header); + prologue_len = ctx->epilogue_offset - ctx->body_len; + for (i = 0; i < prog->len; i++) + ctx->offset[i] = ninsns_rvoff(prologue_len + + ctx->offset[i]); + bpf_prog_fill_jited_linfo(prog, ctx->offset); out_offset: kfree(ctx->offset); kfree(jit_data); -- cgit v1.2.3 From 38a71fc048955c5c9d8bd14351d0f8cbcfef4f5b Mon Sep 17 00:00:00 2001 From: Conor Dooley Date: Tue, 7 Jun 2022 07:55:00 +0100 Subject: riscv: dts: microchip: add mpfs's CAN controllers PolarFire SoC has a pair of CAN controllers, but as they were undocumented there were omitted from the device tree. Add them. Link: https://lore.kernel.org/all/20220607065459.2035746-3-conor.dooley@microchip.com Signed-off-by: Conor Dooley Signed-off-by: Marc Kleine-Budde --- arch/riscv/boot/dts/microchip/mpfs.dtsi | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'arch') diff --git a/arch/riscv/boot/dts/microchip/mpfs.dtsi b/arch/riscv/boot/dts/microchip/mpfs.dtsi index 8c3259134194..737e0e70c432 100644 --- a/arch/riscv/boot/dts/microchip/mpfs.dtsi +++ b/arch/riscv/boot/dts/microchip/mpfs.dtsi @@ -330,6 +330,24 @@ status = "disabled"; }; + can0: can@2010c000 { + compatible = "microchip,mpfs-can"; + reg = <0x0 0x2010c000 0x0 0x1000>; + clocks = <&clkcfg CLK_CAN0>; + interrupt-parent = <&plic>; + interrupts = <56>; + status = "disabled"; + }; + + can1: can@2010d000 { + compatible = "microchip,mpfs-can"; + reg = <0x0 0x2010d000 0x0 0x1000>; + clocks = <&clkcfg CLK_CAN1>; + interrupt-parent = <&plic>; + interrupts = <57>; + status = "disabled"; + }; + mac0: ethernet@20110000 { compatible = "cdns,macb"; reg = <0x0 0x20110000 0x0 0x2000>; -- cgit v1.2.3 From fc386ba7211d514f2c20285cb6b9b502618634e0 Mon Sep 17 00:00:00 2001 From: YueHaibing Date: Sat, 11 Jun 2022 12:09:04 +0800 Subject: bpf, arm: Remove unused function emit_a32_alu_r() Since commit b18bea2a45b1 ("ARM: net: bpf: improve 64-bit ALU implementation") this is unused anymore, so can remove it. Signed-off-by: YueHaibing Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20220611040904.8976-1-yuehaibing@huawei.com --- arch/arm/net/bpf_jit_32.c | 16 ---------------- 1 file changed, 16 deletions(-) (limited to 'arch') diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c index 9e457156ad4d..6a1c9fca5260 100644 --- a/arch/arm/net/bpf_jit_32.c +++ b/arch/arm/net/bpf_jit_32.c @@ -712,22 +712,6 @@ static inline void emit_alu_r(const u8 dst, const u8 src, const bool is64, } } -/* ALU operation (32 bit) - * dst = dst (op) src - */ -static inline void emit_a32_alu_r(const s8 dst, const s8 src, - struct jit_ctx *ctx, const bool is64, - const bool hi, const u8 op) { - const s8 *tmp = bpf2a32[TMP_REG_1]; - s8 rn, rd; - - rn = arm_bpf_get_reg32(src, tmp[1], ctx); - rd = arm_bpf_get_reg32(dst, tmp[0], ctx); - /* ALU operation */ - emit_alu_r(rd, rn, is64, hi, op, ctx); - arm_bpf_put_reg32(dst, rd, ctx); -} - /* ALU operation (64 bit) */ static inline void emit_a32_alu_r64(const bool is64, const s8 dst[], const s8 src[], struct jit_ctx *ctx, -- cgit v1.2.3 From c1804463e5c6a2bf5ab3e278b9d360c25fa49af2 Mon Sep 17 00:00:00 2001 From: Frank Wunderlich Date: Fri, 10 Jun 2022 19:05:41 +0200 Subject: arm64: dts: rockchip: Add mt7531 dsa node to BPI-R2-Pro board Add Device Tree node for mt7531 switch connected to gmac0. Signed-off-by: Frank Wunderlich Reviewed-by: Florian Fainelli Signed-off-by: Jakub Kicinski --- arch/arm64/boot/dts/rockchip/rk3568-bpi-r2-pro.dts | 48 ++++++++++++++++++++++ 1 file changed, 48 insertions(+) (limited to 'arch') diff --git a/arch/arm64/boot/dts/rockchip/rk3568-bpi-r2-pro.dts b/arch/arm64/boot/dts/rockchip/rk3568-bpi-r2-pro.dts index 40cf2236c0b6..7df8cfb1d3b9 100644 --- a/arch/arm64/boot/dts/rockchip/rk3568-bpi-r2-pro.dts +++ b/arch/arm64/boot/dts/rockchip/rk3568-bpi-r2-pro.dts @@ -394,6 +394,54 @@ status = "disabled"; }; +&mdio0 { + #address-cells = <1>; + #size-cells = <0>; + + switch@0 { + compatible = "mediatek,mt7531"; + reg = <0>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + label = "lan0"; + }; + + port@2 { + reg = <2>; + label = "lan1"; + }; + + port@3 { + reg = <3>; + label = "lan2"; + }; + + port@4 { + reg = <4>; + label = "lan3"; + }; + + port@5 { + reg = <5>; + label = "cpu"; + ethernet = <&gmac0>; + phy-mode = "rgmii"; + + fixed-link { + speed = <1000>; + full-duplex; + pause; + }; + }; + }; + }; +}; + &mdio1 { rgmii_phy1: ethernet-phy@0 { compatible = "ethernet-phy-ieee802.3-c22"; -- cgit v1.2.3 From 95acd8817e66d031d2e6ee7def3f1e1874819317 Mon Sep 17 00:00:00 2001 From: Tony Ambardar Date: Fri, 17 Jun 2022 12:57:34 +0200 Subject: bpf, x64: Add predicate for bpf2bpf with tailcalls support in JIT The BPF core/verifier is hard-coded to permit mixing bpf2bpf and tail calls for only x86-64. Change the logic to instead rely on a new weak function 'bool bpf_jit_supports_subprog_tailcalls(void)', which a capable JIT backend can override. Update the x86-64 eBPF JIT to reflect this. Signed-off-by: Tony Ambardar [jakub: drop MIPS bits and tweak patch subject] Signed-off-by: Jakub Sitnicki Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20220617105735.733938-2-jakub@cloudflare.com --- arch/x86/net/bpf_jit_comp.c | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'arch') diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index f298b18a9a3d..2c51ca9f7cec 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -2491,3 +2491,9 @@ void *bpf_arch_text_copy(void *dst, void *src, size_t len) return ERR_PTR(-EINVAL); return dst; } + +/* Indicate the JIT backend supports mixing bpf2bpf and tailcalls. */ +bool bpf_jit_supports_subprog_tailcalls(void) +{ + return true; +} -- cgit v1.2.3 From d4609a5d8c70d21b4a3f801cf896a3c16c613fe1 Mon Sep 17 00:00:00 2001 From: Jakub Sitnicki Date: Fri, 17 Jun 2022 12:57:35 +0200 Subject: bpf, arm64: Keep tail call count across bpf2bpf calls Today doing a BPF tail call after a BPF to BPF call, that is from a subprogram, is allowed only by the x86-64 BPF JIT. Mixing these features requires support from JIT. Tail call count has to be tracked through BPF to BPF calls, as well as through BPF tail calls to prevent unbounded chains of tail calls. arm64 BPF JIT stores the tail call count (TCC) in a dedicated register (X26). This makes it easier to support bpf2bpf calls mixed with tail calls than on x86 platform. In order to keep the tail call count in tact throughout bpf2bpf calls, all we need to do is tweak the program prologue generator. When emitting prologue for a subprogram, we skip the block that initializes the tail call count and emits a jump pad for the tail call. With this change, a sample execution flow where a bpf2bpf call is followed by a tail call would look like so: int entry(struct __sk_buff *skb): 0xffffffc0090151d4: paciasp 0xffffffc0090151d8: stp x29, x30, [sp, #-16]! 0xffffffc0090151dc: mov x29, sp 0xffffffc0090151e0: stp x19, x20, [sp, #-16]! 0xffffffc0090151e4: stp x21, x22, [sp, #-16]! 0xffffffc0090151e8: stp x25, x26, [sp, #-16]! 0xffffffc0090151ec: stp x27, x28, [sp, #-16]! 0xffffffc0090151f0: mov x25, sp 0xffffffc0090151f4: mov x26, #0x0 // <- init TCC only 0xffffffc0090151f8: bti j // in main prog 0xffffffc0090151fc: sub x27, x25, #0x0 0xffffffc009015200: sub sp, sp, #0x10 0xffffffc009015204: mov w1, #0x0 0xffffffc009015208: mov x10, #0xffffffffffffffff 0xffffffc00901520c: strb w1, [x25, x10] 0xffffffc009015210: mov x10, #0xffffffffffffd25c 0xffffffc009015214: movk x10, #0x902, lsl #16 0xffffffc009015218: movk x10, #0xffc0, lsl #32 0xffffffc00901521c: blr x10 -------------------. // bpf2bpf call 0xffffffc009015220: add x7, x0, #0x0 <-------------. 0xffffffc009015224: add sp, sp, #0x10 | | 0xffffffc009015228: ldp x27, x28, [sp], #16 | | 0xffffffc00901522c: ldp x25, x26, [sp], #16 | | 0xffffffc009015230: ldp x21, x22, [sp], #16 | | 0xffffffc009015234: ldp x19, x20, [sp], #16 | | 0xffffffc009015238: ldp x29, x30, [sp], #16 | | 0xffffffc00901523c: add x0, x7, #0x0 | | 0xffffffc009015240: autiasp | | 0xffffffc009015244: ret | | | | int subprog_tail(struct __sk_buff *skb): | | 0xffffffc00902d25c: paciasp <----------------------' | 0xffffffc00902d260: stp x29, x30, [sp, #-16]! | 0xffffffc00902d264: mov x29, sp | 0xffffffc00902d268: stp x19, x20, [sp, #-16]! | 0xffffffc00902d26c: stp x21, x22, [sp, #-16]! | 0xffffffc00902d270: stp x25, x26, [sp, #-16]! | 0xffffffc00902d274: stp x27, x28, [sp, #-16]! | 0xffffffc00902d278: mov x25, sp | 0xffffffc00902d27c: sub x27, x25, #0x0 | 0xffffffc00902d280: sub sp, sp, #0x10 | // <- end of prologue, notice: 0xffffffc00902d284: add x19, x0, #0x0 | // 1) TCC not touched, and 0xffffffc00902d288: mov w0, #0x1 | // 2) no tail call jump pad 0xffffffc00902d28c: mov x10, #0xfffffffffffffffc | 0xffffffc00902d290: str w0, [x25, x10] | 0xffffffc00902d294: mov x20, #0xffffff80ffffffff | 0xffffffc00902d298: movk x20, #0xc033, lsl #16 | 0xffffffc00902d29c: movk x20, #0x4e00 | 0xffffffc00902d2a0: add x0, x19, #0x0 | 0xffffffc00902d2a4: add x1, x20, #0x0 | 0xffffffc00902d2a8: mov x2, #0x0 | 0xffffffc00902d2ac: mov w10, #0x24 | 0xffffffc00902d2b0: ldr w10, [x1, x10] | 0xffffffc00902d2b4: add w2, w2, #0x0 | 0xffffffc00902d2b8: cmp w2, w10 | 0xffffffc00902d2bc: b.cs 0xffffffc00902d2f8 | 0xffffffc00902d2c0: mov w10, #0x21 | 0xffffffc00902d2c4: cmp x26, x10 | // TCC >= MAX_TAIL_CALL_CNT? 0xffffffc00902d2c8: b.cs 0xffffffc00902d2f8 | 0xffffffc00902d2cc: add x26, x26, #0x1 | // TCC++ 0xffffffc00902d2d0: mov w10, #0x110 | 0xffffffc00902d2d4: add x10, x1, x10 | 0xffffffc00902d2d8: lsl x11, x2, #3 | 0xffffffc00902d2dc: ldr x11, [x10, x11] | 0xffffffc00902d2e0: cbz x11, 0xffffffc00902d2f8 | 0xffffffc00902d2e4: mov w10, #0x30 | 0xffffffc00902d2e8: ldr x10, [x11, x10] | 0xffffffc00902d2ec: add x10, x10, #0x24 | 0xffffffc00902d2f0: add sp, sp, #0x10 | // <- destroy just current 0xffffffc00902d2f4: br x10 ---------------------. | // BPF stack frame 0xffffffc00902d2f8: mov x10, #0xfffffffffffffffc | | // before the tail call 0xffffffc00902d2fc: ldr w7, [x25, x10] | | 0xffffffc00902d300: add sp, sp, #0x10 | | 0xffffffc00902d304: ldp x27, x28, [sp], #16 | | 0xffffffc00902d308: ldp x25, x26, [sp], #16 | | 0xffffffc00902d30c: ldp x21, x22, [sp], #16 | | 0xffffffc00902d310: ldp x19, x20, [sp], #16 | | 0xffffffc00902d314: ldp x29, x30, [sp], #16 | | 0xffffffc00902d318: add x0, x7, #0x0 | | 0xffffffc00902d31c: autiasp | | 0xffffffc00902d320: ret | | | | int classifier_0(struct __sk_buff *skb): | | 0xffffffc008ff5874: paciasp | | 0xffffffc008ff5878: stp x29, x30, [sp, #-16]! | | 0xffffffc008ff587c: mov x29, sp | | 0xffffffc008ff5880: stp x19, x20, [sp, #-16]! | | 0xffffffc008ff5884: stp x21, x22, [sp, #-16]! | | 0xffffffc008ff5888: stp x25, x26, [sp, #-16]! | | 0xffffffc008ff588c: stp x27, x28, [sp, #-16]! | | 0xffffffc008ff5890: mov x25, sp | | 0xffffffc008ff5894: mov x26, #0x0 | | 0xffffffc008ff5898: bti j <----------------------' | 0xffffffc008ff589c: sub x27, x25, #0x0 | 0xffffffc008ff58a0: sub sp, sp, #0x0 | 0xffffffc008ff58a4: mov x0, #0xffffffc0ffffffff | 0xffffffc008ff58a8: movk x0, #0x8fc, lsl #16 | 0xffffffc008ff58ac: movk x0, #0x6000 | 0xffffffc008ff58b0: mov w1, #0x1 | 0xffffffc008ff58b4: str w1, [x0] | 0xffffffc008ff58b8: mov w7, #0x0 | 0xffffffc008ff58bc: mov sp, sp | 0xffffffc008ff58c0: ldp x27, x28, [sp], #16 | 0xffffffc008ff58c4: ldp x25, x26, [sp], #16 | 0xffffffc008ff58c8: ldp x21, x22, [sp], #16 | 0xffffffc008ff58cc: ldp x19, x20, [sp], #16 | 0xffffffc008ff58d0: ldp x29, x30, [sp], #16 | 0xffffffc008ff58d4: add x0, x7, #0x0 | 0xffffffc008ff58d8: autiasp | 0xffffffc008ff58dc: ret -------------------------------' Signed-off-by: Jakub Sitnicki Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20220617105735.733938-3-jakub@cloudflare.com --- arch/arm64/net/bpf_jit_comp.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'arch') diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 42f2e9a8616c..f08a4447d363 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -246,6 +246,7 @@ static bool is_lsi_offset(int offset, int scale) static int build_prologue(struct jit_ctx *ctx, bool ebpf_from_cbpf) { const struct bpf_prog *prog = ctx->prog; + const bool is_main_prog = prog->aux->func_idx == 0; const u8 r6 = bpf2a64[BPF_REG_6]; const u8 r7 = bpf2a64[BPF_REG_7]; const u8 r8 = bpf2a64[BPF_REG_8]; @@ -299,7 +300,7 @@ static int build_prologue(struct jit_ctx *ctx, bool ebpf_from_cbpf) /* Set up BPF prog stack base register */ emit(A64_MOV(1, fp, A64_SP), ctx); - if (!ebpf_from_cbpf) { + if (!ebpf_from_cbpf && is_main_prog) { /* Initialize tail_call_cnt */ emit(A64_MOVZ(1, tcc, 0, 0), ctx); @@ -1530,3 +1531,9 @@ void bpf_jit_free_exec(void *addr) { return vfree(addr); } + +/* Indicate the JIT backend supports mixing bpf2bpf and tailcalls. */ +bool bpf_jit_supports_subprog_tailcalls(void) +{ + return true; +} -- cgit v1.2.3 From 066c3bd358355185d9313358281fe03113c0a9ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20L=C3=A9ger?= Date: Fri, 24 Jun 2022 16:39:57 +0200 Subject: ARM: dts: r9a06g032: describe MII converter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the MII converter node which describes the MII converter that is present on the RZ/N1 SoC. Signed-off-by: Clément Léger Reviewed-by: Florian Fainelli Signed-off-by: David S. Miller --- arch/arm/boot/dts/r9a06g032.dtsi | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) (limited to 'arch') diff --git a/arch/arm/boot/dts/r9a06g032.dtsi b/arch/arm/boot/dts/r9a06g032.dtsi index d3665910958b..f6241af33112 100644 --- a/arch/arm/boot/dts/r9a06g032.dtsi +++ b/arch/arm/boot/dts/r9a06g032.dtsi @@ -304,6 +304,45 @@ data-width = <8>; }; + eth_miic: eth-miic@44030000 { + compatible = "renesas,r9a06g032-miic", "renesas,rzn1-miic"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0x44030000 0x10000>; + clocks = <&sysctrl R9A06G032_CLK_MII_REF>, + <&sysctrl R9A06G032_CLK_RGMII_REF>, + <&sysctrl R9A06G032_CLK_RMII_REF>, + <&sysctrl R9A06G032_HCLK_SWITCH_RG>; + clock-names = "mii_ref", "rgmii_ref", "rmii_ref", "hclk"; + power-domains = <&sysctrl>; + status = "disabled"; + + mii_conv1: mii-conv@1 { + reg = <1>; + status = "disabled"; + }; + + mii_conv2: mii-conv@2 { + reg = <2>; + status = "disabled"; + }; + + mii_conv3: mii-conv@3 { + reg = <3>; + status = "disabled"; + }; + + mii_conv4: mii-conv@4 { + reg = <4>; + status = "disabled"; + }; + + mii_conv5: mii-conv@5 { + reg = <5>; + status = "disabled"; + }; + }; + gic: interrupt-controller@44101000 { compatible = "arm,gic-400", "arm,cortex-a7-gic"; interrupt-controller; -- cgit v1.2.3 From 3f5261f1c2a8d7b178f9f65c6dda92523329486e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20L=C3=A9ger?= Date: Fri, 24 Jun 2022 16:39:58 +0200 Subject: ARM: dts: r9a06g032: describe GMAC2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RZ/N1 SoC includes two MAC named GMACx that are compatible with the "snps,dwmac" driver. GMAC1 is connected directly to the MII converter port 1. GMAC2 however can be used as the MAC for the switch CPU management port or can be muxed to be connected directly to the MII converter port 2. This commit add description for the GMAC2 which will be used by the switch description. Signed-off-by: Clément Léger Reviewed-by: Florian Fainelli Signed-off-by: David S. Miller --- arch/arm/boot/dts/r9a06g032.dtsi | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'arch') diff --git a/arch/arm/boot/dts/r9a06g032.dtsi b/arch/arm/boot/dts/r9a06g032.dtsi index f6241af33112..42ce02e51e8d 100644 --- a/arch/arm/boot/dts/r9a06g032.dtsi +++ b/arch/arm/boot/dts/r9a06g032.dtsi @@ -304,6 +304,24 @@ data-width = <8>; }; + gmac2: ethernet@44002000 { + compatible = "renesas,r9a06g032-gmac", "renesas,rzn1-gmac", "snps,dwmac"; + reg = <0x44002000 0x2000>; + interrupt-parent = <&gic>; + interrupts = , + , + ; + interrupt-names = "macirq", "eth_wake_irq", "eth_lpi"; + clocks = <&sysctrl R9A06G032_HCLK_GMAC1>; + clock-names = "stmmaceth"; + power-domains = <&sysctrl>; + snps,multicast-filter-bins = <256>; + snps,perfect-filter-entries = <128>; + tx-fifo-depth = <2048>; + rx-fifo-depth = <4096>; + status = "disabled"; + }; + eth_miic: eth-miic@44030000 { compatible = "renesas,r9a06g032-miic", "renesas,rzn1-miic"; #address-cells = <1>; -- cgit v1.2.3 From cf9695d8a7e927f7563ce6ea0a4e54b8214a12f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20L=C3=A9ger?= Date: Fri, 24 Jun 2022 16:39:59 +0200 Subject: ARM: dts: r9a06g032: describe switch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add description of the switch that is present on the RZ/N1 SoC. This description includes ethernet-ports description for all the ports that are present on the switch along with their connection to the MII converter ports and to the GMAC for the CPU port. Signed-off-by: Clément Léger Reviewed-by: Vladimir Oltean Reviewed-by: Florian Fainelli Signed-off-by: David S. Miller --- arch/arm/boot/dts/r9a06g032.dtsi | 51 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) (limited to 'arch') diff --git a/arch/arm/boot/dts/r9a06g032.dtsi b/arch/arm/boot/dts/r9a06g032.dtsi index 42ce02e51e8d..5b97fa85474f 100644 --- a/arch/arm/boot/dts/r9a06g032.dtsi +++ b/arch/arm/boot/dts/r9a06g032.dtsi @@ -361,6 +361,57 @@ }; }; + switch: switch@44050000 { + compatible = "renesas,r9a06g032-a5psw", "renesas,rzn1-a5psw"; + reg = <0x44050000 0x10000>; + clocks = <&sysctrl R9A06G032_HCLK_SWITCH>, + <&sysctrl R9A06G032_CLK_SWITCH>; + clock-names = "hclk", "clk"; + power-domains = <&sysctrl>; + status = "disabled"; + + ethernet-ports { + #address-cells = <1>; + #size-cells = <0>; + + switch_port0: port@0 { + reg = <0>; + pcs-handle = <&mii_conv5>; + status = "disabled"; + }; + + switch_port1: port@1 { + reg = <1>; + pcs-handle = <&mii_conv4>; + status = "disabled"; + }; + + switch_port2: port@2 { + reg = <2>; + pcs-handle = <&mii_conv3>; + status = "disabled"; + }; + + switch_port3: port@3 { + reg = <3>; + pcs-handle = <&mii_conv2>; + status = "disabled"; + }; + + switch_port4: port@4 { + reg = <4>; + ethernet = <&gmac2>; + label = "cpu"; + phy-mode = "internal"; + status = "disabled"; + fixed-link { + speed = <1000>; + full-duplex; + }; + }; + }; + }; + gic: interrupt-controller@44101000 { compatible = "arm,gic-400", "arm,cortex-a7-gic"; interrupt-controller; -- cgit v1.2.3 From 9aab31d66ec97d7047e42feacc356bc9c21a5bf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20L=C3=A9ger?= Date: Fri, 24 Jun 2022 16:40:00 +0200 Subject: ARM: dts: r9a06g032-rzn1d400-db: add switch description MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add description for the switch, GMAC2 and MII converter. With these definitions, the switch port 0 and 1 (MII port 5 and 4) are working on RZ/N1D-DB board. Signed-off-by: Clément Léger Reviewed-by: Vladimir Oltean Reviewed-by: Florian Fainelli Signed-off-by: David S. Miller --- arch/arm/boot/dts/r9a06g032-rzn1d400-db.dts | 117 ++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) (limited to 'arch') diff --git a/arch/arm/boot/dts/r9a06g032-rzn1d400-db.dts b/arch/arm/boot/dts/r9a06g032-rzn1d400-db.dts index 3f8f3ce87e12..4227aba70c30 100644 --- a/arch/arm/boot/dts/r9a06g032-rzn1d400-db.dts +++ b/arch/arm/boot/dts/r9a06g032-rzn1d400-db.dts @@ -8,6 +8,8 @@ /dts-v1/; +#include +#include #include "r9a06g032.dtsi" / { @@ -31,3 +33,118 @@ timeout-sec = <60>; status = "okay"; }; + +&gmac2 { + status = "okay"; + phy-mode = "gmii"; + fixed-link { + speed = <1000>; + full-duplex; + }; +}; + +&switch { + status = "okay"; + #address-cells = <1>; + #size-cells = <0>; + + pinctrl-names = "default"; + pinctrl-0 = <&pins_mdio1>, <&pins_eth3>, <&pins_eth4>; + + dsa,member = <0 0>; + + mdio { + clock-frequency = <2500000>; + + #address-cells = <1>; + #size-cells = <0>; + + switch0phy4: ethernet-phy@4 { + reg = <4>; + micrel,led-mode = <1>; + }; + + switch0phy5: ethernet-phy@5 { + reg = <5>; + micrel,led-mode = <1>; + }; + }; +}; + +&switch_port0 { + label = "lan0"; + phy-mode = "mii"; + phy-handle = <&switch0phy5>; + status = "okay"; +}; + +&switch_port1 { + label = "lan1"; + phy-mode = "mii"; + phy-handle = <&switch0phy4>; + status = "okay"; +}; + +&switch_port4 { + status = "okay"; +}; + +ð_miic { + status = "okay"; + renesas,miic-switch-portin = ; +}; + +&mii_conv4 { + renesas,miic-input = ; + status = "okay"; +}; + +&mii_conv5 { + renesas,miic-input = ; + status = "okay"; +}; + +&pinctrl{ + pins_mdio1: pins_mdio1 { + pinmux = < + RZN1_PINMUX(152, RZN1_FUNC_MDIO1_SWITCH) + RZN1_PINMUX(153, RZN1_FUNC_MDIO1_SWITCH) + >; + }; + pins_eth3: pins_eth3 { + pinmux = < + RZN1_PINMUX(36, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) + RZN1_PINMUX(37, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) + RZN1_PINMUX(38, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) + RZN1_PINMUX(39, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) + RZN1_PINMUX(40, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) + RZN1_PINMUX(41, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) + RZN1_PINMUX(42, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) + RZN1_PINMUX(43, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) + RZN1_PINMUX(44, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) + RZN1_PINMUX(45, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) + RZN1_PINMUX(46, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) + RZN1_PINMUX(47, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) + >; + drive-strength = <6>; + bias-disable; + }; + pins_eth4: pins_eth4 { + pinmux = < + RZN1_PINMUX(48, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) + RZN1_PINMUX(49, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) + RZN1_PINMUX(50, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) + RZN1_PINMUX(51, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) + RZN1_PINMUX(52, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) + RZN1_PINMUX(53, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) + RZN1_PINMUX(54, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) + RZN1_PINMUX(55, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) + RZN1_PINMUX(56, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) + RZN1_PINMUX(57, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) + RZN1_PINMUX(58, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) + RZN1_PINMUX(59, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) + >; + drive-strength = <6>; + bias-disable; + }; +}; -- cgit v1.2.3 From eba3a9816ad11d4bba9e14acbc8737bdce6c067c Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Mon, 27 Jun 2022 10:39:00 -0700 Subject: Revert the ARM/dts changes for Renesas RZ/N1 Based on a request from Geert: Revert "ARM: dts: r9a06g032-rzn1d400-db: add switch description" This reverts commit 9aab31d66ec97d7047e42feacc356bc9c21a5bf5. Revert "ARM: dts: r9a06g032: describe switch" This reverts commit cf9695d8a7e927f7563ce6ea0a4e54b8214a12f1. Revert "ARM: dts: r9a06g032: describe GMAC2" This reverts commit 3f5261f1c2a8d7b178f9f65c6dda92523329486e. Revert "ARM: dts: r9a06g032: describe MII converter" This reverts commit 066c3bd358355185d9313358281fe03113c0a9ad. to let these changes flow thru the platform and SoC trees. Link: https://lore.kernel.org/r/CAMuHMdUvSLFU56gsp1a9isOiP9otdCJ2-BqhbrffcoHuA6JNig@mail.gmail.com/ Link: https://lore.kernel.org/r/20220627173900.3136386-1-kuba@kernel.org Signed-off-by: Jakub Kicinski --- arch/arm/boot/dts/r9a06g032-rzn1d400-db.dts | 117 ---------------------------- arch/arm/boot/dts/r9a06g032.dtsi | 108 ------------------------- 2 files changed, 225 deletions(-) (limited to 'arch') diff --git a/arch/arm/boot/dts/r9a06g032-rzn1d400-db.dts b/arch/arm/boot/dts/r9a06g032-rzn1d400-db.dts index 4227aba70c30..3f8f3ce87e12 100644 --- a/arch/arm/boot/dts/r9a06g032-rzn1d400-db.dts +++ b/arch/arm/boot/dts/r9a06g032-rzn1d400-db.dts @@ -8,8 +8,6 @@ /dts-v1/; -#include -#include #include "r9a06g032.dtsi" / { @@ -33,118 +31,3 @@ timeout-sec = <60>; status = "okay"; }; - -&gmac2 { - status = "okay"; - phy-mode = "gmii"; - fixed-link { - speed = <1000>; - full-duplex; - }; -}; - -&switch { - status = "okay"; - #address-cells = <1>; - #size-cells = <0>; - - pinctrl-names = "default"; - pinctrl-0 = <&pins_mdio1>, <&pins_eth3>, <&pins_eth4>; - - dsa,member = <0 0>; - - mdio { - clock-frequency = <2500000>; - - #address-cells = <1>; - #size-cells = <0>; - - switch0phy4: ethernet-phy@4 { - reg = <4>; - micrel,led-mode = <1>; - }; - - switch0phy5: ethernet-phy@5 { - reg = <5>; - micrel,led-mode = <1>; - }; - }; -}; - -&switch_port0 { - label = "lan0"; - phy-mode = "mii"; - phy-handle = <&switch0phy5>; - status = "okay"; -}; - -&switch_port1 { - label = "lan1"; - phy-mode = "mii"; - phy-handle = <&switch0phy4>; - status = "okay"; -}; - -&switch_port4 { - status = "okay"; -}; - -ð_miic { - status = "okay"; - renesas,miic-switch-portin = ; -}; - -&mii_conv4 { - renesas,miic-input = ; - status = "okay"; -}; - -&mii_conv5 { - renesas,miic-input = ; - status = "okay"; -}; - -&pinctrl{ - pins_mdio1: pins_mdio1 { - pinmux = < - RZN1_PINMUX(152, RZN1_FUNC_MDIO1_SWITCH) - RZN1_PINMUX(153, RZN1_FUNC_MDIO1_SWITCH) - >; - }; - pins_eth3: pins_eth3 { - pinmux = < - RZN1_PINMUX(36, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) - RZN1_PINMUX(37, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) - RZN1_PINMUX(38, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) - RZN1_PINMUX(39, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) - RZN1_PINMUX(40, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) - RZN1_PINMUX(41, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) - RZN1_PINMUX(42, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) - RZN1_PINMUX(43, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) - RZN1_PINMUX(44, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) - RZN1_PINMUX(45, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) - RZN1_PINMUX(46, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) - RZN1_PINMUX(47, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) - >; - drive-strength = <6>; - bias-disable; - }; - pins_eth4: pins_eth4 { - pinmux = < - RZN1_PINMUX(48, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) - RZN1_PINMUX(49, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) - RZN1_PINMUX(50, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) - RZN1_PINMUX(51, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) - RZN1_PINMUX(52, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) - RZN1_PINMUX(53, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) - RZN1_PINMUX(54, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) - RZN1_PINMUX(55, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) - RZN1_PINMUX(56, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) - RZN1_PINMUX(57, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) - RZN1_PINMUX(58, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) - RZN1_PINMUX(59, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII) - >; - drive-strength = <6>; - bias-disable; - }; -}; diff --git a/arch/arm/boot/dts/r9a06g032.dtsi b/arch/arm/boot/dts/r9a06g032.dtsi index 5b97fa85474f..d3665910958b 100644 --- a/arch/arm/boot/dts/r9a06g032.dtsi +++ b/arch/arm/boot/dts/r9a06g032.dtsi @@ -304,114 +304,6 @@ data-width = <8>; }; - gmac2: ethernet@44002000 { - compatible = "renesas,r9a06g032-gmac", "renesas,rzn1-gmac", "snps,dwmac"; - reg = <0x44002000 0x2000>; - interrupt-parent = <&gic>; - interrupts = , - , - ; - interrupt-names = "macirq", "eth_wake_irq", "eth_lpi"; - clocks = <&sysctrl R9A06G032_HCLK_GMAC1>; - clock-names = "stmmaceth"; - power-domains = <&sysctrl>; - snps,multicast-filter-bins = <256>; - snps,perfect-filter-entries = <128>; - tx-fifo-depth = <2048>; - rx-fifo-depth = <4096>; - status = "disabled"; - }; - - eth_miic: eth-miic@44030000 { - compatible = "renesas,r9a06g032-miic", "renesas,rzn1-miic"; - #address-cells = <1>; - #size-cells = <0>; - reg = <0x44030000 0x10000>; - clocks = <&sysctrl R9A06G032_CLK_MII_REF>, - <&sysctrl R9A06G032_CLK_RGMII_REF>, - <&sysctrl R9A06G032_CLK_RMII_REF>, - <&sysctrl R9A06G032_HCLK_SWITCH_RG>; - clock-names = "mii_ref", "rgmii_ref", "rmii_ref", "hclk"; - power-domains = <&sysctrl>; - status = "disabled"; - - mii_conv1: mii-conv@1 { - reg = <1>; - status = "disabled"; - }; - - mii_conv2: mii-conv@2 { - reg = <2>; - status = "disabled"; - }; - - mii_conv3: mii-conv@3 { - reg = <3>; - status = "disabled"; - }; - - mii_conv4: mii-conv@4 { - reg = <4>; - status = "disabled"; - }; - - mii_conv5: mii-conv@5 { - reg = <5>; - status = "disabled"; - }; - }; - - switch: switch@44050000 { - compatible = "renesas,r9a06g032-a5psw", "renesas,rzn1-a5psw"; - reg = <0x44050000 0x10000>; - clocks = <&sysctrl R9A06G032_HCLK_SWITCH>, - <&sysctrl R9A06G032_CLK_SWITCH>; - clock-names = "hclk", "clk"; - power-domains = <&sysctrl>; - status = "disabled"; - - ethernet-ports { - #address-cells = <1>; - #size-cells = <0>; - - switch_port0: port@0 { - reg = <0>; - pcs-handle = <&mii_conv5>; - status = "disabled"; - }; - - switch_port1: port@1 { - reg = <1>; - pcs-handle = <&mii_conv4>; - status = "disabled"; - }; - - switch_port2: port@2 { - reg = <2>; - pcs-handle = <&mii_conv3>; - status = "disabled"; - }; - - switch_port3: port@3 { - reg = <3>; - pcs-handle = <&mii_conv2>; - status = "disabled"; - }; - - switch_port4: port@4 { - reg = <4>; - ethernet = <&gmac2>; - label = "cpu"; - phy-mode = "internal"; - status = "disabled"; - fixed-link { - speed = <1000>; - full-duplex; - }; - }; - }; - }; - gic: interrupt-controller@44101000 { compatible = "arm,gic-400", "arm,cortex-a7-gic"; interrupt-controller; -- cgit v1.2.3 From 69fd337a975c7e690dfe49d9cb4fe5ba1e6db44e Mon Sep 17 00:00:00 2001 From: Stanislav Fomichev Date: Tue, 28 Jun 2022 10:43:06 -0700 Subject: bpf: per-cgroup lsm flavor Allow attaching to lsm hooks in the cgroup context. Attaching to per-cgroup LSM works exactly like attaching to other per-cgroup hooks. New BPF_LSM_CGROUP is added to trigger new mode; the actual lsm hook we attach to is signaled via existing attach_btf_id. For the hooks that have 'struct socket' or 'struct sock' as its first argument, we use the cgroup associated with that socket. For the rest, we use 'current' cgroup (this is all on default hierarchy == v2 only). Note that for some hooks that work on 'struct sock' we still take the cgroup from 'current' because some of them work on the socket that hasn't been properly initialized yet. Behind the scenes, we allocate a shim program that is attached to the trampoline and runs cgroup effective BPF programs array. This shim has some rudimentary ref counting and can be shared between several programs attaching to the same lsm hook from different cgroups. Note that this patch bloats cgroup size because we add 211 cgroup_bpf_attach_type(s) for simplicity sake. This will be addressed in the subsequent patch. Also note that we only add non-sleepable flavor for now. To enable sleepable use-cases, bpf_prog_run_array_cg has to grab trace rcu, shim programs have to be freed via trace rcu, cgroup_bpf.effective should be also trace-rcu-managed + maybe some other changes that I'm not aware of. Reviewed-by: Martin KaFai Lau Signed-off-by: Stanislav Fomichev Link: https://lore.kernel.org/r/20220628174314.1216643-4-sdf@google.com Signed-off-by: Alexei Starovoitov --- arch/x86/net/bpf_jit_comp.c | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) (limited to 'arch') diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 2c51ca9f7cec..2f460c67f9c7 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -1770,6 +1770,10 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, struct bpf_tramp_link *l, int stack_size, int run_ctx_off, bool save_ret) { + void (*exit)(struct bpf_prog *prog, u64 start, + struct bpf_tramp_run_ctx *run_ctx) = __bpf_prog_exit; + u64 (*enter)(struct bpf_prog *prog, + struct bpf_tramp_run_ctx *run_ctx) = __bpf_prog_enter; u8 *prog = *pprog; u8 *jmp_insn; int ctx_cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie); @@ -1788,15 +1792,21 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, */ emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_1, -run_ctx_off + ctx_cookie_off); + if (p->aux->sleepable) { + enter = __bpf_prog_enter_sleepable; + exit = __bpf_prog_exit_sleepable; + } else if (p->expected_attach_type == BPF_LSM_CGROUP) { + enter = __bpf_prog_enter_lsm_cgroup; + exit = __bpf_prog_exit_lsm_cgroup; + } + /* arg1: mov rdi, progs[i] */ emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long) p); /* arg2: lea rsi, [rbp - ctx_cookie_off] */ EMIT4(0x48, 0x8D, 0x75, -run_ctx_off); - if (emit_call(&prog, - p->aux->sleepable ? __bpf_prog_enter_sleepable : - __bpf_prog_enter, prog)) - return -EINVAL; + if (emit_call(&prog, enter, prog)) + return -EINVAL; /* remember prog start time returned by __bpf_prog_enter */ emit_mov_reg(&prog, true, BPF_REG_6, BPF_REG_0); @@ -1840,10 +1850,8 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, emit_mov_reg(&prog, true, BPF_REG_2, BPF_REG_6); /* arg3: lea rdx, [rbp - run_ctx_off] */ EMIT4(0x48, 0x8D, 0x55, -run_ctx_off); - if (emit_call(&prog, - p->aux->sleepable ? __bpf_prog_exit_sleepable : - __bpf_prog_exit, prog)) - return -EINVAL; + if (emit_call(&prog, exit, prog)) + return -EINVAL; *pprog = prog; return 0; -- cgit v1.2.3 From 535a57a7ffc04932ad83c1a5649b09ba6c93ce83 Mon Sep 17 00:00:00 2001 From: Xu Kuohai Date: Mon, 11 Jul 2022 11:08:20 -0400 Subject: bpf: Remove is_valid_bpf_tramp_flags() Before generating bpf trampoline, x86 calls is_valid_bpf_tramp_flags() to check the input flags. This check is architecture independent. So, to be consistent with x86, arm64 should also do this check before generating bpf trampoline. However, the BPF_TRAMP_F_XXX flags are not used by user code and the flags argument is almost constant at compile time, so this run time check is a bit redundant. Remove is_valid_bpf_tramp_flags() and add some comments to the usage of BPF_TRAMP_F_XXX flags, as suggested by Alexei. Signed-off-by: Xu Kuohai Signed-off-by: Daniel Borkmann Reviewed-by: Jean-Philippe Brucker Acked-by: Song Liu Link: https://lore.kernel.org/bpf/20220711150823.2128542-2-xukuohai@huawei.com --- arch/x86/net/bpf_jit_comp.c | 20 -------------------- 1 file changed, 20 deletions(-) (limited to 'arch') diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index b88f43c9f050..d2614f1bf838 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -1936,23 +1936,6 @@ static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog, return 0; } -static bool is_valid_bpf_tramp_flags(unsigned int flags) -{ - if ((flags & BPF_TRAMP_F_RESTORE_REGS) && - (flags & BPF_TRAMP_F_SKIP_FRAME)) - return false; - - /* - * BPF_TRAMP_F_RET_FENTRY_RET is only used by bpf_struct_ops, - * and it must be used alone. - */ - if ((flags & BPF_TRAMP_F_RET_FENTRY_RET) && - (flags & ~BPF_TRAMP_F_RET_FENTRY_RET)) - return false; - - return true; -} - /* Example: * __be16 eth_type_trans(struct sk_buff *skb, struct net_device *dev); * its 'struct btf_func_model' will be nr_args=2 @@ -2031,9 +2014,6 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i if (nr_args > 6) return -ENOTSUPP; - if (!is_valid_bpf_tramp_flags(flags)) - return -EINVAL; - /* Generated trampoline stack layout: * * RBP + 8 [ return address ] -- cgit v1.2.3 From f1e8a24ed2cab1c907bb47ca5f8dee684896456e Mon Sep 17 00:00:00 2001 From: Xu Kuohai Date: Mon, 11 Jul 2022 11:08:21 -0400 Subject: arm64: Add LDR (literal) instruction Add LDR (literal) instruction to load data from address relative to PC. This instruction will be used to implement long jump from bpf prog to bpf trampoline in the follow-up patch. The instruction encoding: 3 2 2 2 0 0 0 7 6 4 5 0 +-----+-------+---+-----+-------------------------------------+--------+ | 0 x | 0 1 1 | 0 | 0 0 | imm19 | Rt | +-----+-------+---+-----+-------------------------------------+--------+ for 32-bit, variant x == 0; for 64-bit, x == 1. branch_imm_common() is used to check the distance between pc and target address, since it's reused by this patch and LDR (literal) is not a branch instruction, rename it to label_imm_common(). Signed-off-by: Xu Kuohai Signed-off-by: Daniel Borkmann Reviewed-by: Jean-Philippe Brucker Acked-by: Will Deacon Link: https://lore.kernel.org/bpf/20220711150823.2128542-3-xukuohai@huawei.com --- arch/arm64/include/asm/insn.h | 3 +++ arch/arm64/lib/insn.c | 30 ++++++++++++++++++++++++++---- 2 files changed, 29 insertions(+), 4 deletions(-) (limited to 'arch') diff --git a/arch/arm64/include/asm/insn.h b/arch/arm64/include/asm/insn.h index 6aa2dc836db1..834bff720582 100644 --- a/arch/arm64/include/asm/insn.h +++ b/arch/arm64/include/asm/insn.h @@ -510,6 +510,9 @@ u32 aarch64_insn_gen_load_store_imm(enum aarch64_insn_register reg, unsigned int imm, enum aarch64_insn_size_type size, enum aarch64_insn_ldst_type type); +u32 aarch64_insn_gen_load_literal(unsigned long pc, unsigned long addr, + enum aarch64_insn_register reg, + bool is64bit); u32 aarch64_insn_gen_load_store_pair(enum aarch64_insn_register reg1, enum aarch64_insn_register reg2, enum aarch64_insn_register base, diff --git a/arch/arm64/lib/insn.c b/arch/arm64/lib/insn.c index 695d7368fadc..49e972beeac7 100644 --- a/arch/arm64/lib/insn.c +++ b/arch/arm64/lib/insn.c @@ -323,7 +323,7 @@ static u32 aarch64_insn_encode_ldst_size(enum aarch64_insn_size_type type, return insn; } -static inline long branch_imm_common(unsigned long pc, unsigned long addr, +static inline long label_imm_common(unsigned long pc, unsigned long addr, long range) { long offset; @@ -354,7 +354,7 @@ u32 __kprobes aarch64_insn_gen_branch_imm(unsigned long pc, unsigned long addr, * ARM64 virtual address arrangement guarantees all kernel and module * texts are within +/-128M. */ - offset = branch_imm_common(pc, addr, SZ_128M); + offset = label_imm_common(pc, addr, SZ_128M); if (offset >= SZ_128M) return AARCH64_BREAK_FAULT; @@ -382,7 +382,7 @@ u32 aarch64_insn_gen_comp_branch_imm(unsigned long pc, unsigned long addr, u32 insn; long offset; - offset = branch_imm_common(pc, addr, SZ_1M); + offset = label_imm_common(pc, addr, SZ_1M); if (offset >= SZ_1M) return AARCH64_BREAK_FAULT; @@ -421,7 +421,7 @@ u32 aarch64_insn_gen_cond_branch_imm(unsigned long pc, unsigned long addr, u32 insn; long offset; - offset = branch_imm_common(pc, addr, SZ_1M); + offset = label_imm_common(pc, addr, SZ_1M); insn = aarch64_insn_get_bcond_value(); @@ -543,6 +543,28 @@ u32 aarch64_insn_gen_load_store_imm(enum aarch64_insn_register reg, return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_12, insn, imm); } +u32 aarch64_insn_gen_load_literal(unsigned long pc, unsigned long addr, + enum aarch64_insn_register reg, + bool is64bit) +{ + u32 insn; + long offset; + + offset = label_imm_common(pc, addr, SZ_1M); + if (offset >= SZ_1M) + return AARCH64_BREAK_FAULT; + + insn = aarch64_insn_get_ldr_lit_value(); + + if (is64bit) + insn |= BIT(30); + + insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn, reg); + + return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_19, insn, + offset >> 2); +} + u32 aarch64_insn_gen_load_store_pair(enum aarch64_insn_register reg1, enum aarch64_insn_register reg2, enum aarch64_insn_register base, -- cgit v1.2.3 From b2ad54e1533e91449cb2a371e034942bd7882b58 Mon Sep 17 00:00:00 2001 From: Xu Kuohai Date: Mon, 11 Jul 2022 11:08:22 -0400 Subject: bpf, arm64: Implement bpf_arch_text_poke() for arm64 Implement bpf_arch_text_poke() for arm64, so bpf prog or bpf trampoline can be patched with it. When the target address is NULL, the original instruction is patched to a NOP. When the target address and the source address are within the branch range, the original instruction is patched to a bl instruction to the target address directly. To support attaching bpf trampoline to both regular kernel function and bpf prog, we follow the ftrace patchsite way for bpf prog. That is, two instructions are inserted at the beginning of bpf prog, the first one saves the return address to x9, and the second is a nop which will be patched to a bl instruction when a bpf trampoline is attached. However, when a bpf trampoline is attached to bpf prog, the distance between target address and source address may exceed 128MB, the maximum branch range, because bpf trampoline and bpf prog are allocated separately with vmalloc. So long jump should be handled. When a bpf prog is constructed, a plt pointing to empty trampoline dummy_tramp is placed at the end: bpf_prog: mov x9, lr nop // patchsite ... ret plt: ldr x10, target br x10 target: .quad dummy_tramp // plt target This is also the state when no trampoline is attached. When a short-jump bpf trampoline is attached, the patchsite is patched to a bl instruction to the trampoline directly: bpf_prog: mov x9, lr bl // patchsite ... ret plt: ldr x10, target br x10 target: .quad dummy_tramp // plt target When a long-jump bpf trampoline is attached, the plt target is filled with the trampoline address and the patchsite is patched to a bl instruction to the plt: bpf_prog: mov x9, lr bl plt // patchsite ... ret plt: ldr x10, target br x10 target: .quad dummy_tramp is used to prevent another CPU from jumping to an unknown location during the patching process, making the patching process easier. The patching process is as follows: 1. when neither the old address or the new address is a long jump, the patchsite is replaced with a bl to the new address, or nop if the new address is NULL; 2. when the old address is not long jump but the new one is, the branch target address is written to plt first, then the patchsite is replaced with a bl instruction to the plt; 3. when the old address is long jump but the new one is not, the address of dummy_tramp is written to plt first, then the patchsite is replaced with a bl to the new address, or a nop if the new address is NULL; 4. when both the old address and the new address are long jump, the new address is written to plt and the patchsite is not changed. Signed-off-by: Xu Kuohai Signed-off-by: Daniel Borkmann Reviewed-by: Jakub Sitnicki Reviewed-by: KP Singh Reviewed-by: Jean-Philippe Brucker Acked-by: Song Liu Link: https://lore.kernel.org/bpf/20220711150823.2128542-4-xukuohai@huawei.com --- arch/arm64/net/bpf_jit.h | 7 + arch/arm64/net/bpf_jit_comp.c | 329 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 322 insertions(+), 14 deletions(-) (limited to 'arch') diff --git a/arch/arm64/net/bpf_jit.h b/arch/arm64/net/bpf_jit.h index 194c95ccc1cf..a6acb94ea3d6 100644 --- a/arch/arm64/net/bpf_jit.h +++ b/arch/arm64/net/bpf_jit.h @@ -80,6 +80,12 @@ #define A64_STR64I(Xt, Xn, imm) A64_LS_IMM(Xt, Xn, imm, 64, STORE) #define A64_LDR64I(Xt, Xn, imm) A64_LS_IMM(Xt, Xn, imm, 64, LOAD) +/* LDR (literal) */ +#define A64_LDR32LIT(Wt, offset) \ + aarch64_insn_gen_load_literal(0, offset, Wt, false) +#define A64_LDR64LIT(Xt, offset) \ + aarch64_insn_gen_load_literal(0, offset, Xt, true) + /* Load/store register pair */ #define A64_LS_PAIR(Rt, Rt2, Rn, offset, ls, type) \ aarch64_insn_gen_load_store_pair(Rt, Rt2, Rn, offset, \ @@ -270,6 +276,7 @@ #define A64_BTI_C A64_HINT(AARCH64_INSN_HINT_BTIC) #define A64_BTI_J A64_HINT(AARCH64_INSN_HINT_BTIJ) #define A64_BTI_JC A64_HINT(AARCH64_INSN_HINT_BTIJC) +#define A64_NOP A64_HINT(AARCH64_INSN_HINT_NOP) /* DMB */ #define A64_DMB_ISH aarch64_insn_gen_dmb(AARCH64_INSN_MB_ISH) diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index f08a4447d363..0ef35ec30d4e 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -18,6 +19,7 @@ #include #include #include +#include #include #include "bpf_jit.h" @@ -78,6 +80,15 @@ struct jit_ctx { int fpb_offset; }; +struct bpf_plt { + u32 insn_ldr; /* load target */ + u32 insn_br; /* branch to target */ + u64 target; /* target value */ +}; + +#define PLT_TARGET_SIZE sizeof_field(struct bpf_plt, target) +#define PLT_TARGET_OFFSET offsetof(struct bpf_plt, target) + static inline void emit(const u32 insn, struct jit_ctx *ctx) { if (ctx->image != NULL) @@ -140,6 +151,12 @@ static inline void emit_a64_mov_i64(const int reg, const u64 val, } } +static inline void emit_bti(u32 insn, struct jit_ctx *ctx) +{ + if (IS_ENABLED(CONFIG_ARM64_BTI_KERNEL)) + emit(insn, ctx); +} + /* * Kernel addresses in the vmalloc space use at most 48 bits, and the * remaining bits are guaranteed to be 0x1. So we can compose the address @@ -235,13 +252,30 @@ static bool is_lsi_offset(int offset, int scale) return true; } +/* generated prologue: + * bti c // if CONFIG_ARM64_BTI_KERNEL + * mov x9, lr + * nop // POKE_OFFSET + * paciasp // if CONFIG_ARM64_PTR_AUTH_KERNEL + * stp x29, lr, [sp, #-16]! + * mov x29, sp + * stp x19, x20, [sp, #-16]! + * stp x21, x22, [sp, #-16]! + * stp x25, x26, [sp, #-16]! + * stp x27, x28, [sp, #-16]! + * mov x25, sp + * mov tcc, #0 + * // PROLOGUE_OFFSET + */ + +#define BTI_INSNS (IS_ENABLED(CONFIG_ARM64_BTI_KERNEL) ? 1 : 0) +#define PAC_INSNS (IS_ENABLED(CONFIG_ARM64_PTR_AUTH_KERNEL) ? 1 : 0) + +/* Offset of nop instruction in bpf prog entry to be poked */ +#define POKE_OFFSET (BTI_INSNS + 1) + /* Tail call offset to jump into */ -#if IS_ENABLED(CONFIG_ARM64_BTI_KERNEL) || \ - IS_ENABLED(CONFIG_ARM64_PTR_AUTH_KERNEL) -#define PROLOGUE_OFFSET 9 -#else -#define PROLOGUE_OFFSET 8 -#endif +#define PROLOGUE_OFFSET (BTI_INSNS + 2 + PAC_INSNS + 8) static int build_prologue(struct jit_ctx *ctx, bool ebpf_from_cbpf) { @@ -280,12 +314,14 @@ static int build_prologue(struct jit_ctx *ctx, bool ebpf_from_cbpf) * */ + emit_bti(A64_BTI_C, ctx); + + emit(A64_MOV(1, A64_R(9), A64_LR), ctx); + emit(A64_NOP, ctx); + /* Sign lr */ if (IS_ENABLED(CONFIG_ARM64_PTR_AUTH_KERNEL)) emit(A64_PACIASP, ctx); - /* BTI landing pad */ - else if (IS_ENABLED(CONFIG_ARM64_BTI_KERNEL)) - emit(A64_BTI_C, ctx); /* Save FP and LR registers to stay align with ARM64 AAPCS */ emit(A64_PUSH(A64_FP, A64_LR, A64_SP), ctx); @@ -312,8 +348,7 @@ static int build_prologue(struct jit_ctx *ctx, bool ebpf_from_cbpf) } /* BTI landing pad for the tail call, done with a BR */ - if (IS_ENABLED(CONFIG_ARM64_BTI_KERNEL)) - emit(A64_BTI_J, ctx); + emit_bti(A64_BTI_J, ctx); } emit(A64_SUB_I(1, fpb, fp, ctx->fpb_offset), ctx); @@ -557,6 +592,52 @@ static int emit_ll_sc_atomic(const struct bpf_insn *insn, struct jit_ctx *ctx) return 0; } +void dummy_tramp(void); + +asm ( +" .pushsection .text, \"ax\", @progbits\n" +" .type dummy_tramp, %function\n" +"dummy_tramp:" +#if IS_ENABLED(CONFIG_ARM64_BTI_KERNEL) +" bti j\n" /* dummy_tramp is called via "br x10" */ +#endif +" mov x10, lr\n" +" mov lr, x9\n" +" ret x10\n" +" .size dummy_tramp, .-dummy_tramp\n" +" .popsection\n" +); + +/* build a plt initialized like this: + * + * plt: + * ldr tmp, target + * br tmp + * target: + * .quad dummy_tramp + * + * when a long jump trampoline is attached, target is filled with the + * trampoline address, and when the trampoline is removed, target is + * restored to dummy_tramp address. + */ +static void build_plt(struct jit_ctx *ctx) +{ + const u8 tmp = bpf2a64[TMP_REG_1]; + struct bpf_plt *plt = NULL; + + /* make sure target is 64-bit aligned */ + if ((ctx->idx + PLT_TARGET_OFFSET / AARCH64_INSN_SIZE) % 2) + emit(A64_NOP, ctx); + + plt = (struct bpf_plt *)(ctx->image + ctx->idx); + /* plt is called via bl, no BTI needed here */ + emit(A64_LDR64LIT(tmp, 2 * AARCH64_INSN_SIZE), ctx); + emit(A64_BR(tmp), ctx); + + if (ctx->image) + plt->target = (u64)&dummy_tramp; +} + static void build_epilogue(struct jit_ctx *ctx) { const u8 r0 = bpf2a64[BPF_REG_0]; @@ -1356,7 +1437,7 @@ struct arm64_jit_data { struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) { - int image_size, prog_size, extable_size; + int image_size, prog_size, extable_size, extable_align, extable_offset; struct bpf_prog *tmp, *orig_prog = prog; struct bpf_binary_header *header; struct arm64_jit_data *jit_data; @@ -1426,13 +1507,17 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) ctx.epilogue_offset = ctx.idx; build_epilogue(&ctx); + build_plt(&ctx); + extable_align = __alignof__(struct exception_table_entry); extable_size = prog->aux->num_exentries * sizeof(struct exception_table_entry); /* Now we know the actual image size. */ prog_size = sizeof(u32) * ctx.idx; - image_size = prog_size + extable_size; + /* also allocate space for plt target */ + extable_offset = round_up(prog_size + PLT_TARGET_SIZE, extable_align); + image_size = extable_offset + extable_size; header = bpf_jit_binary_alloc(image_size, &image_ptr, sizeof(u32), jit_fill_hole); if (header == NULL) { @@ -1444,7 +1529,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) ctx.image = (__le32 *)image_ptr; if (extable_size) - prog->aux->extable = (void *)image_ptr + prog_size; + prog->aux->extable = (void *)image_ptr + extable_offset; skip_init_ctx: ctx.idx = 0; ctx.exentry_idx = 0; @@ -1458,6 +1543,7 @@ skip_init_ctx: } build_epilogue(&ctx); + build_plt(&ctx); /* 3. Extra pass to validate JITed code. */ if (validate_code(&ctx)) { @@ -1537,3 +1623,218 @@ bool bpf_jit_supports_subprog_tailcalls(void) { return true; } + +static bool is_long_jump(void *ip, void *target) +{ + long offset; + + /* NULL target means this is a NOP */ + if (!target) + return false; + + offset = (long)target - (long)ip; + return offset < -SZ_128M || offset >= SZ_128M; +} + +static int gen_branch_or_nop(enum aarch64_insn_branch_type type, void *ip, + void *addr, void *plt, u32 *insn) +{ + void *target; + + if (!addr) { + *insn = aarch64_insn_gen_nop(); + return 0; + } + + if (is_long_jump(ip, addr)) + target = plt; + else + target = addr; + + *insn = aarch64_insn_gen_branch_imm((unsigned long)ip, + (unsigned long)target, + type); + + return *insn != AARCH64_BREAK_FAULT ? 0 : -EFAULT; +} + +/* Replace the branch instruction from @ip to @old_addr in a bpf prog or a bpf + * trampoline with the branch instruction from @ip to @new_addr. If @old_addr + * or @new_addr is NULL, the old or new instruction is NOP. + * + * When @ip is the bpf prog entry, a bpf trampoline is being attached or + * detached. Since bpf trampoline and bpf prog are allocated separately with + * vmalloc, the address distance may exceed 128MB, the maximum branch range. + * So long jump should be handled. + * + * When a bpf prog is constructed, a plt pointing to empty trampoline + * dummy_tramp is placed at the end: + * + * bpf_prog: + * mov x9, lr + * nop // patchsite + * ... + * ret + * + * plt: + * ldr x10, target + * br x10 + * target: + * .quad dummy_tramp // plt target + * + * This is also the state when no trampoline is attached. + * + * When a short-jump bpf trampoline is attached, the patchsite is patched + * to a bl instruction to the trampoline directly: + * + * bpf_prog: + * mov x9, lr + * bl // patchsite + * ... + * ret + * + * plt: + * ldr x10, target + * br x10 + * target: + * .quad dummy_tramp // plt target + * + * When a long-jump bpf trampoline is attached, the plt target is filled with + * the trampoline address and the patchsite is patched to a bl instruction to + * the plt: + * + * bpf_prog: + * mov x9, lr + * bl plt // patchsite + * ... + * ret + * + * plt: + * ldr x10, target + * br x10 + * target: + * .quad // plt target + * + * The dummy_tramp is used to prevent another CPU from jumping to unknown + * locations during the patching process, making the patching process easier. + */ +int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type poke_type, + void *old_addr, void *new_addr) +{ + int ret; + u32 old_insn; + u32 new_insn; + u32 replaced; + struct bpf_plt *plt = NULL; + unsigned long size = 0UL; + unsigned long offset = ~0UL; + enum aarch64_insn_branch_type branch_type; + char namebuf[KSYM_NAME_LEN]; + void *image = NULL; + u64 plt_target = 0ULL; + bool poking_bpf_entry; + + if (!__bpf_address_lookup((unsigned long)ip, &size, &offset, namebuf)) + /* Only poking bpf text is supported. Since kernel function + * entry is set up by ftrace, we reply on ftrace to poke kernel + * functions. + */ + return -ENOTSUPP; + + image = ip - offset; + /* zero offset means we're poking bpf prog entry */ + poking_bpf_entry = (offset == 0UL); + + /* bpf prog entry, find plt and the real patchsite */ + if (poking_bpf_entry) { + /* plt locates at the end of bpf prog */ + plt = image + size - PLT_TARGET_OFFSET; + + /* skip to the nop instruction in bpf prog entry: + * bti c // if BTI enabled + * mov x9, x30 + * nop + */ + ip = image + POKE_OFFSET * AARCH64_INSN_SIZE; + } + + /* long jump is only possible at bpf prog entry */ + if (WARN_ON((is_long_jump(ip, new_addr) || is_long_jump(ip, old_addr)) && + !poking_bpf_entry)) + return -EINVAL; + + if (poke_type == BPF_MOD_CALL) + branch_type = AARCH64_INSN_BRANCH_LINK; + else + branch_type = AARCH64_INSN_BRANCH_NOLINK; + + if (gen_branch_or_nop(branch_type, ip, old_addr, plt, &old_insn) < 0) + return -EFAULT; + + if (gen_branch_or_nop(branch_type, ip, new_addr, plt, &new_insn) < 0) + return -EFAULT; + + if (is_long_jump(ip, new_addr)) + plt_target = (u64)new_addr; + else if (is_long_jump(ip, old_addr)) + /* if the old target is a long jump and the new target is not, + * restore the plt target to dummy_tramp, so there is always a + * legal and harmless address stored in plt target, and we'll + * never jump from plt to an unknown place. + */ + plt_target = (u64)&dummy_tramp; + + if (plt_target) { + /* non-zero plt_target indicates we're patching a bpf prog, + * which is read only. + */ + if (set_memory_rw(PAGE_MASK & ((uintptr_t)&plt->target), 1)) + return -EFAULT; + WRITE_ONCE(plt->target, plt_target); + set_memory_ro(PAGE_MASK & ((uintptr_t)&plt->target), 1); + /* since plt target points to either the new trampoline + * or dummy_tramp, even if another CPU reads the old plt + * target value before fetching the bl instruction to plt, + * it will be brought back by dummy_tramp, so no barrier is + * required here. + */ + } + + /* if the old target and the new target are both long jumps, no + * patching is required + */ + if (old_insn == new_insn) + return 0; + + mutex_lock(&text_mutex); + if (aarch64_insn_read(ip, &replaced)) { + ret = -EFAULT; + goto out; + } + + if (replaced != old_insn) { + ret = -EFAULT; + goto out; + } + + /* We call aarch64_insn_patch_text_nosync() to replace instruction + * atomically, so no other CPUs will fetch a half-new and half-old + * instruction. But there is chance that another CPU executes the + * old instruction after the patching operation finishes (e.g., + * pipeline not flushed, or icache not synchronized yet). + * + * 1. when a new trampoline is attached, it is not a problem for + * different CPUs to jump to different trampolines temporarily. + * + * 2. when an old trampoline is freed, we should wait for all other + * CPUs to exit the trampoline and make sure the trampoline is no + * longer reachable, since bpf_tramp_image_put() function already + * uses percpu_ref and task-based rcu to do the sync, no need to call + * the sync version here, see bpf_tramp_image_put() for details. + */ + ret = aarch64_insn_patch_text_nosync(ip, new_insn); +out: + mutex_unlock(&text_mutex); + + return ret; +} -- cgit v1.2.3 From efc9909fdce00a827a37609628223cd45bf95d0b Mon Sep 17 00:00:00 2001 From: Xu Kuohai Date: Mon, 11 Jul 2022 11:08:23 -0400 Subject: bpf, arm64: Add bpf trampoline for arm64 This is arm64 version of commit fec56f5890d9 ("bpf: Introduce BPF trampoline"). A bpf trampoline converts native calling convention to bpf calling convention and is used to implement various bpf features, such as fentry, fexit, fmod_ret and struct_ops. This patch does essentially the same thing that bpf trampoline does on x86. Tested on Raspberry Pi 4B and qemu: #18 /1 bpf_tcp_ca/dctcp:OK #18 /2 bpf_tcp_ca/cubic:OK #18 /3 bpf_tcp_ca/invalid_license:OK #18 /4 bpf_tcp_ca/dctcp_fallback:OK #18 /5 bpf_tcp_ca/rel_setsockopt:OK #18 bpf_tcp_ca:OK #51 /1 dummy_st_ops/dummy_st_ops_attach:OK #51 /2 dummy_st_ops/dummy_init_ret_value:OK #51 /3 dummy_st_ops/dummy_init_ptr_arg:OK #51 /4 dummy_st_ops/dummy_multiple_args:OK #51 dummy_st_ops:OK #57 /1 fexit_bpf2bpf/target_no_callees:OK #57 /2 fexit_bpf2bpf/target_yes_callees:OK #57 /3 fexit_bpf2bpf/func_replace:OK #57 /4 fexit_bpf2bpf/func_replace_verify:OK #57 /5 fexit_bpf2bpf/func_sockmap_update:OK #57 /6 fexit_bpf2bpf/func_replace_return_code:OK #57 /7 fexit_bpf2bpf/func_map_prog_compatibility:OK #57 /8 fexit_bpf2bpf/func_replace_multi:OK #57 /9 fexit_bpf2bpf/fmod_ret_freplace:OK #57 fexit_bpf2bpf:OK #237 xdp_bpf2bpf:OK Signed-off-by: Xu Kuohai Signed-off-by: Daniel Borkmann Reviewed-by: Jean-Philippe Brucker Acked-by: Song Liu Acked-by: KP Singh Link: https://lore.kernel.org/bpf/20220711150823.2128542-5-xukuohai@huawei.com --- arch/arm64/net/bpf_jit_comp.c | 385 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 382 insertions(+), 3 deletions(-) (limited to 'arch') diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 0ef35ec30d4e..fd1cb0d2aaa6 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -176,6 +176,14 @@ static inline void emit_addr_mov_i64(const int reg, const u64 val, } } +static inline void emit_call(u64 target, struct jit_ctx *ctx) +{ + u8 tmp = bpf2a64[TMP_REG_1]; + + emit_addr_mov_i64(tmp, target, ctx); + emit(A64_BLR(tmp), ctx); +} + static inline int bpf2a64_offset(int bpf_insn, int off, const struct jit_ctx *ctx) { @@ -1072,8 +1080,7 @@ emit_cond_jmp: &func_addr, &func_addr_fixed); if (ret < 0) return ret; - emit_addr_mov_i64(tmp, func_addr, ctx); - emit(A64_BLR(tmp), ctx); + emit_call(func_addr, ctx); emit(A64_MOV(1, r0, A64_R(0)), ctx); break; } @@ -1417,6 +1424,13 @@ static int validate_code(struct jit_ctx *ctx) if (a64_insn == AARCH64_BREAK_FAULT) return -1; } + return 0; +} + +static int validate_ctx(struct jit_ctx *ctx) +{ + if (validate_code(ctx)) + return -1; if (WARN_ON_ONCE(ctx->exentry_idx != ctx->prog->aux->num_exentries)) return -1; @@ -1546,7 +1560,7 @@ skip_init_ctx: build_plt(&ctx); /* 3. Extra pass to validate JITed code. */ - if (validate_code(&ctx)) { + if (validate_ctx(&ctx)) { bpf_jit_binary_free(header); prog = orig_prog; goto out_off; @@ -1624,6 +1638,371 @@ bool bpf_jit_supports_subprog_tailcalls(void) return true; } +static void invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_link *l, + int args_off, int retval_off, int run_ctx_off, + bool save_ret) +{ + u32 *branch; + u64 enter_prog; + u64 exit_prog; + struct bpf_prog *p = l->link.prog; + int cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie); + + if (p->aux->sleepable) { + enter_prog = (u64)__bpf_prog_enter_sleepable; + exit_prog = (u64)__bpf_prog_exit_sleepable; + } else { + enter_prog = (u64)__bpf_prog_enter; + exit_prog = (u64)__bpf_prog_exit; + } + + if (l->cookie == 0) { + /* if cookie is zero, one instruction is enough to store it */ + emit(A64_STR64I(A64_ZR, A64_SP, run_ctx_off + cookie_off), ctx); + } else { + emit_a64_mov_i64(A64_R(10), l->cookie, ctx); + emit(A64_STR64I(A64_R(10), A64_SP, run_ctx_off + cookie_off), + ctx); + } + + /* save p to callee saved register x19 to avoid loading p with mov_i64 + * each time. + */ + emit_addr_mov_i64(A64_R(19), (const u64)p, ctx); + + /* arg1: prog */ + emit(A64_MOV(1, A64_R(0), A64_R(19)), ctx); + /* arg2: &run_ctx */ + emit(A64_ADD_I(1, A64_R(1), A64_SP, run_ctx_off), ctx); + + emit_call(enter_prog, ctx); + + /* if (__bpf_prog_enter(prog) == 0) + * goto skip_exec_of_prog; + */ + branch = ctx->image + ctx->idx; + emit(A64_NOP, ctx); + + /* save return value to callee saved register x20 */ + emit(A64_MOV(1, A64_R(20), A64_R(0)), ctx); + + emit(A64_ADD_I(1, A64_R(0), A64_SP, args_off), ctx); + if (!p->jited) + emit_addr_mov_i64(A64_R(1), (const u64)p->insnsi, ctx); + + emit_call((const u64)p->bpf_func, ctx); + + if (save_ret) + emit(A64_STR64I(A64_R(0), A64_SP, retval_off), ctx); + + if (ctx->image) { + int offset = &ctx->image[ctx->idx] - branch; + *branch = A64_CBZ(1, A64_R(0), offset); + } + + /* arg1: prog */ + emit(A64_MOV(1, A64_R(0), A64_R(19)), ctx); + /* arg2: start time */ + emit(A64_MOV(1, A64_R(1), A64_R(20)), ctx); + /* arg3: &run_ctx */ + emit(A64_ADD_I(1, A64_R(2), A64_SP, run_ctx_off), ctx); + + emit_call(exit_prog, ctx); +} + +static void invoke_bpf_mod_ret(struct jit_ctx *ctx, struct bpf_tramp_links *tl, + int args_off, int retval_off, int run_ctx_off, + u32 **branches) +{ + int i; + + /* The first fmod_ret program will receive a garbage return value. + * Set this to 0 to avoid confusing the program. + */ + emit(A64_STR64I(A64_ZR, A64_SP, retval_off), ctx); + for (i = 0; i < tl->nr_links; i++) { + invoke_bpf_prog(ctx, tl->links[i], args_off, retval_off, + run_ctx_off, true); + /* if (*(u64 *)(sp + retval_off) != 0) + * goto do_fexit; + */ + emit(A64_LDR64I(A64_R(10), A64_SP, retval_off), ctx); + /* Save the location of branch, and generate a nop. + * This nop will be replaced with a cbnz later. + */ + branches[i] = ctx->image + ctx->idx; + emit(A64_NOP, ctx); + } +} + +static void save_args(struct jit_ctx *ctx, int args_off, int nargs) +{ + int i; + + for (i = 0; i < nargs; i++) { + emit(A64_STR64I(i, A64_SP, args_off), ctx); + args_off += 8; + } +} + +static void restore_args(struct jit_ctx *ctx, int args_off, int nargs) +{ + int i; + + for (i = 0; i < nargs; i++) { + emit(A64_LDR64I(i, A64_SP, args_off), ctx); + args_off += 8; + } +} + +/* Based on the x86's implementation of arch_prepare_bpf_trampoline(). + * + * bpf prog and function entry before bpf trampoline hooked: + * mov x9, lr + * nop + * + * bpf prog and function entry after bpf trampoline hooked: + * mov x9, lr + * bl + * + */ +static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, + struct bpf_tramp_links *tlinks, void *orig_call, + int nargs, u32 flags) +{ + int i; + int stack_size; + int retaddr_off; + int regs_off; + int retval_off; + int args_off; + int nargs_off; + int ip_off; + int run_ctx_off; + struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY]; + struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT]; + struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN]; + bool save_ret; + u32 **branches = NULL; + + /* trampoline stack layout: + * [ parent ip ] + * [ FP ] + * SP + retaddr_off [ self ip ] + * [ FP ] + * + * [ padding ] align SP to multiples of 16 + * + * [ x20 ] callee saved reg x20 + * SP + regs_off [ x19 ] callee saved reg x19 + * + * SP + retval_off [ return value ] BPF_TRAMP_F_CALL_ORIG or + * BPF_TRAMP_F_RET_FENTRY_RET + * + * [ argN ] + * [ ... ] + * SP + args_off [ arg1 ] + * + * SP + nargs_off [ args count ] + * + * SP + ip_off [ traced function ] BPF_TRAMP_F_IP_ARG flag + * + * SP + run_ctx_off [ bpf_tramp_run_ctx ] + */ + + stack_size = 0; + run_ctx_off = stack_size; + /* room for bpf_tramp_run_ctx */ + stack_size += round_up(sizeof(struct bpf_tramp_run_ctx), 8); + + ip_off = stack_size; + /* room for IP address argument */ + if (flags & BPF_TRAMP_F_IP_ARG) + stack_size += 8; + + nargs_off = stack_size; + /* room for args count */ + stack_size += 8; + + args_off = stack_size; + /* room for args */ + stack_size += nargs * 8; + + /* room for return value */ + retval_off = stack_size; + save_ret = flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET); + if (save_ret) + stack_size += 8; + + /* room for callee saved registers, currently x19 and x20 are used */ + regs_off = stack_size; + stack_size += 16; + + /* round up to multiples of 16 to avoid SPAlignmentFault */ + stack_size = round_up(stack_size, 16); + + /* return address locates above FP */ + retaddr_off = stack_size + 8; + + /* bpf trampoline may be invoked by 3 instruction types: + * 1. bl, attached to bpf prog or kernel function via short jump + * 2. br, attached to bpf prog or kernel function via long jump + * 3. blr, working as a function pointer, used by struct_ops. + * So BTI_JC should used here to support both br and blr. + */ + emit_bti(A64_BTI_JC, ctx); + + /* frame for parent function */ + emit(A64_PUSH(A64_FP, A64_R(9), A64_SP), ctx); + emit(A64_MOV(1, A64_FP, A64_SP), ctx); + + /* frame for patched function */ + emit(A64_PUSH(A64_FP, A64_LR, A64_SP), ctx); + emit(A64_MOV(1, A64_FP, A64_SP), ctx); + + /* allocate stack space */ + emit(A64_SUB_I(1, A64_SP, A64_SP, stack_size), ctx); + + if (flags & BPF_TRAMP_F_IP_ARG) { + /* save ip address of the traced function */ + emit_addr_mov_i64(A64_R(10), (const u64)orig_call, ctx); + emit(A64_STR64I(A64_R(10), A64_SP, ip_off), ctx); + } + + /* save args count*/ + emit(A64_MOVZ(1, A64_R(10), nargs, 0), ctx); + emit(A64_STR64I(A64_R(10), A64_SP, nargs_off), ctx); + + /* save args */ + save_args(ctx, args_off, nargs); + + /* save callee saved registers */ + emit(A64_STR64I(A64_R(19), A64_SP, regs_off), ctx); + emit(A64_STR64I(A64_R(20), A64_SP, regs_off + 8), ctx); + + if (flags & BPF_TRAMP_F_CALL_ORIG) { + emit_addr_mov_i64(A64_R(0), (const u64)im, ctx); + emit_call((const u64)__bpf_tramp_enter, ctx); + } + + for (i = 0; i < fentry->nr_links; i++) + invoke_bpf_prog(ctx, fentry->links[i], args_off, + retval_off, run_ctx_off, + flags & BPF_TRAMP_F_RET_FENTRY_RET); + + if (fmod_ret->nr_links) { + branches = kcalloc(fmod_ret->nr_links, sizeof(u32 *), + GFP_KERNEL); + if (!branches) + return -ENOMEM; + + invoke_bpf_mod_ret(ctx, fmod_ret, args_off, retval_off, + run_ctx_off, branches); + } + + if (flags & BPF_TRAMP_F_CALL_ORIG) { + restore_args(ctx, args_off, nargs); + /* call original func */ + emit(A64_LDR64I(A64_R(10), A64_SP, retaddr_off), ctx); + emit(A64_BLR(A64_R(10)), ctx); + /* store return value */ + emit(A64_STR64I(A64_R(0), A64_SP, retval_off), ctx); + /* reserve a nop for bpf_tramp_image_put */ + im->ip_after_call = ctx->image + ctx->idx; + emit(A64_NOP, ctx); + } + + /* update the branches saved in invoke_bpf_mod_ret with cbnz */ + for (i = 0; i < fmod_ret->nr_links && ctx->image != NULL; i++) { + int offset = &ctx->image[ctx->idx] - branches[i]; + *branches[i] = A64_CBNZ(1, A64_R(10), offset); + } + + for (i = 0; i < fexit->nr_links; i++) + invoke_bpf_prog(ctx, fexit->links[i], args_off, retval_off, + run_ctx_off, false); + + if (flags & BPF_TRAMP_F_CALL_ORIG) { + im->ip_epilogue = ctx->image + ctx->idx; + emit_addr_mov_i64(A64_R(0), (const u64)im, ctx); + emit_call((const u64)__bpf_tramp_exit, ctx); + } + + if (flags & BPF_TRAMP_F_RESTORE_REGS) + restore_args(ctx, args_off, nargs); + + /* restore callee saved register x19 and x20 */ + emit(A64_LDR64I(A64_R(19), A64_SP, regs_off), ctx); + emit(A64_LDR64I(A64_R(20), A64_SP, regs_off + 8), ctx); + + if (save_ret) + emit(A64_LDR64I(A64_R(0), A64_SP, retval_off), ctx); + + /* reset SP */ + emit(A64_MOV(1, A64_SP, A64_FP), ctx); + + /* pop frames */ + emit(A64_POP(A64_FP, A64_LR, A64_SP), ctx); + emit(A64_POP(A64_FP, A64_R(9), A64_SP), ctx); + + if (flags & BPF_TRAMP_F_SKIP_FRAME) { + /* skip patched function, return to parent */ + emit(A64_MOV(1, A64_LR, A64_R(9)), ctx); + emit(A64_RET(A64_R(9)), ctx); + } else { + /* return to patched function */ + emit(A64_MOV(1, A64_R(10), A64_LR), ctx); + emit(A64_MOV(1, A64_LR, A64_R(9)), ctx); + emit(A64_RET(A64_R(10)), ctx); + } + + if (ctx->image) + bpf_flush_icache(ctx->image, ctx->image + ctx->idx); + + kfree(branches); + + return ctx->idx; +} + +int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, + void *image_end, const struct btf_func_model *m, + u32 flags, struct bpf_tramp_links *tlinks, + void *orig_call) +{ + int ret; + int nargs = m->nr_args; + int max_insns = ((long)image_end - (long)image) / AARCH64_INSN_SIZE; + struct jit_ctx ctx = { + .image = NULL, + .idx = 0, + }; + + /* the first 8 arguments are passed by registers */ + if (nargs > 8) + return -ENOTSUPP; + + ret = prepare_trampoline(&ctx, im, tlinks, orig_call, nargs, flags); + if (ret < 0) + return ret; + + if (ret > max_insns) + return -EFBIG; + + ctx.image = image; + ctx.idx = 0; + + jit_fill_hole(image, (unsigned int)(image_end - image)); + ret = prepare_trampoline(&ctx, im, tlinks, orig_call, nargs, flags); + + if (ret > 0 && validate_code(&ctx) < 0) + ret = -EINVAL; + + if (ret > 0) + ret *= AARCH64_INSN_SIZE; + + return ret; +} + static bool is_long_jump(void *ip, void *target) { long offset; -- cgit v1.2.3 From dfa2854cbf92831ef492bd848eb4c2b0834c332c Mon Sep 17 00:00:00 2001 From: Ioana Ciornei Date: Thu, 7 Jul 2022 12:14:36 +0300 Subject: arch: arm64: dts: lx2160a-clearfog-itx: rename the sfp GPIO properties Rename the 'mod-def0-gpio' property to 'mod-def0-gpios' so that we use the preferred -gpios suffix. Also, with this change the dtb_check will not complain when trying to verify the DTS against the sff,sfp.yaml binding. Signed-off-by: Ioana Ciornei Signed-off-by: Jakub Kicinski --- arch/arm64/boot/dts/freescale/fsl-lx2160a-clearfog-itx.dtsi | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'arch') diff --git a/arch/arm64/boot/dts/freescale/fsl-lx2160a-clearfog-itx.dtsi b/arch/arm64/boot/dts/freescale/fsl-lx2160a-clearfog-itx.dtsi index 41702e7386e3..a7dcbecc1f41 100644 --- a/arch/arm64/boot/dts/freescale/fsl-lx2160a-clearfog-itx.dtsi +++ b/arch/arm64/boot/dts/freescale/fsl-lx2160a-clearfog-itx.dtsi @@ -34,28 +34,28 @@ sfp0: sfp-0 { compatible = "sff,sfp"; i2c-bus = <&sfp0_i2c>; - mod-def0-gpio = <&gpio2 0 GPIO_ACTIVE_LOW>; + mod-def0-gpios = <&gpio2 0 GPIO_ACTIVE_LOW>; maximum-power-milliwatt = <2000>; }; sfp1: sfp-1 { compatible = "sff,sfp"; i2c-bus = <&sfp1_i2c>; - mod-def0-gpio = <&gpio2 9 GPIO_ACTIVE_LOW>; + mod-def0-gpios = <&gpio2 9 GPIO_ACTIVE_LOW>; maximum-power-milliwatt = <2000>; }; sfp2: sfp-2 { compatible = "sff,sfp"; i2c-bus = <&sfp2_i2c>; - mod-def0-gpio = <&gpio2 10 GPIO_ACTIVE_LOW>; + mod-def0-gpios = <&gpio2 10 GPIO_ACTIVE_LOW>; maximum-power-milliwatt = <2000>; }; sfp3: sfp-3 { compatible = "sff,sfp"; i2c-bus = <&sfp3_i2c>; - mod-def0-gpio = <&gpio2 11 GPIO_ACTIVE_LOW>; + mod-def0-gpios = <&gpio2 11 GPIO_ACTIVE_LOW>; maximum-power-milliwatt = <2000>; }; }; -- cgit v1.2.3 From 4ce223e5ef70844505aee9780a9153b8d7866158 Mon Sep 17 00:00:00 2001 From: Ioana Ciornei Date: Thu, 7 Jul 2022 12:14:37 +0300 Subject: arch: arm64: dts: marvell: rename the sfp GPIO properties Rename the GPIO related sfp properties to include the preffered -gpios suffix. Also, with this change the dtb_check will no longer complain when trying to verify the DTS against the sff,sfp.yaml binding. Signed-off-by: Ioana Ciornei Signed-off-by: Jakub Kicinski --- .../boot/dts/marvell/armada-3720-turris-mox.dts | 10 ++++----- arch/arm64/boot/dts/marvell/armada-3720-uDPU.dts | 16 +++++++-------- .../boot/dts/marvell/armada-7040-mochabin.dts | 16 +++++++-------- .../dts/marvell/armada-8040-clearfog-gt-8k.dts | 4 ++-- arch/arm64/boot/dts/marvell/armada-8040-mcbin.dtsi | 24 +++++++++++----------- .../boot/dts/marvell/armada-8040-puzzle-m801.dts | 16 +++++++-------- arch/arm64/boot/dts/marvell/cn9130-crb.dtsi | 6 +++--- arch/arm64/boot/dts/marvell/cn9130-db.dtsi | 8 ++++---- arch/arm64/boot/dts/marvell/cn9131-db.dtsi | 8 ++++---- arch/arm64/boot/dts/marvell/cn9132-db.dtsi | 8 ++++---- 10 files changed, 58 insertions(+), 58 deletions(-) (limited to 'arch') diff --git a/arch/arm64/boot/dts/marvell/armada-3720-turris-mox.dts b/arch/arm64/boot/dts/marvell/armada-3720-turris-mox.dts index caf9c8529fca..cbf75ddd6857 100644 --- a/arch/arm64/boot/dts/marvell/armada-3720-turris-mox.dts +++ b/arch/arm64/boot/dts/marvell/armada-3720-turris-mox.dts @@ -100,11 +100,11 @@ sfp: sfp { compatible = "sff,sfp"; i2c-bus = <&i2c0>; - los-gpio = <&moxtet_sfp 0 GPIO_ACTIVE_HIGH>; - tx-fault-gpio = <&moxtet_sfp 1 GPIO_ACTIVE_HIGH>; - mod-def0-gpio = <&moxtet_sfp 2 GPIO_ACTIVE_LOW>; - tx-disable-gpio = <&moxtet_sfp 4 GPIO_ACTIVE_HIGH>; - rate-select0-gpio = <&moxtet_sfp 5 GPIO_ACTIVE_HIGH>; + los-gpios = <&moxtet_sfp 0 GPIO_ACTIVE_HIGH>; + tx-fault-gpios = <&moxtet_sfp 1 GPIO_ACTIVE_HIGH>; + mod-def0-gpios = <&moxtet_sfp 2 GPIO_ACTIVE_LOW>; + tx-disable-gpios = <&moxtet_sfp 4 GPIO_ACTIVE_HIGH>; + rate-select0-gpios = <&moxtet_sfp 5 GPIO_ACTIVE_HIGH>; maximum-power-milliwatt = <3000>; /* enabled by U-Boot if SFP module is present */ diff --git a/arch/arm64/boot/dts/marvell/armada-3720-uDPU.dts b/arch/arm64/boot/dts/marvell/armada-3720-uDPU.dts index a35317d24d6c..b20c8e7d923b 100644 --- a/arch/arm64/boot/dts/marvell/armada-3720-uDPU.dts +++ b/arch/arm64/boot/dts/marvell/armada-3720-uDPU.dts @@ -65,20 +65,20 @@ sfp_eth0: sfp-eth0 { compatible = "sff,sfp"; i2c-bus = <&i2c0>; - los-gpio = <&gpiosb 2 GPIO_ACTIVE_HIGH>; - mod-def0-gpio = <&gpiosb 3 GPIO_ACTIVE_LOW>; - tx-disable-gpio = <&gpiosb 4 GPIO_ACTIVE_HIGH>; - tx-fault-gpio = <&gpiosb 5 GPIO_ACTIVE_HIGH>; + los-gpios = <&gpiosb 2 GPIO_ACTIVE_HIGH>; + mod-def0-gpios = <&gpiosb 3 GPIO_ACTIVE_LOW>; + tx-disable-gpios = <&gpiosb 4 GPIO_ACTIVE_HIGH>; + tx-fault-gpios = <&gpiosb 5 GPIO_ACTIVE_HIGH>; maximum-power-milliwatt = <3000>; }; sfp_eth1: sfp-eth1 { compatible = "sff,sfp"; i2c-bus = <&i2c1>; - los-gpio = <&gpiosb 7 GPIO_ACTIVE_HIGH>; - mod-def0-gpio = <&gpiosb 8 GPIO_ACTIVE_LOW>; - tx-disable-gpio = <&gpiosb 9 GPIO_ACTIVE_HIGH>; - tx-fault-gpio = <&gpiosb 10 GPIO_ACTIVE_HIGH>; + los-gpios = <&gpiosb 7 GPIO_ACTIVE_HIGH>; + mod-def0-gpios = <&gpiosb 8 GPIO_ACTIVE_LOW>; + tx-disable-gpios = <&gpiosb 9 GPIO_ACTIVE_HIGH>; + tx-fault-gpios = <&gpiosb 10 GPIO_ACTIVE_HIGH>; maximum-power-milliwatt = <3000>; }; }; diff --git a/arch/arm64/boot/dts/marvell/armada-7040-mochabin.dts b/arch/arm64/boot/dts/marvell/armada-7040-mochabin.dts index 39a8e5e99d79..5f6ed735e31a 100644 --- a/arch/arm64/boot/dts/marvell/armada-7040-mochabin.dts +++ b/arch/arm64/boot/dts/marvell/armada-7040-mochabin.dts @@ -34,20 +34,20 @@ sfp_eth0: sfp-eth0 { compatible = "sff,sfp"; i2c-bus = <&cp0_i2c1>; - los-gpio = <&sfp_gpio 3 GPIO_ACTIVE_HIGH>; - mod-def0-gpio = <&sfp_gpio 2 GPIO_ACTIVE_LOW>; - tx-disable-gpio = <&sfp_gpio 1 GPIO_ACTIVE_HIGH>; - tx-fault-gpio = <&sfp_gpio 0 GPIO_ACTIVE_HIGH>; + los-gpios = <&sfp_gpio 3 GPIO_ACTIVE_HIGH>; + mod-def0-gpios = <&sfp_gpio 2 GPIO_ACTIVE_LOW>; + tx-disable-gpios = <&sfp_gpio 1 GPIO_ACTIVE_HIGH>; + tx-fault-gpios = <&sfp_gpio 0 GPIO_ACTIVE_HIGH>; }; /* SFP 1G */ sfp_eth2: sfp-eth2 { compatible = "sff,sfp"; i2c-bus = <&cp0_i2c0>; - los-gpio = <&sfp_gpio 7 GPIO_ACTIVE_HIGH>; - mod-def0-gpio = <&sfp_gpio 6 GPIO_ACTIVE_LOW>; - tx-disable-gpio = <&sfp_gpio 5 GPIO_ACTIVE_HIGH>; - tx-fault-gpio = <&sfp_gpio 4 GPIO_ACTIVE_HIGH>; + los-gpios = <&sfp_gpio 7 GPIO_ACTIVE_HIGH>; + mod-def0-gpios = <&sfp_gpio 6 GPIO_ACTIVE_LOW>; + tx-disable-gpios = <&sfp_gpio 5 GPIO_ACTIVE_HIGH>; + tx-fault-gpios = <&sfp_gpio 4 GPIO_ACTIVE_HIGH>; }; }; diff --git a/arch/arm64/boot/dts/marvell/armada-8040-clearfog-gt-8k.dts b/arch/arm64/boot/dts/marvell/armada-8040-clearfog-gt-8k.dts index 871f84b4a6ed..079c2745070a 100644 --- a/arch/arm64/boot/dts/marvell/armada-8040-clearfog-gt-8k.dts +++ b/arch/arm64/boot/dts/marvell/armada-8040-clearfog-gt-8k.dts @@ -64,8 +64,8 @@ sfp_cp0_eth0: sfp-cp0-eth0 { compatible = "sff,sfp"; i2c-bus = <&cp0_i2c1>; - mod-def0-gpio = <&cp0_gpio2 17 GPIO_ACTIVE_LOW>; - tx-disable-gpio = <&cp1_gpio1 29 GPIO_ACTIVE_HIGH>; + mod-def0-gpios = <&cp0_gpio2 17 GPIO_ACTIVE_LOW>; + tx-disable-gpios = <&cp1_gpio1 29 GPIO_ACTIVE_HIGH>; pinctrl-names = "default"; pinctrl-0 = <&cp0_sfp_present_pins &cp1_sfp_tx_disable_pins>; maximum-power-milliwatt = <2000>; diff --git a/arch/arm64/boot/dts/marvell/armada-8040-mcbin.dtsi b/arch/arm64/boot/dts/marvell/armada-8040-mcbin.dtsi index 779cf167c33e..33c179838e24 100644 --- a/arch/arm64/boot/dts/marvell/armada-8040-mcbin.dtsi +++ b/arch/arm64/boot/dts/marvell/armada-8040-mcbin.dtsi @@ -65,10 +65,10 @@ /* CON15,16 - CPM lane 4 */ compatible = "sff,sfp"; i2c-bus = <&sfpp0_i2c>; - los-gpio = <&cp1_gpio1 28 GPIO_ACTIVE_HIGH>; - mod-def0-gpio = <&cp1_gpio1 27 GPIO_ACTIVE_LOW>; - tx-disable-gpio = <&cp1_gpio1 29 GPIO_ACTIVE_HIGH>; - tx-fault-gpio = <&cp1_gpio1 26 GPIO_ACTIVE_HIGH>; + los-gpios = <&cp1_gpio1 28 GPIO_ACTIVE_HIGH>; + mod-def0-gpios = <&cp1_gpio1 27 GPIO_ACTIVE_LOW>; + tx-disable-gpios = <&cp1_gpio1 29 GPIO_ACTIVE_HIGH>; + tx-fault-gpios = <&cp1_gpio1 26 GPIO_ACTIVE_HIGH>; pinctrl-names = "default"; pinctrl-0 = <&cp1_sfpp0_pins>; maximum-power-milliwatt = <2000>; @@ -78,10 +78,10 @@ /* CON17,18 - CPS lane 4 */ compatible = "sff,sfp"; i2c-bus = <&sfpp1_i2c>; - los-gpio = <&cp1_gpio1 8 GPIO_ACTIVE_HIGH>; - mod-def0-gpio = <&cp1_gpio1 11 GPIO_ACTIVE_LOW>; - tx-disable-gpio = <&cp1_gpio1 10 GPIO_ACTIVE_HIGH>; - tx-fault-gpio = <&cp0_gpio2 30 GPIO_ACTIVE_HIGH>; + los-gpios = <&cp1_gpio1 8 GPIO_ACTIVE_HIGH>; + mod-def0-gpios = <&cp1_gpio1 11 GPIO_ACTIVE_LOW>; + tx-disable-gpios = <&cp1_gpio1 10 GPIO_ACTIVE_HIGH>; + tx-fault-gpios = <&cp0_gpio2 30 GPIO_ACTIVE_HIGH>; pinctrl-names = "default"; pinctrl-0 = <&cp1_sfpp1_pins &cp0_sfpp1_pins>; maximum-power-milliwatt = <2000>; @@ -91,10 +91,10 @@ /* CON13,14 - CPS lane 5 */ compatible = "sff,sfp"; i2c-bus = <&sfp_1g_i2c>; - los-gpio = <&cp0_gpio2 22 GPIO_ACTIVE_HIGH>; - mod-def0-gpio = <&cp0_gpio2 21 GPIO_ACTIVE_LOW>; - tx-disable-gpio = <&cp1_gpio1 24 GPIO_ACTIVE_HIGH>; - tx-fault-gpio = <&cp0_gpio2 19 GPIO_ACTIVE_HIGH>; + los-gpios = <&cp0_gpio2 22 GPIO_ACTIVE_HIGH>; + mod-def0-gpios = <&cp0_gpio2 21 GPIO_ACTIVE_LOW>; + tx-disable-gpios = <&cp1_gpio1 24 GPIO_ACTIVE_HIGH>; + tx-fault-gpios = <&cp0_gpio2 19 GPIO_ACTIVE_HIGH>; pinctrl-names = "default"; pinctrl-0 = <&cp0_sfp_1g_pins &cp1_sfp_1g_pins>; maximum-power-milliwatt = <2000>; diff --git a/arch/arm64/boot/dts/marvell/armada-8040-puzzle-m801.dts b/arch/arm64/boot/dts/marvell/armada-8040-puzzle-m801.dts index 74bed79e4f5e..72e9b0f671a9 100644 --- a/arch/arm64/boot/dts/marvell/armada-8040-puzzle-m801.dts +++ b/arch/arm64/boot/dts/marvell/armada-8040-puzzle-m801.dts @@ -67,20 +67,20 @@ sfp_cp0_eth0: sfp-cp0-eth0 { compatible = "sff,sfp"; i2c-bus = <&sfpplus0_i2c>; - los-gpio = <&sfpplus_gpio 11 GPIO_ACTIVE_HIGH>; - mod-def0-gpio = <&sfpplus_gpio 10 GPIO_ACTIVE_LOW>; - tx-disable-gpio = <&sfpplus_gpio 9 GPIO_ACTIVE_HIGH>; - tx-fault-gpio = <&sfpplus_gpio 8 GPIO_ACTIVE_HIGH>; + los-gpios = <&sfpplus_gpio 11 GPIO_ACTIVE_HIGH>; + mod-def0-gpios = <&sfpplus_gpio 10 GPIO_ACTIVE_LOW>; + tx-disable-gpios = <&sfpplus_gpio 9 GPIO_ACTIVE_HIGH>; + tx-fault-gpios = <&sfpplus_gpio 8 GPIO_ACTIVE_HIGH>; maximum-power-milliwatt = <3000>; }; sfp_cp1_eth0: sfp-cp1-eth0 { compatible = "sff,sfp"; i2c-bus = <&sfpplus1_i2c>; - los-gpio = <&sfpplus_gpio 3 GPIO_ACTIVE_HIGH>; - mod-def0-gpio = <&sfpplus_gpio 2 GPIO_ACTIVE_LOW>; - tx-disable-gpio = <&sfpplus_gpio 1 GPIO_ACTIVE_HIGH>; - tx-fault-gpio = <&sfpplus_gpio 0 GPIO_ACTIVE_HIGH>; + los-gpios = <&sfpplus_gpio 3 GPIO_ACTIVE_HIGH>; + mod-def0-gpios = <&sfpplus_gpio 2 GPIO_ACTIVE_LOW>; + tx-disable-gpios = <&sfpplus_gpio 1 GPIO_ACTIVE_HIGH>; + tx-fault-gpios = <&sfpplus_gpio 0 GPIO_ACTIVE_HIGH>; maximum-power-milliwatt = <3000>; }; diff --git a/arch/arm64/boot/dts/marvell/cn9130-crb.dtsi b/arch/arm64/boot/dts/marvell/cn9130-crb.dtsi index 1acd746284dc..8e4ec243fb8f 100644 --- a/arch/arm64/boot/dts/marvell/cn9130-crb.dtsi +++ b/arch/arm64/boot/dts/marvell/cn9130-crb.dtsi @@ -78,9 +78,9 @@ compatible = "sff,sfp"; i2c-bus = <&cp0_i2c1>; mod-def0-gpios = <&expander0 3 GPIO_ACTIVE_LOW>; - los-gpio = <&expander0 15 GPIO_ACTIVE_HIGH>; - tx-disable-gpio = <&expander0 2 GPIO_ACTIVE_HIGH>; - tx-fault-gpio = <&cp0_gpio1 24 GPIO_ACTIVE_HIGH>; + los-gpios = <&expander0 15 GPIO_ACTIVE_HIGH>; + tx-disable-gpios = <&expander0 2 GPIO_ACTIVE_HIGH>; + tx-fault-gpios = <&cp0_gpio1 24 GPIO_ACTIVE_HIGH>; maximum-power-milliwatt = <3000>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/marvell/cn9130-db.dtsi b/arch/arm64/boot/dts/marvell/cn9130-db.dtsi index 7e20987253a3..85d7ce13e70a 100644 --- a/arch/arm64/boot/dts/marvell/cn9130-db.dtsi +++ b/arch/arm64/boot/dts/marvell/cn9130-db.dtsi @@ -90,10 +90,10 @@ cp0_sfp_eth0: sfp-eth@0 { compatible = "sff,sfp"; i2c-bus = <&cp0_sfpp0_i2c>; - los-gpio = <&cp0_module_expander1 11 GPIO_ACTIVE_HIGH>; - mod-def0-gpio = <&cp0_module_expander1 10 GPIO_ACTIVE_LOW>; - tx-disable-gpio = <&cp0_module_expander1 9 GPIO_ACTIVE_HIGH>; - tx-fault-gpio = <&cp0_module_expander1 8 GPIO_ACTIVE_HIGH>; + los-gpios = <&cp0_module_expander1 11 GPIO_ACTIVE_HIGH>; + mod-def0-gpios = <&cp0_module_expander1 10 GPIO_ACTIVE_LOW>; + tx-disable-gpios = <&cp0_module_expander1 9 GPIO_ACTIVE_HIGH>; + tx-fault-gpios = <&cp0_module_expander1 8 GPIO_ACTIVE_HIGH>; /* * SFP cages are unconnected on early PCBs because of an the I2C * lanes not being connected. Prevent the port for being diff --git a/arch/arm64/boot/dts/marvell/cn9131-db.dtsi b/arch/arm64/boot/dts/marvell/cn9131-db.dtsi index b7fc241a228c..ff8422fae31b 100644 --- a/arch/arm64/boot/dts/marvell/cn9131-db.dtsi +++ b/arch/arm64/boot/dts/marvell/cn9131-db.dtsi @@ -37,10 +37,10 @@ cp1_sfp_eth1: sfp-eth1 { compatible = "sff,sfp"; i2c-bus = <&cp1_i2c0>; - los-gpio = <&cp1_gpio1 11 GPIO_ACTIVE_HIGH>; - mod-def0-gpio = <&cp1_gpio1 10 GPIO_ACTIVE_LOW>; - tx-disable-gpio = <&cp1_gpio1 9 GPIO_ACTIVE_HIGH>; - tx-fault-gpio = <&cp1_gpio1 8 GPIO_ACTIVE_HIGH>; + los-gpios = <&cp1_gpio1 11 GPIO_ACTIVE_HIGH>; + mod-def0-gpios = <&cp1_gpio1 10 GPIO_ACTIVE_LOW>; + tx-disable-gpios = <&cp1_gpio1 9 GPIO_ACTIVE_HIGH>; + tx-fault-gpios = <&cp1_gpio1 8 GPIO_ACTIVE_HIGH>; pinctrl-names = "default"; pinctrl-0 = <&cp1_sfp_pins>; /* diff --git a/arch/arm64/boot/dts/marvell/cn9132-db.dtsi b/arch/arm64/boot/dts/marvell/cn9132-db.dtsi index 3f1795fb4fe7..512a4fa2861e 100644 --- a/arch/arm64/boot/dts/marvell/cn9132-db.dtsi +++ b/arch/arm64/boot/dts/marvell/cn9132-db.dtsi @@ -57,10 +57,10 @@ cp2_sfp_eth0: sfp-eth0 { compatible = "sff,sfp"; i2c-bus = <&cp2_sfpp0_i2c>; - los-gpio = <&cp2_module_expander1 11 GPIO_ACTIVE_HIGH>; - mod-def0-gpio = <&cp2_module_expander1 10 GPIO_ACTIVE_LOW>; - tx-disable-gpio = <&cp2_module_expander1 9 GPIO_ACTIVE_HIGH>; - tx-fault-gpio = <&cp2_module_expander1 8 GPIO_ACTIVE_HIGH>; + los-gpios = <&cp2_module_expander1 11 GPIO_ACTIVE_HIGH>; + mod-def0-gpios = <&cp2_module_expander1 10 GPIO_ACTIVE_LOW>; + tx-disable-gpios = <&cp2_module_expander1 9 GPIO_ACTIVE_HIGH>; + tx-fault-gpios = <&cp2_module_expander1 8 GPIO_ACTIVE_HIGH>; /* * SFP cages are unconnected on early PCBs because of an the I2C * lanes not being connected. Prevent the port for being -- cgit v1.2.3 From 1d5f82d9dd477d5c66e0214a68c3e4f308eadd6d Mon Sep 17 00:00:00 2001 From: Song Liu Date: Tue, 5 Jul 2022 17:26:12 -0700 Subject: bpf, x86: fix freeing of not-finalized bpf_prog_pack syzbot reported a few issues with bpf_prog_pack [1], [2]. This only happens with multiple subprogs. In jit_subprogs(), we first call bpf_int_jit_compile() on each sub program. And then, we call it on each sub program again. jit_data is not freed in the first call of bpf_int_jit_compile(). Similarly we don't call bpf_jit_binary_pack_finalize() in the first call of bpf_int_jit_compile(). If bpf_int_jit_compile() failed for one sub program, we will call bpf_jit_binary_pack_finalize() for this sub program. However, we don't have a chance to call it for other sub programs. Then we will hit "goto out_free" in jit_subprogs(), and call bpf_jit_free on some subprograms that haven't got bpf_jit_binary_pack_finalize() yet. At this point, bpf_jit_binary_pack_free() is called and the whole 2MB page is freed erroneously. Fix this with a custom bpf_jit_free() for x86_64, which calls bpf_jit_binary_pack_finalize() if necessary. Also, with custom bpf_jit_free(), bpf_prog_aux->use_bpf_prog_pack is not needed any more, remove it. Fixes: 1022a5498f6f ("bpf, x86_64: Use bpf_jit_binary_pack_alloc") [1] https://syzkaller.appspot.com/bug?extid=2f649ec6d2eea1495a8f [2] https://syzkaller.appspot.com/bug?extid=87f65c75f4a72db05445 Reported-by: syzbot+2f649ec6d2eea1495a8f@syzkaller.appspotmail.com Reported-by: syzbot+87f65c75f4a72db05445@syzkaller.appspotmail.com Signed-off-by: Song Liu Link: https://lore.kernel.org/r/20220706002612.4013790-1-song@kernel.org Signed-off-by: Alexei Starovoitov --- arch/x86/net/bpf_jit_comp.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) (limited to 'arch') diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index d2614f1bf838..54c7f46c453f 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -2486,3 +2486,28 @@ bool bpf_jit_supports_subprog_tailcalls(void) { return true; } + +void bpf_jit_free(struct bpf_prog *prog) +{ + if (prog->jited) { + struct x64_jit_data *jit_data = prog->aux->jit_data; + struct bpf_binary_header *hdr; + + /* + * If we fail the final pass of JIT (from jit_subprogs), + * the program may not be finalized yet. Call finalize here + * before freeing it. + */ + if (jit_data) { + bpf_jit_binary_pack_finalize(prog, jit_data->header, + jit_data->rw_header); + kvfree(jit_data->addrs); + kfree(jit_data); + } + hdr = bpf_jit_binary_pack_hdr(prog); + bpf_jit_binary_pack_free(hdr, NULL); + WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(prog)); + } + + bpf_prog_unlock_free(prog); +} -- cgit v1.2.3 From 33f32e5072b6cc84d1b130a3ad485849bcec907a Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Wed, 13 Jul 2022 10:35:03 -0700 Subject: bpf, arm64: Mark dummy_tramp as global When building with clang + CONFIG_CFI_CLANG=y, the following error occurs at link time: ld.lld: error: undefined symbol: dummy_tramp dummy_tramp is declared globally in C but its definition in inline assembly does not use .global, which prevents clang from properly resolving the references to it when creating the CFI jump tables. Mark dummy_tramp as global so that the reference can be properly resolved. Fixes: b2ad54e1533e ("bpf, arm64: Implement bpf_arch_text_poke() for arm64") Suggested-by: Sami Tolvanen Signed-off-by: Nathan Chancellor Signed-off-by: Daniel Borkmann Reviewed-by: Stanislav Fomichev Link: https://github.com/ClangBuiltLinux/linux/issues/1661 Link: https://lore.kernel.org/bpf/20220713173503.3889486-1-nathan@kernel.org --- arch/arm64/net/bpf_jit_comp.c | 1 + 1 file changed, 1 insertion(+) (limited to 'arch') diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index fd1cb0d2aaa6..dcc572b7d4da 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -604,6 +604,7 @@ void dummy_tramp(void); asm ( " .pushsection .text, \"ax\", @progbits\n" +" .global dummy_tramp\n" " .type dummy_tramp, %function\n" "dummy_tramp:" #if IS_ENABLED(CONFIG_ARM64_BTI_KERNEL) -- cgit v1.2.3 From 339ed900b3079b6ea21df55f368b1ccba17cd5e6 Mon Sep 17 00:00:00 2001 From: Xu Kuohai Date: Thu, 21 Jul 2022 08:13:19 -0400 Subject: bpf, arm64: Fix compile error in dummy_tramp() dummy_tramp() uses "lr" to refer to the x30 register, but some assembler does not recognize "lr" and reports a build failure: /tmp/cc52xO0c.s: Assembler messages: /tmp/cc52xO0c.s:8: Error: operand 1 should be an integer register -- `mov lr,x9' /tmp/cc52xO0c.s:7: Error: undefined symbol lr used as an immediate value make[2]: *** [scripts/Makefile.build:250: arch/arm64/net/bpf_jit_comp.o] Error 1 make[1]: *** [scripts/Makefile.build:525: arch/arm64/net] Error 2 So replace "lr" with "x30" to fix it. Fixes: b2ad54e1533e ("bpf, arm64: Implement bpf_arch_text_poke() for arm64") Reported-by: Jon Hunter Signed-off-by: Xu Kuohai Signed-off-by: Daniel Borkmann Tested-by: Jon Hunter Reviewed-by: Jean-Philippe Brucker Link: https://lore.kernel.org/bpf/20220721121319.2999259-1-xukuohai@huaweicloud.com --- arch/arm64/net/bpf_jit_comp.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'arch') diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index dcc572b7d4da..7ca8779ae34f 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -610,8 +610,8 @@ asm ( #if IS_ENABLED(CONFIG_ARM64_BTI_KERNEL) " bti j\n" /* dummy_tramp is called via "br x10" */ #endif -" mov x10, lr\n" -" mov lr, x9\n" +" mov x10, x30\n" +" mov x30, x9\n" " ret x10\n" " .size dummy_tramp, .-dummy_tramp\n" " .popsection\n" -- cgit v1.2.3 From 316cba62dfb7878b7353177e6a7da9cc0c979cde Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Tue, 19 Jul 2022 17:21:25 -0700 Subject: bpf, x64: Allow to use caller address from stack Currently we call the original function by using the absolute address given at the JIT generation. That's not usable when having trampoline attached to multiple functions, or the target address changes dynamically (in case of live patch). In such cases we need to take the return address from the stack. Adding support to retrieve the original function address from the stack by adding new BPF_TRAMP_F_ORIG_STACK flag for arch_prepare_bpf_trampoline function. Basically we take the return address of the 'fentry' call: function + 0: call fentry # stores 'function + 5' address on stack function + 5: ... The 'function + 5' address will be used as the address for the original function to call. Signed-off-by: Jiri Olsa Signed-off-by: Song Liu Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20220720002126.803253-4-song@kernel.org --- arch/x86/net/bpf_jit_comp.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) (limited to 'arch') diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 54c7f46c453f..e1b0c5ed0b7c 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -2119,10 +2119,15 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i if (flags & BPF_TRAMP_F_CALL_ORIG) { restore_regs(m, &prog, nr_args, regs_off); - /* call original function */ - if (emit_call(&prog, orig_call, prog)) { - ret = -EINVAL; - goto cleanup; + if (flags & BPF_TRAMP_F_ORIG_STACK) { + emit_ldx(&prog, BPF_DW, BPF_REG_0, BPF_REG_FP, 8); + EMIT2(0xff, 0xd0); /* call *rax */ + } else { + /* call original function */ + if (emit_call(&prog, orig_call, prog)) { + ret = -EINVAL; + goto cleanup; + } } /* remember return value in a stack for bpf prog to access */ emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8); -- cgit v1.2.3