diff options
| author | Heiko Carstens <hca@linux.ibm.com> | 2023-10-06 15:42:42 +0200 |
|---|---|---|
| committer | Vasily Gorbik <gor@linux.ibm.com> | 2023-10-16 13:04:09 +0200 |
| commit | 802ba53eefc592a6a82231f74e19bafe3256f172 (patch) | |
| tree | 2c9fa712b9d5c17f8a83a7800c64c471534dd143 /arch/s390/include | |
| parent | 76292d7243f809486ec26dd6cd3436dbcb9e5319 (diff) | |
| download | linux-stable-802ba53eefc592a6a82231f74e19bafe3256f172.tar.gz linux-stable-802ba53eefc592a6a82231f74e19bafe3256f172.tar.bz2 linux-stable-802ba53eefc592a6a82231f74e19bafe3256f172.zip | |
s390: add support for DCACHE_WORD_ACCESS
Implement load_unaligned_zeropad() and enable DCACHE_WORD_ACCESS to
speed up string operations in fs/dcache.c and fs/namei.c.
Reviewed-by: Sven Schnelle <svens@linux.ibm.com>
Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
Signed-off-by: Vasily Gorbik <gor@linux.ibm.com>
Diffstat (limited to 'arch/s390/include')
| -rw-r--r-- | arch/s390/include/asm/asm-extable.h | 4 | ||||
| -rw-r--r-- | arch/s390/include/asm/word-at-a-time.h | 22 |
2 files changed, 26 insertions, 0 deletions
diff --git a/arch/s390/include/asm/asm-extable.h b/arch/s390/include/asm/asm-extable.h index d1ecc2b6e5d3..4a6b0a8b6412 100644 --- a/arch/s390/include/asm/asm-extable.h +++ b/arch/s390/include/asm/asm-extable.h @@ -13,6 +13,7 @@ #define EX_TYPE_UA_LOAD_MEM 4 #define EX_TYPE_UA_LOAD_REG 5 #define EX_TYPE_UA_LOAD_REGPAIR 6 +#define EX_TYPE_ZEROPAD 7 #define EX_DATA_REG_ERR_SHIFT 0 #define EX_DATA_REG_ERR GENMASK(3, 0) @@ -80,4 +81,7 @@ #define EX_TABLE_UA_LOAD_REGPAIR(_fault, _target, _regerr, _regzero) \ __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_UA_LOAD_REGPAIR, _regerr, _regzero, 0) +#define EX_TABLE_ZEROPAD(_fault, _target, _regdata, _regaddr) \ + __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_ZEROPAD, _regdata, _regaddr, 0) + #endif /* __ASM_EXTABLE_H */ diff --git a/arch/s390/include/asm/word-at-a-time.h b/arch/s390/include/asm/word-at-a-time.h index 09b37541a175..2579f1694b82 100644 --- a/arch/s390/include/asm/word-at-a-time.h +++ b/arch/s390/include/asm/word-at-a-time.h @@ -3,6 +3,7 @@ #define _ASM_WORD_AT_A_TIME_H #include <linux/kernel.h> +#include <asm/asm-extable.h> #include <asm/bitsperlong.h> struct word_at_a_time { @@ -39,4 +40,25 @@ static inline unsigned long zero_bytemask(unsigned long data) return ~1UL << data; } +/* + * Load an unaligned word from kernel space. + * + * In the (very unlikely) case of the word being a page-crosser + * and the next page not being mapped, take the exception and + * return zeroes in the non-existing part. + */ +static inline unsigned long load_unaligned_zeropad(const void *addr) +{ + unsigned long data; + + asm volatile( + "0: lg %[data],0(%[addr])\n" + "1: nopr %%r7\n" + EX_TABLE_ZEROPAD(0b, 1b, %[data], %[addr]) + EX_TABLE_ZEROPAD(1b, 1b, %[data], %[addr]) + : [data] "=d" (data) + : [addr] "a" (addr), "m" (*(unsigned long *)addr)); + return data; +} + #endif /* _ASM_WORD_AT_A_TIME_H */ |