s390: Use inline qualifier for all EX_TABLE and ALTERNATIVE inline assemblies

Use asm_inline for all inline assemblies which make use of the EX_TABLE or
ALTERNATIVE macros.

These macros expand to many lines and the compiler assumes the number of
lines within an inline assembly is the same as the number of instructions
within an inline assembly. This has an effect on inlining and loop
unrolling decisions.

In order to avoid incorrect assumptions use asm_inline, which tells the
compiler that an inline assembly has the smallest possible size.

In order to avoid confusion when asm_inline should be used or not, since a
couple of inline assemblies are quite large: the rule is to always use
asm_inline whenever the EX_TABLE or ALTERNATIVE macro is used. In specific
cases there may be reasons to not follow this guideline, but that should
be documented with the corresponding code.

Using the inline qualifier everywhere has only a small effect on the kernel
image size:

add/remove: 0/10 grow/shrink: 19/8 up/down: 1492/-1858 (-366)

The only location where this seems to matter is load_unaligned_zeropad()
from word-at-a-time.h where the compiler inlines more functions within the
dcache code, which is indeed code where performance matters.

Suggested-by: Juergen Christ <jchrist@linux.ibm.com>
Reviewed-by: Juergen Christ <jchrist@linux.ibm.com>
Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
Signed-off-by: Vasily Gorbik <gor@linux.ibm.com>

20 files changed, 48 insertions, 44 deletions

diff --git a/arch/s390/boot/ipl_parm.c b/arch/s390/boot/ipl_parm.c
index b49c906b57dc..d04e9b89d14a 100644
--- a/arch/s390/boot/ipl_parm.c
+++ b/arch/s390/boot/ipl_parm.c
@@ -37,7 +37,7 @@ static inline int __diag308(unsigned long subcode, void *addr)
 {
 	union register_pair r1 = { .even = (unsigned long)addr, .odd = 0 };
 
-	asm volatile(
+	asm_inline volatile(
 		"	diag	%[r1],%[subcode],0x308\n"
 		"0:\n"
 		EX_TABLE(0b, 0b)
diff --git a/arch/s390/boot/physmem_info.c b/arch/s390/boot/physmem_info.c
index eb1badb93c40..45e3d057cfaa 100644
--- a/arch/s390/boot/physmem_info.c
+++ b/arch/s390/boot/physmem_info.c
@@ -67,7 +67,7 @@ static int __diag260(unsigned long rx1, unsigned long rx2)
 	rx.odd	= rx2;
 	ry = 0x10; /* storage configuration */
 	exception = 1;
-	asm volatile(
+	asm_inline volatile(
 		"	diag	%[rx],%[ry],0x260\n"
 		"0:	lhi	%[exc],0\n"
 		"1:\n"
@@ -105,7 +105,7 @@ static int diag500_storage_limit(unsigned long *max_physmem_end)
 {
 	unsigned long storage_limit;
 
-	asm volatile(
+	asm_inline volatile(
 		"	lghi	%%r1,%[subcode]\n"
 		"	lghi	%%r2,0\n"
 		"	diag	%%r2,%%r4,0x500\n"
@@ -126,7 +126,7 @@ static int tprot(unsigned long addr)
 	int cc, exception;
 
 	exception = 1;
-	asm volatile(
+	asm_inline volatile(
 		"	tprot	0(%[addr]),0\n"
 		"0:	lhi	%[exc],0\n"
 		"1:\n"
diff --git a/arch/s390/boot/startup.c b/arch/s390/boot/startup.c
index 099b2327c781..06316fb8e0fa 100644
--- a/arch/s390/boot/startup.c
+++ b/arch/s390/boot/startup.c
@@ -77,7 +77,7 @@ static void detect_diag9c(void)
 	int rc = 1;
 
 	cpu = stap();
-	asm volatile(
+	asm_inline volatile(
 		"	diag	%[cpu],%%r0,0x9c\n"
 		"0:	lhi	%[rc],0\n"
 		"1:\n"
@@ -138,7 +138,7 @@ static int cmma_test_essa(void)
 	int rc = 1;
 
 	/* Test ESSA_GET_STATE */
-	asm volatile(
+	asm_inline volatile(
 		"	.insn	rrf,0xb9ab0000,%[tmp],%[tmp],%[cmd],0\n"
 		"0:	lhi	%[rc],0\n"
 		"1:\n"
diff --git a/arch/s390/include/asm/cpu_mf.h b/arch/s390/include/asm/cpu_mf.h
index e1a279e0d6a6..1798fbd59068 100644
--- a/arch/s390/include/asm/cpu_mf.h
+++ b/arch/s390/include/asm/cpu_mf.h
@@ -171,7 +171,7 @@ static inline int qctri(struct cpumf_ctr_info *info)
 {
 	int rc = -EINVAL;
 
-	asm volatile (
+	asm_inline volatile (
 		"0:	qctri	%1\n"
 		"1:	lhi	%0,0\n"
 		"2:\n"
@@ -185,7 +185,7 @@ static inline int lcctl(u64 ctl)
 {
 	int cc;
 
-	asm volatile (
+	asm_inline volatile (
 		"	lcctl	%[ctl]\n"
 		CC_IPM(cc)
 		: CC_OUT(cc, cc)
@@ -200,7 +200,7 @@ static inline int __ecctr(u64 ctr, u64 *content)
 	u64 _content;
 	int cc;
 
-	asm volatile (
+	asm_inline volatile (
 		"	ecctr	%[_content],%[ctr]\n"
 		CC_IPM(cc)
 		: CC_OUT(cc, cc), [_content] "=d" (_content)
diff --git a/arch/s390/include/asm/diag.h b/arch/s390/include/asm/diag.h
index 5790630e31f0..8db8db3b1018 100644
--- a/arch/s390/include/asm/diag.h
+++ b/arch/s390/include/asm/diag.h
@@ -66,7 +66,7 @@ static inline void diag10_range(unsigned long start_pfn, unsigned long num_pfn)
 	end_addr = pfn_to_phys(start_pfn + num_pfn - 1);
 
 	diag_stat_inc(DIAG_STAT_X010);
-	asm volatile(
+	asm_inline volatile(
 		"0:	diag	%0,%1,0x10\n"
 		"1:	nopr	%%r7\n"
 		EX_TABLE(0b, 1b)
diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h
index 4f8d5592c298..d09a92db95f7 100644
--- a/arch/s390/include/asm/processor.h
+++ b/arch/s390/include/asm/processor.h
@@ -416,7 +416,11 @@ static __always_inline bool regs_irqs_disabled(struct pt_regs *regs)
 
 static __always_inline void bpon(void)
 {
-	asm volatile(ALTERNATIVE("nop", ".insn	rrf,0xb2e80000,0,0,13,0", ALT_SPEC(82)));
+	asm_inline volatile(
+		ALTERNATIVE("	nop\n",
+			    "	.insn	rrf,0xb2e80000,0,0,13,0\n",
+			    ALT_SPEC(82))
+		);
 }
 
 #endif /* __ASSEMBLY__ */
diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h
index f30ee6ddfece..8629d70ec38b 100644
--- a/arch/s390/include/asm/uaccess.h
+++ b/arch/s390/include/asm/uaccess.h
@@ -147,7 +147,7 @@ __put_user_##type##_noinstr(unsigned type __user *to,			\
 {									\
 	int rc;								\
 									\
-	asm volatile(							\
+	asm_inline volatile(						\
 		"	llilh	%%r0,%[spec]\n"				\
 		"0:	mvcos	%[to],%[from],%[size]\n"		\
 		"1:	lhi	%[rc],0\n"				\
@@ -263,7 +263,7 @@ __get_user_##type##_noinstr(unsigned type *to,				\
 {									\
 	int rc;								\
 									\
-	asm volatile(							\
+	asm_inline volatile(						\
 		"	lhi	%%r0,%[spec]\n"				\
 		"0:	mvcos	%[to],%[from],%[size]\n"		\
 		"1:	lhi	%[rc],0\n"				\
@@ -490,7 +490,7 @@ static __always_inline int __cmpxchg_user_key(unsigned long address, void *uval,
 		_old = ((unsigned int)old & 0xff) << shift;
 		_new = ((unsigned int)new & 0xff) << shift;
 		mask = ~(0xff << shift);
-		asm volatile(
+		asm_inline volatile(
 			"	spka	0(%[key])\n"
 			"	sacf	256\n"
 			"	llill	%[count],%[max_loops]\n"
@@ -538,7 +538,7 @@ static __always_inline int __cmpxchg_user_key(unsigned long address, void *uval,
 		_old = ((unsigned int)old & 0xffff) << shift;
 		_new = ((unsigned int)new & 0xffff) << shift;
 		mask = ~(0xffff << shift);
-		asm volatile(
+		asm_inline volatile(
 			"	spka	0(%[key])\n"
 			"	sacf	256\n"
 			"	llill	%[count],%[max_loops]\n"
@@ -580,7 +580,7 @@ static __always_inline int __cmpxchg_user_key(unsigned long address, void *uval,
 	case 4:	{
 		unsigned int prev = old;
 
-		asm volatile(
+		asm_inline volatile(
 			"	spka	0(%[key])\n"
 			"	sacf	256\n"
 			"0:	cs	%[prev],%[new],%[address]\n"
@@ -601,7 +601,7 @@ static __always_inline int __cmpxchg_user_key(unsigned long address, void *uval,
 	case 8: {
 		unsigned long prev = old;
 
-		asm volatile(
+		asm_inline volatile(
 			"	spka	0(%[key])\n"
 			"	sacf	256\n"
 			"0:	csg	%[prev],%[new],%[address]\n"
@@ -622,7 +622,7 @@ static __always_inline int __cmpxchg_user_key(unsigned long address, void *uval,
 	case 16: {
 		__uint128_t prev = old;
 
-		asm volatile(
+		asm_inline volatile(
 			"	spka	0(%[key])\n"
 			"	sacf	256\n"
 			"0:	cdsg	%[prev],%[new],%[address]\n"
diff --git a/arch/s390/include/asm/word-at-a-time.h b/arch/s390/include/asm/word-at-a-time.h
index 203acd6e431b..eaa19dee7699 100644
--- a/arch/s390/include/asm/word-at-a-time.h
+++ b/arch/s390/include/asm/word-at-a-time.h
@@ -52,7 +52,7 @@ static inline unsigned long load_unaligned_zeropad(const void *addr)
 {
 	unsigned long data;
 
-	asm volatile(
+	asm_inline volatile(
 		"0:	lg	%[data],0(%[addr])\n"
 		"1:	nopr	%%r7\n"
 		EX_TABLE_ZEROPAD(0b, 1b, %[data], %[addr])
diff --git a/arch/s390/kernel/cert_store.c b/arch/s390/kernel/cert_store.c
index bf983513dd33..03f3a1e52430 100644
--- a/arch/s390/kernel/cert_store.c
+++ b/arch/s390/kernel/cert_store.c
@@ -235,7 +235,7 @@ static int __diag320(unsigned long subcode, void *addr)
 {
 	union register_pair rp = { .even = (unsigned long)addr, };
 
-	asm volatile(
+	asm_inline volatile(
 		"	diag	%[rp],%[subcode],0x320\n"
 		"0:	nopr	%%r7\n"
 		EX_TABLE(0b, 0b)
diff --git a/arch/s390/kernel/diag/diag.c b/arch/s390/kernel/diag/diag.c
index e15b8dee3228..56b862ba9be8 100644
--- a/arch/s390/kernel/diag/diag.c
+++ b/arch/s390/kernel/diag/diag.c
@@ -195,7 +195,7 @@ static inline int __diag204(unsigned long *subcode, unsigned long size, void *ad
 {
 	union register_pair rp = { .even = *subcode, .odd = size };
 
-	asm volatile(
+	asm_inline volatile(
 		"	diag	%[addr],%[rp],0x204\n"
 		"0:	nopr	%%r7\n"
 		EX_TABLE(0b,0b)
@@ -286,7 +286,7 @@ int diag224(void *ptr)
 	int rc = -EOPNOTSUPP;
 
 	diag_stat_inc(DIAG_STAT_X224);
-	asm volatile("\n"
+	asm_inline volatile("\n"
 		"	diag	%[type],%[addr],0x224\n"
 		"0:	lhi	%[rc],0\n"
 		"1:\n"
diff --git a/arch/s390/kernel/ipl.c b/arch/s390/kernel/ipl.c
index 8990e2dffcdf..3b9d9ccfad63 100644
--- a/arch/s390/kernel/ipl.c
+++ b/arch/s390/kernel/ipl.c
@@ -186,7 +186,7 @@ static inline int __diag308(unsigned long subcode, unsigned long addr)
 
 	r1.even = addr;
 	r1.odd	= 0;
-	asm volatile(
+	asm_inline volatile(
 		"	diag	%[r1],%[subcode],0x308\n"
 		"0:	nopr	%%r7\n"
 		EX_TABLE(0b,0b)
diff --git a/arch/s390/kernel/traps.c b/arch/s390/kernel/traps.c
index 4daa3c76e999..9705929d256f 100644
--- a/arch/s390/kernel/traps.c
+++ b/arch/s390/kernel/traps.c
@@ -257,7 +257,7 @@ static void __init test_monitor_call(void)
 
 	if (!IS_ENABLED(CONFIG_BUG))
 		return;
-	asm volatile(
+	asm_inline volatile(
 		"	mc	0,0\n"
 		"0:	xgr	%0,%0\n"
 		"1:\n"
diff --git a/arch/s390/mm/pfault.c b/arch/s390/mm/pfault.c
index 1aac13bb8f53..b01e73f5b9b8 100644
--- a/arch/s390/mm/pfault.c
+++ b/arch/s390/mm/pfault.c
@@ -56,7 +56,7 @@ int __pfault_init(void)
 	if (pfault_disable)
 		return rc;
 	diag_stat_inc(DIAG_STAT_X258);
-	asm volatile(
+	asm_inline volatile(
 		"	diag	%[refbk],%[rc],0x258\n"
 		"0:	nopr	%%r7\n"
 		EX_TABLE(0b, 0b)
@@ -78,7 +78,7 @@ void __pfault_fini(void)
 	if (pfault_disable)
 		return;
 	diag_stat_inc(DIAG_STAT_X258);
-	asm volatile(
+	asm_inline volatile(
 		"	diag	%[refbk],0,0x258\n"
 		"0:	nopr	%%r7\n"
 		EX_TABLE(0b, 0b)
diff --git a/arch/s390/pci/pci_clp.c b/arch/s390/pci/pci_clp.c
index 14bf7e8d06b7..bf083176076d 100644
--- a/arch/s390/pci/pci_clp.c
+++ b/arch/s390/pci/pci_clp.c
@@ -56,7 +56,7 @@ static inline int clp_get_ilp(unsigned long *ilp)
 	int cc, exception;
 
 	exception = 1;
-	asm volatile (
+	asm_inline volatile (
 		"	.insn	rrf,0xb9a00000,%[mask],%[cmd],8,0\n"
 		"0:	lhi	%[exc],0\n"
 		"1:\n"
@@ -79,7 +79,7 @@ static __always_inline int clp_req(void *data, unsigned int lps)
 	u64 ignored;
 
 	exception = 1;
-	asm volatile (
+	asm_inline volatile (
 		"	.insn	rrf,0xb9a00000,%[ign],%[req],0,%[lps]\n"
 		"0:	lhi	%[exc],0\n"
 		"1:\n"
diff --git a/arch/s390/pci/pci_insn.c b/arch/s390/pci/pci_insn.c
index f5a75ea7629a..eb978c8012be 100644
--- a/arch/s390/pci/pci_insn.c
+++ b/arch/s390/pci/pci_insn.c
@@ -160,7 +160,7 @@ static inline int ____pcilg(u64 *data, u64 req, u64 offset, u8 *status)
 	u64 __data;
 
 	exception = 1;
-	asm volatile (
+	asm_inline volatile (
 		"	.insn	rre,0xb9d20000,%[data],%[req_off]\n"
 		"0:	lhi	%[exc],0\n"
 		"1:\n"
@@ -229,7 +229,7 @@ static inline int __pcilg_mio(u64 *data, u64 ioaddr, u64 len, u8 *status)
 	u64 __data;
 
 	exception = 1;
-	asm volatile (
+	asm_inline volatile (
 		"       .insn   rre,0xb9d60000,%[data],%[ioaddr_len]\n"
 		"0:	lhi	%[exc],0\n"
 		"1:\n"
@@ -267,7 +267,7 @@ static inline int __pcistg(u64 data, u64 req, u64 offset, u8 *status)
 	int cc, exception;
 
 	exception = 1;
-	asm volatile (
+	asm_inline volatile (
 		"	.insn	rre,0xb9d00000,%[data],%[req_off]\n"
 		"0:	lhi	%[exc],0\n"
 		"1:\n"
@@ -321,7 +321,7 @@ static inline int __pcistg_mio(u64 data, u64 ioaddr, u64 len, u8 *status)
 	int cc, exception;
 
 	exception = 1;
-	asm volatile (
+	asm_inline volatile (
 		"       .insn   rre,0xb9d40000,%[data],%[ioaddr_len]\n"
 		"0:	lhi	%[exc],0\n"
 		"1:\n"
@@ -356,7 +356,7 @@ static inline int __pcistb(const u64 *data, u64 req, u64 offset, u8 *status)
 	int cc, exception;
 
 	exception = 1;
-	asm volatile (
+	asm_inline volatile (
 		"	.insn	rsy,0xeb00000000d0,%[req],%[offset],%[data]\n"
 		"0:	lhi	%[exc],0\n"
 		"1:\n"
@@ -410,7 +410,7 @@ static inline int __pcistb_mio(const u64 *data, u64 ioaddr, u64 len, u8 *status)
 	int cc, exception;
 
 	exception = 1;
-	asm volatile (
+	asm_inline volatile (
 		"       .insn   rsy,0xeb00000000d4,%[len],%[ioaddr],%[data]\n"
 		"0:	lhi	%[exc],0\n"
 		"1:\n"
diff --git a/arch/s390/pci/pci_mmio.c b/arch/s390/pci/pci_mmio.c
index 46f99dc164ad..dc162c9951dd 100644
--- a/arch/s390/pci/pci_mmio.c
+++ b/arch/s390/pci/pci_mmio.c
@@ -34,7 +34,7 @@ static inline int __pcistb_mio_inuser(
 	int cc, exception;
 
 	exception = 1;
-	asm volatile (
+	asm_inline volatile (
 		"	sacf	256\n"
 		"0:	.insn	rsy,0xeb00000000d4,%[len],%[ioaddr],%[src]\n"
 		"1:	lhi	%[exc],0\n"
@@ -64,7 +64,7 @@ static inline int __pcistg_mio_inuser(
 	 * address space. pcistg then uses the user mappings.
 	 */
 	exception = 1;
-	asm volatile (
+	asm_inline volatile (
 		"	sacf	256\n"
 		"0:	llgc	%[tmp],0(%[src])\n"
 		"4:	sllg	%[val],%[val],8\n"
@@ -211,7 +211,7 @@ static inline int __pcilg_mio_inuser(
 	 * user address @dst
 	 */
 	exception = 1;
-	asm volatile (
+	asm_inline volatile (
 		"	sacf	256\n"
 		"0:	.insn	rre,0xb9d60000,%[val],%[ioaddr_len]\n"
 		"1:	lhi	%[exc],0\n"
diff --git a/drivers/s390/block/dasd_diag.c b/drivers/s390/block/dasd_diag.c
index 8b39803cf94c..56f1af8a7ddd 100644
--- a/drivers/s390/block/dasd_diag.c
+++ b/drivers/s390/block/dasd_diag.c
@@ -76,7 +76,7 @@ static inline int __dia250(void *iob, int cmd)
 	} addr_type;
 
 	exception = 1;
-	asm volatile(
+	asm_inline volatile(
 		"	diag	%[rx],%[cmd],0x250\n"
 		"0:	lhi	%[exc],0\n"
 		"1:\n"
diff --git a/drivers/s390/char/diag_ftp.c b/drivers/s390/char/diag_ftp.c
index 9418a9270d03..711f6982438e 100644
--- a/drivers/s390/char/diag_ftp.c
+++ b/drivers/s390/char/diag_ftp.c
@@ -106,7 +106,7 @@ static int diag_ftp_2c4(struct diag_ftp_ldfpl *fpl,
 	int rc;
 
 	diag_stat_inc(DIAG_STAT_X2C4);
-	asm volatile(
+	asm_inline volatile(
 		"	diag	%[addr],%[cmd],0x2c4\n"
 		"0:	j	2f\n"
 		"1:	la	%[rc],%[err]\n"
diff --git a/drivers/s390/char/sclp.h b/drivers/s390/char/sclp.h
index f7ca6e2f6d18..b31a680e0871 100644
--- a/drivers/s390/char/sclp.h
+++ b/drivers/s390/char/sclp.h
@@ -318,7 +318,7 @@ static inline int sclp_service_call(sclp_cmdw_t command, void *sccb)
 	int cc, exception;
 
 	exception = 1;
-	asm volatile(
+	asm_inline volatile(
 		"0:	.insn	rre,0xb2200000,%[cmd],%[sccb]\n" /* servc */
 		"1:	lhi	%[exc],0\n"
 		"2:\n"
diff --git a/drivers/s390/cio/ioasm.c b/drivers/s390/cio/ioasm.c
index 5ff1e51cddf3..a540045b64a6 100644
--- a/drivers/s390/cio/ioasm.c
+++ b/drivers/s390/cio/ioasm.c
@@ -22,7 +22,7 @@ static inline int __stsch(struct subchannel_id schid, struct schib *addr)
 	int ccode, exception;
 
 	exception = 1;
-	asm volatile(
+	asm_inline volatile(
 		"	lgr	1,%[r1]\n"
 		"	stsch	%[addr]\n"
 		"0:	lhi	%[exc],0\n"
@@ -52,7 +52,7 @@ static inline int __msch(struct subchannel_id schid, struct schib *addr)
 	int ccode, exception;
 
 	exception = 1;
-	asm volatile(
+	asm_inline volatile(
 		"	lgr	1,%[r1]\n"
 		"	msch	%[addr]\n"
 		"0:	lhi	%[exc],0\n"
@@ -106,7 +106,7 @@ static inline int __ssch(struct subchannel_id schid, union orb *addr)
 	int ccode, exception;
 
 	exception = 1;
-	asm volatile(
+	asm_inline volatile(
 		"	lgr	1,%[r1]\n"
 		"	ssch	%[addr]\n"
 		"0:	lhi	%[exc],0\n"
@@ -178,7 +178,7 @@ int chsc(void *chsc_area)
 	int cc, exception;
 
 	exception = 1;
-	asm volatile(
+	asm_inline volatile(
 		"	.insn	rre,0xb25f0000,%[chsc_area],0\n"
 		"0:	lhi	%[exc],0\n"
 		"1:\n"