4 files changed, 86 insertions, 45 deletions

diff --git a/arch/s390/lib/Makefile b/arch/s390/lib/Makefile
index b068729e50ac..e3fffe1dff51 100644
--- a/arch/s390/lib/Makefile
+++ b/arch/s390/lib/Makefile
@@ -2,8 +2,7 @@
 # Makefile for s390-specific library files..
 #
 
-lib-y += delay.o string.o uaccess_pt.o find.o
+lib-y += delay.o string.o uaccess_pt.o uaccess_mvcos.o find.o
 obj-$(CONFIG_32BIT) += div64.o qrnnd.o ucmpdi2.o mem32.o
 obj-$(CONFIG_64BIT) += mem64.o
-lib-$(CONFIG_64BIT) += uaccess_mvcos.o
 lib-$(CONFIG_SMP) += spinlock.o
diff --git a/arch/s390/lib/uaccess.h b/arch/s390/lib/uaccess.h
index b1a22173d027..e5b9c924b733 100644
--- a/arch/s390/lib/uaccess.h
+++ b/arch/s390/lib/uaccess.h
@@ -6,7 +6,11 @@
 #ifndef __ARCH_S390_LIB_UACCESS_H
 #define __ARCH_S390_LIB_UACCESS_H
 
-extern int futex_atomic_op_pt(int, u32 __user *, int, int *);
-extern int futex_atomic_cmpxchg_pt(u32 *, u32 __user *, u32, u32);
+size_t copy_from_user_pt(void *to, const void __user *from, size_t n);
+size_t copy_to_user_pt(void __user *to, const void *from, size_t n);
+size_t copy_in_user_pt(void __user *to, const void __user *from, size_t n);
+size_t clear_user_pt(void __user *to, size_t n);
+size_t strnlen_user_pt(const char __user *src, size_t count);
+size_t strncpy_from_user_pt(char *dst, const char __user *src, size_t count);
 
 #endif /* __ARCH_S390_LIB_UACCESS_H */
diff --git a/arch/s390/lib/uaccess_mvcos.c b/arch/s390/lib/uaccess_mvcos.c
index 95123f57aaf8..66f35e15db2d 100644
--- a/arch/s390/lib/uaccess_mvcos.c
+++ b/arch/s390/lib/uaccess_mvcos.c
@@ -6,7 +6,9 @@
  *		 Gerald Schaefer (gerald.schaefer@de.ibm.com)
  */
 
+#include <linux/jump_label.h>
 #include <linux/errno.h>
+#include <linux/init.h>
 #include <linux/mm.h>
 #include <asm/uaccess.h>
 #include <asm/futex.h>
@@ -26,7 +28,10 @@
 #define SLR	"slgr"
 #endif
 
-static size_t copy_from_user_mvcos(void *x, const void __user *ptr, size_t size)
+static struct static_key have_mvcos = STATIC_KEY_INIT_TRUE;
+
+static inline size_t copy_from_user_mvcos(void *x, const void __user *ptr,
+					  size_t size)
 {
 	register unsigned long reg0 asm("0") = 0x81UL;
 	unsigned long tmp1, tmp2;
@@ -65,7 +70,16 @@ static size_t copy_from_user_mvcos(void *x, const void __user *ptr, size_t size)
 	return size;
 }
 
-static size_t copy_to_user_mvcos(void __user *ptr, const void *x, size_t size)
+size_t __copy_from_user(void *to, const void __user *from, size_t n)
+{
+	if (static_key_true(&have_mvcos))
+		return copy_from_user_mvcos(to, from, n);
+	return copy_from_user_pt(to, from, n);
+}
+EXPORT_SYMBOL(__copy_from_user);
+
+static inline size_t copy_to_user_mvcos(void __user *ptr, const void *x,
+					size_t size)
 {
 	register unsigned long reg0 asm("0") = 0x810000UL;
 	unsigned long tmp1, tmp2;
@@ -94,8 +108,16 @@ static size_t copy_to_user_mvcos(void __user *ptr, const void *x, size_t size)
 	return size;
 }
 
-static size_t copy_in_user_mvcos(void __user *to, const void __user *from,
-				 size_t size)
+size_t __copy_to_user(void __user *to, const void *from, size_t n)
+{
+	if (static_key_true(&have_mvcos))
+		return copy_to_user_mvcos(to, from, n);
+	return copy_to_user_pt(to, from, n);
+}
+EXPORT_SYMBOL(__copy_to_user);
+
+static inline size_t copy_in_user_mvcos(void __user *to, const void __user *from,
+					size_t size)
 {
 	register unsigned long reg0 asm("0") = 0x810081UL;
 	unsigned long tmp1, tmp2;
@@ -117,7 +139,15 @@ static size_t copy_in_user_mvcos(void __user *to, const void __user *from,
 	return size;
 }
 
-static size_t clear_user_mvcos(void __user *to, size_t size)
+size_t __copy_in_user(void __user *to, const void __user *from, size_t n)
+{
+	if (static_key_true(&have_mvcos))
+		return copy_in_user_mvcos(to, from, n);
+	return copy_in_user_pt(to, from, n);
+}
+EXPORT_SYMBOL(__copy_in_user);
+
+static inline size_t clear_user_mvcos(void __user *to, size_t size)
 {
 	register unsigned long reg0 asm("0") = 0x810000UL;
 	unsigned long tmp1, tmp2;
@@ -145,7 +175,15 @@ static size_t clear_user_mvcos(void __user *to, size_t size)
 	return size;
 }
 
-static size_t strnlen_user_mvcos(const char __user *src, size_t count)
+size_t __clear_user(void __user *to, size_t size)
+{
+	if (static_key_true(&have_mvcos))
+		return clear_user_mvcos(to, size);
+	return clear_user_pt(to, size);
+}
+EXPORT_SYMBOL(__clear_user);
+
+static inline size_t strnlen_user_mvcos(const char __user *src, size_t count)
 {
 	size_t done, len, offset, len_str;
 	char buf[256];
@@ -164,10 +202,18 @@ static size_t strnlen_user_mvcos(const char __user *src, size_t count)
 	return done + 1;
 }
 
-static size_t strncpy_from_user_mvcos(char *dst, const char __user *src,
-				      size_t count)
+size_t __strnlen_user(const char __user *src, size_t count)
 {
-	size_t done, len, offset, len_str;
+	if (static_key_true(&have_mvcos))
+		return strnlen_user_mvcos(src, count);
+	return strnlen_user_pt(src, count);
+}
+EXPORT_SYMBOL(__strnlen_user);
+
+static inline size_t strncpy_from_user_mvcos(char *dst, const char __user *src,
+					     size_t count)
+{
+	unsigned long done, len, offset, len_str;
 
 	if (unlikely(!count))
 		return 0;
@@ -185,13 +231,18 @@ static size_t strncpy_from_user_mvcos(char *dst, const char __user *src,
 	return done;
 }
 
-struct uaccess_ops uaccess_mvcos = {
-	.copy_from_user = copy_from_user_mvcos,
-	.copy_to_user = copy_to_user_mvcos,
-	.copy_in_user = copy_in_user_mvcos,
-	.clear_user = clear_user_mvcos,
-	.strnlen_user = strnlen_user_mvcos,
-	.strncpy_from_user = strncpy_from_user_mvcos,
-	.futex_atomic_op = futex_atomic_op_pt,
-	.futex_atomic_cmpxchg = futex_atomic_cmpxchg_pt,
-};
+long __strncpy_from_user(char *dst, const char __user *src, long count)
+{
+	if (static_key_true(&have_mvcos))
+		return strncpy_from_user_mvcos(dst, src, count);
+	return strncpy_from_user_pt(dst, src, count);
+}
+EXPORT_SYMBOL(__strncpy_from_user);
+
+static int __init uaccess_init(void)
+{
+	if (!MACHINE_HAS_MVCOS)
+		static_key_slow_dec(&have_mvcos);
+	return 0;
+}
+early_initcall(uaccess_init);
diff --git a/arch/s390/lib/uaccess_pt.c b/arch/s390/lib/uaccess_pt.c
index 2fa696b39b56..b49c3a440a24 100644
--- a/arch/s390/lib/uaccess_pt.c
+++ b/arch/s390/lib/uaccess_pt.c
@@ -211,7 +211,7 @@ fault:
 	return 0;
 }
 
-static size_t copy_from_user_pt(void *to, const void __user *from, size_t n)
+size_t copy_from_user_pt(void *to, const void __user *from, size_t n)
 {
 	size_t rc;
 
@@ -223,14 +223,14 @@ static size_t copy_from_user_pt(void *to, const void __user *from, size_t n)
 	return rc;
 }
 
-static size_t copy_to_user_pt(void __user *to, const void *from, size_t n)
+size_t copy_to_user_pt(void __user *to, const void *from, size_t n)
 {
 	if (segment_eq(get_fs(), KERNEL_DS))
 		return copy_in_kernel(to, (void __user *) from, n);
 	return __user_copy_pt((unsigned long) to, (void *) from, n, 1);
 }
 
-static size_t clear_user_pt(void __user *to, size_t n)
+size_t clear_user_pt(void __user *to, size_t n)
 {
 	void *zpage = (void *) empty_zero_page;
 	long done, size, ret;
@@ -253,7 +253,7 @@ static size_t clear_user_pt(void __user *to, size_t n)
 	return 0;
 }
 
-static size_t strnlen_user_pt(const char __user *src, size_t count)
+size_t strnlen_user_pt(const char __user *src, size_t count)
 {
 	unsigned long uaddr = (unsigned long) src;
 	struct mm_struct *mm = current->mm;
@@ -289,8 +289,7 @@ fault:
 	goto retry;
 }
 
-static size_t strncpy_from_user_pt(char *dst, const char __user *src,
-				   size_t count)
+size_t strncpy_from_user_pt(char *dst, const char __user *src, size_t count)
 {
 	size_t done, len, offset, len_str;
 
@@ -315,8 +314,7 @@ static size_t strncpy_from_user_pt(char *dst, const char __user *src,
 	return done;
 }
 
-static size_t copy_in_user_pt(void __user *to, const void __user *from,
-			      size_t n)
+size_t copy_in_user_pt(void __user *to, const void __user *from, size_t n)
 {
 	struct mm_struct *mm = current->mm;
 	unsigned long offset_max, uaddr, done, size, error_code;
@@ -411,7 +409,7 @@ static int __futex_atomic_op_pt(int op, u32 __user *uaddr, int oparg, int *old)
 	return ret;
 }
 
-int futex_atomic_op_pt(int op, u32 __user *uaddr, int oparg, int *old)
+int __futex_atomic_op_inuser(int op, u32 __user *uaddr, int oparg, int *old)
 {
 	int ret;
 
@@ -449,8 +447,8 @@ static int __futex_atomic_cmpxchg_pt(u32 *uval, u32 __user *uaddr,
 	return ret;
 }
 
-int futex_atomic_cmpxchg_pt(u32 *uval, u32 __user *uaddr,
-			    u32 oldval, u32 newval)
+int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr,
+				  u32 oldval, u32 newval)
 {
 	int ret;
 
@@ -471,14 +469,3 @@ int futex_atomic_cmpxchg_pt(u32 *uval, u32 __user *uaddr,
 	put_page(virt_to_page(uaddr));
 	return ret;
 }
-
-struct uaccess_ops uaccess_pt = {
-	.copy_from_user		= copy_from_user_pt,
-	.copy_to_user		= copy_to_user_pt,
-	.copy_in_user		= copy_in_user_pt,
-	.clear_user		= clear_user_pt,
-	.strnlen_user		= strnlen_user_pt,
-	.strncpy_from_user	= strncpy_from_user_pt,
-	.futex_atomic_op	= futex_atomic_op_pt,
-	.futex_atomic_cmpxchg	= futex_atomic_cmpxchg_pt,
-};