#include <linux/uaccess.h>
#include <asm/errno.h>
-static inline int futex_atomic_op_inuser (int encoded_op, u32 __user *uaddr)
+int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, u32 oldval, u32 newval);
+int __futex_atomic_op_inuser(int op, u32 __user *uaddr, int oparg, int *old);
+
+static inline int futex_atomic_op_inuser(int encoded_op, u32 __user *uaddr)
{
int op = (encoded_op >> 28) & 7;
int cmp = (encoded_op >> 24) & 15;
oparg = 1 << oparg;
pagefault_disable();
- ret = uaccess.futex_atomic_op(op, uaddr, oparg, &oldval);
+ ret = __futex_atomic_op_inuser(op, uaddr, oparg, &oldval);
pagefault_enable();
if (!ret) {
return ret;
}
-static inline int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr,
- u32 oldval, u32 newval)
-{
- return uaccess.futex_atomic_cmpxchg(uval, uaddr, oldval, newval);
-}
-
#endif /* _ASM_S390_FUTEX_H */
#define ARCH_HAS_SORT_EXTABLE
#define ARCH_HAS_SEARCH_EXTABLE
-struct uaccess_ops {
- size_t (*copy_from_user)(void *, const void __user *, size_t);
- size_t (*copy_to_user)(void __user *, const void *, size_t);
- size_t (*copy_in_user)(void __user *, const void __user *, size_t);
- size_t (*clear_user)(void __user *, size_t);
- size_t (*strnlen_user)(const char __user *, size_t);
- size_t (*strncpy_from_user)(char *, const char __user *, size_t);
- int (*futex_atomic_op)(int op, u32 __user *, int oparg, int *old);
- int (*futex_atomic_cmpxchg)(u32 *, u32 __user *, u32 old, u32 new);
-};
+int __handle_fault(unsigned long, unsigned long, int);
-extern struct uaccess_ops uaccess;
-extern struct uaccess_ops uaccess_mvcos;
-extern struct uaccess_ops uaccess_pt;
+/**
+ * __copy_from_user: - Copy a block of data from user space, with less checking.
+ * @to: Destination address, in kernel space.
+ * @from: Source address, in user space.
+ * @n: Number of bytes to copy.
+ *
+ * Context: User context only. This function may sleep.
+ *
+ * Copy data from user space to kernel space. Caller must check
+ * the specified block with access_ok() before calling this function.
+ *
+ * Returns number of bytes that could not be copied.
+ * On success, this will be zero.
+ *
+ * If some data could not be copied, this function will pad the copied
+ * data to the requested size using zero bytes.
+ */
+size_t __must_check __copy_from_user(void *to, const void __user *from,
+ size_t n);
+
+/**
+ * __copy_to_user: - Copy a block of data into user space, with less checking.
+ * @to: Destination address, in user space.
+ * @from: Source address, in kernel space.
+ * @n: Number of bytes to copy.
+ *
+ * Context: User context only. This function may sleep.
+ *
+ * Copy data from kernel space to user space. Caller must check
+ * the specified block with access_ok() before calling this function.
+ *
+ * Returns number of bytes that could not be copied.
+ * On success, this will be zero.
+ */
+unsigned long __must_check __copy_to_user(void __user *to, const void *from,
+ unsigned long n);
-extern int __handle_fault(unsigned long, unsigned long, int);
+#define __copy_to_user_inatomic __copy_to_user
+#define __copy_from_user_inatomic __copy_from_user
static inline int __put_user_fn(void *x, void __user *ptr, size_t size)
{
- size = uaccess.copy_to_user(ptr, x, size);
- return size ? -EFAULT : size;
+ size = __copy_to_user(ptr, x, size);
+ return size ? -EFAULT : 0;
}
static inline int __get_user_fn(void *x, const void __user *ptr, size_t size)
{
- size = uaccess.copy_from_user(x, ptr, size);
- return size ? -EFAULT : size;
+ size = __copy_from_user(x, ptr, size);
+ return size ? -EFAULT : 0;
}
/*
})
-extern int __put_user_bad(void) __attribute__((noreturn));
+int __put_user_bad(void) __attribute__((noreturn));
#define __get_user(x, ptr) \
({ \
__get_user(x, ptr); \
})
-extern int __get_user_bad(void) __attribute__((noreturn));
+int __get_user_bad(void) __attribute__((noreturn));
#define __put_user_unaligned __put_user
#define __get_user_unaligned __get_user
-/**
- * __copy_to_user: - Copy a block of data into user space, with less checking.
- * @to: Destination address, in user space.
- * @from: Source address, in kernel space.
- * @n: Number of bytes to copy.
- *
- * Context: User context only. This function may sleep.
- *
- * Copy data from kernel space to user space. Caller must check
- * the specified block with access_ok() before calling this function.
- *
- * Returns number of bytes that could not be copied.
- * On success, this will be zero.
- */
-static inline unsigned long __must_check
-__copy_to_user(void __user *to, const void *from, unsigned long n)
-{
- return uaccess.copy_to_user(to, from, n);
-}
-
-#define __copy_to_user_inatomic __copy_to_user
-#define __copy_from_user_inatomic __copy_from_user
-
/**
* copy_to_user: - Copy a block of data into user space.
* @to: Destination address, in user space.
return __copy_to_user(to, from, n);
}
-/**
- * __copy_from_user: - Copy a block of data from user space, with less checking.
- * @to: Destination address, in kernel space.
- * @from: Source address, in user space.
- * @n: Number of bytes to copy.
- *
- * Context: User context only. This function may sleep.
- *
- * Copy data from user space to kernel space. Caller must check
- * the specified block with access_ok() before calling this function.
- *
- * Returns number of bytes that could not be copied.
- * On success, this will be zero.
- *
- * If some data could not be copied, this function will pad the copied
- * data to the requested size using zero bytes.
- */
-static inline unsigned long __must_check
-__copy_from_user(void *to, const void __user *from, unsigned long n)
-{
- return uaccess.copy_from_user(to, from, n);
-}
-
-extern void copy_from_user_overflow(void)
+void copy_from_user_overflow(void)
#ifdef CONFIG_DEBUG_STRICT_USER_COPY_CHECKS
__compiletime_warning("copy_from_user() buffer size is not provably correct")
#endif
return __copy_from_user(to, from, n);
}
-static inline unsigned long __must_check
-__copy_in_user(void __user *to, const void __user *from, unsigned long n)
-{
- return uaccess.copy_in_user(to, from, n);
-}
+unsigned long __must_check
+__copy_in_user(void __user *to, const void __user *from, unsigned long n);
static inline unsigned long __must_check
copy_in_user(void __user *to, const void __user *from, unsigned long n)
/*
* Copy a null terminated string from userspace.
*/
+
+long __strncpy_from_user(char *dst, const char __user *src, long count);
+
static inline long __must_check
strncpy_from_user(char *dst, const char __user *src, long count)
{
might_fault();
- return uaccess.strncpy_from_user(dst, src, count);
+ return __strncpy_from_user(dst, src, count);
}
-static inline unsigned long
-strnlen_user(const char __user * src, unsigned long n)
+size_t __must_check __strnlen_user(const char __user *src, size_t count);
+
+static inline size_t strnlen_user(const char __user *src, size_t n)
{
might_fault();
- return uaccess.strnlen_user(src, n);
+ return __strnlen_user(src, n);
}
/**
/*
* Zero Userspace
*/
+size_t __must_check __clear_user(void __user *to, size_t size);
-static inline unsigned long __must_check
-__clear_user(void __user *to, unsigned long n)
-{
- return uaccess.clear_user(to, n);
-}
-
-static inline unsigned long __must_check
-clear_user(void __user *to, unsigned long n)
+static inline size_t __must_check clear_user(void __user *to, size_t n)
{
might_fault();
- return uaccess.clear_user(to, n);
+ return __clear_user(to, n);
}
-extern int copy_to_user_real(void __user *dest, void *src, size_t count);
-extern int copy_from_user_real(void *dest, void __user *src, size_t count);
+int copy_to_user_real(void __user *dest, void *src, size_t count);
+int copy_from_user_real(void *dest, void __user *src, size_t count);
#endif /* __S390_UACCESS_H */
#include <linux/compat.h>
#include <asm/ipl.h>
-#include <asm/uaccess.h>
#include <asm/facility.h>
#include <asm/smp.h>
#include <asm/mmu_context.h>
#include <asm/sclp.h>
#include "entry.h"
-/*
- * User copy operations.
- */
-struct uaccess_ops uaccess;
-EXPORT_SYMBOL(uaccess);
-
/*
* Machine setup..
*/
init_mm.end_data = (unsigned long) &_edata;
init_mm.brk = (unsigned long) &_end;
- uaccess = MACHINE_HAS_MVCOS ? uaccess_mvcos : uaccess_pt;
-
parse_early_param();
detect_memory_layout(memory_chunk, memory_end);
os_info_init();
# Makefile for s390-specific library files..
#
-lib-y += delay.o string.o uaccess_pt.o find.o
+lib-y += delay.o string.o uaccess_pt.o uaccess_mvcos.o find.o
obj-$(CONFIG_32BIT) += div64.o qrnnd.o ucmpdi2.o mem32.o
obj-$(CONFIG_64BIT) += mem64.o
-lib-$(CONFIG_64BIT) += uaccess_mvcos.o
lib-$(CONFIG_SMP) += spinlock.o
#ifndef __ARCH_S390_LIB_UACCESS_H
#define __ARCH_S390_LIB_UACCESS_H
-extern int futex_atomic_op_pt(int, u32 __user *, int, int *);
-extern int futex_atomic_cmpxchg_pt(u32 *, u32 __user *, u32, u32);
+size_t copy_from_user_pt(void *to, const void __user *from, size_t n);
+size_t copy_to_user_pt(void __user *to, const void *from, size_t n);
+size_t copy_in_user_pt(void __user *to, const void __user *from, size_t n);
+size_t clear_user_pt(void __user *to, size_t n);
+size_t strnlen_user_pt(const char __user *src, size_t count);
+size_t strncpy_from_user_pt(char *dst, const char __user *src, size_t count);
#endif /* __ARCH_S390_LIB_UACCESS_H */
* Gerald Schaefer (gerald.schaefer@de.ibm.com)
*/
+#include <linux/jump_label.h>
#include <linux/errno.h>
+#include <linux/init.h>
#include <linux/mm.h>
#include <asm/uaccess.h>
#include <asm/futex.h>
#define SLR "slgr"
#endif
-static size_t copy_from_user_mvcos(void *x, const void __user *ptr, size_t size)
+static struct static_key have_mvcos = STATIC_KEY_INIT_TRUE;
+
+static inline size_t copy_from_user_mvcos(void *x, const void __user *ptr,
+ size_t size)
{
register unsigned long reg0 asm("0") = 0x81UL;
unsigned long tmp1, tmp2;
return size;
}
-static size_t copy_to_user_mvcos(void __user *ptr, const void *x, size_t size)
+size_t __copy_from_user(void *to, const void __user *from, size_t n)
+{
+ if (static_key_true(&have_mvcos))
+ return copy_from_user_mvcos(to, from, n);
+ return copy_from_user_pt(to, from, n);
+}
+EXPORT_SYMBOL(__copy_from_user);
+
+static inline size_t copy_to_user_mvcos(void __user *ptr, const void *x,
+ size_t size)
{
register unsigned long reg0 asm("0") = 0x810000UL;
unsigned long tmp1, tmp2;
return size;
}
-static size_t copy_in_user_mvcos(void __user *to, const void __user *from,
- size_t size)
+size_t __copy_to_user(void __user *to, const void *from, size_t n)
+{
+ if (static_key_true(&have_mvcos))
+ return copy_to_user_mvcos(to, from, n);
+ return copy_to_user_pt(to, from, n);
+}
+EXPORT_SYMBOL(__copy_to_user);
+
+static inline size_t copy_in_user_mvcos(void __user *to, const void __user *from,
+ size_t size)
{
register unsigned long reg0 asm("0") = 0x810081UL;
unsigned long tmp1, tmp2;
return size;
}
-static size_t clear_user_mvcos(void __user *to, size_t size)
+size_t __copy_in_user(void __user *to, const void __user *from, size_t n)
+{
+ if (static_key_true(&have_mvcos))
+ return copy_in_user_mvcos(to, from, n);
+ return copy_in_user_pt(to, from, n);
+}
+EXPORT_SYMBOL(__copy_in_user);
+
+static inline size_t clear_user_mvcos(void __user *to, size_t size)
{
register unsigned long reg0 asm("0") = 0x810000UL;
unsigned long tmp1, tmp2;
return size;
}
-static size_t strnlen_user_mvcos(const char __user *src, size_t count)
+size_t __clear_user(void __user *to, size_t size)
+{
+ if (static_key_true(&have_mvcos))
+ return clear_user_mvcos(to, size);
+ return clear_user_pt(to, size);
+}
+EXPORT_SYMBOL(__clear_user);
+
+static inline size_t strnlen_user_mvcos(const char __user *src, size_t count)
{
size_t done, len, offset, len_str;
char buf[256];
return done + 1;
}
-static size_t strncpy_from_user_mvcos(char *dst, const char __user *src,
- size_t count)
+size_t __strnlen_user(const char __user *src, size_t count)
{
- size_t done, len, offset, len_str;
+ if (static_key_true(&have_mvcos))
+ return strnlen_user_mvcos(src, count);
+ return strnlen_user_pt(src, count);
+}
+EXPORT_SYMBOL(__strnlen_user);
+
+static inline size_t strncpy_from_user_mvcos(char *dst, const char __user *src,
+ size_t count)
+{
+ unsigned long done, len, offset, len_str;
if (unlikely(!count))
return 0;
return done;
}
-struct uaccess_ops uaccess_mvcos = {
- .copy_from_user = copy_from_user_mvcos,
- .copy_to_user = copy_to_user_mvcos,
- .copy_in_user = copy_in_user_mvcos,
- .clear_user = clear_user_mvcos,
- .strnlen_user = strnlen_user_mvcos,
- .strncpy_from_user = strncpy_from_user_mvcos,
- .futex_atomic_op = futex_atomic_op_pt,
- .futex_atomic_cmpxchg = futex_atomic_cmpxchg_pt,
-};
+long __strncpy_from_user(char *dst, const char __user *src, long count)
+{
+ if (static_key_true(&have_mvcos))
+ return strncpy_from_user_mvcos(dst, src, count);
+ return strncpy_from_user_pt(dst, src, count);
+}
+EXPORT_SYMBOL(__strncpy_from_user);
+
+static int __init uaccess_init(void)
+{
+ if (!MACHINE_HAS_MVCOS)
+ static_key_slow_dec(&have_mvcos);
+ return 0;
+}
+early_initcall(uaccess_init);
return 0;
}
-static size_t copy_from_user_pt(void *to, const void __user *from, size_t n)
+size_t copy_from_user_pt(void *to, const void __user *from, size_t n)
{
size_t rc;
return rc;
}
-static size_t copy_to_user_pt(void __user *to, const void *from, size_t n)
+size_t copy_to_user_pt(void __user *to, const void *from, size_t n)
{
if (segment_eq(get_fs(), KERNEL_DS))
return copy_in_kernel(to, (void __user *) from, n);
return __user_copy_pt((unsigned long) to, (void *) from, n, 1);
}
-static size_t clear_user_pt(void __user *to, size_t n)
+size_t clear_user_pt(void __user *to, size_t n)
{
void *zpage = (void *) empty_zero_page;
long done, size, ret;
return 0;
}
-static size_t strnlen_user_pt(const char __user *src, size_t count)
+size_t strnlen_user_pt(const char __user *src, size_t count)
{
unsigned long uaddr = (unsigned long) src;
struct mm_struct *mm = current->mm;
goto retry;
}
-static size_t strncpy_from_user_pt(char *dst, const char __user *src,
- size_t count)
+size_t strncpy_from_user_pt(char *dst, const char __user *src, size_t count)
{
size_t done, len, offset, len_str;
return done;
}
-static size_t copy_in_user_pt(void __user *to, const void __user *from,
- size_t n)
+size_t copy_in_user_pt(void __user *to, const void __user *from, size_t n)
{
struct mm_struct *mm = current->mm;
unsigned long offset_max, uaddr, done, size, error_code;
return ret;
}
-int futex_atomic_op_pt(int op, u32 __user *uaddr, int oparg, int *old)
+int __futex_atomic_op_inuser(int op, u32 __user *uaddr, int oparg, int *old)
{
int ret;
return ret;
}
-int futex_atomic_cmpxchg_pt(u32 *uval, u32 __user *uaddr,
- u32 oldval, u32 newval)
+int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr,
+ u32 oldval, u32 newval)
{
int ret;
put_page(virt_to_page(uaddr));
return ret;
}
-
-struct uaccess_ops uaccess_pt = {
- .copy_from_user = copy_from_user_pt,
- .copy_to_user = copy_to_user_pt,
- .copy_in_user = copy_in_user_pt,
- .clear_user = clear_user_pt,
- .strnlen_user = strnlen_user_pt,
- .strncpy_from_user = strncpy_from_user_pt,
- .futex_atomic_op = futex_atomic_op_pt,
- .futex_atomic_cmpxchg = futex_atomic_cmpxchg_pt,
-};
projects / linux-beck.git / commitdiff