include/asm-x86/uaccess_64.h

   1 #ifndef __X86_64_UACCESS_H
   2 #define __X86_64_UACCESS_H
   3
   4 /*
   5  * User space memory access functions
   6  */
   7 #include <linux/compiler.h>
   8 #include <linux/errno.h>
   9 #include <linux/prefetch.h>
  10 #include <asm/page.h>
  11
  12 #define ARCH_HAS_SEARCH_EXTABLE
  13
  14 extern void __put_user_1(void);
  15 extern void __put_user_2(void);
  16 extern void __put_user_4(void);
  17 extern void __put_user_8(void);
  18 extern void __put_user_bad(void);
  19
  20 #define __put_user_x(size, ret, x, ptr)                                 \
  21         asm volatile("call __put_user_" #size                           \
  22                      :"=a" (ret)                                        \
  23                      :"c" (ptr),"a" (x)                                 \
  24                      :"ebx")
  25
  26 #define put_user(x, ptr)                                                \
  27         __put_user_check((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))
  28
  29 #define __get_user(x, ptr)                                              \
  30         __get_user_nocheck((x), (ptr), sizeof(*(ptr)))
  31 #define __put_user(x, ptr)                                              \
  32         __put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))
  33
  34 #define __get_user_unaligned __get_user
  35 #define __put_user_unaligned __put_user
  36
  37 #define __put_user_check(x, ptr, size)                          \
  38 ({                                                              \
  39         int __pu_err;                                           \
  40         typeof(*(ptr)) __user *__pu_addr = (ptr);               \
  41         switch (size) {                                         \
  42         case 1:                                                 \
  43                 __put_user_x(1, __pu_err, x, __pu_addr);        \
  44                 break;                                          \
  45         case 2:                                                 \
  46                 __put_user_x(2, __pu_err, x, __pu_addr);        \
  47                 break;                                          \
  48         case 4:                                                 \
  49                 __put_user_x(4, __pu_err, x, __pu_addr);        \
  50                 break;                                          \
  51         case 8:                                                 \
  52                 __put_user_x(8, __pu_err, x, __pu_addr);        \
  53                 break;                                          \
  54         default:                                                \
  55                 __put_user_bad();                               \
  56         }                                                       \
  57         __pu_err;                                               \
  58 })
  59
  60 /*
  61  * Copy To/From Userspace
  62  */
  63
  64 /* Handles exceptions in both to and from, but doesn't do access_ok */
  65 __must_check unsigned long
  66 copy_user_generic(void *to, const void *from, unsigned len);
  67
  68 __must_check unsigned long
  69 copy_to_user(void __user *to, const void *from, unsigned len);
  70 __must_check unsigned long
  71 copy_from_user(void *to, const void __user *from, unsigned len);
  72 __must_check unsigned long
  73 copy_in_user(void __user *to, const void __user *from, unsigned len);
  74
  75 static __always_inline __must_check
  76 int __copy_from_user(void *dst, const void __user *src, unsigned size)
  77 {
  78         int ret = 0;
  79         if (!__builtin_constant_p(size))
  80                 return copy_user_generic(dst, (__force void *)src, size);
  81         switch (size) {
  82         case 1:__get_user_asm(*(u8 *)dst, (u8 __user *)src,
  83                               ret, "b", "b", "=q", 1);
  84                 return ret;
  85         case 2:__get_user_asm(*(u16 *)dst, (u16 __user *)src,
  86                               ret, "w", "w", "=r", 2);
  87                 return ret;
  88         case 4:__get_user_asm(*(u32 *)dst, (u32 __user *)src,
  89                               ret, "l", "k", "=r", 4);
  90                 return ret;
  91         case 8:__get_user_asm(*(u64 *)dst, (u64 __user *)src,
  92                               ret, "q", "", "=r", 8);
  93                 return ret;
  94         case 10:
  95                 __get_user_asm(*(u64 *)dst, (u64 __user *)src,
  96                                ret, "q", "", "=r", 16);
  97                 if (unlikely(ret))
  98                         return ret;
  99                 __get_user_asm(*(u16 *)(8 + (char *)dst),
 100                                (u16 __user *)(8 + (char __user *)src),
 101                                ret, "w", "w", "=r", 2);
 102                 return ret;
 103         case 16:
 104                 __get_user_asm(*(u64 *)dst, (u64 __user *)src,
 105                                ret, "q", "", "=r", 16);
 106                 if (unlikely(ret))
 107                         return ret;
 108                 __get_user_asm(*(u64 *)(8 + (char *)dst),
 109                                (u64 __user *)(8 + (char __user *)src),
 110                                ret, "q", "", "=r", 8);
 111                 return ret;
 112         default:
 113                 return copy_user_generic(dst, (__force void *)src, size);
 114         }
 115 }
 116
 117 static __always_inline __must_check
 118 int __copy_to_user(void __user *dst, const void *src, unsigned size)
 119 {
 120         int ret = 0;
 121         if (!__builtin_constant_p(size))
 122                 return copy_user_generic((__force void *)dst, src, size);
 123         switch (size) {
 124         case 1:__put_user_asm(*(u8 *)src, (u8 __user *)dst,
 125                               ret, "b", "b", "iq", 1);
 126                 return ret;
 127         case 2:__put_user_asm(*(u16 *)src, (u16 __user *)dst,
 128                               ret, "w", "w", "ir", 2);
 129                 return ret;
 130         case 4:__put_user_asm(*(u32 *)src, (u32 __user *)dst,
 131                               ret, "l", "k", "ir", 4);
 132                 return ret;
 133         case 8:__put_user_asm(*(u64 *)src, (u64 __user *)dst,
 134                               ret, "q", "", "ir", 8);
 135                 return ret;
 136         case 10:
 137                 __put_user_asm(*(u64 *)src, (u64 __user *)dst,
 138                                ret, "q", "", "ir", 10);
 139                 if (unlikely(ret))
 140                         return ret;
 141                 asm("":::"memory");
 142                 __put_user_asm(4[(u16 *)src], 4 + (u16 __user *)dst,
 143                                ret, "w", "w", "ir", 2);
 144                 return ret;
 145         case 16:
 146                 __put_user_asm(*(u64 *)src, (u64 __user *)dst,
 147                                ret, "q", "", "ir", 16);
 148                 if (unlikely(ret))
 149                         return ret;
 150                 asm("":::"memory");
 151                 __put_user_asm(1[(u64 *)src], 1 + (u64 __user *)dst,
 152                                ret, "q", "", "ir", 8);
 153                 return ret;
 154         default:
 155                 return copy_user_generic((__force void *)dst, src, size);
 156         }
 157 }
 158
 159 static __always_inline __must_check
 160 int __copy_in_user(void __user *dst, const void __user *src, unsigned size)
 161 {
 162         int ret = 0;
 163         if (!__builtin_constant_p(size))
 164                 return copy_user_generic((__force void *)dst,
 165                                          (__force void *)src, size);
 166         switch (size) {
 167         case 1: {
 168                 u8 tmp;
 169                 __get_user_asm(tmp, (u8 __user *)src,
 170                                ret, "b", "b", "=q", 1);
 171                 if (likely(!ret))
 172                         __put_user_asm(tmp, (u8 __user *)dst,
 173                                        ret, "b", "b", "iq", 1);
 174                 return ret;
 175         }
 176         case 2: {
 177                 u16 tmp;
 178                 __get_user_asm(tmp, (u16 __user *)src,
 179                                ret, "w", "w", "=r", 2);
 180                 if (likely(!ret))
 181                         __put_user_asm(tmp, (u16 __user *)dst,
 182                                        ret, "w", "w", "ir", 2);
 183                 return ret;
 184         }
 185
 186         case 4: {
 187                 u32 tmp;
 188                 __get_user_asm(tmp, (u32 __user *)src,
 189                                ret, "l", "k", "=r", 4);
 190                 if (likely(!ret))
 191                         __put_user_asm(tmp, (u32 __user *)dst,
 192                                        ret, "l", "k", "ir", 4);
 193                 return ret;
 194         }
 195         case 8: {
 196                 u64 tmp;
 197                 __get_user_asm(tmp, (u64 __user *)src,
 198                                ret, "q", "", "=r", 8);
 199                 if (likely(!ret))
 200                         __put_user_asm(tmp, (u64 __user *)dst,
 201                                        ret, "q", "", "ir", 8);
 202                 return ret;
 203         }
 204         default:
 205                 return copy_user_generic((__force void *)dst,
 206                                          (__force void *)src, size);
 207         }
 208 }
 209
 210 __must_check long
 211 strncpy_from_user(char *dst, const char __user *src, long count);
 212 __must_check long
 213 __strncpy_from_user(char *dst, const char __user *src, long count);
 214 __must_check long strnlen_user(const char __user *str, long n);
 215 __must_check long __strnlen_user(const char __user *str, long n);
 216 __must_check long strlen_user(const char __user *str);
 217 __must_check unsigned long clear_user(void __user *mem, unsigned long len);
 218 __must_check unsigned long __clear_user(void __user *mem, unsigned long len);
 219
 220 __must_check long __copy_from_user_inatomic(void *dst, const void __user *src,
 221                                             unsigned size);
 222
 223 static __must_check __always_inline int
 224 __copy_to_user_inatomic(void __user *dst, const void *src, unsigned size)
 225 {
 226         return copy_user_generic((__force void *)dst, src, size);
 227 }
 228
 229 #define ARCH_HAS_NOCACHE_UACCESS 1
 230 extern long __copy_user_nocache(void *dst, const void __user *src,
 231                                 unsigned size, int zerorest);
 232
 233 static inline int __copy_from_user_nocache(void *dst, const void __user *src,
 234                                            unsigned size)
 235 {
 236         might_sleep();
 237         return __copy_user_nocache(dst, src, size, 1);
 238 }
 239
 240 static inline int __copy_from_user_inatomic_nocache(void *dst,
 241                                                     const void __user *src,
 242                                                     unsigned size)
 243 {
 244         return __copy_user_nocache(dst, src, size, 0);
 245 }
 246
 247 #endif /* __X86_64_UACCESS_H */