arch/x86/kernel/fpu/xstate.c

   1 /*
   2  * xsave/xrstor support.
   3  *
   4  * Author: Suresh Siddha <suresh.b.siddha@intel.com>
   5  */
   6 #include <linux/compat.h>
   7 #include <linux/cpu.h>
   8
   9 #include <asm/fpu/api.h>
  10 #include <asm/fpu/internal.h>
  11 #include <asm/fpu/signal.h>
  12 #include <asm/fpu/regset.h>
  13
  14 #include <asm/tlbflush.h>
  15
  16 static const char *xfeature_names[] =
  17 {
  18         "x87 floating point registers"  ,
  19         "SSE registers"                 ,
  20         "AVX registers"                 ,
  21         "MPX bounds registers"          ,
  22         "MPX CSR"                       ,
  23         "AVX-512 opmask"                ,
  24         "AVX-512 Hi256"                 ,
  25         "AVX-512 ZMM_Hi256"             ,
  26         "unknown xstate feature"        ,
  27 };
  28
  29 /*
  30  * Mask of xstate features supported by the CPU and the kernel:
  31  */
  32 u64 xfeatures_mask __read_mostly;
  33
  34 static unsigned int xstate_offsets[XFEATURES_NR_MAX] = { [ 0 ... XFEATURES_NR_MAX - 1] = -1};
  35 static unsigned int xstate_sizes[XFEATURES_NR_MAX]   = { [ 0 ... XFEATURES_NR_MAX - 1] = -1};
  36 static unsigned int xstate_comp_offsets[sizeof(xfeatures_mask)*8];
  37
  38 /* The number of supported xfeatures in xfeatures_mask: */
  39 static unsigned int xfeatures_nr;
  40
  41 /*
  42  * Clear all of the X86_FEATURE_* bits that are unavailable
  43  * when the CPU has no XSAVE support.
  44  */
  45 void fpu__xstate_clear_all_cpu_caps(void)
  46 {
  47         setup_clear_cpu_cap(X86_FEATURE_XSAVE);
  48         setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT);
  49         setup_clear_cpu_cap(X86_FEATURE_XSAVEC);
  50         setup_clear_cpu_cap(X86_FEATURE_XSAVES);
  51         setup_clear_cpu_cap(X86_FEATURE_AVX);
  52         setup_clear_cpu_cap(X86_FEATURE_AVX2);
  53         setup_clear_cpu_cap(X86_FEATURE_AVX512F);
  54         setup_clear_cpu_cap(X86_FEATURE_AVX512PF);
  55         setup_clear_cpu_cap(X86_FEATURE_AVX512ER);
  56         setup_clear_cpu_cap(X86_FEATURE_AVX512CD);
  57         setup_clear_cpu_cap(X86_FEATURE_MPX);
  58 }
  59
  60 /*
  61  * Return whether the system supports a given xfeature.
  62  *
  63  * Also return the name of the (most advanced) feature that the caller requested:
  64  */
  65 int cpu_has_xfeatures(u64 xfeatures_needed, const char **feature_name)
  66 {
  67         u64 xfeatures_missing = xfeatures_needed & ~xfeatures_mask;
  68
  69         if (unlikely(feature_name)) {
  70                 long xfeature_idx, max_idx;
  71                 u64 xfeatures_print;
  72                 /*
  73                  * So we use FLS here to be able to print the most advanced
  74                  * feature that was requested but is missing. So if a driver
  75                  * asks about "XFEATURE_MASK_SSE | XFEATURE_MASK_YMM" we'll print the
  76                  * missing AVX feature - this is the most informative message
  77                  * to users:
  78                  */
  79                 if (xfeatures_missing)
  80                         xfeatures_print = xfeatures_missing;
  81                 else
  82                         xfeatures_print = xfeatures_needed;
  83
  84                 xfeature_idx = fls64(xfeatures_print)-1;
  85                 max_idx = ARRAY_SIZE(xfeature_names)-1;
  86                 xfeature_idx = min(xfeature_idx, max_idx);
  87
  88                 *feature_name = xfeature_names[xfeature_idx];
  89         }
  90
  91         if (xfeatures_missing)
  92                 return 0;
  93
  94         return 1;
  95 }
  96 EXPORT_SYMBOL_GPL(cpu_has_xfeatures);
  97
  98 /*
  99  * When executing XSAVEOPT (or other optimized XSAVE instructions), if
 100  * a processor implementation detects that an FPU state component is still
 101  * (or is again) in its initialized state, it may clear the corresponding
 102  * bit in the header.xfeatures field, and can skip the writeout of registers
 103  * to the corresponding memory layout.
 104  *
 105  * This means that when the bit is zero, the state component might still contain
 106  * some previous - non-initialized register state.
 107  *
 108  * Before writing xstate information to user-space we sanitize those components,
 109  * to always ensure that the memory layout of a feature will be in the init state
 110  * if the corresponding header bit is zero. This is to ensure that user-space doesn't
 111  * see some stale state in the memory layout during signal handling, debugging etc.
 112  */
 113 void fpstate_sanitize_xstate(struct fpu *fpu)
 114 {
 115         struct fxregs_state *fx = &fpu->state.fxsave;
 116         int feature_bit;
 117         u64 xfeatures;
 118
 119         if (!use_xsaveopt())
 120                 return;
 121
 122         xfeatures = fpu->state.xsave.header.xfeatures;
 123
 124         /*
 125          * None of the feature bits are in init state. So nothing else
 126          * to do for us, as the memory layout is up to date.
 127          */
 128         if ((xfeatures & xfeatures_mask) == xfeatures_mask)
 129                 return;
 130
 131         /*
 132          * FP is in init state
 133          */
 134         if (!(xfeatures & XFEATURE_MASK_FP)) {
 135                 fx->cwd = 0x37f;
 136                 fx->swd = 0;
 137                 fx->twd = 0;
 138                 fx->fop = 0;
 139                 fx->rip = 0;
 140                 fx->rdp = 0;
 141                 memset(&fx->st_space[0], 0, 128);
 142         }
 143
 144         /*
 145          * SSE is in init state
 146          */
 147         if (!(xfeatures & XFEATURE_MASK_SSE))
 148                 memset(&fx->xmm_space[0], 0, 256);
 149
 150         /*
 151          * First two features are FPU and SSE, which above we handled
 152          * in a special way already:
 153          */
 154         feature_bit = 0x2;
 155         xfeatures = (xfeatures_mask & ~xfeatures) >> 2;
 156
 157         /*
 158          * Update all the remaining memory layouts according to their
 159          * standard xstate layout, if their header bit is in the init
 160          * state:
 161          */
 162         while (xfeatures) {
 163                 if (xfeatures & 0x1) {
 164                         int offset = xstate_offsets[feature_bit];
 165                         int size = xstate_sizes[feature_bit];
 166
 167                         memcpy((void *)fx + offset,
 168                                (void *)&init_fpstate.xsave + offset,
 169                                size);
 170                 }
 171
 172                 xfeatures >>= 1;
 173                 feature_bit++;
 174         }
 175 }
 176
 177 /*
 178  * Enable the extended processor state save/restore feature.
 179  * Called once per CPU onlining.
 180  */
 181 void fpu__init_cpu_xstate(void)
 182 {
 183         if (!cpu_has_xsave || !xfeatures_mask)
 184                 return;
 185
 186         cr4_set_bits(X86_CR4_OSXSAVE);
 187         xsetbv(XCR_XFEATURE_ENABLED_MASK, xfeatures_mask);
 188 }
 189
 190 /*
 191  * Record the offsets and sizes of various xstates contained
 192  * in the XSAVE state memory layout.
 193  *
 194  * ( Note that certain features might be non-present, for them
 195  *   we'll have 0 offset and 0 size. )
 196  */
 197 static void __init setup_xstate_features(void)
 198 {
 199         u32 eax, ebx, ecx, edx, leaf;
 200
 201         xfeatures_nr = fls64(xfeatures_mask);
 202
 203         for (leaf = 2; leaf < xfeatures_nr; leaf++) {
 204                 cpuid_count(XSTATE_CPUID, leaf, &eax, &ebx, &ecx, &edx);
 205
 206                 xstate_offsets[leaf] = ebx;
 207                 xstate_sizes[leaf] = eax;
 208
 209                 printk(KERN_INFO "x86/fpu: xstate_offset[%d]: %4d, xstate_sizes[%d]: %4d\n", leaf, ebx, leaf, eax);
 210         }
 211 }
 212
 213 static void __init print_xstate_feature(u64 xstate_mask)
 214 {
 215         const char *feature_name;
 216
 217         if (cpu_has_xfeatures(xstate_mask, &feature_name))
 218                 pr_info("x86/fpu: Supporting XSAVE feature 0x%02Lx: '%s'\n", xstate_mask, feature_name);
 219 }
 220
 221 /*
 222  * Print out all the supported xstate features:
 223  */
 224 static void __init print_xstate_features(void)
 225 {
 226         print_xstate_feature(XFEATURE_MASK_FP);
 227         print_xstate_feature(XFEATURE_MASK_SSE);
 228         print_xstate_feature(XFEATURE_MASK_YMM);
 229         print_xstate_feature(XFEATURE_MASK_BNDREGS);
 230         print_xstate_feature(XFEATURE_MASK_BNDCSR);
 231         print_xstate_feature(XFEATURE_MASK_OPMASK);
 232         print_xstate_feature(XFEATURE_MASK_ZMM_Hi256);
 233         print_xstate_feature(XFEATURE_MASK_Hi16_ZMM);
 234 }
 235
 236 /*
 237  * This function sets up offsets and sizes of all extended states in
 238  * xsave area. This supports both standard format and compacted format
 239  * of the xsave aread.
 240  */
 241 static void __init setup_xstate_comp(void)
 242 {
 243         unsigned int xstate_comp_sizes[sizeof(xfeatures_mask)*8];
 244         int i;
 245
 246         /*
 247          * The FP xstates and SSE xstates are legacy states. They are always
 248          * in the fixed offsets in the xsave area in either compacted form
 249          * or standard form.
 250          */
 251         xstate_comp_offsets[0] = 0;
 252         xstate_comp_offsets[1] = offsetof(struct fxregs_state, xmm_space);
 253
 254         if (!cpu_has_xsaves) {
 255                 for (i = 2; i < xfeatures_nr; i++) {
 256                         if (test_bit(i, (unsigned long *)&xfeatures_mask)) {
 257                                 xstate_comp_offsets[i] = xstate_offsets[i];
 258                                 xstate_comp_sizes[i] = xstate_sizes[i];
 259                         }
 260                 }
 261                 return;
 262         }
 263
 264         xstate_comp_offsets[2] = FXSAVE_SIZE + XSAVE_HDR_SIZE;
 265
 266         for (i = 2; i < xfeatures_nr; i++) {
 267                 if (test_bit(i, (unsigned long *)&xfeatures_mask))
 268                         xstate_comp_sizes[i] = xstate_sizes[i];
 269                 else
 270                         xstate_comp_sizes[i] = 0;
 271
 272                 if (i > 2)
 273                         xstate_comp_offsets[i] = xstate_comp_offsets[i-1]
 274                                         + xstate_comp_sizes[i-1];
 275
 276         }
 277 }
 278
 279 /*
 280  * setup the xstate image representing the init state
 281  */
 282 static void __init setup_init_fpu_buf(void)
 283 {
 284         static int on_boot_cpu = 1;
 285
 286         WARN_ON_FPU(!on_boot_cpu);
 287         on_boot_cpu = 0;
 288
 289         if (!cpu_has_xsave)
 290                 return;
 291
 292         setup_xstate_features();
 293         print_xstate_features();
 294
 295         if (cpu_has_xsaves) {
 296                 init_fpstate.xsave.header.xcomp_bv = (u64)1 << 63 | xfeatures_mask;
 297                 init_fpstate.xsave.header.xfeatures = xfeatures_mask;
 298         }
 299
 300         /*
 301          * Init all the features state with header_bv being 0x0
 302          */
 303         copy_kernel_to_xregs_booting(&init_fpstate.xsave);
 304
 305         /*
 306          * Dump the init state again. This is to identify the init state
 307          * of any feature which is not represented by all zero's.
 308          */
 309         copy_xregs_to_kernel_booting(&init_fpstate.xsave);
 310 }
 311
 312 /*
 313  * Calculate total size of enabled xstates in XCR0/xfeatures_mask.
 314  */
 315 static unsigned int __init calculate_xstate_size(void)
 316 {
 317         unsigned int eax, ebx, ecx, edx;
 318         unsigned int calculated_xstate_size;
 319         int i;
 320
 321         if (!cpu_has_xsaves) {
 322                 cpuid_count(XSTATE_CPUID, 0, &eax, &ebx, &ecx, &edx);
 323                 calculated_xstate_size = ebx;
 324                 return calculated_xstate_size;
 325         }
 326
 327         calculated_xstate_size = FXSAVE_SIZE + XSAVE_HDR_SIZE;
 328         for (i = 2; i < 64; i++) {
 329                 if (test_bit(i, (unsigned long *)&xfeatures_mask)) {
 330                         cpuid_count(XSTATE_CPUID, i, &eax, &ebx, &ecx, &edx);
 331                         calculated_xstate_size += eax;
 332                 }
 333         }
 334         return calculated_xstate_size;
 335 }
 336
 337 /*
 338  * Will the runtime-enumerated 'xstate_size' fit in the init
 339  * task's statically-allocated buffer?
 340  */
 341 static bool is_supported_xstate_size(unsigned int test_xstate_size)
 342 {
 343         if (test_xstate_size <= sizeof(union fpregs_state))
 344                 return true;
 345
 346         pr_warn("x86/fpu: xstate buffer too small (%zu < %d), disabling xsave\n",
 347                         sizeof(union fpregs_state), test_xstate_size);
 348         return false;
 349 }
 350
 351 static int init_xstate_size(void)
 352 {
 353         /* Recompute the context size for enabled features: */
 354         unsigned int possible_xstate_size = calculate_xstate_size();
 355
 356         /* Ensure we have the space to store all enabled: */
 357         if (!is_supported_xstate_size(possible_xstate_size))
 358                 return -EINVAL;
 359
 360         /*
 361          * The size is OK, we are definitely going to use xsave,
 362          * make it known to the world that we need more space.
 363          */
 364         xstate_size = possible_xstate_size;
 365         return 0;
 366 }
 367
 368 /*
 369  * We enabled the XSAVE hardware, but something went wrong and
 370  * we can not use it.  Disable it.
 371  */
 372 static void fpu__init_disable_system_xstate(void)
 373 {
 374         xfeatures_mask = 0;
 375         cr4_clear_bits(X86_CR4_OSXSAVE);
 376         fpu__xstate_clear_all_cpu_caps();
 377 }
 378
 379 /*
 380  * Enable and initialize the xsave feature.
 381  * Called once per system bootup.
 382  */
 383 void __init fpu__init_system_xstate(void)
 384 {
 385         unsigned int eax, ebx, ecx, edx;
 386         static int on_boot_cpu = 1;
 387         int err;
 388
 389         WARN_ON_FPU(!on_boot_cpu);
 390         on_boot_cpu = 0;
 391
 392         if (!cpu_has_xsave) {
 393                 pr_info("x86/fpu: Legacy x87 FPU detected.\n");
 394                 return;
 395         }
 396
 397         if (boot_cpu_data.cpuid_level < XSTATE_CPUID) {
 398                 WARN_ON_FPU(1);
 399                 return;
 400         }
 401
 402         cpuid_count(XSTATE_CPUID, 0, &eax, &ebx, &ecx, &edx);
 403         xfeatures_mask = eax + ((u64)edx << 32);
 404
 405         if ((xfeatures_mask & XFEATURE_MASK_FPSSE) != XFEATURE_MASK_FPSSE) {
 406                 pr_err("x86/fpu: FP/SSE not present amongst the CPU's xstate features: 0x%llx.\n", xfeatures_mask);
 407                 BUG();
 408         }
 409
 410         /* Support only the state known to the OS: */
 411         xfeatures_mask = xfeatures_mask & XCNTXT_MASK;
 412
 413         /* Enable xstate instructions to be able to continue with initialization: */
 414         fpu__init_cpu_xstate();
 415         err = init_xstate_size();
 416         if (err) {
 417                 /* something went wrong, boot without any XSAVE support */
 418                 fpu__init_disable_system_xstate();
 419                 return;
 420         }
 421
 422         update_regset_xstate_info(xstate_size, xfeatures_mask);
 423         fpu__init_prepare_fx_sw_frame();
 424         setup_init_fpu_buf();
 425         setup_xstate_comp();
 426
 427         pr_info("x86/fpu: Enabled xstate features 0x%llx, context size is %d bytes, using '%s' format.\n",
 428                 xfeatures_mask,
 429                 xstate_size,
 430                 cpu_has_xsaves ? "compacted" : "standard");
 431 }
 432
 433 /*
 434  * Restore minimal FPU state after suspend:
 435  */
 436 void fpu__resume_cpu(void)
 437 {
 438         /*
 439          * Restore XCR0 on xsave capable CPUs:
 440          */
 441         if (cpu_has_xsave)
 442                 xsetbv(XCR_XFEATURE_ENABLED_MASK, xfeatures_mask);
 443 }
 444
 445 /*
 446  * Given the xsave area and a state inside, this function returns the
 447  * address of the state.
 448  *
 449  * This is the API that is called to get xstate address in either
 450  * standard format or compacted format of xsave area.
 451  *
 452  * Note that if there is no data for the field in the xsave buffer
 453  * this will return NULL.
 454  *
 455  * Inputs:
 456  *      xstate: the thread's storage area for all FPU data
 457  *      xstate_feature: state which is defined in xsave.h (e.g.
 458  *      XFEATURE_MASK_FP, XFEATURE_MASK_SSE, etc...)
 459  * Output:
 460  *      address of the state in the xsave area, or NULL if the
 461  *      field is not present in the xsave buffer.
 462  */
 463 void *get_xsave_addr(struct xregs_state *xsave, int xstate_feature)
 464 {
 465         int feature_nr = fls64(xstate_feature) - 1;
 466         /*
 467          * Do we even *have* xsave state?
 468          */
 469         if (!boot_cpu_has(X86_FEATURE_XSAVE))
 470                 return NULL;
 471
 472         xsave = &current->thread.fpu.state.xsave;
 473         /*
 474          * We should not ever be requesting features that we
 475          * have not enabled.  Remember that pcntxt_mask is
 476          * what we write to the XCR0 register.
 477          */
 478         WARN_ONCE(!(xfeatures_mask & xstate_feature),
 479                   "get of unsupported state");
 480         /*
 481          * This assumes the last 'xsave*' instruction to
 482          * have requested that 'xstate_feature' be saved.
 483          * If it did not, we might be seeing and old value
 484          * of the field in the buffer.
 485          *
 486          * This can happen because the last 'xsave' did not
 487          * request that this feature be saved (unlikely)
 488          * or because the "init optimization" caused it
 489          * to not be saved.
 490          */
 491         if (!(xsave->header.xfeatures & xstate_feature))
 492                 return NULL;
 493
 494         return (void *)xsave + xstate_comp_offsets[feature_nr];
 495 }
 496 EXPORT_SYMBOL_GPL(get_xsave_addr);
 497
 498 /*
 499  * This wraps up the common operations that need to occur when retrieving
 500  * data from xsave state.  It first ensures that the current task was
 501  * using the FPU and retrieves the data in to a buffer.  It then calculates
 502  * the offset of the requested field in the buffer.
 503  *
 504  * This function is safe to call whether the FPU is in use or not.
 505  *
 506  * Note that this only works on the current task.
 507  *
 508  * Inputs:
 509  *      @xsave_state: state which is defined in xsave.h (e.g. XFEATURE_MASK_FP,
 510  *      XFEATURE_MASK_SSE, etc...)
 511  * Output:
 512  *      address of the state in the xsave area or NULL if the state
 513  *      is not present or is in its 'init state'.
 514  */
 515 const void *get_xsave_field_ptr(int xsave_state)
 516 {
 517         struct fpu *fpu = &current->thread.fpu;
 518
 519         if (!fpu->fpstate_active)
 520                 return NULL;
 521         /*
 522          * fpu__save() takes the CPU's xstate registers
 523          * and saves them off to the 'fpu memory buffer.
 524          */
 525         fpu__save(fpu);
 526
 527         return get_xsave_addr(&fpu->state.xsave, xsave_state);
 528 }