2 * x86 FPU boot time init code:
4 #include <asm/fpu/internal.h>
5 #include <asm/tlbflush.h>
7 #include <asm/cmdline.h>
9 #include <linux/sched.h>
10 #include <linux/init.h>
13 * Initialize the TS bit in CR0 according to the style of context-switches
16 static void fpu__init_cpu_ctx_switch(void)
18 if (!boot_cpu_has(X86_FEATURE_EAGER_FPU))
25 * Initialize the registers found in all CPUs, CR0 and CR4:
27 static void fpu__init_cpu_generic(void)
30 unsigned long cr4_mask = 0;
33 cr4_mask |= X86_CR4_OSFXSR;
35 cr4_mask |= X86_CR4_OSXMMEXCPT;
37 cr4_set_bits(cr4_mask);
40 cr0 &= ~(X86_CR0_TS|X86_CR0_EM); /* clear TS and EM */
45 /* Flush out any pending x87 state: */
46 #ifdef CONFIG_MATH_EMULATION
48 fpstate_init_soft(¤t->thread.fpu.state.soft);
51 asm volatile ("fninit");
55 * Enable all supported FPU features. Called when a CPU is brought online:
57 void fpu__init_cpu(void)
59 fpu__init_cpu_generic();
60 fpu__init_cpu_xstate();
61 fpu__init_cpu_ctx_switch();
65 * The earliest FPU detection code.
67 * Set the X86_FEATURE_FPU CPU-capability bit based on
68 * trying to execute an actual sequence of FPU instructions:
70 static void fpu__init_system_early_generic(struct cpuinfo_x86 *c)
78 cr0 &= ~(X86_CR0_TS | X86_CR0_EM);
81 asm volatile("fninit ; fnstsw %0 ; fnstcw %1"
82 : "+m" (fsw), "+m" (fcw));
84 if (fsw == 0 && (fcw & 0x103f) == 0x003f)
85 set_cpu_cap(c, X86_FEATURE_FPU);
87 clear_cpu_cap(c, X86_FEATURE_FPU);
89 #ifndef CONFIG_MATH_EMULATION
91 pr_emerg("x86/fpu: Giving up, no FPU found and no math emulation present\n");
99 * Boot time FPU feature detection code:
101 unsigned int mxcsr_feature_mask __read_mostly = 0xffffffffu;
103 static void __init fpu__init_system_mxcsr(void)
105 unsigned int mask = 0;
108 /* Static because GCC does not get 16-byte stack alignment right: */
109 static struct fxregs_state fxregs __initdata;
111 asm volatile("fxsave %0" : "+m" (fxregs));
113 mask = fxregs.mxcsr_mask;
116 * If zero then use the default features mask,
117 * which has all features set, except the
118 * denormals-are-zero feature bit:
123 mxcsr_feature_mask &= mask;
127 * Once per bootup FPU initialization sequences that will run on most x86 CPUs:
129 static void __init fpu__init_system_generic(void)
132 * Set up the legacy init FPU context. (xstate init might overwrite this
133 * with a more modern format, if the CPU supports it.)
135 fpstate_init_fxstate(&init_fpstate.fxsave);
137 fpu__init_system_mxcsr();
141 * Size of the FPU context state. All tasks in the system use the
142 * same context size, regardless of what portion they use.
143 * This is inherent to the XSAVE architecture which puts all state
144 * components into a single, continuous memory block:
146 unsigned int xstate_size;
147 EXPORT_SYMBOL_GPL(xstate_size);
149 /* Get alignment of the TYPE. */
150 #define TYPE_ALIGN(TYPE) offsetof(struct { char x; TYPE test; }, test)
153 * Enforce that 'MEMBER' is the last field of 'TYPE'.
155 * Align the computed size with alignment of the TYPE,
156 * because that's how C aligns structs.
158 #define CHECK_MEMBER_AT_END_OF(TYPE, MEMBER) \
159 BUILD_BUG_ON(sizeof(TYPE) != ALIGN(offsetofend(TYPE, MEMBER), \
163 * We append the 'struct fpu' to the task_struct:
165 static void __init fpu__init_task_struct_size(void)
167 int task_size = sizeof(struct task_struct);
170 * Subtract off the static size of the register state.
171 * It potentially has a bunch of padding.
173 task_size -= sizeof(((struct task_struct *)0)->thread.fpu.state);
176 * Add back the dynamically-calculated register state
179 task_size += xstate_size;
182 * We dynamically size 'struct fpu', so we require that
183 * it be at the end of 'thread_struct' and that
184 * 'thread_struct' be at the end of 'task_struct'. If
185 * you hit a compile error here, check the structure to
186 * see if something got added to the end.
188 CHECK_MEMBER_AT_END_OF(struct fpu, state);
189 CHECK_MEMBER_AT_END_OF(struct thread_struct, fpu);
190 CHECK_MEMBER_AT_END_OF(struct task_struct, thread);
192 arch_task_struct_size = task_size;
196 * Set up the xstate_size based on the legacy FPU context size.
198 * We set this up first, and later it will be overwritten by
199 * fpu__init_system_xstate() if the CPU knows about xstates.
201 static void __init fpu__init_system_xstate_size_legacy(void)
203 static int on_boot_cpu __initdata = 1;
205 WARN_ON_FPU(!on_boot_cpu);
209 * Note that xstate_size might be overwriten later during
210 * fpu__init_system_xstate().
215 * Disable xsave as we do not support it if i387
216 * emulation is enabled.
218 setup_clear_cpu_cap(X86_FEATURE_XSAVE);
219 setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT);
220 xstate_size = sizeof(struct swregs_state);
223 xstate_size = sizeof(struct fxregs_state);
225 xstate_size = sizeof(struct fregs_state);
228 * Quirk: we don't yet handle the XSAVES* instructions
229 * correctly, as we don't correctly convert between
230 * standard and compacted format when interfacing
231 * with user-space - so disable it for now.
233 * The difference is small: with recent CPUs the
234 * compacted format is only marginally smaller than
235 * the standard FPU state format.
237 * ( This is easy to backport while we are fixing
240 setup_clear_cpu_cap(X86_FEATURE_XSAVES);
244 * FPU context switching strategies:
246 * Against popular belief, we don't do lazy FPU saves, due to the
247 * task migration complications it brings on SMP - we only do
250 * 'lazy' is the traditional strategy, which is based on setting
251 * CR0::TS to 1 during context-switch (instead of doing a full
252 * restore of the FPU state), which causes the first FPU instruction
253 * after the context switch (whenever it is executed) to fault - at
254 * which point we lazily restore the FPU state into FPU registers.
256 * Tasks are of course under no obligation to execute FPU instructions,
257 * so it can easily happen that another context-switch occurs without
258 * a single FPU instruction being executed. If we eventually switch
259 * back to the original task (that still owns the FPU) then we have
260 * not only saved the restores along the way, but we also have the
261 * FPU ready to be used for the original task.
263 * 'lazy' is deprecated because it's almost never a performance win
264 * and it's much more complicated than 'eager'.
266 * 'eager' switching is by default on all CPUs, there we switch the FPU
267 * state during every context switch, regardless of whether the task
268 * has used FPU instructions in that time slice or not. This is done
269 * because modern FPU context saving instructions are able to optimize
270 * state saving and restoration in hardware: they can detect both
271 * unused and untouched FPU state and optimize accordingly.
273 * [ Note that even in 'lazy' mode we might optimize context switches
274 * to use 'eager' restores, if we detect that a task is using the FPU
275 * frequently. See the fpu->counter logic in fpu/internal.h for that. ]
277 static enum { ENABLE, DISABLE } eagerfpu = ENABLE;
280 * Find supported xfeatures based on cpu features and command-line input.
281 * This must be called after fpu__init_parse_early_param() is called and
282 * xfeatures_mask is enumerated.
284 u64 __init fpu__get_supported_xfeatures_mask(void)
286 /* Support all xfeatures known to us */
287 if (eagerfpu != DISABLE)
290 /* Warning of xfeatures being disabled for no eagerfpu mode */
291 if (xfeatures_mask & XFEATURE_MASK_EAGER) {
292 pr_err("x86/fpu: eagerfpu switching disabled, disabling the following xstate features: 0x%llx.\n",
293 xfeatures_mask & XFEATURE_MASK_EAGER);
296 /* Return a mask that masks out all features requiring eagerfpu mode */
297 return ~XFEATURE_MASK_EAGER;
301 * Disable features dependent on eagerfpu.
303 static void __init fpu__clear_eager_fpu_features(void)
305 setup_clear_cpu_cap(X86_FEATURE_MPX);
306 setup_clear_cpu_cap(X86_FEATURE_AVX);
307 setup_clear_cpu_cap(X86_FEATURE_AVX2);
308 setup_clear_cpu_cap(X86_FEATURE_AVX512F);
309 setup_clear_cpu_cap(X86_FEATURE_AVX512PF);
310 setup_clear_cpu_cap(X86_FEATURE_AVX512ER);
311 setup_clear_cpu_cap(X86_FEATURE_AVX512CD);
315 * Pick the FPU context switching strategy:
317 * When eagerfpu is AUTO or ENABLE, we ensure it is ENABLE if either of
318 * the following is true:
320 * (1) the cpu has xsaveopt, as it has the optimization and doing eager
321 * FPU switching has a relatively low cost compared to a plain xsave;
322 * (2) the cpu has xsave features (e.g. MPX) that depend on eager FPU
323 * switching. Should the kernel boot with noxsaveopt, we support MPX
324 * with eager FPU switching at a higher cost.
326 static void __init fpu__init_system_ctx_switch(void)
328 static bool on_boot_cpu __initdata = 1;
330 WARN_ON_FPU(!on_boot_cpu);
333 WARN_ON_FPU(current->thread.fpu.fpstate_active);
334 current_thread_info()->status = 0;
336 if (boot_cpu_has(X86_FEATURE_XSAVEOPT) && eagerfpu != DISABLE)
339 if (xfeatures_mask & XFEATURE_MASK_EAGER)
342 if (eagerfpu == ENABLE)
343 setup_force_cpu_cap(X86_FEATURE_EAGER_FPU);
345 printk(KERN_INFO "x86/fpu: Using '%s' FPU context switches.\n", eagerfpu == ENABLE ? "eager" : "lazy");
349 * We parse fpu parameters early because fpu__init_system() is executed
350 * before parse_early_param().
352 static void __init fpu__init_parse_early_param(void)
354 if (cmdline_find_option_bool(boot_command_line, "eagerfpu=off")) {
356 fpu__clear_eager_fpu_features();
359 if (cmdline_find_option_bool(boot_command_line, "no387"))
360 setup_clear_cpu_cap(X86_FEATURE_FPU);
362 if (cmdline_find_option_bool(boot_command_line, "nofxsr")) {
363 setup_clear_cpu_cap(X86_FEATURE_FXSR);
364 setup_clear_cpu_cap(X86_FEATURE_FXSR_OPT);
365 setup_clear_cpu_cap(X86_FEATURE_XMM);
368 if (cmdline_find_option_bool(boot_command_line, "noxsave"))
369 fpu__xstate_clear_all_cpu_caps();
371 if (cmdline_find_option_bool(boot_command_line, "noxsaveopt"))
372 setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT);
374 if (cmdline_find_option_bool(boot_command_line, "noxsaves"))
375 setup_clear_cpu_cap(X86_FEATURE_XSAVES);
379 * Called on the boot CPU once per system bootup, to set up the initial
380 * FPU state that is later cloned into all processes:
382 void __init fpu__init_system(struct cpuinfo_x86 *c)
384 fpu__init_parse_early_param();
385 fpu__init_system_early_generic(c);
388 * The FPU has to be operational for some of the
389 * later FPU init activities:
394 * But don't leave CR0::TS set yet, as some of the FPU setup
395 * methods depend on being able to execute FPU instructions
396 * that will fault on a set TS, such as the FXSAVE in
397 * fpu__init_system_mxcsr().
401 fpu__init_system_generic();
402 fpu__init_system_xstate_size_legacy();
403 fpu__init_system_xstate();
404 fpu__init_task_struct_size();
406 fpu__init_system_ctx_switch();