arch/x86/xen/smp.c

   1 /*
   2  * Xen SMP support
   3  *
   4  * This file implements the Xen versions of smp_ops.  SMP under Xen is
   5  * very straightforward.  Bringing a CPU up is simply a matter of
   6  * loading its initial context and setting it running.
   7  *
   8  * IPIs are handled through the Xen event mechanism.
   9  *
  10  * Because virtual CPUs can be scheduled onto any real CPU, there's no
  11  * useful topology information for the kernel to make use of.  As a
  12  * result, all CPUs are treated as if they're single-core and
  13  * single-threaded.
  14  */
  15 #include <linux/sched.h>
  16 #include <linux/err.h>
  17 #include <linux/slab.h>
  18 #include <linux/smp.h>
  19 #include <linux/irq_work.h>
  20 #include <linux/tick.h>
  21
  22 #include <asm/paravirt.h>
  23 #include <asm/desc.h>
  24 #include <asm/pgtable.h>
  25 #include <asm/cpu.h>
  26
  27 #include <xen/interface/xen.h>
  28 #include <xen/interface/vcpu.h>
  29 #include <xen/interface/xenpmu.h>
  30
  31 #include <asm/xen/interface.h>
  32 #include <asm/xen/hypercall.h>
  33
  34 #include <xen/xen.h>
  35 #include <xen/page.h>
  36 #include <xen/events.h>
  37
  38 #include <xen/hvc-console.h>
  39 #include "xen-ops.h"
  40 #include "mmu.h"
  41 #include "smp.h"
  42 #include "pmu.h"
  43
  44 cpumask_var_t xen_cpu_initialized_map;
  45
  46 struct xen_common_irq {
  47         int irq;
  48         char *name;
  49 };
  50 static DEFINE_PER_CPU(struct xen_common_irq, xen_resched_irq) = { .irq = -1 };
  51 static DEFINE_PER_CPU(struct xen_common_irq, xen_callfunc_irq) = { .irq = -1 };
  52 static DEFINE_PER_CPU(struct xen_common_irq, xen_callfuncsingle_irq) = { .irq = -1 };
  53 static DEFINE_PER_CPU(struct xen_common_irq, xen_irq_work) = { .irq = -1 };
  54 static DEFINE_PER_CPU(struct xen_common_irq, xen_debug_irq) = { .irq = -1 };
  55 static DEFINE_PER_CPU(struct xen_common_irq, xen_pmu_irq) = { .irq = -1 };
  56
  57 static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id);
  58 static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id);
  59 static irqreturn_t xen_irq_work_interrupt(int irq, void *dev_id);
  60
  61 /*
  62  * Reschedule call back.
  63  */
  64 static irqreturn_t xen_reschedule_interrupt(int irq, void *dev_id)
  65 {
  66         inc_irq_stat(irq_resched_count);
  67         scheduler_ipi();
  68
  69         return IRQ_HANDLED;
  70 }
  71
  72 static void cpu_bringup(void)
  73 {
  74         int cpu;
  75
  76         cpu_init();
  77         touch_softlockup_watchdog();
  78         preempt_disable();
  79
  80         /* PVH runs in ring 0 and allows us to do native syscalls. Yay! */
  81         if (!xen_feature(XENFEAT_supervisor_mode_kernel)) {
  82                 xen_enable_sysenter();
  83                 xen_enable_syscall();
  84         }
  85         cpu = smp_processor_id();
  86         smp_store_cpu_info(cpu);
  87         cpu_data(cpu).x86_max_cores = 1;
  88         set_cpu_sibling_map(cpu);
  89
  90         xen_setup_cpu_clockevents();
  91
  92         notify_cpu_starting(cpu);
  93
  94         set_cpu_online(cpu, true);
  95
  96         cpu_set_state_online(cpu);  /* Implies full memory barrier. */
  97
  98         /* We can take interrupts now: we're officially "up". */
  99         local_irq_enable();
 100 }
 101
 102 asmlinkage __visible void cpu_bringup_and_idle(void)
 103 {
 104         cpu_bringup();
 105         cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
 106 }
 107
 108 void xen_smp_intr_free(unsigned int cpu)
 109 {
 110         if (per_cpu(xen_resched_irq, cpu).irq >= 0) {
 111                 unbind_from_irqhandler(per_cpu(xen_resched_irq, cpu).irq, NULL);
 112                 per_cpu(xen_resched_irq, cpu).irq = -1;
 113                 kfree(per_cpu(xen_resched_irq, cpu).name);
 114                 per_cpu(xen_resched_irq, cpu).name = NULL;
 115         }
 116         if (per_cpu(xen_callfunc_irq, cpu).irq >= 0) {
 117                 unbind_from_irqhandler(per_cpu(xen_callfunc_irq, cpu).irq, NULL);
 118                 per_cpu(xen_callfunc_irq, cpu).irq = -1;
 119                 kfree(per_cpu(xen_callfunc_irq, cpu).name);
 120                 per_cpu(xen_callfunc_irq, cpu).name = NULL;
 121         }
 122         if (per_cpu(xen_debug_irq, cpu).irq >= 0) {
 123                 unbind_from_irqhandler(per_cpu(xen_debug_irq, cpu).irq, NULL);
 124                 per_cpu(xen_debug_irq, cpu).irq = -1;
 125                 kfree(per_cpu(xen_debug_irq, cpu).name);
 126                 per_cpu(xen_debug_irq, cpu).name = NULL;
 127         }
 128         if (per_cpu(xen_callfuncsingle_irq, cpu).irq >= 0) {
 129                 unbind_from_irqhandler(per_cpu(xen_callfuncsingle_irq, cpu).irq,
 130                                        NULL);
 131                 per_cpu(xen_callfuncsingle_irq, cpu).irq = -1;
 132                 kfree(per_cpu(xen_callfuncsingle_irq, cpu).name);
 133                 per_cpu(xen_callfuncsingle_irq, cpu).name = NULL;
 134         }
 135         if (xen_hvm_domain())
 136                 return;
 137
 138         if (per_cpu(xen_irq_work, cpu).irq >= 0) {
 139                 unbind_from_irqhandler(per_cpu(xen_irq_work, cpu).irq, NULL);
 140                 per_cpu(xen_irq_work, cpu).irq = -1;
 141                 kfree(per_cpu(xen_irq_work, cpu).name);
 142                 per_cpu(xen_irq_work, cpu).name = NULL;
 143         }
 144
 145         if (per_cpu(xen_pmu_irq, cpu).irq >= 0) {
 146                 unbind_from_irqhandler(per_cpu(xen_pmu_irq, cpu).irq, NULL);
 147                 per_cpu(xen_pmu_irq, cpu).irq = -1;
 148                 kfree(per_cpu(xen_pmu_irq, cpu).name);
 149                 per_cpu(xen_pmu_irq, cpu).name = NULL;
 150         }
 151 };
 152 int xen_smp_intr_init(unsigned int cpu)
 153 {
 154         int rc;
 155         char *resched_name, *callfunc_name, *debug_name, *pmu_name;
 156
 157         resched_name = kasprintf(GFP_KERNEL, "resched%d", cpu);
 158         rc = bind_ipi_to_irqhandler(XEN_RESCHEDULE_VECTOR,
 159                                     cpu,
 160                                     xen_reschedule_interrupt,
 161                                     IRQF_PERCPU|IRQF_NOBALANCING,
 162                                     resched_name,
 163                                     NULL);
 164         if (rc < 0)
 165                 goto fail;
 166         per_cpu(xen_resched_irq, cpu).irq = rc;
 167         per_cpu(xen_resched_irq, cpu).name = resched_name;
 168
 169         callfunc_name = kasprintf(GFP_KERNEL, "callfunc%d", cpu);
 170         rc = bind_ipi_to_irqhandler(XEN_CALL_FUNCTION_VECTOR,
 171                                     cpu,
 172                                     xen_call_function_interrupt,
 173                                     IRQF_PERCPU|IRQF_NOBALANCING,
 174                                     callfunc_name,
 175                                     NULL);
 176         if (rc < 0)
 177                 goto fail;
 178         per_cpu(xen_callfunc_irq, cpu).irq = rc;
 179         per_cpu(xen_callfunc_irq, cpu).name = callfunc_name;
 180
 181         debug_name = kasprintf(GFP_KERNEL, "debug%d", cpu);
 182         rc = bind_virq_to_irqhandler(VIRQ_DEBUG, cpu, xen_debug_interrupt,
 183                                      IRQF_PERCPU | IRQF_NOBALANCING,
 184                                      debug_name, NULL);
 185         if (rc < 0)
 186                 goto fail;
 187         per_cpu(xen_debug_irq, cpu).irq = rc;
 188         per_cpu(xen_debug_irq, cpu).name = debug_name;
 189
 190         callfunc_name = kasprintf(GFP_KERNEL, "callfuncsingle%d", cpu);
 191         rc = bind_ipi_to_irqhandler(XEN_CALL_FUNCTION_SINGLE_VECTOR,
 192                                     cpu,
 193                                     xen_call_function_single_interrupt,
 194                                     IRQF_PERCPU|IRQF_NOBALANCING,
 195                                     callfunc_name,
 196                                     NULL);
 197         if (rc < 0)
 198                 goto fail;
 199         per_cpu(xen_callfuncsingle_irq, cpu).irq = rc;
 200         per_cpu(xen_callfuncsingle_irq, cpu).name = callfunc_name;
 201
 202         /*
 203          * The IRQ worker on PVHVM goes through the native path and uses the
 204          * IPI mechanism.
 205          */
 206         if (xen_hvm_domain())
 207                 return 0;
 208
 209         callfunc_name = kasprintf(GFP_KERNEL, "irqwork%d", cpu);
 210         rc = bind_ipi_to_irqhandler(XEN_IRQ_WORK_VECTOR,
 211                                     cpu,
 212                                     xen_irq_work_interrupt,
 213                                     IRQF_PERCPU|IRQF_NOBALANCING,
 214                                     callfunc_name,
 215                                     NULL);
 216         if (rc < 0)
 217                 goto fail;
 218         per_cpu(xen_irq_work, cpu).irq = rc;
 219         per_cpu(xen_irq_work, cpu).name = callfunc_name;
 220
 221         if (is_xen_pmu(cpu)) {
 222                 pmu_name = kasprintf(GFP_KERNEL, "pmu%d", cpu);
 223                 rc = bind_virq_to_irqhandler(VIRQ_XENPMU, cpu,
 224                                              xen_pmu_irq_handler,
 225                                              IRQF_PERCPU|IRQF_NOBALANCING,
 226                                              pmu_name, NULL);
 227                 if (rc < 0)
 228                         goto fail;
 229                 per_cpu(xen_pmu_irq, cpu).irq = rc;
 230                 per_cpu(xen_pmu_irq, cpu).name = pmu_name;
 231         }
 232
 233         return 0;
 234
 235  fail:
 236         xen_smp_intr_free(cpu);
 237         return rc;
 238 }
 239
 240 static void __init xen_fill_possible_map(void)
 241 {
 242         int i, rc;
 243
 244         if (xen_initial_domain())
 245                 return;
 246
 247         for (i = 0; i < nr_cpu_ids; i++) {
 248                 rc = HYPERVISOR_vcpu_op(VCPUOP_is_up, i, NULL);
 249                 if (rc >= 0) {
 250                         num_processors++;
 251                         set_cpu_possible(i, true);
 252                 }
 253         }
 254 }
 255
 256 static void __init xen_filter_cpu_maps(void)
 257 {
 258         int i, rc;
 259         unsigned int subtract = 0;
 260
 261         if (!xen_initial_domain())
 262                 return;
 263
 264         num_processors = 0;
 265         disabled_cpus = 0;
 266         for (i = 0; i < nr_cpu_ids; i++) {
 267                 rc = HYPERVISOR_vcpu_op(VCPUOP_is_up, i, NULL);
 268                 if (rc >= 0) {
 269                         num_processors++;
 270                         set_cpu_possible(i, true);
 271                 } else {
 272                         set_cpu_possible(i, false);
 273                         set_cpu_present(i, false);
 274                         subtract++;
 275                 }
 276         }
 277 #ifdef CONFIG_HOTPLUG_CPU
 278         /* This is akin to using 'nr_cpus' on the Linux command line.
 279          * Which is OK as when we use 'dom0_max_vcpus=X' we can only
 280          * have up to X, while nr_cpu_ids is greater than X. This
 281          * normally is not a problem, except when CPU hotplugging
 282          * is involved and then there might be more than X CPUs
 283          * in the guest - which will not work as there is no
 284          * hypercall to expand the max number of VCPUs an already
 285          * running guest has. So cap it up to X. */
 286         if (subtract)
 287                 nr_cpu_ids = nr_cpu_ids - subtract;
 288 #endif
 289
 290 }
 291
 292 static void __init xen_smp_prepare_boot_cpu(void)
 293 {
 294         BUG_ON(smp_processor_id() != 0);
 295         native_smp_prepare_boot_cpu();
 296
 297         if (xen_pv_domain()) {
 298                 if (!xen_feature(XENFEAT_writable_page_tables))
 299                         /* We've switched to the "real" per-cpu gdt, so make
 300                          * sure the old memory can be recycled. */
 301                         make_lowmem_page_readwrite(xen_initial_gdt);
 302
 303 #ifdef CONFIG_X86_32
 304                 /*
 305                  * Xen starts us with XEN_FLAT_RING1_DS, but linux code
 306                  * expects __USER_DS
 307                  */
 308                 loadsegment(ds, __USER_DS);
 309                 loadsegment(es, __USER_DS);
 310 #endif
 311
 312                 xen_filter_cpu_maps();
 313                 xen_setup_vcpu_info_placement();
 314         }
 315
 316         /*
 317          * Setup vcpu_info for boot CPU.
 318          */
 319         if (xen_hvm_domain())
 320                 xen_vcpu_setup(0);
 321
 322         /*
 323          * The alternative logic (which patches the unlock/lock) runs before
 324          * the smp bootup up code is activated. Hence we need to set this up
 325          * the core kernel is being patched. Otherwise we will have only
 326          * modules patched but not core code.
 327          */
 328         xen_init_spinlocks();
 329 }
 330
 331 static void __init xen_smp_prepare_cpus(unsigned int max_cpus)
 332 {
 333         unsigned cpu;
 334         unsigned int i;
 335
 336         if (skip_ioapic_setup) {
 337                 char *m = (max_cpus == 0) ?
 338                         "The nosmp parameter is incompatible with Xen; " \
 339                         "use Xen dom0_max_vcpus=1 parameter" :
 340                         "The noapic parameter is incompatible with Xen";
 341
 342                 xen_raw_printk(m);
 343                 panic(m);
 344         }
 345         xen_init_lock_cpu(0);
 346
 347         smp_store_boot_cpu_info();
 348         cpu_data(0).x86_max_cores = 1;
 349
 350         for_each_possible_cpu(i) {
 351                 zalloc_cpumask_var(&per_cpu(cpu_sibling_map, i), GFP_KERNEL);
 352                 zalloc_cpumask_var(&per_cpu(cpu_core_map, i), GFP_KERNEL);
 353                 zalloc_cpumask_var(&per_cpu(cpu_llc_shared_map, i), GFP_KERNEL);
 354         }
 355         set_cpu_sibling_map(0);
 356
 357         xen_pmu_init(0);
 358
 359         if (xen_smp_intr_init(0))
 360                 BUG();
 361
 362         if (!alloc_cpumask_var(&xen_cpu_initialized_map, GFP_KERNEL))
 363                 panic("could not allocate xen_cpu_initialized_map\n");
 364
 365         cpumask_copy(xen_cpu_initialized_map, cpumask_of(0));
 366
 367         /* Restrict the possible_map according to max_cpus. */
 368         while ((num_possible_cpus() > 1) && (num_possible_cpus() > max_cpus)) {
 369                 for (cpu = nr_cpu_ids - 1; !cpu_possible(cpu); cpu--)
 370                         continue;
 371                 set_cpu_possible(cpu, false);
 372         }
 373
 374         for_each_possible_cpu(cpu)
 375                 set_cpu_present(cpu, true);
 376 }
 377
 378 static int
 379 cpu_initialize_context(unsigned int cpu, struct task_struct *idle)
 380 {
 381         struct vcpu_guest_context *ctxt;
 382         struct desc_struct *gdt;
 383         unsigned long gdt_mfn;
 384
 385         /* used to tell cpu_init() that it can proceed with initialization */
 386         cpumask_set_cpu(cpu, cpu_callout_mask);
 387         if (cpumask_test_and_set_cpu(cpu, xen_cpu_initialized_map))
 388                 return 0;
 389
 390         ctxt = kzalloc(sizeof(*ctxt), GFP_KERNEL);
 391         if (ctxt == NULL)
 392                 return -ENOMEM;
 393
 394         gdt = get_cpu_gdt_table(cpu);
 395
 396 #ifdef CONFIG_X86_32
 397         ctxt->user_regs.fs = __KERNEL_PERCPU;
 398         ctxt->user_regs.gs = __KERNEL_STACK_CANARY;
 399 #endif
 400         memset(&ctxt->fpu_ctxt, 0, sizeof(ctxt->fpu_ctxt));
 401
 402         ctxt->user_regs.eip = (unsigned long)cpu_bringup_and_idle;
 403         ctxt->flags = VGCF_IN_KERNEL;
 404         ctxt->user_regs.eflags = 0x1000; /* IOPL_RING1 */
 405         ctxt->user_regs.ds = __USER_DS;
 406         ctxt->user_regs.es = __USER_DS;
 407         ctxt->user_regs.ss = __KERNEL_DS;
 408
 409         xen_copy_trap_info(ctxt->trap_ctxt);
 410
 411         ctxt->ldt_ents = 0;
 412
 413         BUG_ON((unsigned long)gdt & ~PAGE_MASK);
 414
 415         gdt_mfn = arbitrary_virt_to_mfn(gdt);
 416         make_lowmem_page_readonly(gdt);
 417         make_lowmem_page_readonly(mfn_to_virt(gdt_mfn));
 418
 419         ctxt->gdt_frames[0] = gdt_mfn;
 420         ctxt->gdt_ents      = GDT_ENTRIES;
 421
 422         ctxt->kernel_ss = __KERNEL_DS;
 423         ctxt->kernel_sp = idle->thread.sp0;
 424
 425 #ifdef CONFIG_X86_32
 426         ctxt->event_callback_cs     = __KERNEL_CS;
 427         ctxt->failsafe_callback_cs  = __KERNEL_CS;
 428 #else
 429         ctxt->gs_base_kernel = per_cpu_offset(cpu);
 430 #endif
 431         ctxt->event_callback_eip    =
 432                 (unsigned long)xen_hypervisor_callback;
 433         ctxt->failsafe_callback_eip =
 434                 (unsigned long)xen_failsafe_callback;
 435         ctxt->user_regs.cs = __KERNEL_CS;
 436         per_cpu(xen_cr3, cpu) = __pa(swapper_pg_dir);
 437
 438         ctxt->user_regs.esp = idle->thread.sp0 - sizeof(struct pt_regs);
 439         ctxt->ctrlreg[3] = xen_pfn_to_cr3(virt_to_gfn(swapper_pg_dir));
 440         if (HYPERVISOR_vcpu_op(VCPUOP_initialise, xen_vcpu_nr(cpu), ctxt))
 441                 BUG();
 442
 443         kfree(ctxt);
 444         return 0;
 445 }
 446
 447 static int xen_cpu_up(unsigned int cpu, struct task_struct *idle)
 448 {
 449         int rc;
 450
 451         common_cpu_up(cpu, idle);
 452
 453         xen_setup_runstate_info(cpu);
 454
 455         /*
 456          * PV VCPUs are always successfully taken down (see 'while' loop
 457          * in xen_cpu_die()), so -EBUSY is an error.
 458          */
 459         rc = cpu_check_up_prepare(cpu);
 460         if (rc)
 461                 return rc;
 462
 463         /* make sure interrupts start blocked */
 464         per_cpu(xen_vcpu, cpu)->evtchn_upcall_mask = 1;
 465
 466         rc = cpu_initialize_context(cpu, idle);
 467         if (rc)
 468                 return rc;
 469
 470         xen_pmu_init(cpu);
 471
 472         rc = HYPERVISOR_vcpu_op(VCPUOP_up, xen_vcpu_nr(cpu), NULL);
 473         BUG_ON(rc);
 474
 475         while (cpu_report_state(cpu) != CPU_ONLINE)
 476                 HYPERVISOR_sched_op(SCHEDOP_yield, NULL);
 477
 478         return 0;
 479 }
 480
 481 static void xen_smp_cpus_done(unsigned int max_cpus)
 482 {
 483 }
 484
 485 #ifdef CONFIG_HOTPLUG_CPU
 486 static int xen_cpu_disable(void)
 487 {
 488         unsigned int cpu = smp_processor_id();
 489         if (cpu == 0)
 490                 return -EBUSY;
 491
 492         cpu_disable_common();
 493
 494         load_cr3(swapper_pg_dir);
 495         return 0;
 496 }
 497
 498 static void xen_cpu_die(unsigned int cpu)
 499 {
 500         while (xen_pv_domain() && HYPERVISOR_vcpu_op(VCPUOP_is_up,
 501                                                      xen_vcpu_nr(cpu), NULL)) {
 502                 __set_current_state(TASK_UNINTERRUPTIBLE);
 503                 schedule_timeout(HZ/10);
 504         }
 505
 506         if (common_cpu_die(cpu) == 0) {
 507                 xen_smp_intr_free(cpu);
 508                 xen_uninit_lock_cpu(cpu);
 509                 xen_teardown_timer(cpu);
 510                 xen_pmu_finish(cpu);
 511         }
 512 }
 513
 514 static void xen_play_dead(void) /* used only with HOTPLUG_CPU */
 515 {
 516         play_dead_common();
 517         HYPERVISOR_vcpu_op(VCPUOP_down, xen_vcpu_nr(smp_processor_id()), NULL);
 518         cpu_bringup();
 519         /*
 520          * commit 4b0c0f294 (tick: Cleanup NOHZ per cpu data on cpu down)
 521          * clears certain data that the cpu_idle loop (which called us
 522          * and that we return from) expects. The only way to get that
 523          * data back is to call:
 524          */
 525         tick_nohz_idle_enter();
 526
 527         cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
 528 }
 529
 530 #else /* !CONFIG_HOTPLUG_CPU */
 531 static int xen_cpu_disable(void)
 532 {
 533         return -ENOSYS;
 534 }
 535
 536 static void xen_cpu_die(unsigned int cpu)
 537 {
 538         BUG();
 539 }
 540
 541 static void xen_play_dead(void)
 542 {
 543         BUG();
 544 }
 545
 546 #endif
 547 static void stop_self(void *v)
 548 {
 549         int cpu = smp_processor_id();
 550
 551         /* make sure we're not pinning something down */
 552         load_cr3(swapper_pg_dir);
 553         /* should set up a minimal gdt */
 554
 555         set_cpu_online(cpu, false);
 556
 557         HYPERVISOR_vcpu_op(VCPUOP_down, xen_vcpu_nr(cpu), NULL);
 558         BUG();
 559 }
 560
 561 static void xen_stop_other_cpus(int wait)
 562 {
 563         smp_call_function(stop_self, NULL, wait);
 564 }
 565
 566 static void xen_smp_send_reschedule(int cpu)
 567 {
 568         xen_send_IPI_one(cpu, XEN_RESCHEDULE_VECTOR);
 569 }
 570
 571 static void __xen_send_IPI_mask(const struct cpumask *mask,
 572                               int vector)
 573 {
 574         unsigned cpu;
 575
 576         for_each_cpu_and(cpu, mask, cpu_online_mask)
 577                 xen_send_IPI_one(cpu, vector);
 578 }
 579
 580 static void xen_smp_send_call_function_ipi(const struct cpumask *mask)
 581 {
 582         int cpu;
 583
 584         __xen_send_IPI_mask(mask, XEN_CALL_FUNCTION_VECTOR);
 585
 586         /* Make sure other vcpus get a chance to run if they need to. */
 587         for_each_cpu(cpu, mask) {
 588                 if (xen_vcpu_stolen(cpu)) {
 589                         HYPERVISOR_sched_op(SCHEDOP_yield, NULL);
 590                         break;
 591                 }
 592         }
 593 }
 594
 595 static void xen_smp_send_call_function_single_ipi(int cpu)
 596 {
 597         __xen_send_IPI_mask(cpumask_of(cpu),
 598                           XEN_CALL_FUNCTION_SINGLE_VECTOR);
 599 }
 600
 601 static inline int xen_map_vector(int vector)
 602 {
 603         int xen_vector;
 604
 605         switch (vector) {
 606         case RESCHEDULE_VECTOR:
 607                 xen_vector = XEN_RESCHEDULE_VECTOR;
 608                 break;
 609         case CALL_FUNCTION_VECTOR:
 610                 xen_vector = XEN_CALL_FUNCTION_VECTOR;
 611                 break;
 612         case CALL_FUNCTION_SINGLE_VECTOR:
 613                 xen_vector = XEN_CALL_FUNCTION_SINGLE_VECTOR;
 614                 break;
 615         case IRQ_WORK_VECTOR:
 616                 xen_vector = XEN_IRQ_WORK_VECTOR;
 617                 break;
 618 #ifdef CONFIG_X86_64
 619         case NMI_VECTOR:
 620         case APIC_DM_NMI: /* Some use that instead of NMI_VECTOR */
 621                 xen_vector = XEN_NMI_VECTOR;
 622                 break;
 623 #endif
 624         default:
 625                 xen_vector = -1;
 626                 printk(KERN_ERR "xen: vector 0x%x is not implemented\n",
 627                         vector);
 628         }
 629
 630         return xen_vector;
 631 }
 632
 633 void xen_send_IPI_mask(const struct cpumask *mask,
 634                               int vector)
 635 {
 636         int xen_vector = xen_map_vector(vector);
 637
 638         if (xen_vector >= 0)
 639                 __xen_send_IPI_mask(mask, xen_vector);
 640 }
 641
 642 void xen_send_IPI_all(int vector)
 643 {
 644         int xen_vector = xen_map_vector(vector);
 645
 646         if (xen_vector >= 0)
 647                 __xen_send_IPI_mask(cpu_online_mask, xen_vector);
 648 }
 649
 650 void xen_send_IPI_self(int vector)
 651 {
 652         int xen_vector = xen_map_vector(vector);
 653
 654         if (xen_vector >= 0)
 655                 xen_send_IPI_one(smp_processor_id(), xen_vector);
 656 }
 657
 658 void xen_send_IPI_mask_allbutself(const struct cpumask *mask,
 659                                 int vector)
 660 {
 661         unsigned cpu;
 662         unsigned int this_cpu = smp_processor_id();
 663         int xen_vector = xen_map_vector(vector);
 664
 665         if (!(num_online_cpus() > 1) || (xen_vector < 0))
 666                 return;
 667
 668         for_each_cpu_and(cpu, mask, cpu_online_mask) {
 669                 if (this_cpu == cpu)
 670                         continue;
 671
 672                 xen_send_IPI_one(cpu, xen_vector);
 673         }
 674 }
 675
 676 void xen_send_IPI_allbutself(int vector)
 677 {
 678         xen_send_IPI_mask_allbutself(cpu_online_mask, vector);
 679 }
 680
 681 static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id)
 682 {
 683         irq_enter();
 684         generic_smp_call_function_interrupt();
 685         inc_irq_stat(irq_call_count);
 686         irq_exit();
 687
 688         return IRQ_HANDLED;
 689 }
 690
 691 static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id)
 692 {
 693         irq_enter();
 694         generic_smp_call_function_single_interrupt();
 695         inc_irq_stat(irq_call_count);
 696         irq_exit();
 697
 698         return IRQ_HANDLED;
 699 }
 700
 701 static irqreturn_t xen_irq_work_interrupt(int irq, void *dev_id)
 702 {
 703         irq_enter();
 704         irq_work_run();
 705         inc_irq_stat(apic_irq_work_irqs);
 706         irq_exit();
 707
 708         return IRQ_HANDLED;
 709 }
 710
 711 static const struct smp_ops xen_smp_ops __initconst = {
 712         .smp_prepare_boot_cpu = xen_smp_prepare_boot_cpu,
 713         .smp_prepare_cpus = xen_smp_prepare_cpus,
 714         .smp_cpus_done = xen_smp_cpus_done,
 715
 716         .cpu_up = xen_cpu_up,
 717         .cpu_die = xen_cpu_die,
 718         .cpu_disable = xen_cpu_disable,
 719         .play_dead = xen_play_dead,
 720
 721         .stop_other_cpus = xen_stop_other_cpus,
 722         .smp_send_reschedule = xen_smp_send_reschedule,
 723
 724         .send_call_func_ipi = xen_smp_send_call_function_ipi,
 725         .send_call_func_single_ipi = xen_smp_send_call_function_single_ipi,
 726 };
 727
 728 void __init xen_smp_init(void)
 729 {
 730         smp_ops = xen_smp_ops;
 731         xen_fill_possible_map();
 732 }
 733
 734 static void __init xen_hvm_smp_prepare_cpus(unsigned int max_cpus)
 735 {
 736         native_smp_prepare_cpus(max_cpus);
 737         WARN_ON(xen_smp_intr_init(0));
 738
 739         xen_init_lock_cpu(0);
 740 }
 741
 742 void __init xen_hvm_smp_init(void)
 743 {
 744         smp_ops.smp_prepare_cpus = xen_hvm_smp_prepare_cpus;
 745         smp_ops.smp_send_reschedule = xen_smp_send_reschedule;
 746         smp_ops.cpu_die = xen_cpu_die;
 747         smp_ops.send_call_func_ipi = xen_smp_send_call_function_ipi;
 748         smp_ops.send_call_func_single_ipi = xen_smp_send_call_function_single_ipi;
 749         smp_ops.smp_prepare_boot_cpu = xen_smp_prepare_boot_cpu;
 750 }