arch/arm/kernel/smp_tlb.c

   1 /*
   2  *  linux/arch/arm/kernel/smp_tlb.c
   3  *
   4  *  Copyright (C) 2002 ARM Limited, All Rights Reserved.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 as
   8  * published by the Free Software Foundation.
   9  */
  10 #include <linux/preempt.h>
  11 #include <linux/smp.h>
  12
  13 #include <asm/smp_plat.h>
  14 #include <asm/tlbflush.h>
  15 #include <asm/mmu_context.h>
  16
  17 /**********************************************************************/
  18
  19 /*
  20  * TLB operations
  21  */
  22 struct tlb_args {
  23         struct vm_area_struct *ta_vma;
  24         unsigned long ta_start;
  25         unsigned long ta_end;
  26 };
  27
  28 static inline void ipi_flush_tlb_all(void *ignored)
  29 {
  30         local_flush_tlb_all();
  31 }
  32
  33 static inline void ipi_flush_tlb_mm(void *arg)
  34 {
  35         struct mm_struct *mm = (struct mm_struct *)arg;
  36
  37         local_flush_tlb_mm(mm);
  38 }
  39
  40 static inline void ipi_flush_tlb_page(void *arg)
  41 {
  42         struct tlb_args *ta = (struct tlb_args *)arg;
  43
  44         local_flush_tlb_page(ta->ta_vma, ta->ta_start);
  45 }
  46
  47 static inline void ipi_flush_tlb_kernel_page(void *arg)
  48 {
  49         struct tlb_args *ta = (struct tlb_args *)arg;
  50
  51         local_flush_tlb_kernel_page(ta->ta_start);
  52 }
  53
  54 static inline void ipi_flush_tlb_range(void *arg)
  55 {
  56         struct tlb_args *ta = (struct tlb_args *)arg;
  57
  58         local_flush_tlb_range(ta->ta_vma, ta->ta_start, ta->ta_end);
  59 }
  60
  61 static inline void ipi_flush_tlb_kernel_range(void *arg)
  62 {
  63         struct tlb_args *ta = (struct tlb_args *)arg;
  64
  65         local_flush_tlb_kernel_range(ta->ta_start, ta->ta_end);
  66 }
  67
  68 static inline void ipi_flush_bp_all(void *ignored)
  69 {
  70         local_flush_bp_all();
  71 }
  72
  73 static void ipi_flush_tlb_a15_erratum(void *arg)
  74 {
  75         dmb();
  76 }
  77
  78 static void broadcast_tlb_a15_erratum(void)
  79 {
  80         if (!erratum_a15_798181())
  81                 return;
  82
  83         dummy_flush_tlb_a15_erratum();
  84         smp_call_function(ipi_flush_tlb_a15_erratum, NULL, 1);
  85 }
  86
  87 static void broadcast_tlb_mm_a15_erratum(struct mm_struct *mm)
  88 {
  89         int this_cpu;
  90         cpumask_t mask = { CPU_BITS_NONE };
  91
  92         if (!erratum_a15_798181())
  93                 return;
  94
  95         dummy_flush_tlb_a15_erratum();
  96         this_cpu = get_cpu();
  97         a15_erratum_get_cpumask(this_cpu, mm, &mask);
  98         smp_call_function_many(&mask, ipi_flush_tlb_a15_erratum, NULL, 1);
  99         put_cpu();
 100 }
 101
 102 void flush_tlb_all(void)
 103 {
 104         if (tlb_ops_need_broadcast())
 105                 on_each_cpu(ipi_flush_tlb_all, NULL, 1);
 106         else
 107                 __flush_tlb_all();
 108         broadcast_tlb_a15_erratum();
 109 }
 110
 111 void flush_tlb_mm(struct mm_struct *mm)
 112 {
 113         if (tlb_ops_need_broadcast())
 114                 on_each_cpu_mask(mm_cpumask(mm), ipi_flush_tlb_mm, mm, 1);
 115         else
 116                 __flush_tlb_mm(mm);
 117         broadcast_tlb_mm_a15_erratum(mm);
 118 }
 119
 120 void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
 121 {
 122         if (tlb_ops_need_broadcast()) {
 123                 struct tlb_args ta;
 124                 ta.ta_vma = vma;
 125                 ta.ta_start = uaddr;
 126                 on_each_cpu_mask(mm_cpumask(vma->vm_mm), ipi_flush_tlb_page,
 127                                         &ta, 1);
 128         } else
 129                 __flush_tlb_page(vma, uaddr);
 130         broadcast_tlb_mm_a15_erratum(vma->vm_mm);
 131 }
 132
 133 void flush_tlb_kernel_page(unsigned long kaddr)
 134 {
 135         if (tlb_ops_need_broadcast()) {
 136                 struct tlb_args ta;
 137                 ta.ta_start = kaddr;
 138                 on_each_cpu(ipi_flush_tlb_kernel_page, &ta, 1);
 139         } else
 140                 __flush_tlb_kernel_page(kaddr);
 141         broadcast_tlb_a15_erratum();
 142 }
 143
 144 void flush_tlb_range(struct vm_area_struct *vma,
 145                      unsigned long start, unsigned long end)
 146 {
 147         if (tlb_ops_need_broadcast()) {
 148                 struct tlb_args ta;
 149                 ta.ta_vma = vma;
 150                 ta.ta_start = start;
 151                 ta.ta_end = end;
 152                 on_each_cpu_mask(mm_cpumask(vma->vm_mm), ipi_flush_tlb_range,
 153                                         &ta, 1);
 154         } else
 155                 local_flush_tlb_range(vma, start, end);
 156         broadcast_tlb_mm_a15_erratum(vma->vm_mm);
 157 }
 158
 159 void flush_tlb_kernel_range(unsigned long start, unsigned long end)
 160 {
 161         if (tlb_ops_need_broadcast()) {
 162                 struct tlb_args ta;
 163                 ta.ta_start = start;
 164                 ta.ta_end = end;
 165                 on_each_cpu(ipi_flush_tlb_kernel_range, &ta, 1);
 166         } else
 167                 local_flush_tlb_kernel_range(start, end);
 168         broadcast_tlb_a15_erratum();
 169 }
 170
 171 void flush_bp_all(void)
 172 {
 173         if (tlb_ops_need_broadcast())
 174                 on_each_cpu(ipi_flush_bp_all, NULL, 1);
 175         else
 176                 __flush_bp_all();
 177 }