ARM: 7659/1: mm: make mm->context.id an atomic64_t variable

commit 8a4e3a9ead7e37ce1505602b564c15da09ac039f upstream.

mm->context.id is updated under asid_lock when a new ASID is allocated
to an mm_struct. However, it is also read without the lock when a task
is being scheduled and checking whether or not the current ASID
generation is up-to-date.

If two threads of the same process are being scheduled in parallel and
the bottom bits of the generation in their mm->context.id match the
current generation (that is, the mm_struct has not been used for ~2^24
rollovers) then the non-atomic, lockless access to mm->context.id may
yield the incorrect ASID.

This patch fixes this issue by making mm->context.id and atomic64_t,
ensuring that the generation is always read consistently. For code that
only requires access to the ASID bits (e.g. TLB flushing by mm), then
the value is accessed directly, which GCC converts to an ldrb.

Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/arch/arm/include/asm/mmu.h b/arch/arm/include/asm/mmu.h
index 9f77e7804f3b66e7c08e7ece91780b2df178c9ca..e3d55547e75592a9eb98fa1704ee8df9e872c2c5 100644 (file)
--- a/arch/arm/include/asm/mmu.h
+++ b/arch/arm/include/asm/mmu.h
@@ -5,15 +5,15 @@
 
 typedef struct {
 #ifdef CONFIG_CPU_HAS_ASID
-       u64 id;
+       atomic64_t      id;
 #endif
-       unsigned int vmalloc_seq;
+       unsigned int    vmalloc_seq;
 } mm_context_t;
 
 #ifdef CONFIG_CPU_HAS_ASID
 #define ASID_BITS      8
 #define ASID_MASK      ((~0ULL) << ASID_BITS)
-#define ASID(mm)       ((mm)->context.id & ~ASID_MASK)
+#define ASID(mm)       ((mm)->context.id.counter & ~ASID_MASK)
 #else
 #define ASID(mm)       (0)
 #endif
@@ -26,7 +26,7 @@ typedef struct {
  *  modified for 2.6 by Hyok S. Choi <hyok.choi@samsung.com>
  */
 typedef struct {
-       unsigned long           end_brk;
+       unsigned long   end_brk;
 } mm_context_t;
 
 #endif

diff --git a/arch/arm/include/asm/mmu_context.h b/arch/arm/include/asm/mmu_context.h
index e1f644bc7cc504412613a08489dffbfc4ecc63bd..863a6611323c70077a9428b198a10d59758ff919 100644 (file)
--- a/arch/arm/include/asm/mmu_context.h
+++ b/arch/arm/include/asm/mmu_context.h
@@ -25,7 +25,7 @@ void __check_vmalloc_seq(struct mm_struct *mm);
 #ifdef CONFIG_CPU_HAS_ASID
 
 void check_and_switch_context(struct mm_struct *mm, struct task_struct *tsk);
-#define init_new_context(tsk,mm)       ({ mm->context.id = 0; })
+#define init_new_context(tsk,mm)       ({ atomic64_set(&mm->context.id, 0); 0; })
 
 #else  /* !CONFIG_CPU_HAS_ASID */
 

diff --git a/arch/arm/kernel/asm-offsets.c b/arch/arm/kernel/asm-offsets.c
index c985b481192c2643e938425a57e3e6fc9866ba90..cf10d188624430b7f741ecd95dde7488e3bce7ae 100644 (file)
--- a/arch/arm/kernel/asm-offsets.c
+++ b/arch/arm/kernel/asm-offsets.c
@@ -107,7 +107,7 @@ int main(void)
   BLANK();
 #endif
 #ifdef CONFIG_CPU_HAS_ASID
-  DEFINE(MM_CONTEXT_ID,                offsetof(struct mm_struct, context.id));
+  DEFINE(MM_CONTEXT_ID,                offsetof(struct mm_struct, context.id.counter));
   BLANK();
 #endif
   DEFINE(VMA_VM_MM,            offsetof(struct vm_area_struct, vm_mm));

diff --git a/arch/arm/mm/context.c b/arch/arm/mm/context.c
index 6a6b8bf2958f799ddaa28975b0b08086532feb75..d07df17f6d6ddf6346900289e7174c060e0789cf 100644 (file)
--- a/arch/arm/mm/context.c
+++ b/arch/arm/mm/context.c
@@ -149,9 +149,9 @@ static int is_reserved_asid(u64 asid)
        return 0;
 }
 
-static void new_context(struct mm_struct *mm, unsigned int cpu)
+static u64 new_context(struct mm_struct *mm, unsigned int cpu)
 {
-       u64 asid = mm->context.id;
+       u64 asid = atomic64_read(&mm->context.id);
        u64 generation = atomic64_read(&asid_generation);
 
        if (asid != 0 && is_reserved_asid(asid)) {
@@ -178,13 +178,14 @@ static void new_context(struct mm_struct *mm, unsigned int cpu)
                cpumask_clear(mm_cpumask(mm));
        }
 
-       mm->context.id = asid;
+       return asid;
 }
 
 void check_and_switch_context(struct mm_struct *mm, struct task_struct *tsk)
 {
        unsigned long flags;
        unsigned int cpu = smp_processor_id();
+       u64 asid;
 
        if (unlikely(mm->context.vmalloc_seq != init_mm.context.vmalloc_seq))
                __check_vmalloc_seq(mm);
@@ -195,19 +196,23 @@ void check_and_switch_context(struct mm_struct *mm, struct task_struct *tsk)
         */
        cpu_set_reserved_ttbr0();
 
-       if (!((mm->context.id ^ atomic64_read(&asid_generation)) >> ASID_BITS)
-           && atomic64_xchg(&per_cpu(active_asids, cpu), mm->context.id))
+       asid = atomic64_read(&mm->context.id);
+       if (!((asid ^ atomic64_read(&asid_generation)) >> ASID_BITS)
+           && atomic64_xchg(&per_cpu(active_asids, cpu), asid))
                goto switch_mm_fastpath;
 
        raw_spin_lock_irqsave(&cpu_asid_lock, flags);
        /* Check that our ASID belongs to the current generation. */
-       if ((mm->context.id ^ atomic64_read(&asid_generation)) >> ASID_BITS)
-               new_context(mm, cpu);
+       asid = atomic64_read(&mm->context.id);
+       if ((asid ^ atomic64_read(&asid_generation)) >> ASID_BITS) {
+               asid = new_context(mm, cpu);
+               atomic64_set(&mm->context.id, asid);
+       }
 
        if (cpumask_test_and_clear_cpu(cpu, &tlb_flush_pending))
                local_flush_tlb_all();
 
-       atomic64_set(&per_cpu(active_asids, cpu), mm->context.id);
+       atomic64_set(&per_cpu(active_asids, cpu), asid);
        cpumask_set_cpu(cpu, mm_cpumask(mm));
        raw_spin_unlock_irqrestore(&cpu_asid_lock, flags);