arch/sparc64/mm/init.c

   1 /*  $Id: init.c,v 1.209 2002/02/09 19:49:31 davem Exp $
   2  *  arch/sparc64/mm/init.c
   3  *
   4  *  Copyright (C) 1996-1999 David S. Miller (davem@caip.rutgers.edu)
   5  *  Copyright (C) 1997-1999 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/kernel.h>
  10 #include <linux/sched.h>
  11 #include <linux/string.h>
  12 #include <linux/init.h>
  13 #include <linux/bootmem.h>
  14 #include <linux/mm.h>
  15 #include <linux/hugetlb.h>
  16 #include <linux/slab.h>
  17 #include <linux/initrd.h>
  18 #include <linux/swap.h>
  19 #include <linux/pagemap.h>
  20 #include <linux/poison.h>
  21 #include <linux/fs.h>
  22 #include <linux/seq_file.h>
  23 #include <linux/kprobes.h>
  24 #include <linux/cache.h>
  25 #include <linux/sort.h>
  26
  27 #include <asm/head.h>
  28 #include <asm/system.h>
  29 #include <asm/page.h>
  30 #include <asm/pgalloc.h>
  31 #include <asm/pgtable.h>
  32 #include <asm/oplib.h>
  33 #include <asm/iommu.h>
  34 #include <asm/io.h>
  35 #include <asm/uaccess.h>
  36 #include <asm/mmu_context.h>
  37 #include <asm/tlbflush.h>
  38 #include <asm/dma.h>
  39 #include <asm/starfire.h>
  40 #include <asm/tlb.h>
  41 #include <asm/spitfire.h>
  42 #include <asm/sections.h>
  43 #include <asm/tsb.h>
  44 #include <asm/hypervisor.h>
  45 #include <asm/prom.h>
  46
  47 extern void device_scan(void);
  48
  49 #define MAX_PHYS_ADDRESS        (1UL << 42UL)
  50 #define KPTE_BITMAP_CHUNK_SZ    (256UL * 1024UL * 1024UL)
  51 #define KPTE_BITMAP_BYTES       \
  52         ((MAX_PHYS_ADDRESS / KPTE_BITMAP_CHUNK_SZ) / 8)
  53
  54 unsigned long kern_linear_pte_xor[2] __read_mostly;
  55
  56 /* A bitmap, one bit for every 256MB of physical memory.  If the bit
  57  * is clear, we should use a 4MB page (via kern_linear_pte_xor[0]) else
  58  * if set we should use a 256MB page (via kern_linear_pte_xor[1]).
  59  */
  60 unsigned long kpte_linear_bitmap[KPTE_BITMAP_BYTES / sizeof(unsigned long)];
  61
  62 #ifndef CONFIG_DEBUG_PAGEALLOC
  63 /* A special kernel TSB for 4MB and 256MB linear mappings.  */
  64 struct tsb swapper_4m_tsb[KERNEL_TSB4M_NENTRIES];
  65 #endif
  66
  67 #define MAX_BANKS       32
  68
  69 static struct linux_prom64_registers pavail[MAX_BANKS] __initdata;
  70 static struct linux_prom64_registers pavail_rescan[MAX_BANKS] __initdata;
  71 static int pavail_ents __initdata;
  72 static int pavail_rescan_ents __initdata;
  73
  74 static int cmp_p64(const void *a, const void *b)
  75 {
  76         const struct linux_prom64_registers *x = a, *y = b;
  77
  78         if (x->phys_addr > y->phys_addr)
  79                 return 1;
  80         if (x->phys_addr < y->phys_addr)
  81                 return -1;
  82         return 0;
  83 }
  84
  85 static void __init read_obp_memory(const char *property,
  86                                    struct linux_prom64_registers *regs,
  87                                    int *num_ents)
  88 {
  89         int node = prom_finddevice("/memory");
  90         int prop_size = prom_getproplen(node, property);
  91         int ents, ret, i;
  92
  93         ents = prop_size / sizeof(struct linux_prom64_registers);
  94         if (ents > MAX_BANKS) {
  95                 prom_printf("The machine has more %s property entries than "
  96                             "this kernel can support (%d).\n",
  97                             property, MAX_BANKS);
  98                 prom_halt();
  99         }
 100
 101         ret = prom_getproperty(node, property, (char *) regs, prop_size);
 102         if (ret == -1) {
 103                 prom_printf("Couldn't get %s property from /memory.\n");
 104                 prom_halt();
 105         }
 106
 107         /* Sanitize what we got from the firmware, by page aligning
 108          * everything.
 109          */
 110         for (i = 0; i < ents; i++) {
 111                 unsigned long base, size;
 112
 113                 base = regs[i].phys_addr;
 114                 size = regs[i].reg_size;
 115
 116                 size &= PAGE_MASK;
 117                 if (base & ~PAGE_MASK) {
 118                         unsigned long new_base = PAGE_ALIGN(base);
 119
 120                         size -= new_base - base;
 121                         if ((long) size < 0L)
 122                                 size = 0UL;
 123                         base = new_base;
 124                 }
 125                 if (size == 0UL) {
 126                         /* If it is empty, simply get rid of it.
 127                          * This simplifies the logic of the other
 128                          * functions that process these arrays.
 129                          */
 130                         memmove(&regs[i], &regs[i + 1],
 131                                 (ents - i - 1) * sizeof(regs[0]));
 132                         i--;
 133                         ents--;
 134                         continue;
 135                 }
 136                 regs[i].phys_addr = base;
 137                 regs[i].reg_size = size;
 138         }
 139
 140         *num_ents = ents;
 141
 142         sort(regs, ents, sizeof(struct linux_prom64_registers),
 143              cmp_p64, NULL);
 144 }
 145
 146 unsigned long *sparc64_valid_addr_bitmap __read_mostly;
 147
 148 /* Kernel physical address base and size in bytes.  */
 149 unsigned long kern_base __read_mostly;
 150 unsigned long kern_size __read_mostly;
 151
 152 /* get_new_mmu_context() uses "cache + 1".  */
 153 DEFINE_SPINLOCK(ctx_alloc_lock);
 154 unsigned long tlb_context_cache = CTX_FIRST_VERSION - 1;
 155 #define CTX_BMAP_SLOTS (1UL << (CTX_NR_BITS - 6))
 156 unsigned long mmu_context_bmap[CTX_BMAP_SLOTS];
 157
 158 /* Initial ramdisk setup */
 159 extern unsigned long sparc_ramdisk_image64;
 160 extern unsigned int sparc_ramdisk_image;
 161 extern unsigned int sparc_ramdisk_size;
 162
 163 struct page *mem_map_zero __read_mostly;
 164
 165 unsigned int sparc64_highest_unlocked_tlb_ent __read_mostly;
 166
 167 unsigned long sparc64_kern_pri_context __read_mostly;
 168 unsigned long sparc64_kern_pri_nuc_bits __read_mostly;
 169 unsigned long sparc64_kern_sec_context __read_mostly;
 170
 171 int bigkernel = 0;
 172
 173 struct kmem_cache *pgtable_cache __read_mostly;
 174
 175 static void zero_ctor(void *addr, struct kmem_cache *cache, unsigned long flags)
 176 {
 177         clear_page(addr);
 178 }
 179
 180 extern void tsb_cache_init(void);
 181
 182 void pgtable_cache_init(void)
 183 {
 184         pgtable_cache = kmem_cache_create("pgtable_cache",
 185                                           PAGE_SIZE, PAGE_SIZE,
 186                                           SLAB_HWCACHE_ALIGN |
 187                                           SLAB_MUST_HWCACHE_ALIGN,
 188                                           zero_ctor,
 189                                           NULL);
 190         if (!pgtable_cache) {
 191                 prom_printf("Could not create pgtable_cache\n");
 192                 prom_halt();
 193         }
 194         tsb_cache_init();
 195 }
 196
 197 #ifdef CONFIG_DEBUG_DCFLUSH
 198 atomic_t dcpage_flushes = ATOMIC_INIT(0);
 199 #ifdef CONFIG_SMP
 200 atomic_t dcpage_flushes_xcall = ATOMIC_INIT(0);
 201 #endif
 202 #endif
 203
 204 inline void flush_dcache_page_impl(struct page *page)
 205 {
 206         BUG_ON(tlb_type == hypervisor);
 207 #ifdef CONFIG_DEBUG_DCFLUSH
 208         atomic_inc(&dcpage_flushes);
 209 #endif
 210
 211 #ifdef DCACHE_ALIASING_POSSIBLE
 212         __flush_dcache_page(page_address(page),
 213                             ((tlb_type == spitfire) &&
 214                              page_mapping(page) != NULL));
 215 #else
 216         if (page_mapping(page) != NULL &&
 217             tlb_type == spitfire)
 218                 __flush_icache_page(__pa(page_address(page)));
 219 #endif
 220 }
 221
 222 #define PG_dcache_dirty         PG_arch_1
 223 #define PG_dcache_cpu_shift     24UL
 224 #define PG_dcache_cpu_mask      (256UL - 1UL)
 225
 226 #if NR_CPUS > 256
 227 #error D-cache dirty tracking and thread_info->cpu need fixing for > 256 cpus
 228 #endif
 229
 230 #define dcache_dirty_cpu(page) \
 231         (((page)->flags >> PG_dcache_cpu_shift) & PG_dcache_cpu_mask)
 232
 233 static __inline__ void set_dcache_dirty(struct page *page, int this_cpu)
 234 {
 235         unsigned long mask = this_cpu;
 236         unsigned long non_cpu_bits;
 237
 238         non_cpu_bits = ~(PG_dcache_cpu_mask << PG_dcache_cpu_shift);
 239         mask = (mask << PG_dcache_cpu_shift) | (1UL << PG_dcache_dirty);
 240
 241         __asm__ __volatile__("1:\n\t"
 242                              "ldx       [%2], %%g7\n\t"
 243                              "and       %%g7, %1, %%g1\n\t"
 244                              "or        %%g1, %0, %%g1\n\t"
 245                              "casx      [%2], %%g7, %%g1\n\t"
 246                              "cmp       %%g7, %%g1\n\t"
 247                              "membar    #StoreLoad | #StoreStore\n\t"
 248                              "bne,pn    %%xcc, 1b\n\t"
 249                              " nop"
 250                              : /* no outputs */
 251                              : "r" (mask), "r" (non_cpu_bits), "r" (&page->flags)
 252                              : "g1", "g7");
 253 }
 254
 255 static __inline__ void clear_dcache_dirty_cpu(struct page *page, unsigned long cpu)
 256 {
 257         unsigned long mask = (1UL << PG_dcache_dirty);
 258
 259         __asm__ __volatile__("! test_and_clear_dcache_dirty\n"
 260                              "1:\n\t"
 261                              "ldx       [%2], %%g7\n\t"
 262                              "srlx      %%g7, %4, %%g1\n\t"
 263                              "and       %%g1, %3, %%g1\n\t"
 264                              "cmp       %%g1, %0\n\t"
 265                              "bne,pn    %%icc, 2f\n\t"
 266                              " andn     %%g7, %1, %%g1\n\t"
 267                              "casx      [%2], %%g7, %%g1\n\t"
 268                              "cmp       %%g7, %%g1\n\t"
 269                              "membar    #StoreLoad | #StoreStore\n\t"
 270                              "bne,pn    %%xcc, 1b\n\t"
 271                              " nop\n"
 272                              "2:"
 273                              : /* no outputs */
 274                              : "r" (cpu), "r" (mask), "r" (&page->flags),
 275                                "i" (PG_dcache_cpu_mask),
 276                                "i" (PG_dcache_cpu_shift)
 277                              : "g1", "g7");
 278 }
 279
 280 static inline void tsb_insert(struct tsb *ent, unsigned long tag, unsigned long pte)
 281 {
 282         unsigned long tsb_addr = (unsigned long) ent;
 283
 284         if (tlb_type == cheetah_plus || tlb_type == hypervisor)
 285                 tsb_addr = __pa(tsb_addr);
 286
 287         __tsb_insert(tsb_addr, tag, pte);
 288 }
 289
 290 unsigned long _PAGE_ALL_SZ_BITS __read_mostly;
 291 unsigned long _PAGE_SZBITS __read_mostly;
 292
 293 void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t pte)
 294 {
 295         struct mm_struct *mm;
 296         struct tsb *tsb;
 297         unsigned long tag, flags;
 298         unsigned long tsb_index, tsb_hash_shift;
 299
 300         if (tlb_type != hypervisor) {
 301                 unsigned long pfn = pte_pfn(pte);
 302                 unsigned long pg_flags;
 303                 struct page *page;
 304
 305                 if (pfn_valid(pfn) &&
 306                     (page = pfn_to_page(pfn), page_mapping(page)) &&
 307                     ((pg_flags = page->flags) & (1UL << PG_dcache_dirty))) {
 308                         int cpu = ((pg_flags >> PG_dcache_cpu_shift) &
 309                                    PG_dcache_cpu_mask);
 310                         int this_cpu = get_cpu();
 311
 312                         /* This is just to optimize away some function calls
 313                          * in the SMP case.
 314                          */
 315                         if (cpu == this_cpu)
 316                                 flush_dcache_page_impl(page);
 317                         else
 318                                 smp_flush_dcache_page_impl(page, cpu);
 319
 320                         clear_dcache_dirty_cpu(page, cpu);
 321
 322                         put_cpu();
 323                 }
 324         }
 325
 326         mm = vma->vm_mm;
 327
 328         tsb_index = MM_TSB_BASE;
 329         tsb_hash_shift = PAGE_SHIFT;
 330
 331         spin_lock_irqsave(&mm->context.lock, flags);
 332
 333 #ifdef CONFIG_HUGETLB_PAGE
 334         if (mm->context.tsb_block[MM_TSB_HUGE].tsb != NULL) {
 335                 if ((tlb_type == hypervisor &&
 336                      (pte_val(pte) & _PAGE_SZALL_4V) == _PAGE_SZHUGE_4V) ||
 337                     (tlb_type != hypervisor &&
 338                      (pte_val(pte) & _PAGE_SZALL_4U) == _PAGE_SZHUGE_4U)) {
 339                         tsb_index = MM_TSB_HUGE;
 340                         tsb_hash_shift = HPAGE_SHIFT;
 341                 }
 342         }
 343 #endif
 344
 345         tsb = mm->context.tsb_block[tsb_index].tsb;
 346         tsb += ((address >> tsb_hash_shift) &
 347                 (mm->context.tsb_block[tsb_index].tsb_nentries - 1UL));
 348         tag = (address >> 22UL);
 349         tsb_insert(tsb, tag, pte_val(pte));
 350
 351         spin_unlock_irqrestore(&mm->context.lock, flags);
 352 }
 353
 354 void flush_dcache_page(struct page *page)
 355 {
 356         struct address_space *mapping;
 357         int this_cpu;
 358
 359         if (tlb_type == hypervisor)
 360                 return;
 361
 362         /* Do not bother with the expensive D-cache flush if it
 363          * is merely the zero page.  The 'bigcore' testcase in GDB
 364          * causes this case to run millions of times.
 365          */
 366         if (page == ZERO_PAGE(0))
 367                 return;
 368
 369         this_cpu = get_cpu();
 370
 371         mapping = page_mapping(page);
 372         if (mapping && !mapping_mapped(mapping)) {
 373                 int dirty = test_bit(PG_dcache_dirty, &page->flags);
 374                 if (dirty) {
 375                         int dirty_cpu = dcache_dirty_cpu(page);
 376
 377                         if (dirty_cpu == this_cpu)
 378                                 goto out;
 379                         smp_flush_dcache_page_impl(page, dirty_cpu);
 380                 }
 381                 set_dcache_dirty(page, this_cpu);
 382         } else {
 383                 /* We could delay the flush for the !page_mapping
 384                  * case too.  But that case is for exec env/arg
 385                  * pages and those are %99 certainly going to get
 386                  * faulted into the tlb (and thus flushed) anyways.
 387                  */
 388                 flush_dcache_page_impl(page);
 389         }
 390
 391 out:
 392         put_cpu();
 393 }
 394
 395 void __kprobes flush_icache_range(unsigned long start, unsigned long end)
 396 {
 397         /* Cheetah and Hypervisor platform cpus have coherent I-cache. */
 398         if (tlb_type == spitfire) {
 399                 unsigned long kaddr;
 400
 401                 /* This code only runs on Spitfire cpus so this is
 402                  * why we can assume _PAGE_PADDR_4U.
 403                  */
 404                 for (kaddr = start; kaddr < end; kaddr += PAGE_SIZE) {
 405                         unsigned long paddr, mask = _PAGE_PADDR_4U;
 406
 407                         if (kaddr >= PAGE_OFFSET)
 408                                 paddr = kaddr & mask;
 409                         else {
 410                                 pgd_t *pgdp = pgd_offset_k(kaddr);
 411                                 pud_t *pudp = pud_offset(pgdp, kaddr);
 412                                 pmd_t *pmdp = pmd_offset(pudp, kaddr);
 413                                 pte_t *ptep = pte_offset_kernel(pmdp, kaddr);
 414
 415                                 paddr = pte_val(*ptep) & mask;
 416                         }
 417                         __flush_icache_page(paddr);
 418                 }
 419         }
 420 }
 421
 422 void show_mem(void)
 423 {
 424         unsigned long total = 0, reserved = 0;
 425         unsigned long shared = 0, cached = 0;
 426         pg_data_t *pgdat;
 427
 428         printk(KERN_INFO "Mem-info:\n");
 429         show_free_areas();
 430         printk(KERN_INFO "Free swap:       %6ldkB\n",
 431                nr_swap_pages << (PAGE_SHIFT-10));
 432         for_each_online_pgdat(pgdat) {
 433                 unsigned long i, flags;
 434
 435                 pgdat_resize_lock(pgdat, &flags);
 436                 for (i = 0; i < pgdat->node_spanned_pages; i++) {
 437                         struct page *page = pgdat_page_nr(pgdat, i);
 438                         total++;
 439                         if (PageReserved(page))
 440                                 reserved++;
 441                         else if (PageSwapCache(page))
 442                                 cached++;
 443                         else if (page_count(page))
 444                                 shared += page_count(page) - 1;
 445                 }
 446                 pgdat_resize_unlock(pgdat, &flags);
 447         }
 448
 449         printk(KERN_INFO "%lu pages of RAM\n", total);
 450         printk(KERN_INFO "%lu reserved pages\n", reserved);
 451         printk(KERN_INFO "%lu pages shared\n", shared);
 452         printk(KERN_INFO "%lu pages swap cached\n", cached);
 453
 454         printk(KERN_INFO "%lu pages dirty\n",
 455                global_page_state(NR_FILE_DIRTY));
 456         printk(KERN_INFO "%lu pages writeback\n",
 457                global_page_state(NR_WRITEBACK));
 458         printk(KERN_INFO "%lu pages mapped\n",
 459                global_page_state(NR_FILE_MAPPED));
 460         printk(KERN_INFO "%lu pages slab\n",
 461                 global_page_state(NR_SLAB_RECLAIMABLE) +
 462                 global_page_state(NR_SLAB_UNRECLAIMABLE));
 463         printk(KERN_INFO "%lu pages pagetables\n",
 464                global_page_state(NR_PAGETABLE));
 465 }
 466
 467 void mmu_info(struct seq_file *m)
 468 {
 469         if (tlb_type == cheetah)
 470                 seq_printf(m, "MMU Type\t: Cheetah\n");
 471         else if (tlb_type == cheetah_plus)
 472                 seq_printf(m, "MMU Type\t: Cheetah+\n");
 473         else if (tlb_type == spitfire)
 474                 seq_printf(m, "MMU Type\t: Spitfire\n");
 475         else if (tlb_type == hypervisor)
 476                 seq_printf(m, "MMU Type\t: Hypervisor (sun4v)\n");
 477         else
 478                 seq_printf(m, "MMU Type\t: ???\n");
 479
 480 #ifdef CONFIG_DEBUG_DCFLUSH
 481         seq_printf(m, "DCPageFlushes\t: %d\n",
 482                    atomic_read(&dcpage_flushes));
 483 #ifdef CONFIG_SMP
 484         seq_printf(m, "DCPageFlushesXC\t: %d\n",
 485                    atomic_read(&dcpage_flushes_xcall));
 486 #endif /* CONFIG_SMP */
 487 #endif /* CONFIG_DEBUG_DCFLUSH */
 488 }
 489
 490 struct linux_prom_translation {
 491         unsigned long virt;
 492         unsigned long size;
 493         unsigned long data;
 494 };
 495
 496 /* Exported for kernel TLB miss handling in ktlb.S */
 497 struct linux_prom_translation prom_trans[512] __read_mostly;
 498 unsigned int prom_trans_ents __read_mostly;
 499
 500 /* Exported for SMP bootup purposes. */
 501 unsigned long kern_locked_tte_data;
 502
 503 /* The obp translations are saved based on 8k pagesize, since obp can
 504  * use a mixture of pagesizes. Misses to the LOW_OBP_ADDRESS ->
 505  * HI_OBP_ADDRESS range are handled in ktlb.S.
 506  */
 507 static inline int in_obp_range(unsigned long vaddr)
 508 {
 509         return (vaddr >= LOW_OBP_ADDRESS &&
 510                 vaddr < HI_OBP_ADDRESS);
 511 }
 512
 513 static int cmp_ptrans(const void *a, const void *b)
 514 {
 515         const struct linux_prom_translation *x = a, *y = b;
 516
 517         if (x->virt > y->virt)
 518                 return 1;
 519         if (x->virt < y->virt)
 520                 return -1;
 521         return 0;
 522 }
 523
 524 /* Read OBP translations property into 'prom_trans[]'.  */
 525 static void __init read_obp_translations(void)
 526 {
 527         int n, node, ents, first, last, i;
 528
 529         node = prom_finddevice("/virtual-memory");
 530         n = prom_getproplen(node, "translations");
 531         if (unlikely(n == 0 || n == -1)) {
 532                 prom_printf("prom_mappings: Couldn't get size.\n");
 533                 prom_halt();
 534         }
 535         if (unlikely(n > sizeof(prom_trans))) {
 536                 prom_printf("prom_mappings: Size %Zd is too big.\n", n);
 537                 prom_halt();
 538         }
 539
 540         if ((n = prom_getproperty(node, "translations",
 541                                   (char *)&prom_trans[0],
 542                                   sizeof(prom_trans))) == -1) {
 543                 prom_printf("prom_mappings: Couldn't get property.\n");
 544                 prom_halt();
 545         }
 546
 547         n = n / sizeof(struct linux_prom_translation);
 548
 549         ents = n;
 550
 551         sort(prom_trans, ents, sizeof(struct linux_prom_translation),
 552              cmp_ptrans, NULL);
 553
 554         /* Now kick out all the non-OBP entries.  */
 555         for (i = 0; i < ents; i++) {
 556                 if (in_obp_range(prom_trans[i].virt))
 557                         break;
 558         }
 559         first = i;
 560         for (; i < ents; i++) {
 561                 if (!in_obp_range(prom_trans[i].virt))
 562                         break;
 563         }
 564         last = i;
 565
 566         for (i = 0; i < (last - first); i++) {
 567                 struct linux_prom_translation *src = &prom_trans[i + first];
 568                 struct linux_prom_translation *dest = &prom_trans[i];
 569
 570                 *dest = *src;
 571         }
 572         for (; i < ents; i++) {
 573                 struct linux_prom_translation *dest = &prom_trans[i];
 574                 dest->virt = dest->size = dest->data = 0x0UL;
 575         }
 576
 577         prom_trans_ents = last - first;
 578
 579         if (tlb_type == spitfire) {
 580                 /* Clear diag TTE bits. */
 581                 for (i = 0; i < prom_trans_ents; i++)
 582                         prom_trans[i].data &= ~0x0003fe0000000000UL;
 583         }
 584 }
 585
 586 static void __init hypervisor_tlb_lock(unsigned long vaddr,
 587                                        unsigned long pte,
 588                                        unsigned long mmu)
 589 {
 590         register unsigned long func asm("%o5");
 591         register unsigned long arg0 asm("%o0");
 592         register unsigned long arg1 asm("%o1");
 593         register unsigned long arg2 asm("%o2");
 594         register unsigned long arg3 asm("%o3");
 595
 596         func = HV_FAST_MMU_MAP_PERM_ADDR;
 597         arg0 = vaddr;
 598         arg1 = 0;
 599         arg2 = pte;
 600         arg3 = mmu;
 601         __asm__ __volatile__("ta        0x80"
 602                              : "=&r" (func), "=&r" (arg0),
 603                                "=&r" (arg1), "=&r" (arg2),
 604                                "=&r" (arg3)
 605                              : "0" (func), "1" (arg0), "2" (arg1),
 606                                "3" (arg2), "4" (arg3));
 607         if (arg0 != 0) {
 608                 prom_printf("hypervisor_tlb_lock[%lx:%lx:%lx:%lx]: "
 609                             "errors with %lx\n", vaddr, 0, pte, mmu, arg0);
 610                 prom_halt();
 611         }
 612 }
 613
 614 static unsigned long kern_large_tte(unsigned long paddr);
 615
 616 static void __init remap_kernel(void)
 617 {
 618         unsigned long phys_page, tte_vaddr, tte_data;
 619         int tlb_ent = sparc64_highest_locked_tlbent();
 620
 621         tte_vaddr = (unsigned long) KERNBASE;
 622         phys_page = (prom_boot_mapping_phys_low >> 22UL) << 22UL;
 623         tte_data = kern_large_tte(phys_page);
 624
 625         kern_locked_tte_data = tte_data;
 626
 627         /* Now lock us into the TLBs via Hypervisor or OBP. */
 628         if (tlb_type == hypervisor) {
 629                 hypervisor_tlb_lock(tte_vaddr, tte_data, HV_MMU_DMMU);
 630                 hypervisor_tlb_lock(tte_vaddr, tte_data, HV_MMU_IMMU);
 631                 if (bigkernel) {
 632                         tte_vaddr += 0x400000;
 633                         tte_data += 0x400000;
 634                         hypervisor_tlb_lock(tte_vaddr, tte_data, HV_MMU_DMMU);
 635                         hypervisor_tlb_lock(tte_vaddr, tte_data, HV_MMU_IMMU);
 636                 }
 637         } else {
 638                 prom_dtlb_load(tlb_ent, tte_data, tte_vaddr);
 639                 prom_itlb_load(tlb_ent, tte_data, tte_vaddr);
 640                 if (bigkernel) {
 641                         tlb_ent -= 1;
 642                         prom_dtlb_load(tlb_ent,
 643                                        tte_data + 0x400000,
 644                                        tte_vaddr + 0x400000);
 645                         prom_itlb_load(tlb_ent,
 646                                        tte_data + 0x400000,
 647                                        tte_vaddr + 0x400000);
 648                 }
 649                 sparc64_highest_unlocked_tlb_ent = tlb_ent - 1;
 650         }
 651         if (tlb_type == cheetah_plus) {
 652                 sparc64_kern_pri_context = (CTX_CHEETAH_PLUS_CTX0 |
 653                                             CTX_CHEETAH_PLUS_NUC);
 654                 sparc64_kern_pri_nuc_bits = CTX_CHEETAH_PLUS_NUC;
 655                 sparc64_kern_sec_context = CTX_CHEETAH_PLUS_CTX0;
 656         }
 657 }
 658
 659
 660 static void __init inherit_prom_mappings(void)
 661 {
 662         read_obp_translations();
 663
 664         /* Now fixup OBP's idea about where we really are mapped. */
 665         prom_printf("Remapping the kernel... ");
 666         remap_kernel();
 667         prom_printf("done.\n");
 668 }
 669
 670 void prom_world(int enter)
 671 {
 672         if (!enter)
 673                 set_fs((mm_segment_t) { get_thread_current_ds() });
 674
 675         __asm__ __volatile__("flushw");
 676 }
 677
 678 #ifdef DCACHE_ALIASING_POSSIBLE
 679 void __flush_dcache_range(unsigned long start, unsigned long end)
 680 {
 681         unsigned long va;
 682
 683         if (tlb_type == spitfire) {
 684                 int n = 0;
 685
 686                 for (va = start; va < end; va += 32) {
 687                         spitfire_put_dcache_tag(va & 0x3fe0, 0x0);
 688                         if (++n >= 512)
 689                                 break;
 690                 }
 691         } else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
 692                 start = __pa(start);
 693                 end = __pa(end);
 694                 for (va = start; va < end; va += 32)
 695                         __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
 696                                              "membar #Sync"
 697                                              : /* no outputs */
 698                                              : "r" (va),
 699                                                "i" (ASI_DCACHE_INVALIDATE));
 700         }
 701 }
 702 #endif /* DCACHE_ALIASING_POSSIBLE */
 703
 704 /* Caller does TLB context flushing on local CPU if necessary.
 705  * The caller also ensures that CTX_VALID(mm->context) is false.
 706  *
 707  * We must be careful about boundary cases so that we never
 708  * let the user have CTX 0 (nucleus) or we ever use a CTX
 709  * version of zero (and thus NO_CONTEXT would not be caught
 710  * by version mis-match tests in mmu_context.h).
 711  *
 712  * Always invoked with interrupts disabled.
 713  */
 714 void get_new_mmu_context(struct mm_struct *mm)
 715 {
 716         unsigned long ctx, new_ctx;
 717         unsigned long orig_pgsz_bits;
 718         unsigned long flags;
 719         int new_version;
 720
 721         spin_lock_irqsave(&ctx_alloc_lock, flags);
 722         orig_pgsz_bits = (mm->context.sparc64_ctx_val & CTX_PGSZ_MASK);
 723         ctx = (tlb_context_cache + 1) & CTX_NR_MASK;
 724         new_ctx = find_next_zero_bit(mmu_context_bmap, 1 << CTX_NR_BITS, ctx);
 725         new_version = 0;
 726         if (new_ctx >= (1 << CTX_NR_BITS)) {
 727                 new_ctx = find_next_zero_bit(mmu_context_bmap, ctx, 1);
 728                 if (new_ctx >= ctx) {
 729                         int i;
 730                         new_ctx = (tlb_context_cache & CTX_VERSION_MASK) +
 731                                 CTX_FIRST_VERSION;
 732                         if (new_ctx == 1)
 733                                 new_ctx = CTX_FIRST_VERSION;
 734
 735                         /* Don't call memset, for 16 entries that's just
 736                          * plain silly...
 737                          */
 738                         mmu_context_bmap[0] = 3;
 739                         mmu_context_bmap[1] = 0;
 740                         mmu_context_bmap[2] = 0;
 741                         mmu_context_bmap[3] = 0;
 742                         for (i = 4; i < CTX_BMAP_SLOTS; i += 4) {
 743                                 mmu_context_bmap[i + 0] = 0;
 744                                 mmu_context_bmap[i + 1] = 0;
 745                                 mmu_context_bmap[i + 2] = 0;
 746                                 mmu_context_bmap[i + 3] = 0;
 747                         }
 748                         new_version = 1;
 749                         goto out;
 750                 }
 751         }
 752         mmu_context_bmap[new_ctx>>6] |= (1UL << (new_ctx & 63));
 753         new_ctx |= (tlb_context_cache & CTX_VERSION_MASK);
 754 out:
 755         tlb_context_cache = new_ctx;
 756         mm->context.sparc64_ctx_val = new_ctx | orig_pgsz_bits;
 757         spin_unlock_irqrestore(&ctx_alloc_lock, flags);
 758
 759         if (unlikely(new_version))
 760                 smp_new_mmu_context_version();
 761 }
 762
 763 void sparc_ultra_dump_itlb(void)
 764 {
 765         int slot;
 766
 767         if (tlb_type == spitfire) {
 768                 printk ("Contents of itlb: ");
 769                 for (slot = 0; slot < 14; slot++) printk ("    ");
 770                 printk ("%2x:%016lx,%016lx\n",
 771                         0,
 772                         spitfire_get_itlb_tag(0), spitfire_get_itlb_data(0));
 773                 for (slot = 1; slot < 64; slot+=3) {
 774                         printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx %2x:%016lx,%016lx\n",
 775                                 slot,
 776                                 spitfire_get_itlb_tag(slot), spitfire_get_itlb_data(slot),
 777                                 slot+1,
 778                                 spitfire_get_itlb_tag(slot+1), spitfire_get_itlb_data(slot+1),
 779                                 slot+2,
 780                                 spitfire_get_itlb_tag(slot+2), spitfire_get_itlb_data(slot+2));
 781                 }
 782         } else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
 783                 printk ("Contents of itlb0:\n");
 784                 for (slot = 0; slot < 16; slot+=2) {
 785                         printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
 786                                 slot,
 787                                 cheetah_get_litlb_tag(slot), cheetah_get_litlb_data(slot),
 788                                 slot+1,
 789                                 cheetah_get_litlb_tag(slot+1), cheetah_get_litlb_data(slot+1));
 790                 }
 791                 printk ("Contents of itlb2:\n");
 792                 for (slot = 0; slot < 128; slot+=2) {
 793                         printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
 794                                 slot,
 795                                 cheetah_get_itlb_tag(slot), cheetah_get_itlb_data(slot),
 796                                 slot+1,
 797                                 cheetah_get_itlb_tag(slot+1), cheetah_get_itlb_data(slot+1));
 798                 }
 799         }
 800 }
 801
 802 void sparc_ultra_dump_dtlb(void)
 803 {
 804         int slot;
 805
 806         if (tlb_type == spitfire) {
 807                 printk ("Contents of dtlb: ");
 808                 for (slot = 0; slot < 14; slot++) printk ("    ");
 809                 printk ("%2x:%016lx,%016lx\n", 0,
 810                         spitfire_get_dtlb_tag(0), spitfire_get_dtlb_data(0));
 811                 for (slot = 1; slot < 64; slot+=3) {
 812                         printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx %2x:%016lx,%016lx\n",
 813                                 slot,
 814                                 spitfire_get_dtlb_tag(slot), spitfire_get_dtlb_data(slot),
 815                                 slot+1,
 816                                 spitfire_get_dtlb_tag(slot+1), spitfire_get_dtlb_data(slot+1),
 817                                 slot+2,
 818                                 spitfire_get_dtlb_tag(slot+2), spitfire_get_dtlb_data(slot+2));
 819                 }
 820         } else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
 821                 printk ("Contents of dtlb0:\n");
 822                 for (slot = 0; slot < 16; slot+=2) {
 823                         printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
 824                                 slot,
 825                                 cheetah_get_ldtlb_tag(slot), cheetah_get_ldtlb_data(slot),
 826                                 slot+1,
 827                                 cheetah_get_ldtlb_tag(slot+1), cheetah_get_ldtlb_data(slot+1));
 828                 }
 829                 printk ("Contents of dtlb2:\n");
 830                 for (slot = 0; slot < 512; slot+=2) {
 831                         printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
 832                                 slot,
 833                                 cheetah_get_dtlb_tag(slot, 2), cheetah_get_dtlb_data(slot, 2),
 834                                 slot+1,
 835                                 cheetah_get_dtlb_tag(slot+1, 2), cheetah_get_dtlb_data(slot+1, 2));
 836                 }
 837                 if (tlb_type == cheetah_plus) {
 838                         printk ("Contents of dtlb3:\n");
 839                         for (slot = 0; slot < 512; slot+=2) {
 840                                 printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
 841                                         slot,
 842                                         cheetah_get_dtlb_tag(slot, 3), cheetah_get_dtlb_data(slot, 3),
 843                                         slot+1,
 844                                         cheetah_get_dtlb_tag(slot+1, 3), cheetah_get_dtlb_data(slot+1, 3));
 845                         }
 846                 }
 847         }
 848 }
 849
 850 extern unsigned long cmdline_memory_size;
 851
 852 /* Find a free area for the bootmem map, avoiding the kernel image
 853  * and the initial ramdisk.
 854  */
 855 static unsigned long __init choose_bootmap_pfn(unsigned long start_pfn,
 856                                                unsigned long end_pfn)
 857 {
 858         unsigned long avoid_start, avoid_end, bootmap_size;
 859         int i;
 860
 861         bootmap_size = ((end_pfn - start_pfn) + 7) / 8;
 862         bootmap_size = ALIGN(bootmap_size, sizeof(long));
 863
 864         avoid_start = avoid_end = 0;
 865 #ifdef CONFIG_BLK_DEV_INITRD
 866         avoid_start = initrd_start;
 867         avoid_end = PAGE_ALIGN(initrd_end);
 868 #endif
 869
 870 #ifdef CONFIG_DEBUG_BOOTMEM
 871         prom_printf("choose_bootmap_pfn: kern[%lx:%lx] avoid[%lx:%lx]\n",
 872                     kern_base, PAGE_ALIGN(kern_base + kern_size),
 873                     avoid_start, avoid_end);
 874 #endif
 875         for (i = 0; i < pavail_ents; i++) {
 876                 unsigned long start, end;
 877
 878                 start = pavail[i].phys_addr;
 879                 end = start + pavail[i].reg_size;
 880
 881                 while (start < end) {
 882                         if (start >= kern_base &&
 883                             start < PAGE_ALIGN(kern_base + kern_size)) {
 884                                 start = PAGE_ALIGN(kern_base + kern_size);
 885                                 continue;
 886                         }
 887                         if (start >= avoid_start && start < avoid_end) {
 888                                 start = avoid_end;
 889                                 continue;
 890                         }
 891
 892                         if ((end - start) < bootmap_size)
 893                                 break;
 894
 895                         if (start < kern_base &&
 896                             (start + bootmap_size) > kern_base) {
 897                                 start = PAGE_ALIGN(kern_base + kern_size);
 898                                 continue;
 899                         }
 900
 901                         if (start < avoid_start &&
 902                             (start + bootmap_size) > avoid_start) {
 903                                 start = avoid_end;
 904                                 continue;
 905                         }
 906
 907                         /* OK, it doesn't overlap anything, use it.  */
 908 #ifdef CONFIG_DEBUG_BOOTMEM
 909                         prom_printf("choose_bootmap_pfn: Using %lx [%lx]\n",
 910                                     start >> PAGE_SHIFT, start);
 911 #endif
 912                         return start >> PAGE_SHIFT;
 913                 }
 914         }
 915
 916         prom_printf("Cannot find free area for bootmap, aborting.\n");
 917         prom_halt();
 918 }
 919
 920 static void __init trim_pavail(unsigned long *cur_size_p,
 921                                unsigned long *end_of_phys_p)
 922 {
 923         unsigned long to_trim = *cur_size_p - cmdline_memory_size;
 924         unsigned long avoid_start, avoid_end;
 925         int i;
 926
 927         to_trim = PAGE_ALIGN(to_trim);
 928
 929         avoid_start = avoid_end = 0;
 930 #ifdef CONFIG_BLK_DEV_INITRD
 931         avoid_start = initrd_start;
 932         avoid_end = PAGE_ALIGN(initrd_end);
 933 #endif
 934
 935         /* Trim some pavail[] entries in order to satisfy the
 936          * requested "mem=xxx" kernel command line specification.
 937          *
 938          * We must not trim off the kernel image area nor the
 939          * initial ramdisk range (if any).  Also, we must not trim
 940          * any pavail[] entry down to zero in order to preserve
 941          * the invariant that all pavail[] entries have a non-zero
 942          * size which is assumed by all of the code in here.
 943          */
 944         for (i = 0; i < pavail_ents; i++) {
 945                 unsigned long start, end, kern_end;
 946                 unsigned long trim_low, trim_high, n;
 947
 948                 kern_end = PAGE_ALIGN(kern_base + kern_size);
 949
 950                 trim_low = start = pavail[i].phys_addr;
 951                 trim_high = end = start + pavail[i].reg_size;
 952
 953                 if (kern_base >= start &&
 954                     kern_base < end) {
 955                         trim_low = kern_base;
 956                         if (kern_end >= end)
 957                                 continue;
 958                 }
 959                 if (kern_end >= start &&
 960                     kern_end < end) {
 961                         trim_high = kern_end;
 962                 }
 963                 if (avoid_start &&
 964                     avoid_start >= start &&
 965                     avoid_start < end) {
 966                         if (trim_low > avoid_start)
 967                                 trim_low = avoid_start;
 968                         if (avoid_end >= end)
 969                                 continue;
 970                 }
 971                 if (avoid_end &&
 972                     avoid_end >= start &&
 973                     avoid_end < end) {
 974                         if (trim_high < avoid_end)
 975                                 trim_high = avoid_end;
 976                 }
 977
 978                 if (trim_high <= trim_low)
 979                         continue;
 980
 981                 if (trim_low == start && trim_high == end) {
 982                         /* Whole chunk is available for trimming.
 983                          * Trim all except one page, in order to keep
 984                          * entry non-empty.
 985                          */
 986                         n = (end - start) - PAGE_SIZE;
 987                         if (n > to_trim)
 988                                 n = to_trim;
 989
 990                         if (n) {
 991                                 pavail[i].phys_addr += n;
 992                                 pavail[i].reg_size -= n;
 993                                 to_trim -= n;
 994                         }
 995                 } else {
 996                         n = (trim_low - start);
 997                         if (n > to_trim)
 998                                 n = to_trim;
 999
1000                         if (n) {
1001                                 pavail[i].phys_addr += n;
1002                                 pavail[i].reg_size -= n;
1003                                 to_trim -= n;
1004                         }
1005                         if (to_trim) {
1006                                 n = end - trim_high;
1007                                 if (n > to_trim)
1008                                         n = to_trim;
1009                                 if (n) {
1010                                         pavail[i].reg_size -= n;
1011                                         to_trim -= n;
1012                                 }
1013                         }
1014                 }
1015
1016                 if (!to_trim)
1017                         break;
1018         }
1019
1020         /* Recalculate.  */
1021         *cur_size_p = 0UL;
1022         for (i = 0; i < pavail_ents; i++) {
1023                 *end_of_phys_p = pavail[i].phys_addr +
1024                         pavail[i].reg_size;
1025                 *cur_size_p += pavail[i].reg_size;
1026         }
1027 }
1028
1029 static unsigned long __init bootmem_init(unsigned long *pages_avail,
1030                                          unsigned long phys_base)
1031 {
1032         unsigned long bootmap_size, end_pfn;
1033         unsigned long end_of_phys_memory = 0UL;
1034         unsigned long bootmap_pfn, bytes_avail, size;
1035         int i;
1036
1037 #ifdef CONFIG_DEBUG_BOOTMEM
1038         prom_printf("bootmem_init: Scan pavail, ");
1039 #endif
1040
1041         bytes_avail = 0UL;
1042         for (i = 0; i < pavail_ents; i++) {
1043                 end_of_phys_memory = pavail[i].phys_addr +
1044                         pavail[i].reg_size;
1045                 bytes_avail += pavail[i].reg_size;
1046         }
1047
1048         /* Determine the location of the initial ramdisk before trying
1049          * to honor the "mem=xxx" command line argument.  We must know
1050          * where the kernel image and the ramdisk image are so that we
1051          * do not trim those two areas from the physical memory map.
1052          */
1053
1054 #ifdef CONFIG_BLK_DEV_INITRD
1055         /* Now have to check initial ramdisk, so that bootmap does not overwrite it */
1056         if (sparc_ramdisk_image || sparc_ramdisk_image64) {
1057                 unsigned long ramdisk_image = sparc_ramdisk_image ?
1058                         sparc_ramdisk_image : sparc_ramdisk_image64;
1059                 ramdisk_image -= KERNBASE;
1060                 initrd_start = ramdisk_image + phys_base;
1061                 initrd_end = initrd_start + sparc_ramdisk_size;
1062                 if (initrd_end > end_of_phys_memory) {
1063                         printk(KERN_CRIT "initrd extends beyond end of memory "
1064                                          "(0x%016lx > 0x%016lx)\ndisabling initrd\n",
1065                                initrd_end, end_of_phys_memory);
1066                         initrd_start = 0;
1067                         initrd_end = 0;
1068                 }
1069         }
1070 #endif
1071
1072         if (cmdline_memory_size &&
1073             bytes_avail > cmdline_memory_size)
1074                 trim_pavail(&bytes_avail,
1075                             &end_of_phys_memory);
1076
1077         *pages_avail = bytes_avail >> PAGE_SHIFT;
1078
1079         end_pfn = end_of_phys_memory >> PAGE_SHIFT;
1080
1081         /* Initialize the boot-time allocator. */
1082         max_pfn = max_low_pfn = end_pfn;
1083         min_low_pfn = (phys_base >> PAGE_SHIFT);
1084
1085         bootmap_pfn = choose_bootmap_pfn(min_low_pfn, end_pfn);
1086
1087 #ifdef CONFIG_DEBUG_BOOTMEM
1088         prom_printf("init_bootmem(min[%lx], bootmap[%lx], max[%lx])\n",
1089                     min_low_pfn, bootmap_pfn, max_low_pfn);
1090 #endif
1091         bootmap_size = init_bootmem_node(NODE_DATA(0), bootmap_pfn,
1092                                          min_low_pfn, end_pfn);
1093
1094         /* Now register the available physical memory with the
1095          * allocator.
1096          */
1097         for (i = 0; i < pavail_ents; i++) {
1098 #ifdef CONFIG_DEBUG_BOOTMEM
1099                 prom_printf("free_bootmem(pavail:%d): base[%lx] size[%lx]\n",
1100                             i, pavail[i].phys_addr, pavail[i].reg_size);
1101 #endif
1102                 free_bootmem(pavail[i].phys_addr, pavail[i].reg_size);
1103         }
1104
1105 #ifdef CONFIG_BLK_DEV_INITRD
1106         if (initrd_start) {
1107                 size = initrd_end - initrd_start;
1108
1109                 /* Resert the initrd image area. */
1110 #ifdef CONFIG_DEBUG_BOOTMEM
1111                 prom_printf("reserve_bootmem(initrd): base[%llx] size[%lx]\n",
1112                         initrd_start, initrd_end);
1113 #endif
1114                 reserve_bootmem(initrd_start, size);
1115                 *pages_avail -= PAGE_ALIGN(size) >> PAGE_SHIFT;
1116
1117                 initrd_start += PAGE_OFFSET;
1118                 initrd_end += PAGE_OFFSET;
1119         }
1120 #endif
1121         /* Reserve the kernel text/data/bss. */
1122 #ifdef CONFIG_DEBUG_BOOTMEM
1123         prom_printf("reserve_bootmem(kernel): base[%lx] size[%lx]\n", kern_base, kern_size);
1124 #endif
1125         reserve_bootmem(kern_base, kern_size);
1126         *pages_avail -= PAGE_ALIGN(kern_size) >> PAGE_SHIFT;
1127
1128         /* Reserve the bootmem map.   We do not account for it
1129          * in pages_avail because we will release that memory
1130          * in free_all_bootmem.
1131          */
1132         size = bootmap_size;
1133 #ifdef CONFIG_DEBUG_BOOTMEM
1134         prom_printf("reserve_bootmem(bootmap): base[%lx] size[%lx]\n",
1135                     (bootmap_pfn << PAGE_SHIFT), size);
1136 #endif
1137         reserve_bootmem((bootmap_pfn << PAGE_SHIFT), size);
1138         *pages_avail -= PAGE_ALIGN(size) >> PAGE_SHIFT;
1139
1140         for (i = 0; i < pavail_ents; i++) {
1141                 unsigned long start_pfn, end_pfn;
1142
1143                 start_pfn = pavail[i].phys_addr >> PAGE_SHIFT;
1144                 end_pfn = (start_pfn + (pavail[i].reg_size >> PAGE_SHIFT));
1145 #ifdef CONFIG_DEBUG_BOOTMEM
1146                 prom_printf("memory_present(0, %lx, %lx)\n",
1147                             start_pfn, end_pfn);
1148 #endif
1149                 memory_present(0, start_pfn, end_pfn);
1150         }
1151
1152         sparse_init();
1153
1154         return end_pfn;
1155 }
1156
1157 static struct linux_prom64_registers pall[MAX_BANKS] __initdata;
1158 static int pall_ents __initdata;
1159
1160 #ifdef CONFIG_DEBUG_PAGEALLOC
1161 static unsigned long kernel_map_range(unsigned long pstart, unsigned long pend, pgprot_t prot)
1162 {
1163         unsigned long vstart = PAGE_OFFSET + pstart;
1164         unsigned long vend = PAGE_OFFSET + pend;
1165         unsigned long alloc_bytes = 0UL;
1166
1167         if ((vstart & ~PAGE_MASK) || (vend & ~PAGE_MASK)) {
1168                 prom_printf("kernel_map: Unaligned physmem[%lx:%lx]\n",
1169                             vstart, vend);
1170                 prom_halt();
1171         }
1172
1173         while (vstart < vend) {
1174                 unsigned long this_end, paddr = __pa(vstart);
1175                 pgd_t *pgd = pgd_offset_k(vstart);
1176                 pud_t *pud;
1177                 pmd_t *pmd;
1178                 pte_t *pte;
1179
1180                 pud = pud_offset(pgd, vstart);
1181                 if (pud_none(*pud)) {
1182                         pmd_t *new;
1183
1184                         new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
1185                         alloc_bytes += PAGE_SIZE;
1186                         pud_populate(&init_mm, pud, new);
1187                 }
1188
1189                 pmd = pmd_offset(pud, vstart);
1190                 if (!pmd_present(*pmd)) {
1191                         pte_t *new;
1192
1193                         new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
1194                         alloc_bytes += PAGE_SIZE;
1195                         pmd_populate_kernel(&init_mm, pmd, new);
1196                 }
1197
1198                 pte = pte_offset_kernel(pmd, vstart);
1199                 this_end = (vstart + PMD_SIZE) & PMD_MASK;
1200                 if (this_end > vend)
1201                         this_end = vend;
1202
1203                 while (vstart < this_end) {
1204                         pte_val(*pte) = (paddr | pgprot_val(prot));
1205
1206                         vstart += PAGE_SIZE;
1207                         paddr += PAGE_SIZE;
1208                         pte++;
1209                 }
1210         }
1211
1212         return alloc_bytes;
1213 }
1214
1215 extern unsigned int kvmap_linear_patch[1];
1216 #endif /* CONFIG_DEBUG_PAGEALLOC */
1217
1218 static void __init mark_kpte_bitmap(unsigned long start, unsigned long end)
1219 {
1220         const unsigned long shift_256MB = 28;
1221         const unsigned long mask_256MB = ((1UL << shift_256MB) - 1UL);
1222         const unsigned long size_256MB = (1UL << shift_256MB);
1223
1224         while (start < end) {
1225                 long remains;
1226
1227                 remains = end - start;
1228                 if (remains < size_256MB)
1229                         break;
1230
1231                 if (start & mask_256MB) {
1232                         start = (start + size_256MB) & ~mask_256MB;
1233                         continue;
1234                 }
1235
1236                 while (remains >= size_256MB) {
1237                         unsigned long index = start >> shift_256MB;
1238
1239                         __set_bit(index, kpte_linear_bitmap);
1240
1241                         start += size_256MB;
1242                         remains -= size_256MB;
1243                 }
1244         }
1245 }
1246
1247 static void __init kernel_physical_mapping_init(void)
1248 {
1249         unsigned long i;
1250 #ifdef CONFIG_DEBUG_PAGEALLOC
1251         unsigned long mem_alloced = 0UL;
1252 #endif
1253
1254         read_obp_memory("reg", &pall[0], &pall_ents);
1255
1256         for (i = 0; i < pall_ents; i++) {
1257                 unsigned long phys_start, phys_end;
1258
1259                 phys_start = pall[i].phys_addr;
1260                 phys_end = phys_start + pall[i].reg_size;
1261
1262                 mark_kpte_bitmap(phys_start, phys_end);
1263
1264 #ifdef CONFIG_DEBUG_PAGEALLOC
1265                 mem_alloced += kernel_map_range(phys_start, phys_end,
1266                                                 PAGE_KERNEL);
1267 #endif
1268         }
1269
1270 #ifdef CONFIG_DEBUG_PAGEALLOC
1271         printk("Allocated %ld bytes for kernel page tables.\n",
1272                mem_alloced);
1273
1274         kvmap_linear_patch[0] = 0x01000000; /* nop */
1275         flushi(&kvmap_linear_patch[0]);
1276
1277         __flush_tlb_all();
1278 #endif
1279 }
1280
1281 #ifdef CONFIG_DEBUG_PAGEALLOC
1282 void kernel_map_pages(struct page *page, int numpages, int enable)
1283 {
1284         unsigned long phys_start = page_to_pfn(page) << PAGE_SHIFT;
1285         unsigned long phys_end = phys_start + (numpages * PAGE_SIZE);
1286
1287         kernel_map_range(phys_start, phys_end,
1288                          (enable ? PAGE_KERNEL : __pgprot(0)));
1289
1290         flush_tsb_kernel_range(PAGE_OFFSET + phys_start,
1291                                PAGE_OFFSET + phys_end);
1292
1293         /* we should perform an IPI and flush all tlbs,
1294          * but that can deadlock->flush only current cpu.
1295          */
1296         __flush_tlb_kernel_range(PAGE_OFFSET + phys_start,
1297                                  PAGE_OFFSET + phys_end);
1298 }
1299 #endif
1300
1301 unsigned long __init find_ecache_flush_span(unsigned long size)
1302 {
1303         int i;
1304
1305         for (i = 0; i < pavail_ents; i++) {
1306                 if (pavail[i].reg_size >= size)
1307                         return pavail[i].phys_addr;
1308         }
1309
1310         return ~0UL;
1311 }
1312
1313 static void __init tsb_phys_patch(void)
1314 {
1315         struct tsb_ldquad_phys_patch_entry *pquad;
1316         struct tsb_phys_patch_entry *p;
1317
1318         pquad = &__tsb_ldquad_phys_patch;
1319         while (pquad < &__tsb_ldquad_phys_patch_end) {
1320                 unsigned long addr = pquad->addr;
1321
1322                 if (tlb_type == hypervisor)
1323                         *(unsigned int *) addr = pquad->sun4v_insn;
1324                 else
1325                         *(unsigned int *) addr = pquad->sun4u_insn;
1326                 wmb();
1327                 __asm__ __volatile__("flush     %0"
1328                                      : /* no outputs */
1329                                      : "r" (addr));
1330
1331                 pquad++;
1332         }
1333
1334         p = &__tsb_phys_patch;
1335         while (p < &__tsb_phys_patch_end) {
1336                 unsigned long addr = p->addr;
1337
1338                 *(unsigned int *) addr = p->insn;
1339                 wmb();
1340                 __asm__ __volatile__("flush     %0"
1341                                      : /* no outputs */
1342                                      : "r" (addr));
1343
1344                 p++;
1345         }
1346 }
1347
1348 /* Don't mark as init, we give this to the Hypervisor.  */
1349 #ifndef CONFIG_DEBUG_PAGEALLOC
1350 #define NUM_KTSB_DESCR  2
1351 #else
1352 #define NUM_KTSB_DESCR  1
1353 #endif
1354 static struct hv_tsb_descr ktsb_descr[NUM_KTSB_DESCR];
1355 extern struct tsb swapper_tsb[KERNEL_TSB_NENTRIES];
1356
1357 static void __init sun4v_ktsb_init(void)
1358 {
1359         unsigned long ktsb_pa;
1360
1361         /* First KTSB for PAGE_SIZE mappings.  */
1362         ktsb_pa = kern_base + ((unsigned long)&swapper_tsb[0] - KERNBASE);
1363
1364         switch (PAGE_SIZE) {
1365         case 8 * 1024:
1366         default:
1367                 ktsb_descr[0].pgsz_idx = HV_PGSZ_IDX_8K;
1368                 ktsb_descr[0].pgsz_mask = HV_PGSZ_MASK_8K;
1369                 break;
1370
1371         case 64 * 1024:
1372                 ktsb_descr[0].pgsz_idx = HV_PGSZ_IDX_64K;
1373                 ktsb_descr[0].pgsz_mask = HV_PGSZ_MASK_64K;
1374                 break;
1375
1376         case 512 * 1024:
1377                 ktsb_descr[0].pgsz_idx = HV_PGSZ_IDX_512K;
1378                 ktsb_descr[0].pgsz_mask = HV_PGSZ_MASK_512K;
1379                 break;
1380
1381         case 4 * 1024 * 1024:
1382                 ktsb_descr[0].pgsz_idx = HV_PGSZ_IDX_4MB;
1383                 ktsb_descr[0].pgsz_mask = HV_PGSZ_MASK_4MB;
1384                 break;
1385         };
1386
1387         ktsb_descr[0].assoc = 1;
1388         ktsb_descr[0].num_ttes = KERNEL_TSB_NENTRIES;
1389         ktsb_descr[0].ctx_idx = 0;
1390         ktsb_descr[0].tsb_base = ktsb_pa;
1391         ktsb_descr[0].resv = 0;
1392
1393 #ifndef CONFIG_DEBUG_PAGEALLOC
1394         /* Second KTSB for 4MB/256MB mappings.  */
1395         ktsb_pa = (kern_base +
1396                    ((unsigned long)&swapper_4m_tsb[0] - KERNBASE));
1397
1398         ktsb_descr[1].pgsz_idx = HV_PGSZ_IDX_4MB;
1399         ktsb_descr[1].pgsz_mask = (HV_PGSZ_MASK_4MB |
1400                                    HV_PGSZ_MASK_256MB);
1401         ktsb_descr[1].assoc = 1;
1402         ktsb_descr[1].num_ttes = KERNEL_TSB4M_NENTRIES;
1403         ktsb_descr[1].ctx_idx = 0;
1404         ktsb_descr[1].tsb_base = ktsb_pa;
1405         ktsb_descr[1].resv = 0;
1406 #endif
1407 }
1408
1409 void __cpuinit sun4v_ktsb_register(void)
1410 {
1411         register unsigned long func asm("%o5");
1412         register unsigned long arg0 asm("%o0");
1413         register unsigned long arg1 asm("%o1");
1414         unsigned long pa;
1415
1416         pa = kern_base + ((unsigned long)&ktsb_descr[0] - KERNBASE);
1417
1418         func = HV_FAST_MMU_TSB_CTX0;
1419         arg0 = NUM_KTSB_DESCR;
1420         arg1 = pa;
1421         __asm__ __volatile__("ta        %6"
1422                              : "=&r" (func), "=&r" (arg0), "=&r" (arg1)
1423                              : "0" (func), "1" (arg0), "2" (arg1),
1424                                "i" (HV_FAST_TRAP));
1425 }
1426
1427 /* paging_init() sets up the page tables */
1428
1429 extern void cheetah_ecache_flush_init(void);
1430 extern void sun4v_patch_tlb_handlers(void);
1431
1432 static unsigned long last_valid_pfn;
1433 pgd_t swapper_pg_dir[2048];
1434
1435 static void sun4u_pgprot_init(void);
1436 static void sun4v_pgprot_init(void);
1437
1438 void __init paging_init(void)
1439 {
1440         unsigned long end_pfn, pages_avail, shift, phys_base;
1441         unsigned long real_end, i;
1442
1443         kern_base = (prom_boot_mapping_phys_low >> 22UL) << 22UL;
1444         kern_size = (unsigned long)&_end - (unsigned long)KERNBASE;
1445
1446         /* Invalidate both kernel TSBs.  */
1447         memset(swapper_tsb, 0x40, sizeof(swapper_tsb));
1448 #ifndef CONFIG_DEBUG_PAGEALLOC
1449         memset(swapper_4m_tsb, 0x40, sizeof(swapper_4m_tsb));
1450 #endif
1451
1452         if (tlb_type == hypervisor)
1453                 sun4v_pgprot_init();
1454         else
1455                 sun4u_pgprot_init();
1456
1457         if (tlb_type == cheetah_plus ||
1458             tlb_type == hypervisor)
1459                 tsb_phys_patch();
1460
1461         if (tlb_type == hypervisor) {
1462                 sun4v_patch_tlb_handlers();
1463                 sun4v_ktsb_init();
1464         }
1465
1466         /* Find available physical memory... */
1467         read_obp_memory("available", &pavail[0], &pavail_ents);
1468
1469         phys_base = 0xffffffffffffffffUL;
1470         for (i = 0; i < pavail_ents; i++)
1471                 phys_base = min(phys_base, pavail[i].phys_addr);
1472
1473         set_bit(0, mmu_context_bmap);
1474
1475         shift = kern_base + PAGE_OFFSET - ((unsigned long)KERNBASE);
1476
1477         real_end = (unsigned long)_end;
1478         if ((real_end > ((unsigned long)KERNBASE + 0x400000)))
1479                 bigkernel = 1;
1480         if ((real_end > ((unsigned long)KERNBASE + 0x800000))) {
1481                 prom_printf("paging_init: Kernel > 8MB, too large.\n");
1482                 prom_halt();
1483         }
1484
1485         /* Set kernel pgd to upper alias so physical page computations
1486          * work.
1487          */
1488         init_mm.pgd += ((shift) / (sizeof(pgd_t)));
1489
1490         memset(swapper_low_pmd_dir, 0, sizeof(swapper_low_pmd_dir));
1491
1492         /* Now can init the kernel/bad page tables. */
1493         pud_set(pud_offset(&swapper_pg_dir[0], 0),
1494                 swapper_low_pmd_dir + (shift / sizeof(pgd_t)));
1495
1496         inherit_prom_mappings();
1497
1498         /* Ok, we can use our TLB miss and window trap handlers safely.  */
1499         setup_tba();
1500
1501         __flush_tlb_all();
1502
1503         if (tlb_type == hypervisor)
1504                 sun4v_ktsb_register();
1505
1506         /* Setup bootmem... */
1507         pages_avail = 0;
1508         last_valid_pfn = end_pfn = bootmem_init(&pages_avail, phys_base);
1509
1510         max_mapnr = last_valid_pfn;
1511
1512         kernel_physical_mapping_init();
1513
1514         prom_build_devicetree();
1515
1516         {
1517                 unsigned long zones_size[MAX_NR_ZONES];
1518                 unsigned long zholes_size[MAX_NR_ZONES];
1519                 int znum;
1520
1521                 for (znum = 0; znum < MAX_NR_ZONES; znum++)
1522                         zones_size[znum] = zholes_size[znum] = 0;
1523
1524                 zones_size[ZONE_NORMAL] = end_pfn;
1525                 zholes_size[ZONE_NORMAL] = end_pfn - pages_avail;
1526
1527                 free_area_init_node(0, &contig_page_data, zones_size,
1528                                     __pa(PAGE_OFFSET) >> PAGE_SHIFT,
1529                                     zholes_size);
1530         }
1531
1532         device_scan();
1533 }
1534
1535 static void __init taint_real_pages(void)
1536 {
1537         int i;
1538
1539         read_obp_memory("available", &pavail_rescan[0], &pavail_rescan_ents);
1540
1541         /* Find changes discovered in the physmem available rescan and
1542          * reserve the lost portions in the bootmem maps.
1543          */
1544         for (i = 0; i < pavail_ents; i++) {
1545                 unsigned long old_start, old_end;
1546
1547                 old_start = pavail[i].phys_addr;
1548                 old_end = old_start +
1549                         pavail[i].reg_size;
1550                 while (old_start < old_end) {
1551                         int n;
1552
1553                         for (n = 0; n < pavail_rescan_ents; n++) {
1554                                 unsigned long new_start, new_end;
1555
1556                                 new_start = pavail_rescan[n].phys_addr;
1557                                 new_end = new_start +
1558                                         pavail_rescan[n].reg_size;
1559
1560                                 if (new_start <= old_start &&
1561                                     new_end >= (old_start + PAGE_SIZE)) {
1562                                         set_bit(old_start >> 22,
1563                                                 sparc64_valid_addr_bitmap);
1564                                         goto do_next_page;
1565                                 }
1566                         }
1567                         reserve_bootmem(old_start, PAGE_SIZE);
1568
1569                 do_next_page:
1570                         old_start += PAGE_SIZE;
1571                 }
1572         }
1573 }
1574
1575 int __init page_in_phys_avail(unsigned long paddr)
1576 {
1577         int i;
1578
1579         paddr &= PAGE_MASK;
1580
1581         for (i = 0; i < pavail_rescan_ents; i++) {
1582                 unsigned long start, end;
1583
1584                 start = pavail_rescan[i].phys_addr;
1585                 end = start + pavail_rescan[i].reg_size;
1586
1587                 if (paddr >= start && paddr < end)
1588                         return 1;
1589         }
1590         if (paddr >= kern_base && paddr < (kern_base + kern_size))
1591                 return 1;
1592 #ifdef CONFIG_BLK_DEV_INITRD
1593         if (paddr >= __pa(initrd_start) &&
1594             paddr < __pa(PAGE_ALIGN(initrd_end)))
1595                 return 1;
1596 #endif
1597
1598         return 0;
1599 }
1600
1601 void __init mem_init(void)
1602 {
1603         unsigned long codepages, datapages, initpages;
1604         unsigned long addr, last;
1605         int i;
1606
1607         i = last_valid_pfn >> ((22 - PAGE_SHIFT) + 6);
1608         i += 1;
1609         sparc64_valid_addr_bitmap = (unsigned long *) alloc_bootmem(i << 3);
1610         if (sparc64_valid_addr_bitmap == NULL) {
1611                 prom_printf("mem_init: Cannot alloc valid_addr_bitmap.\n");
1612                 prom_halt();
1613         }
1614         memset(sparc64_valid_addr_bitmap, 0, i << 3);
1615
1616         addr = PAGE_OFFSET + kern_base;
1617         last = PAGE_ALIGN(kern_size) + addr;
1618         while (addr < last) {
1619                 set_bit(__pa(addr) >> 22, sparc64_valid_addr_bitmap);
1620                 addr += PAGE_SIZE;
1621         }
1622
1623         taint_real_pages();
1624
1625         high_memory = __va(last_valid_pfn << PAGE_SHIFT);
1626
1627 #ifdef CONFIG_DEBUG_BOOTMEM
1628         prom_printf("mem_init: Calling free_all_bootmem().\n");
1629 #endif
1630         totalram_pages = num_physpages = free_all_bootmem() - 1;
1631
1632         /*
1633          * Set up the zero page, mark it reserved, so that page count
1634          * is not manipulated when freeing the page from user ptes.
1635          */
1636         mem_map_zero = alloc_pages(GFP_KERNEL|__GFP_ZERO, 0);
1637         if (mem_map_zero == NULL) {
1638                 prom_printf("paging_init: Cannot alloc zero page.\n");
1639                 prom_halt();
1640         }
1641         SetPageReserved(mem_map_zero);
1642
1643         codepages = (((unsigned long) _etext) - ((unsigned long) _start));
1644         codepages = PAGE_ALIGN(codepages) >> PAGE_SHIFT;
1645         datapages = (((unsigned long) _edata) - ((unsigned long) _etext));
1646         datapages = PAGE_ALIGN(datapages) >> PAGE_SHIFT;
1647         initpages = (((unsigned long) __init_end) - ((unsigned long) __init_begin));
1648         initpages = PAGE_ALIGN(initpages) >> PAGE_SHIFT;
1649
1650         printk("Memory: %luk available (%ldk kernel code, %ldk data, %ldk init) [%016lx,%016lx]\n",
1651                nr_free_pages() << (PAGE_SHIFT-10),
1652                codepages << (PAGE_SHIFT-10),
1653                datapages << (PAGE_SHIFT-10),
1654                initpages << (PAGE_SHIFT-10),
1655                PAGE_OFFSET, (last_valid_pfn << PAGE_SHIFT));
1656
1657         if (tlb_type == cheetah || tlb_type == cheetah_plus)
1658                 cheetah_ecache_flush_init();
1659 }
1660
1661 void free_initmem(void)
1662 {
1663         unsigned long addr, initend;
1664
1665         /*
1666          * The init section is aligned to 8k in vmlinux.lds. Page align for >8k pagesizes.
1667          */
1668         addr = PAGE_ALIGN((unsigned long)(__init_begin));
1669         initend = (unsigned long)(__init_end) & PAGE_MASK;
1670         for (; addr < initend; addr += PAGE_SIZE) {
1671                 unsigned long page;
1672                 struct page *p;
1673
1674                 page = (addr +
1675                         ((unsigned long) __va(kern_base)) -
1676                         ((unsigned long) KERNBASE));
1677                 memset((void *)addr, POISON_FREE_INITMEM, PAGE_SIZE);
1678                 p = virt_to_page(page);
1679
1680                 ClearPageReserved(p);
1681                 init_page_count(p);
1682                 __free_page(p);
1683                 num_physpages++;
1684                 totalram_pages++;
1685         }
1686 }
1687
1688 #ifdef CONFIG_BLK_DEV_INITRD
1689 void free_initrd_mem(unsigned long start, unsigned long end)
1690 {
1691         if (start < end)
1692                 printk ("Freeing initrd memory: %ldk freed\n", (end - start) >> 10);
1693         for (; start < end; start += PAGE_SIZE) {
1694                 struct page *p = virt_to_page(start);
1695
1696                 ClearPageReserved(p);
1697                 init_page_count(p);
1698                 __free_page(p);
1699                 num_physpages++;
1700                 totalram_pages++;
1701         }
1702 }
1703 #endif
1704
1705 #define _PAGE_CACHE_4U  (_PAGE_CP_4U | _PAGE_CV_4U)
1706 #define _PAGE_CACHE_4V  (_PAGE_CP_4V | _PAGE_CV_4V)
1707 #define __DIRTY_BITS_4U  (_PAGE_MODIFIED_4U | _PAGE_WRITE_4U | _PAGE_W_4U)
1708 #define __DIRTY_BITS_4V  (_PAGE_MODIFIED_4V | _PAGE_WRITE_4V | _PAGE_W_4V)
1709 #define __ACCESS_BITS_4U (_PAGE_ACCESSED_4U | _PAGE_READ_4U | _PAGE_R)
1710 #define __ACCESS_BITS_4V (_PAGE_ACCESSED_4V | _PAGE_READ_4V | _PAGE_R)
1711
1712 pgprot_t PAGE_KERNEL __read_mostly;
1713 EXPORT_SYMBOL(PAGE_KERNEL);
1714
1715 pgprot_t PAGE_KERNEL_LOCKED __read_mostly;
1716 pgprot_t PAGE_COPY __read_mostly;
1717
1718 pgprot_t PAGE_SHARED __read_mostly;
1719 EXPORT_SYMBOL(PAGE_SHARED);
1720
1721 pgprot_t PAGE_EXEC __read_mostly;
1722 unsigned long pg_iobits __read_mostly;
1723
1724 unsigned long _PAGE_IE __read_mostly;
1725 EXPORT_SYMBOL(_PAGE_IE);
1726
1727 unsigned long _PAGE_E __read_mostly;
1728 EXPORT_SYMBOL(_PAGE_E);
1729
1730 unsigned long _PAGE_CACHE __read_mostly;
1731 EXPORT_SYMBOL(_PAGE_CACHE);
1732
1733 static void prot_init_common(unsigned long page_none,
1734                              unsigned long page_shared,
1735                              unsigned long page_copy,
1736                              unsigned long page_readonly,
1737                              unsigned long page_exec_bit)
1738 {
1739         PAGE_COPY = __pgprot(page_copy);
1740         PAGE_SHARED = __pgprot(page_shared);
1741
1742         protection_map[0x0] = __pgprot(page_none);
1743         protection_map[0x1] = __pgprot(page_readonly & ~page_exec_bit);
1744         protection_map[0x2] = __pgprot(page_copy & ~page_exec_bit);
1745         protection_map[0x3] = __pgprot(page_copy & ~page_exec_bit);
1746         protection_map[0x4] = __pgprot(page_readonly);
1747         protection_map[0x5] = __pgprot(page_readonly);
1748         protection_map[0x6] = __pgprot(page_copy);
1749         protection_map[0x7] = __pgprot(page_copy);
1750         protection_map[0x8] = __pgprot(page_none);
1751         protection_map[0x9] = __pgprot(page_readonly & ~page_exec_bit);
1752         protection_map[0xa] = __pgprot(page_shared & ~page_exec_bit);
1753         protection_map[0xb] = __pgprot(page_shared & ~page_exec_bit);
1754         protection_map[0xc] = __pgprot(page_readonly);
1755         protection_map[0xd] = __pgprot(page_readonly);
1756         protection_map[0xe] = __pgprot(page_shared);
1757         protection_map[0xf] = __pgprot(page_shared);
1758 }
1759
1760 static void __init sun4u_pgprot_init(void)
1761 {
1762         unsigned long page_none, page_shared, page_copy, page_readonly;
1763         unsigned long page_exec_bit;
1764
1765         PAGE_KERNEL = __pgprot (_PAGE_PRESENT_4U | _PAGE_VALID |
1766                                 _PAGE_CACHE_4U | _PAGE_P_4U |
1767                                 __ACCESS_BITS_4U | __DIRTY_BITS_4U |
1768                                 _PAGE_EXEC_4U);
1769         PAGE_KERNEL_LOCKED = __pgprot (_PAGE_PRESENT_4U | _PAGE_VALID |
1770                                        _PAGE_CACHE_4U | _PAGE_P_4U |
1771                                        __ACCESS_BITS_4U | __DIRTY_BITS_4U |
1772                                        _PAGE_EXEC_4U | _PAGE_L_4U);
1773         PAGE_EXEC = __pgprot(_PAGE_EXEC_4U);
1774
1775         _PAGE_IE = _PAGE_IE_4U;
1776         _PAGE_E = _PAGE_E_4U;
1777         _PAGE_CACHE = _PAGE_CACHE_4U;
1778
1779         pg_iobits = (_PAGE_VALID | _PAGE_PRESENT_4U | __DIRTY_BITS_4U |
1780                      __ACCESS_BITS_4U | _PAGE_E_4U);
1781
1782 #ifdef CONFIG_DEBUG_PAGEALLOC
1783         kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZBITS_4U) ^
1784                 0xfffff80000000000;
1785 #else
1786         kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZ4MB_4U) ^
1787                 0xfffff80000000000;
1788 #endif
1789         kern_linear_pte_xor[0] |= (_PAGE_CP_4U | _PAGE_CV_4U |
1790                                    _PAGE_P_4U | _PAGE_W_4U);
1791
1792         /* XXX Should use 256MB on Panther. XXX */
1793         kern_linear_pte_xor[1] = kern_linear_pte_xor[0];
1794
1795         _PAGE_SZBITS = _PAGE_SZBITS_4U;
1796         _PAGE_ALL_SZ_BITS =  (_PAGE_SZ4MB_4U | _PAGE_SZ512K_4U |
1797                               _PAGE_SZ64K_4U | _PAGE_SZ8K_4U |
1798                               _PAGE_SZ32MB_4U | _PAGE_SZ256MB_4U);
1799
1800
1801         page_none = _PAGE_PRESENT_4U | _PAGE_ACCESSED_4U | _PAGE_CACHE_4U;
1802         page_shared = (_PAGE_VALID | _PAGE_PRESENT_4U | _PAGE_CACHE_4U |
1803                        __ACCESS_BITS_4U | _PAGE_WRITE_4U | _PAGE_EXEC_4U);
1804         page_copy   = (_PAGE_VALID | _PAGE_PRESENT_4U | _PAGE_CACHE_4U |
1805                        __ACCESS_BITS_4U | _PAGE_EXEC_4U);
1806         page_readonly   = (_PAGE_VALID | _PAGE_PRESENT_4U | _PAGE_CACHE_4U |
1807                            __ACCESS_BITS_4U | _PAGE_EXEC_4U);
1808
1809         page_exec_bit = _PAGE_EXEC_4U;
1810
1811         prot_init_common(page_none, page_shared, page_copy, page_readonly,
1812                          page_exec_bit);
1813 }
1814
1815 static void __init sun4v_pgprot_init(void)
1816 {
1817         unsigned long page_none, page_shared, page_copy, page_readonly;
1818         unsigned long page_exec_bit;
1819
1820         PAGE_KERNEL = __pgprot (_PAGE_PRESENT_4V | _PAGE_VALID |
1821                                 _PAGE_CACHE_4V | _PAGE_P_4V |
1822                                 __ACCESS_BITS_4V | __DIRTY_BITS_4V |
1823                                 _PAGE_EXEC_4V);
1824         PAGE_KERNEL_LOCKED = PAGE_KERNEL;
1825         PAGE_EXEC = __pgprot(_PAGE_EXEC_4V);
1826
1827         _PAGE_IE = _PAGE_IE_4V;
1828         _PAGE_E = _PAGE_E_4V;
1829         _PAGE_CACHE = _PAGE_CACHE_4V;
1830
1831 #ifdef CONFIG_DEBUG_PAGEALLOC
1832         kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZBITS_4V) ^
1833                 0xfffff80000000000;
1834 #else
1835         kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZ4MB_4V) ^
1836                 0xfffff80000000000;
1837 #endif
1838         kern_linear_pte_xor[0] |= (_PAGE_CP_4V | _PAGE_CV_4V |
1839                                    _PAGE_P_4V | _PAGE_W_4V);
1840
1841 #ifdef CONFIG_DEBUG_PAGEALLOC
1842         kern_linear_pte_xor[1] = (_PAGE_VALID | _PAGE_SZBITS_4V) ^
1843                 0xfffff80000000000;
1844 #else
1845         kern_linear_pte_xor[1] = (_PAGE_VALID | _PAGE_SZ256MB_4V) ^
1846                 0xfffff80000000000;
1847 #endif
1848         kern_linear_pte_xor[1] |= (_PAGE_CP_4V | _PAGE_CV_4V |
1849                                    _PAGE_P_4V | _PAGE_W_4V);
1850
1851         pg_iobits = (_PAGE_VALID | _PAGE_PRESENT_4V | __DIRTY_BITS_4V |
1852                      __ACCESS_BITS_4V | _PAGE_E_4V);
1853
1854         _PAGE_SZBITS = _PAGE_SZBITS_4V;
1855         _PAGE_ALL_SZ_BITS = (_PAGE_SZ16GB_4V | _PAGE_SZ2GB_4V |
1856                              _PAGE_SZ256MB_4V | _PAGE_SZ32MB_4V |
1857                              _PAGE_SZ4MB_4V | _PAGE_SZ512K_4V |
1858                              _PAGE_SZ64K_4V | _PAGE_SZ8K_4V);
1859
1860         page_none = _PAGE_PRESENT_4V | _PAGE_ACCESSED_4V | _PAGE_CACHE_4V;
1861         page_shared = (_PAGE_VALID | _PAGE_PRESENT_4V | _PAGE_CACHE_4V |
1862                        __ACCESS_BITS_4V | _PAGE_WRITE_4V | _PAGE_EXEC_4V);
1863         page_copy   = (_PAGE_VALID | _PAGE_PRESENT_4V | _PAGE_CACHE_4V |
1864                        __ACCESS_BITS_4V | _PAGE_EXEC_4V);
1865         page_readonly = (_PAGE_VALID | _PAGE_PRESENT_4V | _PAGE_CACHE_4V |
1866                          __ACCESS_BITS_4V | _PAGE_EXEC_4V);
1867
1868         page_exec_bit = _PAGE_EXEC_4V;
1869
1870         prot_init_common(page_none, page_shared, page_copy, page_readonly,
1871                          page_exec_bit);
1872 }
1873
1874 unsigned long pte_sz_bits(unsigned long sz)
1875 {
1876         if (tlb_type == hypervisor) {
1877                 switch (sz) {
1878                 case 8 * 1024:
1879                 default:
1880                         return _PAGE_SZ8K_4V;
1881                 case 64 * 1024:
1882                         return _PAGE_SZ64K_4V;
1883                 case 512 * 1024:
1884                         return _PAGE_SZ512K_4V;
1885                 case 4 * 1024 * 1024:
1886                         return _PAGE_SZ4MB_4V;
1887                 };
1888         } else {
1889                 switch (sz) {
1890                 case 8 * 1024:
1891                 default:
1892                         return _PAGE_SZ8K_4U;
1893                 case 64 * 1024:
1894                         return _PAGE_SZ64K_4U;
1895                 case 512 * 1024:
1896                         return _PAGE_SZ512K_4U;
1897                 case 4 * 1024 * 1024:
1898                         return _PAGE_SZ4MB_4U;
1899                 };
1900         }
1901 }
1902
1903 pte_t mk_pte_io(unsigned long page, pgprot_t prot, int space, unsigned long page_size)
1904 {
1905         pte_t pte;
1906
1907         pte_val(pte)  = page | pgprot_val(pgprot_noncached(prot));
1908         pte_val(pte) |= (((unsigned long)space) << 32);
1909         pte_val(pte) |= pte_sz_bits(page_size);
1910
1911         return pte;
1912 }
1913
1914 static unsigned long kern_large_tte(unsigned long paddr)
1915 {
1916         unsigned long val;
1917
1918         val = (_PAGE_VALID | _PAGE_SZ4MB_4U |
1919                _PAGE_CP_4U | _PAGE_CV_4U | _PAGE_P_4U |
1920                _PAGE_EXEC_4U | _PAGE_L_4U | _PAGE_W_4U);
1921         if (tlb_type == hypervisor)
1922                 val = (_PAGE_VALID | _PAGE_SZ4MB_4V |
1923                        _PAGE_CP_4V | _PAGE_CV_4V | _PAGE_P_4V |
1924                        _PAGE_EXEC_4V | _PAGE_W_4V);
1925
1926         return val | paddr;
1927 }
1928
1929 /* If not locked, zap it. */
1930 void __flush_tlb_all(void)
1931 {
1932         unsigned long pstate;
1933         int i;
1934
1935         __asm__ __volatile__("flushw\n\t"
1936                              "rdpr      %%pstate, %0\n\t"
1937                              "wrpr      %0, %1, %%pstate"
1938                              : "=r" (pstate)
1939                              : "i" (PSTATE_IE));
1940         if (tlb_type == spitfire) {
1941                 for (i = 0; i < 64; i++) {
1942                         /* Spitfire Errata #32 workaround */
1943                         /* NOTE: Always runs on spitfire, so no
1944                          *       cheetah+ page size encodings.
1945                          */
1946                         __asm__ __volatile__("stxa      %0, [%1] %2\n\t"
1947                                              "flush     %%g6"
1948                                              : /* No outputs */
1949                                              : "r" (0),
1950                                              "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
1951
1952                         if (!(spitfire_get_dtlb_data(i) & _PAGE_L_4U)) {
1953                                 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
1954                                                      "membar #Sync"
1955                                                      : /* no outputs */
1956                                                      : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
1957                                 spitfire_put_dtlb_data(i, 0x0UL);
1958                         }
1959
1960                         /* Spitfire Errata #32 workaround */
1961                         /* NOTE: Always runs on spitfire, so no
1962                          *       cheetah+ page size encodings.
1963                          */
1964                         __asm__ __volatile__("stxa      %0, [%1] %2\n\t"
1965                                              "flush     %%g6"
1966                                              : /* No outputs */
1967                                              : "r" (0),
1968                                              "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
1969
1970                         if (!(spitfire_get_itlb_data(i) & _PAGE_L_4U)) {
1971                                 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
1972                                                      "membar #Sync"
1973                                                      : /* no outputs */
1974                                                      : "r" (TLB_TAG_ACCESS), "i" (ASI_IMMU));
1975                                 spitfire_put_itlb_data(i, 0x0UL);
1976                         }
1977                 }
1978         } else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
1979                 cheetah_flush_dtlb_all();
1980                 cheetah_flush_itlb_all();
1981         }
1982         __asm__ __volatile__("wrpr      %0, 0, %%pstate"
1983                              : : "r" (pstate));
1984 }
1985
1986 #ifdef CONFIG_MEMORY_HOTPLUG
1987
1988 void online_page(struct page *page)
1989 {
1990         ClearPageReserved(page);
1991         init_page_count(page);
1992         __free_page(page);
1993         totalram_pages++;
1994         num_physpages++;
1995 }
1996
1997 int remove_memory(u64 start, u64 size)
1998 {
1999         return -EINVAL;
2000 }
2001
2002 #endif /* CONFIG_MEMORY_HOTPLUG */