2 * Copyright (c) 2007, Intel Corporation.
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * You should have received a copy of the GNU General Public License along with
15 * this program; if not, write to the Free Software Foundation, Inc.,
16 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18 * Authors: Thomas Hellstrom <thomas-at-tungstengraphics.com>
19 * Alan Cox <alan@linux.intel.com>
23 #include <linux/shmem_fs.h>
28 * GTT resource allocator - manage page mappings in GTT space
32 * psb_gtt_mask_pte - generate GTT pte entry
33 * @pfn: page number to encode
34 * @type: type of memory in the GTT
36 * Set the GTT entry for the appropriate memory type.
38 static inline uint32_t psb_gtt_mask_pte(uint32_t pfn, int type)
40 uint32_t mask = PSB_PTE_VALID;
42 /* Ensure we explode rather than put an invalid low mapping of
43 a high mapping page into the gtt */
44 BUG_ON(pfn & ~(0xFFFFFFFF >> PAGE_SHIFT));
46 if (type & PSB_MMU_CACHED_MEMORY)
47 mask |= PSB_PTE_CACHED;
48 if (type & PSB_MMU_RO_MEMORY)
50 if (type & PSB_MMU_WO_MEMORY)
53 return (pfn << PAGE_SHIFT) | mask;
57 * psb_gtt_entry - find the GTT entries for a gtt_range
58 * @dev: our DRM device
61 * Given a gtt_range object return the GTT offset of the page table
62 * entries for this gtt_range
64 static u32 __iomem *psb_gtt_entry(struct drm_device *dev, struct gtt_range *r)
66 struct drm_psb_private *dev_priv = dev->dev_private;
69 offset = r->resource.start - dev_priv->gtt_mem->start;
71 return dev_priv->gtt_map + (offset >> PAGE_SHIFT);
75 * psb_gtt_insert - put an object into the GTT
76 * @dev: our DRM device
79 * Take our preallocated GTT range and insert the GEM object into
80 * the GTT. This is protected via the gtt mutex which the caller
83 static int psb_gtt_insert(struct drm_device *dev, struct gtt_range *r,
86 u32 __iomem *gtt_slot;
91 if (r->pages == NULL) {
96 WARN_ON(r->stolen); /* refcount these maybe ? */
98 gtt_slot = psb_gtt_entry(dev, r);
102 /* Make sure changes are visible to the GPU */
103 set_pages_array_wc(pages, r->npage);
106 /* Write our page entries into the GTT itself */
107 for (i = r->roll; i < r->npage; i++) {
108 pte = psb_gtt_mask_pte(page_to_pfn(r->pages[i]), 0);
109 iowrite32(pte, gtt_slot++);
111 for (i = 0; i < r->roll; i++) {
112 pte = psb_gtt_mask_pte(page_to_pfn(r->pages[i]), 0);
113 iowrite32(pte, gtt_slot++);
115 /* Make sure all the entries are set before we return */
116 ioread32(gtt_slot - 1);
122 * psb_gtt_remove - remove an object from the GTT
123 * @dev: our DRM device
126 * Remove a preallocated GTT range from the GTT. Overwrite all the
127 * page table entries with the dummy page. This is protected via the gtt
128 * mutex which the caller must hold.
130 static void psb_gtt_remove(struct drm_device *dev, struct gtt_range *r)
132 struct drm_psb_private *dev_priv = dev->dev_private;
133 u32 __iomem *gtt_slot;
139 gtt_slot = psb_gtt_entry(dev, r);
140 pte = psb_gtt_mask_pte(page_to_pfn(dev_priv->scratch_page), 0);
142 for (i = 0; i < r->npage; i++)
143 iowrite32(pte, gtt_slot++);
144 ioread32(gtt_slot - 1);
145 set_pages_array_wb(r->pages, r->npage);
149 * psb_gtt_roll - set scrolling position
150 * @dev: our DRM device
151 * @r: the gtt mapping we are using
154 * Roll an existing pinned mapping by moving the pages through the GTT.
155 * This allows us to implement hardware scrolling on the consoles without
158 void psb_gtt_roll(struct drm_device *dev, struct gtt_range *r, int roll)
160 u32 __iomem *gtt_slot;
164 if (roll >= r->npage) {
171 /* Not currently in the GTT - no worry we will write the mapping at
172 the right position when it gets pinned */
173 if (!r->stolen && !r->in_gart)
176 gtt_slot = psb_gtt_entry(dev, r);
178 for (i = r->roll; i < r->npage; i++) {
179 pte = psb_gtt_mask_pte(page_to_pfn(r->pages[i]), 0);
180 iowrite32(pte, gtt_slot++);
182 for (i = 0; i < r->roll; i++) {
183 pte = psb_gtt_mask_pte(page_to_pfn(r->pages[i]), 0);
184 iowrite32(pte, gtt_slot++);
186 ioread32(gtt_slot - 1);
190 * psb_gtt_attach_pages - attach and pin GEM pages
193 * Pin and build an in kernel list of the pages that back our GEM object.
194 * While we hold this the pages cannot be swapped out. This is protected
195 * via the gtt mutex which the caller must hold.
197 static int psb_gtt_attach_pages(struct gtt_range *gt)
203 pages = drm_gem_get_pages(>->gem, 0);
205 return PTR_ERR(pages);
213 * psb_gtt_detach_pages - attach and pin GEM pages
216 * Undo the effect of psb_gtt_attach_pages. At this point the pages
217 * must have been removed from the GTT as they could now be paged out
218 * and move bus address. This is protected via the gtt mutex which the
221 static void psb_gtt_detach_pages(struct gtt_range *gt)
223 drm_gem_put_pages(>->gem, gt->pages, true, false);
228 * psb_gtt_pin - pin pages into the GTT
231 * Pin a set of pages into the GTT. The pins are refcounted so that
232 * multiple pins need multiple unpins to undo.
234 * Non GEM backed objects treat this as a no-op as they are always GTT
237 int psb_gtt_pin(struct gtt_range *gt)
240 struct drm_device *dev = gt->gem.dev;
241 struct drm_psb_private *dev_priv = dev->dev_private;
243 mutex_lock(&dev_priv->gtt_mutex);
245 if (gt->in_gart == 0 && gt->stolen == 0) {
246 ret = psb_gtt_attach_pages(gt);
249 ret = psb_gtt_insert(dev, gt, 0);
251 psb_gtt_detach_pages(gt);
257 mutex_unlock(&dev_priv->gtt_mutex);
262 * psb_gtt_unpin - Drop a GTT pin requirement
265 * Undoes the effect of psb_gtt_pin. On the last drop the GEM object
266 * will be removed from the GTT which will also drop the page references
267 * and allow the VM to clean up or page stuff.
269 * Non GEM backed objects treat this as a no-op as they are always GTT
272 void psb_gtt_unpin(struct gtt_range *gt)
274 struct drm_device *dev = gt->gem.dev;
275 struct drm_psb_private *dev_priv = dev->dev_private;
277 mutex_lock(&dev_priv->gtt_mutex);
279 WARN_ON(!gt->in_gart);
282 if (gt->in_gart == 0 && gt->stolen == 0) {
283 psb_gtt_remove(dev, gt);
284 psb_gtt_detach_pages(gt);
286 mutex_unlock(&dev_priv->gtt_mutex);
290 * GTT resource allocator - allocate and manage GTT address space
294 * psb_gtt_alloc_range - allocate GTT address space
295 * @dev: Our DRM device
296 * @len: length (bytes) of address space required
297 * @name: resource name
298 * @backed: resource should be backed by stolen pages
300 * Ask the kernel core to find us a suitable range of addresses
301 * to use for a GTT mapping.
303 * Returns a gtt_range structure describing the object, or NULL on
304 * error. On successful return the resource is both allocated and marked
307 struct gtt_range *psb_gtt_alloc_range(struct drm_device *dev, int len,
308 const char *name, int backed)
310 struct drm_psb_private *dev_priv = dev->dev_private;
311 struct gtt_range *gt;
312 struct resource *r = dev_priv->gtt_mem;
314 unsigned long start, end;
317 /* The start of the GTT is the stolen pages */
319 end = r->start + dev_priv->gtt.stolen_size - 1;
321 /* The rest we will use for GEM backed objects */
322 start = r->start + dev_priv->gtt.stolen_size;
326 gt = kzalloc(sizeof(struct gtt_range), GFP_KERNEL);
329 gt->resource.name = name;
331 gt->in_gart = backed;
333 /* Ensure this is set for non GEM objects */
335 ret = allocate_resource(dev_priv->gtt_mem, >->resource,
336 len, start, end, PAGE_SIZE, NULL, NULL);
338 gt->offset = gt->resource.start - r->start;
346 * psb_gtt_free_range - release GTT address space
347 * @dev: our DRM device
348 * @gt: a mapping created with psb_gtt_alloc_range
350 * Release a resource that was allocated with psb_gtt_alloc_range. If the
351 * object has been pinned by mmap users we clean this up here currently.
353 void psb_gtt_free_range(struct drm_device *dev, struct gtt_range *gt)
355 /* Undo the mmap pin if we are destroying the object */
360 WARN_ON(gt->in_gart && !gt->stolen);
361 release_resource(>->resource);
365 static void psb_gtt_alloc(struct drm_device *dev)
367 struct drm_psb_private *dev_priv = dev->dev_private;
368 init_rwsem(&dev_priv->gtt.sem);
371 void psb_gtt_takedown(struct drm_device *dev)
373 struct drm_psb_private *dev_priv = dev->dev_private;
375 if (dev_priv->gtt_map) {
376 iounmap(dev_priv->gtt_map);
377 dev_priv->gtt_map = NULL;
379 if (dev_priv->gtt_initialized) {
380 pci_write_config_word(dev->pdev, PSB_GMCH_CTRL,
381 dev_priv->gmch_ctrl);
382 PSB_WVDC32(dev_priv->pge_ctl, PSB_PGETBL_CTL);
383 (void) PSB_RVDC32(PSB_PGETBL_CTL);
385 if (dev_priv->vram_addr)
386 iounmap(dev_priv->gtt_map);
389 int psb_gtt_init(struct drm_device *dev, int resume)
391 struct drm_psb_private *dev_priv = dev->dev_private;
393 unsigned long stolen_size, vram_stolen_size;
394 unsigned i, num_pages;
402 mutex_init(&dev_priv->gtt_mutex);
409 pci_read_config_word(dev->pdev, PSB_GMCH_CTRL, &dev_priv->gmch_ctrl);
410 pci_write_config_word(dev->pdev, PSB_GMCH_CTRL,
411 dev_priv->gmch_ctrl | _PSB_GMCH_ENABLED);
413 dev_priv->pge_ctl = PSB_RVDC32(PSB_PGETBL_CTL);
414 PSB_WVDC32(dev_priv->pge_ctl | _PSB_PGETBL_ENABLED, PSB_PGETBL_CTL);
415 (void) PSB_RVDC32(PSB_PGETBL_CTL);
417 /* The root resource we allocate address space from */
418 dev_priv->gtt_initialized = 1;
420 pg->gtt_phys_start = dev_priv->pge_ctl & PAGE_MASK;
423 * The video mmu has a hw bug when accessing 0x0D0000000.
424 * Make gatt start at 0x0e000,0000. This doesn't actually
425 * matter for us but may do if the video acceleration ever
428 pg->mmu_gatt_start = 0xE0000000;
430 pg->gtt_start = pci_resource_start(dev->pdev, PSB_GTT_RESOURCE);
431 gtt_pages = pci_resource_len(dev->pdev, PSB_GTT_RESOURCE)
433 /* CDV doesn't report this. In which case the system has 64 gtt pages */
434 if (pg->gtt_start == 0 || gtt_pages == 0) {
435 dev_dbg(dev->dev, "GTT PCI BAR not initialized.\n");
437 pg->gtt_start = dev_priv->pge_ctl;
440 pg->gatt_start = pci_resource_start(dev->pdev, PSB_GATT_RESOURCE);
441 pg->gatt_pages = pci_resource_len(dev->pdev, PSB_GATT_RESOURCE)
443 dev_priv->gtt_mem = &dev->pdev->resource[PSB_GATT_RESOURCE];
445 if (pg->gatt_pages == 0 || pg->gatt_start == 0) {
446 static struct resource fudge; /* Preferably peppermint */
447 /* This can occur on CDV systems. Fudge it in this case.
448 We really don't care what imaginary space is being allocated
450 dev_dbg(dev->dev, "GATT PCI BAR not initialized.\n");
451 pg->gatt_start = 0x40000000;
452 pg->gatt_pages = (128 * 1024 * 1024) >> PAGE_SHIFT;
453 /* This is a little confusing but in fact the GTT is providing
454 a view from the GPU into memory and not vice versa. As such
455 this is really allocating space that is not the same as the
456 CPU address space on CDV */
457 fudge.start = 0x40000000;
458 fudge.end = 0x40000000 + 128 * 1024 * 1024 - 1;
459 fudge.name = "fudge";
460 fudge.flags = IORESOURCE_MEM;
461 dev_priv->gtt_mem = &fudge;
464 pci_read_config_dword(dev->pdev, PSB_BSM, &dev_priv->stolen_base);
465 vram_stolen_size = pg->gtt_phys_start - dev_priv->stolen_base
468 stolen_size = vram_stolen_size;
470 dev_dbg(dev->dev, "Stolen memory base 0x%x, size %luK\n",
471 dev_priv->stolen_base, vram_stolen_size / 1024);
473 if (resume && (gtt_pages != pg->gtt_pages) &&
474 (stolen_size != pg->stolen_size)) {
475 dev_err(dev->dev, "GTT resume error.\n");
480 pg->gtt_pages = gtt_pages;
481 pg->stolen_size = stolen_size;
482 dev_priv->vram_stolen_size = vram_stolen_size;
485 * Map the GTT and the stolen memory area
488 dev_priv->gtt_map = ioremap_nocache(pg->gtt_phys_start,
489 gtt_pages << PAGE_SHIFT);
490 if (!dev_priv->gtt_map) {
491 dev_err(dev->dev, "Failure to map gtt.\n");
497 dev_priv->vram_addr = ioremap_wc(dev_priv->stolen_base,
499 if (!dev_priv->vram_addr) {
500 dev_err(dev->dev, "Failure to map stolen base.\n");
506 * Insert vram stolen pages into the GTT
509 pfn_base = dev_priv->stolen_base >> PAGE_SHIFT;
510 num_pages = vram_stolen_size >> PAGE_SHIFT;
511 dev_dbg(dev->dev, "Set up %d stolen pages starting at 0x%08x, GTT offset %dK\n",
512 num_pages, pfn_base << PAGE_SHIFT, 0);
513 for (i = 0; i < num_pages; ++i) {
514 pte = psb_gtt_mask_pte(pfn_base + i, 0);
515 iowrite32(pte, dev_priv->gtt_map + i);
519 * Init rest of GTT to the scratch page to avoid accidents or scribbles
522 pfn_base = page_to_pfn(dev_priv->scratch_page);
523 pte = psb_gtt_mask_pte(pfn_base, 0);
524 for (; i < gtt_pages; ++i)
525 iowrite32(pte, dev_priv->gtt_map + i);
527 (void) ioread32(dev_priv->gtt_map + i - 1);
531 psb_gtt_takedown(dev);
535 int psb_gtt_restore(struct drm_device *dev)
537 struct drm_psb_private *dev_priv = dev->dev_private;
538 struct resource *r = dev_priv->gtt_mem->child;
539 struct gtt_range *range;
540 unsigned int restored = 0, total = 0, size = 0;
542 /* On resume, the gtt_mutex is already initialized */
543 mutex_lock(&dev_priv->gtt_mutex);
544 psb_gtt_init(dev, 1);
547 range = container_of(r, struct gtt_range, resource);
549 psb_gtt_insert(dev, range, 1);
550 size += range->resource.end - range->resource.start;
556 mutex_unlock(&dev_priv->gtt_mutex);
557 DRM_DEBUG_DRIVER("Restored %u of %u gtt ranges (%u KB)", restored,
558 total, (size / 1024));