2 * Copyright (C) 2007 Ben Skeggs.
6 * Permission is hereby granted, free of charge, to any person obtaining
7 * a copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sublicense, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial
16 * portions of the Software.
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
19 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
21 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
22 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
23 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
24 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
31 #include "nouveau_drv.h"
32 #include "nouveau_vm.h"
34 struct nv50_instmem_priv {
35 uint32_t save1700[5]; /* 0x1700->0x1710 */
37 struct nouveau_gpuobj *pramin_pt;
38 struct nouveau_gpuobj *pramin_bar;
39 struct nouveau_gpuobj *fb_bar;
43 nv50_channel_del(struct nouveau_channel **pchan)
45 struct nouveau_channel *chan;
52 nouveau_gpuobj_ref(NULL, &chan->ramfc);
53 nouveau_gpuobj_ref(NULL, &chan->vm_pd);
54 if (chan->ramin_heap.free_stack.next)
55 drm_mm_takedown(&chan->ramin_heap);
56 nouveau_gpuobj_ref(NULL, &chan->ramin);
61 nv50_channel_new(struct drm_device *dev, u32 size,
62 struct nouveau_channel **pchan)
64 struct drm_nouveau_private *dev_priv = dev->dev_private;
65 u32 pgd = (dev_priv->chipset == 0x50) ? 0x1400 : 0x0200;
66 u32 fc = (dev_priv->chipset == 0x50) ? 0x0000 : 0x4200;
67 struct nouveau_channel *chan;
70 chan = kzalloc(sizeof(*chan), GFP_KERNEL);
75 ret = nouveau_gpuobj_new(dev, NULL, size, 0x1000, 0, &chan->ramin);
77 nv50_channel_del(&chan);
81 ret = drm_mm_init(&chan->ramin_heap, 0x6000, chan->ramin->size);
83 nv50_channel_del(&chan);
87 ret = nouveau_gpuobj_new_fake(dev, chan->ramin->pinst == ~0 ? ~0 :
88 chan->ramin->pinst + pgd,
89 chan->ramin->vinst + pgd,
90 0x4000, NVOBJ_FLAG_ZERO_ALLOC,
93 nv50_channel_del(&chan);
97 ret = nouveau_gpuobj_new_fake(dev, chan->ramin->pinst == ~0 ? ~0 :
98 chan->ramin->pinst + fc,
99 chan->ramin->vinst + fc, 0x100,
100 NVOBJ_FLAG_ZERO_ALLOC, &chan->ramfc);
102 nv50_channel_del(&chan);
111 nv50_instmem_init(struct drm_device *dev)
113 struct drm_nouveau_private *dev_priv = dev->dev_private;
114 struct nv50_instmem_priv *priv;
115 struct nouveau_channel *chan;
119 priv = kzalloc(sizeof(*priv), GFP_KERNEL);
122 dev_priv->engine.instmem.priv = priv;
124 /* Save state, will restore at takedown. */
125 for (i = 0x1700; i <= 0x1710; i += 4)
126 priv->save1700[(i-0x1700)/4] = nv_rd32(dev, i);
128 /* Global PRAMIN heap */
129 ret = drm_mm_init(&dev_priv->ramin_heap, 0, dev_priv->ramin_size);
131 NV_ERROR(dev, "Failed to init RAMIN heap\n");
135 /* we need a channel to plug into the hw to control the BARs */
136 ret = nv50_channel_new(dev, 128*1024, &dev_priv->channels.ptr[0]);
139 chan = dev_priv->channels.ptr[127] = dev_priv->channels.ptr[0];
141 /* allocate page table for PRAMIN BAR */
142 ret = nouveau_gpuobj_new(dev, chan, (dev_priv->ramin_size >> 12) * 8,
143 0x1000, NVOBJ_FLAG_ZERO_ALLOC,
148 nv_wo32(chan->vm_pd, 0x0000, priv->pramin_pt->vinst | 0x63);
149 nv_wo32(chan->vm_pd, 0x0004, 0);
151 /* DMA object for PRAMIN BAR */
152 ret = nouveau_gpuobj_new(dev, chan, 6*4, 16, 0, &priv->pramin_bar);
155 nv_wo32(priv->pramin_bar, 0x00, 0x7fc00000);
156 nv_wo32(priv->pramin_bar, 0x04, dev_priv->ramin_size - 1);
157 nv_wo32(priv->pramin_bar, 0x08, 0x00000000);
158 nv_wo32(priv->pramin_bar, 0x0c, 0x00000000);
159 nv_wo32(priv->pramin_bar, 0x10, 0x00000000);
160 nv_wo32(priv->pramin_bar, 0x14, 0x00000000);
162 nv50_instmem_map(chan->ramin);
165 nv_wr32(dev, 0x001704, 0x00000000 | (chan->ramin->vinst >> 12));
166 nv_wr32(dev, 0x001704, 0x40000000 | (chan->ramin->vinst >> 12));
167 nv_wr32(dev, 0x00170c, 0x80000000 | (priv->pramin_bar->cinst >> 4));
169 tmp = nv_ri32(dev, 0);
170 nv_wi32(dev, 0, ~tmp);
171 if (nv_ri32(dev, 0) != ~tmp) {
172 NV_ERROR(dev, "PRAMIN readback failed\n");
175 nv_wi32(dev, 0, tmp);
177 dev_priv->ramin_available = true;
179 /* Determine VM layout */
180 dev_priv->vm_gart_base = roundup(NV50_VM_BLOCK, NV50_VM_BLOCK);
181 dev_priv->vm_gart_size = NV50_VM_BLOCK;
183 dev_priv->vm_vram_base = dev_priv->vm_gart_base + dev_priv->vm_gart_size;
184 dev_priv->vm_vram_size = dev_priv->vram_size;
185 if (dev_priv->vm_vram_size > NV50_VM_MAX_VRAM)
186 dev_priv->vm_vram_size = NV50_VM_MAX_VRAM;
187 dev_priv->vm_vram_size = roundup(dev_priv->vm_vram_size, NV50_VM_BLOCK);
188 dev_priv->vm_vram_pt_nr = dev_priv->vm_vram_size / NV50_VM_BLOCK;
190 dev_priv->vm_end = dev_priv->vm_vram_base + dev_priv->vm_vram_size;
192 NV_DEBUG(dev, "NV50VM: GART 0x%016llx-0x%016llx\n",
193 dev_priv->vm_gart_base,
194 dev_priv->vm_gart_base + dev_priv->vm_gart_size - 1);
195 NV_DEBUG(dev, "NV50VM: VRAM 0x%016llx-0x%016llx\n",
196 dev_priv->vm_vram_base,
197 dev_priv->vm_vram_base + dev_priv->vm_vram_size - 1);
199 /* VRAM page table(s), mapped into VM at +1GiB */
200 for (i = 0; i < dev_priv->vm_vram_pt_nr; i++) {
201 ret = nouveau_gpuobj_new(dev, NULL, NV50_VM_BLOCK / 0x10000 * 8,
202 0, NVOBJ_FLAG_ZERO_ALLOC,
203 &chan->vm_vram_pt[i]);
205 NV_ERROR(dev, "Error creating VRAM PGT: %d\n", ret);
206 dev_priv->vm_vram_pt_nr = i;
209 dev_priv->vm_vram_pt[i] = chan->vm_vram_pt[i];
211 nv_wo32(chan->vm_pd, 0x10 + (i*8),
212 chan->vm_vram_pt[i]->vinst | 0x61);
213 nv_wo32(chan->vm_pd, 0x14 + (i*8), 0);
216 /* DMA object for FB BAR */
217 ret = nouveau_gpuobj_new(dev, chan, 6*4, 16, 0, &priv->fb_bar);
220 nv_wo32(priv->fb_bar, 0x00, 0x7fc00000);
221 nv_wo32(priv->fb_bar, 0x04, 0x40000000 +
222 pci_resource_len(dev->pdev, 1) - 1);
223 nv_wo32(priv->fb_bar, 0x08, 0x40000000);
224 nv_wo32(priv->fb_bar, 0x0c, 0x00000000);
225 nv_wo32(priv->fb_bar, 0x10, 0x00000000);
226 nv_wo32(priv->fb_bar, 0x14, 0x00000000);
228 dev_priv->engine.instmem.flush(dev);
230 nv_wr32(dev, 0x001708, 0x80000000 | (priv->fb_bar->cinst >> 4));
231 for (i = 0; i < 8; i++)
232 nv_wr32(dev, 0x1900 + (i*4), 0);
238 nv50_instmem_takedown(struct drm_device *dev)
240 struct drm_nouveau_private *dev_priv = dev->dev_private;
241 struct nv50_instmem_priv *priv = dev_priv->engine.instmem.priv;
242 struct nouveau_channel *chan = dev_priv->channels.ptr[0];
250 dev_priv->ramin_available = false;
252 /* Restore state from before init */
253 for (i = 0x1700; i <= 0x1710; i += 4)
254 nv_wr32(dev, i, priv->save1700[(i - 0x1700) / 4]);
256 nouveau_gpuobj_ref(NULL, &priv->fb_bar);
257 nouveau_gpuobj_ref(NULL, &priv->pramin_bar);
258 nouveau_gpuobj_ref(NULL, &priv->pramin_pt);
260 /* Destroy dummy channel */
262 for (i = 0; i < dev_priv->vm_vram_pt_nr; i++)
263 nouveau_gpuobj_ref(NULL, &chan->vm_vram_pt[i]);
264 dev_priv->vm_vram_pt_nr = 0;
266 nv50_channel_del(&dev_priv->channels.ptr[0]);
267 dev_priv->channels.ptr[127] = NULL;
270 dev_priv->engine.instmem.priv = NULL;
275 nv50_instmem_suspend(struct drm_device *dev)
277 struct drm_nouveau_private *dev_priv = dev->dev_private;
279 dev_priv->ramin_available = false;
284 nv50_instmem_resume(struct drm_device *dev)
286 struct drm_nouveau_private *dev_priv = dev->dev_private;
287 struct nv50_instmem_priv *priv = dev_priv->engine.instmem.priv;
288 struct nouveau_channel *chan = dev_priv->channels.ptr[0];
291 /* Poke the relevant regs, and pray it works :) */
292 nv_wr32(dev, NV50_PUNK_BAR_CFG_BASE, (chan->ramin->vinst >> 12));
293 nv_wr32(dev, NV50_PUNK_UNK1710, 0);
294 nv_wr32(dev, NV50_PUNK_BAR_CFG_BASE, (chan->ramin->vinst >> 12) |
295 NV50_PUNK_BAR_CFG_BASE_VALID);
296 nv_wr32(dev, NV50_PUNK_BAR1_CTXDMA, (priv->fb_bar->cinst >> 4) |
297 NV50_PUNK_BAR1_CTXDMA_VALID);
298 nv_wr32(dev, NV50_PUNK_BAR3_CTXDMA, (priv->pramin_bar->cinst >> 4) |
299 NV50_PUNK_BAR3_CTXDMA_VALID);
301 for (i = 0; i < 8; i++)
302 nv_wr32(dev, 0x1900 + (i*4), 0);
304 dev_priv->ramin_available = true;
307 struct nv50_gpuobj_node {
308 struct nouveau_bo *vram;
309 struct drm_mm_node *ramin;
315 nv50_instmem_get(struct nouveau_gpuobj *gpuobj, u32 size, u32 align)
317 struct drm_device *dev = gpuobj->dev;
318 struct nv50_gpuobj_node *node = NULL;
321 node = kzalloc(sizeof(*node), GFP_KERNEL);
326 ret = nouveau_bo_new(dev, NULL, size, align, TTM_PL_FLAG_VRAM,
327 0, 0x0000, true, false, &node->vram);
329 NV_ERROR(dev, "error getting PRAMIN backing pages: %d\n", ret);
334 ret = nouveau_bo_pin(node->vram, TTM_PL_FLAG_VRAM);
336 NV_ERROR(dev, "error pinning PRAMIN backing VRAM: %d\n", ret);
337 nouveau_bo_ref(NULL, &node->vram);
341 gpuobj->vinst = node->vram->bo.mem.start << PAGE_SHIFT;
342 gpuobj->size = node->vram->bo.mem.num_pages << PAGE_SHIFT;
348 nv50_instmem_put(struct nouveau_gpuobj *gpuobj)
350 struct nv50_gpuobj_node *node;
355 nouveau_bo_unpin(node->vram);
356 nouveau_bo_ref(NULL, &node->vram);
361 nv50_instmem_map(struct nouveau_gpuobj *gpuobj)
363 struct drm_nouveau_private *dev_priv = gpuobj->dev->dev_private;
364 struct nv50_instmem_priv *priv = dev_priv->engine.instmem.priv;
365 struct nv50_gpuobj_node *node = gpuobj->node;
366 struct drm_device *dev = gpuobj->dev;
367 struct drm_mm_node *ramin = NULL;
372 if (drm_mm_pre_get(&dev_priv->ramin_heap))
375 spin_lock(&dev_priv->ramin_lock);
376 ramin = drm_mm_search_free(&dev_priv->ramin_heap, gpuobj->size,
379 spin_unlock(&dev_priv->ramin_lock);
383 ramin = drm_mm_get_block_atomic(ramin, gpuobj->size, node->align);
384 spin_unlock(&dev_priv->ramin_lock);
385 } while (ramin == NULL);
387 pte = (ramin->start >> 12) << 1;
388 pte_end = ((ramin->size >> 12) << 1) + pte;
389 vram = gpuobj->vinst;
391 NV_DEBUG(dev, "pramin=0x%lx, pte=%d, pte_end=%d\n",
392 ramin->start, pte, pte_end);
393 NV_DEBUG(dev, "first vram page: 0x%010llx\n", gpuobj->vinst);
396 if (dev_priv->vram_sys_base) {
397 vram += dev_priv->vram_sys_base;
401 while (pte < pte_end) {
402 nv_wo32(priv->pramin_pt, (pte * 4) + 0, lower_32_bits(vram));
403 nv_wo32(priv->pramin_pt, (pte * 4) + 4, upper_32_bits(vram));
407 dev_priv->engine.instmem.flush(dev);
409 nv50_vm_flush_engine(dev, 6);
412 gpuobj->pinst = ramin->start;
417 nv50_instmem_unmap(struct nouveau_gpuobj *gpuobj)
419 struct drm_nouveau_private *dev_priv = gpuobj->dev->dev_private;
420 struct nv50_instmem_priv *priv = dev_priv->engine.instmem.priv;
421 struct nv50_gpuobj_node *node = gpuobj->node;
424 if (!node->ramin || !dev_priv->ramin_available)
427 pte = (node->ramin->start >> 12) << 1;
428 pte_end = ((node->ramin->size >> 12) << 1) + pte;
430 while (pte < pte_end) {
431 nv_wo32(priv->pramin_pt, (pte * 4) + 0, 0x00000000);
432 nv_wo32(priv->pramin_pt, (pte * 4) + 4, 0x00000000);
435 dev_priv->engine.instmem.flush(gpuobj->dev);
437 spin_lock(&dev_priv->ramin_lock);
438 drm_mm_put_block(node->ramin);
440 spin_unlock(&dev_priv->ramin_lock);
444 nv50_instmem_flush(struct drm_device *dev)
446 nv_wr32(dev, 0x00330c, 0x00000001);
447 if (!nv_wait(dev, 0x00330c, 0x00000002, 0x00000000))
448 NV_ERROR(dev, "PRAMIN flush timeout\n");
452 nv84_instmem_flush(struct drm_device *dev)
454 nv_wr32(dev, 0x070000, 0x00000001);
455 if (!nv_wait(dev, 0x070000, 0x00000002, 0x00000000))
456 NV_ERROR(dev, "PRAMIN flush timeout\n");