1 /**************************************************************************
3 * Copyright © 2012 VMware, Inc., Palo Alto, CA., USA
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **************************************************************************/
28 #include "vmwgfx_drv.h"
31 * If we set up the screen target otable, screen objects stop working.
34 #define VMW_OTABLE_SETUP_SUB ((VMWGFX_ENABLE_SCREEN_TARGET_OTABLE) ? 0 : 1)
37 #define VMW_PPN_SIZE 8
38 #define VMW_MOBFMT_PTDEPTH_0 SVGA3D_MOBFMT_PTDEPTH64_0
39 #define VMW_MOBFMT_PTDEPTH_1 SVGA3D_MOBFMT_PTDEPTH64_1
40 #define VMW_MOBFMT_PTDEPTH_2 SVGA3D_MOBFMT_PTDEPTH64_2
42 #define VMW_PPN_SIZE 4
43 #define VMW_MOBFMT_PTDEPTH_0 SVGA3D_MOBFMT_PTDEPTH_0
44 #define VMW_MOBFMT_PTDEPTH_1 SVGA3D_MOBFMT_PTDEPTH_1
45 #define VMW_MOBFMT_PTDEPTH_2 SVGA3D_MOBFMT_PTDEPTH_2
49 * struct vmw_mob - Structure containing page table and metadata for a
50 * Guest Memory OBject.
52 * @num_pages Number of pages that make up the page table.
53 * @pt_level The indirection level of the page table. 0-2.
54 * @pt_root_page DMA address of the level 0 page of the page table.
57 struct ttm_buffer_object *pt_bo;
58 unsigned long num_pages;
60 dma_addr_t pt_root_page;
65 * struct vmw_otable - Guest Memory OBject table metadata
67 * @size: Size of the table (page-aligned).
68 * @page_table: Pointer to a struct vmw_mob holding the page table.
72 struct vmw_mob *page_table;
75 static int vmw_mob_pt_populate(struct vmw_private *dev_priv,
77 static void vmw_mob_pt_setup(struct vmw_mob *mob,
78 struct vmw_piter data_iter,
79 unsigned long num_data_pages);
82 * vmw_setup_otable_base - Issue an object table base setup command to
85 * @dev_priv: Pointer to a device private structure
86 * @type: Type of object table base
87 * @offset Start of table offset into dev_priv::otable_bo
88 * @otable Pointer to otable metadata;
90 * This function returns -ENOMEM if it fails to reserve fifo space,
91 * and may block waiting for fifo space.
93 static int vmw_setup_otable_base(struct vmw_private *dev_priv,
96 struct vmw_otable *otable)
99 SVGA3dCmdHeader header;
100 SVGA3dCmdSetOTableBase64 body;
103 const struct vmw_sg_table *vsgt;
104 struct vmw_piter iter;
107 BUG_ON(otable->page_table != NULL);
109 vsgt = vmw_bo_sg_table(dev_priv->otable_bo);
110 vmw_piter_start(&iter, vsgt, offset >> PAGE_SHIFT);
111 WARN_ON(!vmw_piter_next(&iter));
113 mob = vmw_mob_create(otable->size >> PAGE_SHIFT);
114 if (unlikely(mob == NULL)) {
115 DRM_ERROR("Failed creating OTable page table.\n");
119 if (otable->size <= PAGE_SIZE) {
120 mob->pt_level = VMW_MOBFMT_PTDEPTH_0;
121 mob->pt_root_page = vmw_piter_dma_addr(&iter);
122 } else if (vsgt->num_regions == 1) {
123 mob->pt_level = SVGA3D_MOBFMT_RANGE;
124 mob->pt_root_page = vmw_piter_dma_addr(&iter);
126 ret = vmw_mob_pt_populate(dev_priv, mob);
127 if (unlikely(ret != 0))
128 goto out_no_populate;
130 vmw_mob_pt_setup(mob, iter, otable->size >> PAGE_SHIFT);
131 mob->pt_level += VMW_MOBFMT_PTDEPTH_1 - SVGA3D_MOBFMT_PTDEPTH_1;
134 cmd = vmw_fifo_reserve(dev_priv, sizeof(*cmd));
135 if (unlikely(cmd == NULL)) {
136 DRM_ERROR("Failed reserving FIFO space for OTable setup.\n");
141 memset(cmd, 0, sizeof(*cmd));
142 cmd->header.id = SVGA_3D_CMD_SET_OTABLE_BASE64;
143 cmd->header.size = sizeof(cmd->body);
144 cmd->body.type = type;
145 cmd->body.baseAddress = cpu_to_le64(mob->pt_root_page >> PAGE_SHIFT);
146 cmd->body.sizeInBytes = otable->size;
147 cmd->body.validSizeInBytes = 0;
148 cmd->body.ptDepth = mob->pt_level;
151 * The device doesn't support this, But the otable size is
152 * determined at compile-time, so this BUG shouldn't trigger
155 BUG_ON(mob->pt_level == VMW_MOBFMT_PTDEPTH_2);
157 vmw_fifo_commit(dev_priv, sizeof(*cmd));
158 otable->page_table = mob;
164 vmw_mob_destroy(mob);
169 * vmw_takedown_otable_base - Issue an object table base takedown command
172 * @dev_priv: Pointer to a device private structure
173 * @type: Type of object table base
176 static void vmw_takedown_otable_base(struct vmw_private *dev_priv,
178 struct vmw_otable *otable)
181 SVGA3dCmdHeader header;
182 SVGA3dCmdSetOTableBase body;
184 struct ttm_buffer_object *bo;
186 if (otable->page_table == NULL)
189 bo = otable->page_table->pt_bo;
190 cmd = vmw_fifo_reserve(dev_priv, sizeof(*cmd));
191 if (unlikely(cmd == NULL)) {
192 DRM_ERROR("Failed reserving FIFO space for OTable "
195 memset(cmd, 0, sizeof(*cmd));
196 cmd->header.id = SVGA_3D_CMD_SET_OTABLE_BASE;
197 cmd->header.size = sizeof(cmd->body);
198 cmd->body.type = type;
199 cmd->body.baseAddress = 0;
200 cmd->body.sizeInBytes = 0;
201 cmd->body.validSizeInBytes = 0;
202 cmd->body.ptDepth = SVGA3D_MOBFMT_INVALID;
203 vmw_fifo_commit(dev_priv, sizeof(*cmd));
209 ret = ttm_bo_reserve(bo, false, true, false, NULL);
212 vmw_fence_single_bo(bo, NULL);
213 ttm_bo_unreserve(bo);
216 vmw_mob_destroy(otable->page_table);
217 otable->page_table = NULL;
221 * vmw_otables_setup - Set up guest backed memory object tables
223 * @dev_priv: Pointer to a device private structure
225 * Takes care of the device guest backed surface
226 * initialization, by setting up the guest backed memory object tables.
227 * Returns 0 on success and various error codes on failure. A succesful return
228 * means the object tables can be taken down using the vmw_otables_takedown
231 int vmw_otables_setup(struct vmw_private *dev_priv)
233 unsigned long offset;
234 unsigned long bo_size;
235 struct vmw_otable *otables;
239 otables = kzalloc(SVGA_OTABLE_DX9_MAX * sizeof(*otables),
241 if (unlikely(otables == NULL)) {
242 DRM_ERROR("Failed to allocate space for otable "
247 otables[SVGA_OTABLE_MOB].size =
248 VMWGFX_NUM_MOB * SVGA3D_OTABLE_MOB_ENTRY_SIZE;
249 otables[SVGA_OTABLE_SURFACE].size =
250 VMWGFX_NUM_GB_SURFACE * SVGA3D_OTABLE_SURFACE_ENTRY_SIZE;
251 otables[SVGA_OTABLE_CONTEXT].size =
252 VMWGFX_NUM_GB_CONTEXT * SVGA3D_OTABLE_CONTEXT_ENTRY_SIZE;
253 otables[SVGA_OTABLE_SHADER].size =
254 VMWGFX_NUM_GB_SHADER * SVGA3D_OTABLE_SHADER_ENTRY_SIZE;
255 otables[SVGA_OTABLE_SCREEN_TARGET].size =
256 VMWGFX_NUM_GB_SCREEN_TARGET *
257 SVGA3D_OTABLE_SCREEN_TARGET_ENTRY_SIZE;
260 for (i = 0; i < SVGA_OTABLE_DX9_MAX; ++i) {
262 (otables[i].size + PAGE_SIZE - 1) & PAGE_MASK;
263 bo_size += otables[i].size;
266 ret = ttm_bo_create(&dev_priv->bdev, bo_size,
268 &vmw_sys_ne_placement,
270 &dev_priv->otable_bo);
272 if (unlikely(ret != 0))
275 ret = ttm_bo_reserve(dev_priv->otable_bo, false, true, false, NULL);
277 ret = vmw_bo_driver.ttm_tt_populate(dev_priv->otable_bo->ttm);
278 if (unlikely(ret != 0))
280 ret = vmw_bo_map_dma(dev_priv->otable_bo);
281 if (unlikely(ret != 0))
284 ttm_bo_unreserve(dev_priv->otable_bo);
287 for (i = 0; i < SVGA_OTABLE_DX9_MAX - VMW_OTABLE_SETUP_SUB; ++i) {
288 ret = vmw_setup_otable_base(dev_priv, i, offset,
290 if (unlikely(ret != 0))
292 offset += otables[i].size;
295 dev_priv->otables = otables;
299 ttm_bo_unreserve(dev_priv->otable_bo);
301 for (i = 0; i < SVGA_OTABLE_DX9_MAX - VMW_OTABLE_SETUP_SUB; ++i)
302 vmw_takedown_otable_base(dev_priv, i, &otables[i]);
304 ttm_bo_unref(&dev_priv->otable_bo);
312 * vmw_otables_takedown - Take down guest backed memory object tables
314 * @dev_priv: Pointer to a device private structure
316 * Take down the Guest Memory Object tables.
318 void vmw_otables_takedown(struct vmw_private *dev_priv)
321 struct ttm_buffer_object *bo = dev_priv->otable_bo;
324 for (i = 0; i < SVGA_OTABLE_DX9_MAX - VMW_OTABLE_SETUP_SUB; ++i)
325 vmw_takedown_otable_base(dev_priv, i,
326 &dev_priv->otables[i]);
328 ret = ttm_bo_reserve(bo, false, true, false, NULL);
331 vmw_fence_single_bo(bo, NULL);
332 ttm_bo_unreserve(bo);
334 ttm_bo_unref(&dev_priv->otable_bo);
335 kfree(dev_priv->otables);
336 dev_priv->otables = NULL;
341 * vmw_mob_calculate_pt_pages - Calculate the number of page table pages
342 * needed for a guest backed memory object.
344 * @data_pages: Number of data pages in the memory object buffer.
346 static unsigned long vmw_mob_calculate_pt_pages(unsigned long data_pages)
348 unsigned long data_size = data_pages * PAGE_SIZE;
349 unsigned long tot_size = 0;
351 while (likely(data_size > PAGE_SIZE)) {
352 data_size = DIV_ROUND_UP(data_size, PAGE_SIZE);
353 data_size *= VMW_PPN_SIZE;
354 tot_size += (data_size + PAGE_SIZE - 1) & PAGE_MASK;
357 return tot_size >> PAGE_SHIFT;
361 * vmw_mob_create - Create a mob, but don't populate it.
363 * @data_pages: Number of data pages of the underlying buffer object.
365 struct vmw_mob *vmw_mob_create(unsigned long data_pages)
367 struct vmw_mob *mob = kzalloc(sizeof(*mob), GFP_KERNEL);
369 if (unlikely(mob == NULL))
372 mob->num_pages = vmw_mob_calculate_pt_pages(data_pages);
378 * vmw_mob_pt_populate - Populate the mob pagetable
380 * @mob: Pointer to the mob the pagetable of which we want to
383 * This function allocates memory to be used for the pagetable, and
384 * adjusts TTM memory accounting accordingly. Returns ENOMEM if
385 * memory resources aren't sufficient and may cause TTM buffer objects
386 * to be swapped out by using the TTM memory accounting function.
388 static int vmw_mob_pt_populate(struct vmw_private *dev_priv,
392 BUG_ON(mob->pt_bo != NULL);
394 ret = ttm_bo_create(&dev_priv->bdev, mob->num_pages * PAGE_SIZE,
396 &vmw_sys_ne_placement,
397 0, false, NULL, &mob->pt_bo);
398 if (unlikely(ret != 0))
401 ret = ttm_bo_reserve(mob->pt_bo, false, true, false, NULL);
404 ret = vmw_bo_driver.ttm_tt_populate(mob->pt_bo->ttm);
405 if (unlikely(ret != 0))
407 ret = vmw_bo_map_dma(mob->pt_bo);
408 if (unlikely(ret != 0))
411 ttm_bo_unreserve(mob->pt_bo);
416 ttm_bo_unreserve(mob->pt_bo);
417 ttm_bo_unref(&mob->pt_bo);
423 * vmw_mob_assign_ppn - Assign a value to a page table entry
425 * @addr: Pointer to pointer to page table entry.
426 * @val: The page table entry
428 * Assigns a value to a page table entry pointed to by *@addr and increments
429 * *@addr according to the page table entry size.
431 #if (VMW_PPN_SIZE == 8)
432 static void vmw_mob_assign_ppn(__le32 **addr, dma_addr_t val)
434 *((__le64 *) *addr) = cpu_to_le64(val >> PAGE_SHIFT);
438 static void vmw_mob_assign_ppn(__le32 **addr, dma_addr_t val)
440 *(*addr)++ = cpu_to_le32(val >> PAGE_SHIFT);
445 * vmw_mob_build_pt - Build a pagetable
447 * @data_addr: Array of DMA addresses to the underlying buffer
448 * object's data pages.
449 * @num_data_pages: Number of buffer object data pages.
450 * @pt_pages: Array of page pointers to the page table pages.
452 * Returns the number of page table pages actually used.
453 * Uses atomic kmaps of highmem pages to avoid TLB thrashing.
455 static unsigned long vmw_mob_build_pt(struct vmw_piter *data_iter,
456 unsigned long num_data_pages,
457 struct vmw_piter *pt_iter)
459 unsigned long pt_size = num_data_pages * VMW_PPN_SIZE;
460 unsigned long num_pt_pages = DIV_ROUND_UP(pt_size, PAGE_SIZE);
461 unsigned long pt_page;
462 __le32 *addr, *save_addr;
466 for (pt_page = 0; pt_page < num_pt_pages; ++pt_page) {
467 page = vmw_piter_page(pt_iter);
469 save_addr = addr = kmap_atomic(page);
471 for (i = 0; i < PAGE_SIZE / VMW_PPN_SIZE; ++i) {
472 vmw_mob_assign_ppn(&addr,
473 vmw_piter_dma_addr(data_iter));
474 if (unlikely(--num_data_pages == 0))
476 WARN_ON(!vmw_piter_next(data_iter));
478 kunmap_atomic(save_addr);
479 vmw_piter_next(pt_iter);
486 * vmw_mob_build_pt - Set up a multilevel mob pagetable
488 * @mob: Pointer to a mob whose page table needs setting up.
489 * @data_addr Array of DMA addresses to the buffer object's data
491 * @num_data_pages: Number of buffer object data pages.
493 * Uses tail recursion to set up a multilevel mob page table.
495 static void vmw_mob_pt_setup(struct vmw_mob *mob,
496 struct vmw_piter data_iter,
497 unsigned long num_data_pages)
499 unsigned long num_pt_pages = 0;
500 struct ttm_buffer_object *bo = mob->pt_bo;
501 struct vmw_piter save_pt_iter;
502 struct vmw_piter pt_iter;
503 const struct vmw_sg_table *vsgt;
506 ret = ttm_bo_reserve(bo, false, true, false, NULL);
509 vsgt = vmw_bo_sg_table(bo);
510 vmw_piter_start(&pt_iter, vsgt, 0);
511 BUG_ON(!vmw_piter_next(&pt_iter));
513 while (likely(num_data_pages > 1)) {
515 BUG_ON(mob->pt_level > 2);
516 save_pt_iter = pt_iter;
517 num_pt_pages = vmw_mob_build_pt(&data_iter, num_data_pages,
519 data_iter = save_pt_iter;
520 num_data_pages = num_pt_pages;
523 mob->pt_root_page = vmw_piter_dma_addr(&save_pt_iter);
524 ttm_bo_unreserve(bo);
528 * vmw_mob_destroy - Destroy a mob, unpopulating first if necessary.
530 * @mob: Pointer to a mob to destroy.
532 void vmw_mob_destroy(struct vmw_mob *mob)
535 ttm_bo_unref(&mob->pt_bo);
540 * vmw_mob_unbind - Hide a mob from the device.
542 * @dev_priv: Pointer to a device private.
543 * @mob_id: Device id of the mob to unbind.
545 void vmw_mob_unbind(struct vmw_private *dev_priv,
549 SVGA3dCmdHeader header;
550 SVGA3dCmdDestroyGBMob body;
553 struct ttm_buffer_object *bo = mob->pt_bo;
556 ret = ttm_bo_reserve(bo, false, true, false, NULL);
558 * Noone else should be using this buffer.
563 cmd = vmw_fifo_reserve(dev_priv, sizeof(*cmd));
564 if (unlikely(cmd == NULL)) {
565 DRM_ERROR("Failed reserving FIFO space for Memory "
566 "Object unbinding.\n");
568 cmd->header.id = SVGA_3D_CMD_DESTROY_GB_MOB;
569 cmd->header.size = sizeof(cmd->body);
570 cmd->body.mobid = mob->id;
571 vmw_fifo_commit(dev_priv, sizeof(*cmd));
574 vmw_fence_single_bo(bo, NULL);
575 ttm_bo_unreserve(bo);
577 vmw_3d_resource_dec(dev_priv, false);
581 * vmw_mob_bind - Make a mob visible to the device after first
582 * populating it if necessary.
584 * @dev_priv: Pointer to a device private.
585 * @mob: Pointer to the mob we're making visible.
586 * @data_addr: Array of DMA addresses to the data pages of the underlying
588 * @num_data_pages: Number of data pages of the underlying buffer
590 * @mob_id: Device id of the mob to bind
592 * This function is intended to be interfaced with the ttm_tt backend
595 int vmw_mob_bind(struct vmw_private *dev_priv,
597 const struct vmw_sg_table *vsgt,
598 unsigned long num_data_pages,
602 bool pt_set_up = false;
603 struct vmw_piter data_iter;
605 SVGA3dCmdHeader header;
606 SVGA3dCmdDefineGBMob64 body;
610 vmw_piter_start(&data_iter, vsgt, 0);
611 if (unlikely(!vmw_piter_next(&data_iter)))
614 if (likely(num_data_pages == 1)) {
615 mob->pt_level = VMW_MOBFMT_PTDEPTH_0;
616 mob->pt_root_page = vmw_piter_dma_addr(&data_iter);
617 } else if (vsgt->num_regions == 1) {
618 mob->pt_level = SVGA3D_MOBFMT_RANGE;
619 mob->pt_root_page = vmw_piter_dma_addr(&data_iter);
620 } else if (unlikely(mob->pt_bo == NULL)) {
621 ret = vmw_mob_pt_populate(dev_priv, mob);
622 if (unlikely(ret != 0))
625 vmw_mob_pt_setup(mob, data_iter, num_data_pages);
627 mob->pt_level += VMW_MOBFMT_PTDEPTH_1 - SVGA3D_MOBFMT_PTDEPTH_1;
630 (void) vmw_3d_resource_inc(dev_priv, false);
632 cmd = vmw_fifo_reserve(dev_priv, sizeof(*cmd));
633 if (unlikely(cmd == NULL)) {
634 DRM_ERROR("Failed reserving FIFO space for Memory "
635 "Object binding.\n");
636 goto out_no_cmd_space;
639 cmd->header.id = SVGA_3D_CMD_DEFINE_GB_MOB64;
640 cmd->header.size = sizeof(cmd->body);
641 cmd->body.mobid = mob_id;
642 cmd->body.ptDepth = mob->pt_level;
643 cmd->body.base = cpu_to_le64(mob->pt_root_page >> PAGE_SHIFT);
644 cmd->body.sizeInBytes = num_data_pages * PAGE_SIZE;
646 vmw_fifo_commit(dev_priv, sizeof(*cmd));
651 vmw_3d_resource_dec(dev_priv, false);
653 ttm_bo_unref(&mob->pt_bo);