drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors: Dave Airlie
  25  *          Alex Deucher
  26  *          Jerome Glisse
  27  */
  28 #include <linux/dma-fence-array.h>
  29 #include <drm/drmP.h>
  30 #include <drm/amdgpu_drm.h>
  31 #include "amdgpu.h"
  32 #include "amdgpu_trace.h"
  33
  34 /*
  35  * GPUVM
  36  * GPUVM is similar to the legacy gart on older asics, however
  37  * rather than there being a single global gart table
  38  * for the entire GPU, there are multiple VM page tables active
  39  * at any given time.  The VM page tables can contain a mix
  40  * vram pages and system memory pages and system memory pages
  41  * can be mapped as snooped (cached system pages) or unsnooped
  42  * (uncached system pages).
  43  * Each VM has an ID associated with it and there is a page table
  44  * associated with each VMID.  When execting a command buffer,
  45  * the kernel tells the the ring what VMID to use for that command
  46  * buffer.  VMIDs are allocated dynamically as commands are submitted.
  47  * The userspace drivers maintain their own address space and the kernel
  48  * sets up their pages tables accordingly when they submit their
  49  * command buffers and a VMID is assigned.
  50  * Cayman/Trinity support up to 8 active VMs at any given time;
  51  * SI supports 16.
  52  */
  53
  54 /* Local structure. Encapsulate some VM table update parameters to reduce
  55  * the number of function parameters
  56  */
  57 struct amdgpu_pte_update_params {
  58         /* amdgpu device we do this update for */
  59         struct amdgpu_device *adev;
  60         /* address where to copy page table entries from */
  61         uint64_t src;
  62         /* indirect buffer to fill with commands */
  63         struct amdgpu_ib *ib;
  64         /* Function which actually does the update */
  65         void (*func)(struct amdgpu_pte_update_params *params, uint64_t pe,
  66                      uint64_t addr, unsigned count, uint32_t incr,
  67                      uint64_t flags);
  68         /* indicate update pt or its shadow */
  69         bool shadow;
  70 };
  71
  72 /* Helper to disable partial resident texture feature from a fence callback */
  73 struct amdgpu_prt_cb {
  74         struct amdgpu_device *adev;
  75         struct dma_fence_cb cb;
  76 };
  77
  78 /**
  79  * amdgpu_vm_num_pde - return the number of page directory entries
  80  *
  81  * @adev: amdgpu_device pointer
  82  *
  83  * Calculate the number of page directory entries.
  84  */
  85 static unsigned amdgpu_vm_num_pdes(struct amdgpu_device *adev)
  86 {
  87         return adev->vm_manager.max_pfn >> amdgpu_vm_block_size;
  88 }
  89
  90 /**
  91  * amdgpu_vm_directory_size - returns the size of the page directory in bytes
  92  *
  93  * @adev: amdgpu_device pointer
  94  *
  95  * Calculate the size of the page directory in bytes.
  96  */
  97 static unsigned amdgpu_vm_directory_size(struct amdgpu_device *adev)
  98 {
  99         return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_num_pdes(adev) * 8);
 100 }
 101
 102 /**
 103  * amdgpu_vm_get_pd_bo - add the VM PD to a validation list
 104  *
 105  * @vm: vm providing the BOs
 106  * @validated: head of validation list
 107  * @entry: entry to add
 108  *
 109  * Add the page directory to the list of BOs to
 110  * validate for command submission.
 111  */
 112 void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
 113                          struct list_head *validated,
 114                          struct amdgpu_bo_list_entry *entry)
 115 {
 116         entry->robj = vm->page_directory;
 117         entry->priority = 0;
 118         entry->tv.bo = &vm->page_directory->tbo;
 119         entry->tv.shared = true;
 120         entry->user_pages = NULL;
 121         list_add(&entry->tv.head, validated);
 122 }
 123
 124 /**
 125  * amdgpu_vm_validate_pt_bos - validate the page table BOs
 126  *
 127  * @adev: amdgpu device pointer
 128  * @vm: vm providing the BOs
 129  * @validate: callback to do the validation
 130  * @param: parameter for the validation callback
 131  *
 132  * Validate the page table BOs on command submission if neccessary.
 133  */
 134 int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 135                               int (*validate)(void *p, struct amdgpu_bo *bo),
 136                               void *param)
 137 {
 138         uint64_t num_evictions;
 139         unsigned i;
 140         int r;
 141
 142         /* We only need to validate the page tables
 143          * if they aren't already valid.
 144          */
 145         num_evictions = atomic64_read(&adev->num_evictions);
 146         if (num_evictions == vm->last_eviction_counter)
 147                 return 0;
 148
 149         /* add the vm page table to the list */
 150         for (i = 0; i <= vm->max_pde_used; ++i) {
 151                 struct amdgpu_bo *bo = vm->page_tables[i].bo;
 152
 153                 if (!bo)
 154                         continue;
 155
 156                 r = validate(param, bo);
 157                 if (r)
 158                         return r;
 159         }
 160
 161         return 0;
 162 }
 163
 164 /**
 165  * amdgpu_vm_move_pt_bos_in_lru - move the PT BOs to the LRU tail
 166  *
 167  * @adev: amdgpu device instance
 168  * @vm: vm providing the BOs
 169  *
 170  * Move the PT BOs to the tail of the LRU.
 171  */
 172 void amdgpu_vm_move_pt_bos_in_lru(struct amdgpu_device *adev,
 173                                   struct amdgpu_vm *vm)
 174 {
 175         struct ttm_bo_global *glob = adev->mman.bdev.glob;
 176         unsigned i;
 177
 178         spin_lock(&glob->lru_lock);
 179         for (i = 0; i <= vm->max_pde_used; ++i) {
 180                 struct amdgpu_bo *bo = vm->page_tables[i].bo;
 181
 182                 if (!bo)
 183                         continue;
 184
 185                 ttm_bo_move_to_lru_tail(&bo->tbo);
 186         }
 187         spin_unlock(&glob->lru_lock);
 188 }
 189
 190 static bool amdgpu_vm_is_gpu_reset(struct amdgpu_device *adev,
 191                               struct amdgpu_vm_id *id)
 192 {
 193         return id->current_gpu_reset_count !=
 194                 atomic_read(&adev->gpu_reset_counter) ? true : false;
 195 }
 196
 197 /**
 198  * amdgpu_vm_grab_id - allocate the next free VMID
 199  *
 200  * @vm: vm to allocate id for
 201  * @ring: ring we want to submit job to
 202  * @sync: sync object where we add dependencies
 203  * @fence: fence protecting ID from reuse
 204  *
 205  * Allocate an id for the vm, adding fences to the sync obj as necessary.
 206  */
 207 int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
 208                       struct amdgpu_sync *sync, struct dma_fence *fence,
 209                       struct amdgpu_job *job)
 210 {
 211         struct amdgpu_device *adev = ring->adev;
 212         uint64_t fence_context = adev->fence_context + ring->idx;
 213         struct dma_fence *updates = sync->last_vm_update;
 214         struct amdgpu_vm_id *id, *idle;
 215         struct dma_fence **fences;
 216         unsigned i;
 217         int r = 0;
 218
 219         fences = kmalloc_array(sizeof(void *), adev->vm_manager.num_ids,
 220                                GFP_KERNEL);
 221         if (!fences)
 222                 return -ENOMEM;
 223
 224         mutex_lock(&adev->vm_manager.lock);
 225
 226         /* Check if we have an idle VMID */
 227         i = 0;
 228         list_for_each_entry(idle, &adev->vm_manager.ids_lru, list) {
 229                 fences[i] = amdgpu_sync_peek_fence(&idle->active, ring);
 230                 if (!fences[i])
 231                         break;
 232                 ++i;
 233         }
 234
 235         /* If we can't find a idle VMID to use, wait till one becomes available */
 236         if (&idle->list == &adev->vm_manager.ids_lru) {
 237                 u64 fence_context = adev->vm_manager.fence_context + ring->idx;
 238                 unsigned seqno = ++adev->vm_manager.seqno[ring->idx];
 239                 struct dma_fence_array *array;
 240                 unsigned j;
 241
 242                 for (j = 0; j < i; ++j)
 243                         dma_fence_get(fences[j]);
 244
 245                 array = dma_fence_array_create(i, fences, fence_context,
 246                                            seqno, true);
 247                 if (!array) {
 248                         for (j = 0; j < i; ++j)
 249                                 dma_fence_put(fences[j]);
 250                         kfree(fences);
 251                         r = -ENOMEM;
 252                         goto error;
 253                 }
 254
 255
 256                 r = amdgpu_sync_fence(ring->adev, sync, &array->base);
 257                 dma_fence_put(&array->base);
 258                 if (r)
 259                         goto error;
 260
 261                 mutex_unlock(&adev->vm_manager.lock);
 262                 return 0;
 263
 264         }
 265         kfree(fences);
 266
 267         job->vm_needs_flush = true;
 268         /* Check if we can use a VMID already assigned to this VM */
 269         i = ring->idx;
 270         do {
 271                 struct dma_fence *flushed;
 272
 273                 id = vm->ids[i++];
 274                 if (i == AMDGPU_MAX_RINGS)
 275                         i = 0;
 276
 277                 /* Check all the prerequisites to using this VMID */
 278                 if (!id)
 279                         continue;
 280                 if (amdgpu_vm_is_gpu_reset(adev, id))
 281                         continue;
 282
 283                 if (atomic64_read(&id->owner) != vm->client_id)
 284                         continue;
 285
 286                 if (job->vm_pd_addr != id->pd_gpu_addr)
 287                         continue;
 288
 289                 if (!id->last_flush)
 290                         continue;
 291
 292                 if (id->last_flush->context != fence_context &&
 293                     !dma_fence_is_signaled(id->last_flush))
 294                         continue;
 295
 296                 flushed  = id->flushed_updates;
 297                 if (updates &&
 298                     (!flushed || dma_fence_is_later(updates, flushed)))
 299                         continue;
 300
 301                 /* Good we can use this VMID. Remember this submission as
 302                  * user of the VMID.
 303                  */
 304                 r = amdgpu_sync_fence(ring->adev, &id->active, fence);
 305                 if (r)
 306                         goto error;
 307
 308                 id->current_gpu_reset_count = atomic_read(&adev->gpu_reset_counter);
 309                 list_move_tail(&id->list, &adev->vm_manager.ids_lru);
 310                 vm->ids[ring->idx] = id;
 311
 312                 job->vm_id = id - adev->vm_manager.ids;
 313                 job->vm_needs_flush = false;
 314                 trace_amdgpu_vm_grab_id(vm, ring->idx, job);
 315
 316                 mutex_unlock(&adev->vm_manager.lock);
 317                 return 0;
 318
 319         } while (i != ring->idx);
 320
 321         /* Still no ID to use? Then use the idle one found earlier */
 322         id = idle;
 323
 324         /* Remember this submission as user of the VMID */
 325         r = amdgpu_sync_fence(ring->adev, &id->active, fence);
 326         if (r)
 327                 goto error;
 328
 329         dma_fence_put(id->first);
 330         id->first = dma_fence_get(fence);
 331
 332         dma_fence_put(id->last_flush);
 333         id->last_flush = NULL;
 334
 335         dma_fence_put(id->flushed_updates);
 336         id->flushed_updates = dma_fence_get(updates);
 337
 338         id->pd_gpu_addr = job->vm_pd_addr;
 339         id->current_gpu_reset_count = atomic_read(&adev->gpu_reset_counter);
 340         list_move_tail(&id->list, &adev->vm_manager.ids_lru);
 341         atomic64_set(&id->owner, vm->client_id);
 342         vm->ids[ring->idx] = id;
 343
 344         job->vm_id = id - adev->vm_manager.ids;
 345         trace_amdgpu_vm_grab_id(vm, ring->idx, job);
 346
 347 error:
 348         mutex_unlock(&adev->vm_manager.lock);
 349         return r;
 350 }
 351
 352 static bool amdgpu_vm_ring_has_compute_vm_bug(struct amdgpu_ring *ring)
 353 {
 354         struct amdgpu_device *adev = ring->adev;
 355         const struct amdgpu_ip_block *ip_block;
 356
 357         if (ring->funcs->type != AMDGPU_RING_TYPE_COMPUTE)
 358                 /* only compute rings */
 359                 return false;
 360
 361         ip_block = amdgpu_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX);
 362         if (!ip_block)
 363                 return false;
 364
 365         if (ip_block->version->major <= 7) {
 366                 /* gfx7 has no workaround */
 367                 return true;
 368         } else if (ip_block->version->major == 8) {
 369                 if (adev->gfx.mec_fw_version >= 673)
 370                         /* gfx8 is fixed in MEC firmware 673 */
 371                         return false;
 372                 else
 373                         return true;
 374         }
 375         return false;
 376 }
 377
 378 /**
 379  * amdgpu_vm_flush - hardware flush the vm
 380  *
 381  * @ring: ring to use for flush
 382  * @vm_id: vmid number to use
 383  * @pd_addr: address of the page directory
 384  *
 385  * Emit a VM flush when it is necessary.
 386  */
 387 int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job)
 388 {
 389         struct amdgpu_device *adev = ring->adev;
 390         struct amdgpu_vm_id *id = &adev->vm_manager.ids[job->vm_id];
 391         bool gds_switch_needed = ring->funcs->emit_gds_switch && (
 392                 id->gds_base != job->gds_base ||
 393                 id->gds_size != job->gds_size ||
 394                 id->gws_base != job->gws_base ||
 395                 id->gws_size != job->gws_size ||
 396                 id->oa_base != job->oa_base ||
 397                 id->oa_size != job->oa_size);
 398         int r;
 399
 400         if (ring->funcs->emit_pipeline_sync && (
 401             job->vm_needs_flush || gds_switch_needed ||
 402             amdgpu_vm_ring_has_compute_vm_bug(ring)))
 403                 amdgpu_ring_emit_pipeline_sync(ring);
 404
 405         if (ring->funcs->emit_vm_flush && (job->vm_needs_flush ||
 406             amdgpu_vm_is_gpu_reset(adev, id))) {
 407                 struct dma_fence *fence;
 408
 409                 trace_amdgpu_vm_flush(job->vm_pd_addr, ring->idx, job->vm_id);
 410                 amdgpu_ring_emit_vm_flush(ring, job->vm_id, job->vm_pd_addr);
 411
 412                 r = amdgpu_fence_emit(ring, &fence);
 413                 if (r)
 414                         return r;
 415
 416                 mutex_lock(&adev->vm_manager.lock);
 417                 dma_fence_put(id->last_flush);
 418                 id->last_flush = fence;
 419                 mutex_unlock(&adev->vm_manager.lock);
 420         }
 421
 422         if (gds_switch_needed) {
 423                 id->gds_base = job->gds_base;
 424                 id->gds_size = job->gds_size;
 425                 id->gws_base = job->gws_base;
 426                 id->gws_size = job->gws_size;
 427                 id->oa_base = job->oa_base;
 428                 id->oa_size = job->oa_size;
 429                 amdgpu_ring_emit_gds_switch(ring, job->vm_id,
 430                                             job->gds_base, job->gds_size,
 431                                             job->gws_base, job->gws_size,
 432                                             job->oa_base, job->oa_size);
 433         }
 434
 435         return 0;
 436 }
 437
 438 /**
 439  * amdgpu_vm_reset_id - reset VMID to zero
 440  *
 441  * @adev: amdgpu device structure
 442  * @vm_id: vmid number to use
 443  *
 444  * Reset saved GDW, GWS and OA to force switch on next flush.
 445  */
 446 void amdgpu_vm_reset_id(struct amdgpu_device *adev, unsigned vm_id)
 447 {
 448         struct amdgpu_vm_id *id = &adev->vm_manager.ids[vm_id];
 449
 450         id->gds_base = 0;
 451         id->gds_size = 0;
 452         id->gws_base = 0;
 453         id->gws_size = 0;
 454         id->oa_base = 0;
 455         id->oa_size = 0;
 456 }
 457
 458 /**
 459  * amdgpu_vm_bo_find - find the bo_va for a specific vm & bo
 460  *
 461  * @vm: requested vm
 462  * @bo: requested buffer object
 463  *
 464  * Find @bo inside the requested vm.
 465  * Search inside the @bos vm list for the requested vm
 466  * Returns the found bo_va or NULL if none is found
 467  *
 468  * Object has to be reserved!
 469  */
 470 struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm,
 471                                        struct amdgpu_bo *bo)
 472 {
 473         struct amdgpu_bo_va *bo_va;
 474
 475         list_for_each_entry(bo_va, &bo->va, bo_list) {
 476                 if (bo_va->vm == vm) {
 477                         return bo_va;
 478                 }
 479         }
 480         return NULL;
 481 }
 482
 483 /**
 484  * amdgpu_vm_do_set_ptes - helper to call the right asic function
 485  *
 486  * @params: see amdgpu_pte_update_params definition
 487  * @pe: addr of the page entry
 488  * @addr: dst addr to write into pe
 489  * @count: number of page entries to update
 490  * @incr: increase next addr by incr bytes
 491  * @flags: hw access flags
 492  *
 493  * Traces the parameters and calls the right asic functions
 494  * to setup the page table using the DMA.
 495  */
 496 static void amdgpu_vm_do_set_ptes(struct amdgpu_pte_update_params *params,
 497                                   uint64_t pe, uint64_t addr,
 498                                   unsigned count, uint32_t incr,
 499                                   uint64_t flags)
 500 {
 501         trace_amdgpu_vm_set_ptes(pe, addr, count, incr, flags);
 502
 503         if (count < 3) {
 504                 amdgpu_vm_write_pte(params->adev, params->ib, pe,
 505                                     addr | flags, count, incr);
 506
 507         } else {
 508                 amdgpu_vm_set_pte_pde(params->adev, params->ib, pe, addr,
 509                                       count, incr, flags);
 510         }
 511 }
 512
 513 /**
 514  * amdgpu_vm_do_copy_ptes - copy the PTEs from the GART
 515  *
 516  * @params: see amdgpu_pte_update_params definition
 517  * @pe: addr of the page entry
 518  * @addr: dst addr to write into pe
 519  * @count: number of page entries to update
 520  * @incr: increase next addr by incr bytes
 521  * @flags: hw access flags
 522  *
 523  * Traces the parameters and calls the DMA function to copy the PTEs.
 524  */
 525 static void amdgpu_vm_do_copy_ptes(struct amdgpu_pte_update_params *params,
 526                                    uint64_t pe, uint64_t addr,
 527                                    unsigned count, uint32_t incr,
 528                                    uint64_t flags)
 529 {
 530         uint64_t src = (params->src + (addr >> 12) * 8);
 531
 532
 533         trace_amdgpu_vm_copy_ptes(pe, src, count);
 534
 535         amdgpu_vm_copy_pte(params->adev, params->ib, pe, src, count);
 536 }
 537
 538 /**
 539  * amdgpu_vm_map_gart - Resolve gart mapping of addr
 540  *
 541  * @pages_addr: optional DMA address to use for lookup
 542  * @addr: the unmapped addr
 543  *
 544  * Look up the physical address of the page that the pte resolves
 545  * to and return the pointer for the page table entry.
 546  */
 547 static uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr)
 548 {
 549         uint64_t result;
 550
 551         /* page table offset */
 552         result = pages_addr[addr >> PAGE_SHIFT];
 553
 554         /* in case cpu page size != gpu page size*/
 555         result |= addr & (~PAGE_MASK);
 556
 557         result &= 0xFFFFFFFFFFFFF000ULL;
 558
 559         return result;
 560 }
 561
 562 /*
 563  * amdgpu_vm_update_pdes - make sure that page directory is valid
 564  *
 565  * @adev: amdgpu_device pointer
 566  * @vm: requested vm
 567  * @start: start of GPU address range
 568  * @end: end of GPU address range
 569  *
 570  * Allocates new page tables if necessary
 571  * and updates the page directory.
 572  * Returns 0 for success, error for failure.
 573  */
 574 int amdgpu_vm_update_page_directory(struct amdgpu_device *adev,
 575                                     struct amdgpu_vm *vm)
 576 {
 577         struct amdgpu_bo *shadow;
 578         struct amdgpu_ring *ring;
 579         uint64_t pd_addr, shadow_addr;
 580         uint32_t incr = AMDGPU_VM_PTE_COUNT * 8;
 581         uint64_t last_pde = ~0, last_pt = ~0, last_shadow = ~0;
 582         unsigned count = 0, pt_idx, ndw;
 583         struct amdgpu_job *job;
 584         struct amdgpu_pte_update_params params;
 585         struct dma_fence *fence = NULL;
 586
 587         int r;
 588
 589         ring = container_of(vm->entity.sched, struct amdgpu_ring, sched);
 590         shadow = vm->page_directory->shadow;
 591
 592         /* padding, etc. */
 593         ndw = 64;
 594
 595         /* assume the worst case */
 596         ndw += vm->max_pde_used * 6;
 597
 598         pd_addr = amdgpu_bo_gpu_offset(vm->page_directory);
 599         if (shadow) {
 600                 r = amdgpu_ttm_bind(&shadow->tbo, &shadow->tbo.mem);
 601                 if (r)
 602                         return r;
 603                 shadow_addr = amdgpu_bo_gpu_offset(shadow);
 604                 ndw *= 2;
 605         } else {
 606                 shadow_addr = 0;
 607         }
 608
 609         r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
 610         if (r)
 611                 return r;
 612
 613         memset(&params, 0, sizeof(params));
 614         params.adev = adev;
 615         params.ib = &job->ibs[0];
 616
 617         /* walk over the address space and update the page directory */
 618         for (pt_idx = 0; pt_idx <= vm->max_pde_used; ++pt_idx) {
 619                 struct amdgpu_bo *bo = vm->page_tables[pt_idx].bo;
 620                 uint64_t pde, pt;
 621
 622                 if (bo == NULL)
 623                         continue;
 624
 625                 if (bo->shadow) {
 626                         struct amdgpu_bo *pt_shadow = bo->shadow;
 627
 628                         r = amdgpu_ttm_bind(&pt_shadow->tbo,
 629                                             &pt_shadow->tbo.mem);
 630                         if (r)
 631                                 return r;
 632                 }
 633
 634                 pt = amdgpu_bo_gpu_offset(bo);
 635                 if (vm->page_tables[pt_idx].addr == pt)
 636                         continue;
 637
 638                 vm->page_tables[pt_idx].addr = pt;
 639
 640                 pde = pd_addr + pt_idx * 8;
 641                 if (((last_pde + 8 * count) != pde) ||
 642                     ((last_pt + incr * count) != pt) ||
 643                     (count == AMDGPU_VM_MAX_UPDATE_SIZE)) {
 644
 645                         if (count) {
 646                                 if (shadow)
 647                                         amdgpu_vm_do_set_ptes(&params,
 648                                                               last_shadow,
 649                                                               last_pt, count,
 650                                                               incr,
 651                                                               AMDGPU_PTE_VALID);
 652
 653                                 amdgpu_vm_do_set_ptes(&params, last_pde,
 654                                                       last_pt, count, incr,
 655                                                       AMDGPU_PTE_VALID);
 656                         }
 657
 658                         count = 1;
 659                         last_pde = pde;
 660                         last_shadow = shadow_addr + pt_idx * 8;
 661                         last_pt = pt;
 662                 } else {
 663                         ++count;
 664                 }
 665         }
 666
 667         if (count) {
 668                 if (vm->page_directory->shadow)
 669                         amdgpu_vm_do_set_ptes(&params, last_shadow, last_pt,
 670                                               count, incr, AMDGPU_PTE_VALID);
 671
 672                 amdgpu_vm_do_set_ptes(&params, last_pde, last_pt,
 673                                       count, incr, AMDGPU_PTE_VALID);
 674         }
 675
 676         if (params.ib->length_dw == 0) {
 677                 amdgpu_job_free(job);
 678                 return 0;
 679         }
 680
 681         amdgpu_ring_pad_ib(ring, params.ib);
 682         amdgpu_sync_resv(adev, &job->sync, vm->page_directory->tbo.resv,
 683                          AMDGPU_FENCE_OWNER_VM);
 684         if (shadow)
 685                 amdgpu_sync_resv(adev, &job->sync, shadow->tbo.resv,
 686                                  AMDGPU_FENCE_OWNER_VM);
 687
 688         WARN_ON(params.ib->length_dw > ndw);
 689         r = amdgpu_job_submit(job, ring, &vm->entity,
 690                               AMDGPU_FENCE_OWNER_VM, &fence);
 691         if (r)
 692                 goto error_free;
 693
 694         amdgpu_bo_fence(vm->page_directory, fence, true);
 695         dma_fence_put(vm->page_directory_fence);
 696         vm->page_directory_fence = dma_fence_get(fence);
 697         dma_fence_put(fence);
 698
 699         return 0;
 700
 701 error_free:
 702         amdgpu_job_free(job);
 703         return r;
 704 }
 705
 706 /**
 707  * amdgpu_vm_update_ptes - make sure that page tables are valid
 708  *
 709  * @params: see amdgpu_pte_update_params definition
 710  * @vm: requested vm
 711  * @start: start of GPU address range
 712  * @end: end of GPU address range
 713  * @dst: destination address to map to, the next dst inside the function
 714  * @flags: mapping flags
 715  *
 716  * Update the page tables in the range @start - @end.
 717  */
 718 static void amdgpu_vm_update_ptes(struct amdgpu_pte_update_params *params,
 719                                   struct amdgpu_vm *vm,
 720                                   uint64_t start, uint64_t end,
 721                                   uint64_t dst, uint64_t flags)
 722 {
 723         const uint64_t mask = AMDGPU_VM_PTE_COUNT - 1;
 724
 725         uint64_t cur_pe_start, cur_nptes, cur_dst;
 726         uint64_t addr; /* next GPU address to be updated */
 727         uint64_t pt_idx;
 728         struct amdgpu_bo *pt;
 729         unsigned nptes; /* next number of ptes to be updated */
 730         uint64_t next_pe_start;
 731
 732         /* initialize the variables */
 733         addr = start;
 734         pt_idx = addr >> amdgpu_vm_block_size;
 735         pt = vm->page_tables[pt_idx].bo;
 736         if (params->shadow) {
 737                 if (!pt->shadow)
 738                         return;
 739                 pt = pt->shadow;
 740         }
 741         if ((addr & ~mask) == (end & ~mask))
 742                 nptes = end - addr;
 743         else
 744                 nptes = AMDGPU_VM_PTE_COUNT - (addr & mask);
 745
 746         cur_pe_start = amdgpu_bo_gpu_offset(pt);
 747         cur_pe_start += (addr & mask) * 8;
 748         cur_nptes = nptes;
 749         cur_dst = dst;
 750
 751         /* for next ptb*/
 752         addr += nptes;
 753         dst += nptes * AMDGPU_GPU_PAGE_SIZE;
 754
 755         /* walk over the address space and update the page tables */
 756         while (addr < end) {
 757                 pt_idx = addr >> amdgpu_vm_block_size;
 758                 pt = vm->page_tables[pt_idx].bo;
 759                 if (params->shadow) {
 760                         if (!pt->shadow)
 761                                 return;
 762                         pt = pt->shadow;
 763                 }
 764
 765                 if ((addr & ~mask) == (end & ~mask))
 766                         nptes = end - addr;
 767                 else
 768                         nptes = AMDGPU_VM_PTE_COUNT - (addr & mask);
 769
 770                 next_pe_start = amdgpu_bo_gpu_offset(pt);
 771                 next_pe_start += (addr & mask) * 8;
 772
 773                 if ((cur_pe_start + 8 * cur_nptes) == next_pe_start &&
 774                     ((cur_nptes + nptes) <= AMDGPU_VM_MAX_UPDATE_SIZE)) {
 775                         /* The next ptb is consecutive to current ptb.
 776                          * Don't call the update function now.
 777                          * Will update two ptbs together in future.
 778                         */
 779                         cur_nptes += nptes;
 780                 } else {
 781                         params->func(params, cur_pe_start, cur_dst, cur_nptes,
 782                                      AMDGPU_GPU_PAGE_SIZE, flags);
 783
 784                         cur_pe_start = next_pe_start;
 785                         cur_nptes = nptes;
 786                         cur_dst = dst;
 787                 }
 788
 789                 /* for next ptb*/
 790                 addr += nptes;
 791                 dst += nptes * AMDGPU_GPU_PAGE_SIZE;
 792         }
 793
 794         params->func(params, cur_pe_start, cur_dst, cur_nptes,
 795                      AMDGPU_GPU_PAGE_SIZE, flags);
 796 }
 797
 798 /*
 799  * amdgpu_vm_frag_ptes - add fragment information to PTEs
 800  *
 801  * @params: see amdgpu_pte_update_params definition
 802  * @vm: requested vm
 803  * @start: first PTE to handle
 804  * @end: last PTE to handle
 805  * @dst: addr those PTEs should point to
 806  * @flags: hw mapping flags
 807  */
 808 static void amdgpu_vm_frag_ptes(struct amdgpu_pte_update_params *params,
 809                                 struct amdgpu_vm *vm,
 810                                 uint64_t start, uint64_t end,
 811                                 uint64_t dst, uint64_t flags)
 812 {
 813         /**
 814          * The MC L1 TLB supports variable sized pages, based on a fragment
 815          * field in the PTE. When this field is set to a non-zero value, page
 816          * granularity is increased from 4KB to (1 << (12 + frag)). The PTE
 817          * flags are considered valid for all PTEs within the fragment range
 818          * and corresponding mappings are assumed to be physically contiguous.
 819          *
 820          * The L1 TLB can store a single PTE for the whole fragment,
 821          * significantly increasing the space available for translation
 822          * caching. This leads to large improvements in throughput when the
 823          * TLB is under pressure.
 824          *
 825          * The L2 TLB distributes small and large fragments into two
 826          * asymmetric partitions. The large fragment cache is significantly
 827          * larger. Thus, we try to use large fragments wherever possible.
 828          * Userspace can support this by aligning virtual base address and
 829          * allocation size to the fragment size.
 830          */
 831
 832         /* SI and newer are optimized for 64KB */
 833         uint64_t frag_flags = AMDGPU_PTE_FRAG(AMDGPU_LOG2_PAGES_PER_FRAG);
 834         uint64_t frag_align = 1 << AMDGPU_LOG2_PAGES_PER_FRAG;
 835
 836         uint64_t frag_start = ALIGN(start, frag_align);
 837         uint64_t frag_end = end & ~(frag_align - 1);
 838
 839         /* system pages are non continuously */
 840         if (params->src || !(flags & AMDGPU_PTE_VALID) ||
 841             (frag_start >= frag_end)) {
 842
 843                 amdgpu_vm_update_ptes(params, vm, start, end, dst, flags);
 844                 return;
 845         }
 846
 847         /* handle the 4K area at the beginning */
 848         if (start != frag_start) {
 849                 amdgpu_vm_update_ptes(params, vm, start, frag_start,
 850                                       dst, flags);
 851                 dst += (frag_start - start) * AMDGPU_GPU_PAGE_SIZE;
 852         }
 853
 854         /* handle the area in the middle */
 855         amdgpu_vm_update_ptes(params, vm, frag_start, frag_end, dst,
 856                               flags | frag_flags);
 857
 858         /* handle the 4K area at the end */
 859         if (frag_end != end) {
 860                 dst += (frag_end - frag_start) * AMDGPU_GPU_PAGE_SIZE;
 861                 amdgpu_vm_update_ptes(params, vm, frag_end, end, dst, flags);
 862         }
 863 }
 864
 865 /**
 866  * amdgpu_vm_bo_update_mapping - update a mapping in the vm page table
 867  *
 868  * @adev: amdgpu_device pointer
 869  * @exclusive: fence we need to sync to
 870  * @src: address where to copy page table entries from
 871  * @pages_addr: DMA addresses to use for mapping
 872  * @vm: requested vm
 873  * @start: start of mapped range
 874  * @last: last mapped entry
 875  * @flags: flags for the entries
 876  * @addr: addr to set the area to
 877  * @fence: optional resulting fence
 878  *
 879  * Fill in the page table entries between @start and @last.
 880  * Returns 0 for success, -EINVAL for failure.
 881  */
 882 static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
 883                                        struct dma_fence *exclusive,
 884                                        uint64_t src,
 885                                        dma_addr_t *pages_addr,
 886                                        struct amdgpu_vm *vm,
 887                                        uint64_t start, uint64_t last,
 888                                        uint64_t flags, uint64_t addr,
 889                                        struct dma_fence **fence)
 890 {
 891         struct amdgpu_ring *ring;
 892         void *owner = AMDGPU_FENCE_OWNER_VM;
 893         unsigned nptes, ncmds, ndw;
 894         struct amdgpu_job *job;
 895         struct amdgpu_pte_update_params params;
 896         struct dma_fence *f = NULL;
 897         int r;
 898
 899         memset(&params, 0, sizeof(params));
 900         params.adev = adev;
 901         params.src = src;
 902
 903         ring = container_of(vm->entity.sched, struct amdgpu_ring, sched);
 904
 905         memset(&params, 0, sizeof(params));
 906         params.adev = adev;
 907         params.src = src;
 908
 909         /* sync to everything on unmapping */
 910         if (!(flags & AMDGPU_PTE_VALID))
 911                 owner = AMDGPU_FENCE_OWNER_UNDEFINED;
 912
 913         nptes = last - start + 1;
 914
 915         /*
 916          * reserve space for one command every (1 << BLOCK_SIZE)
 917          *  entries or 2k dwords (whatever is smaller)
 918          */
 919         ncmds = (nptes >> min(amdgpu_vm_block_size, 11)) + 1;
 920
 921         /* padding, etc. */
 922         ndw = 64;
 923
 924         if (src) {
 925                 /* only copy commands needed */
 926                 ndw += ncmds * 7;
 927
 928                 params.func = amdgpu_vm_do_copy_ptes;
 929
 930         } else if (pages_addr) {
 931                 /* copy commands needed */
 932                 ndw += ncmds * 7;
 933
 934                 /* and also PTEs */
 935                 ndw += nptes * 2;
 936
 937                 params.func = amdgpu_vm_do_copy_ptes;
 938
 939         } else {
 940                 /* set page commands needed */
 941                 ndw += ncmds * 10;
 942
 943                 /* two extra commands for begin/end of fragment */
 944                 ndw += 2 * 10;
 945
 946                 params.func = amdgpu_vm_do_set_ptes;
 947         }
 948
 949         r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
 950         if (r)
 951                 return r;
 952
 953         params.ib = &job->ibs[0];
 954
 955         if (!src && pages_addr) {
 956                 uint64_t *pte;
 957                 unsigned i;
 958
 959                 /* Put the PTEs at the end of the IB. */
 960                 i = ndw - nptes * 2;
 961                 pte= (uint64_t *)&(job->ibs->ptr[i]);
 962                 params.src = job->ibs->gpu_addr + i * 4;
 963
 964                 for (i = 0; i < nptes; ++i) {
 965                         pte[i] = amdgpu_vm_map_gart(pages_addr, addr + i *
 966                                                     AMDGPU_GPU_PAGE_SIZE);
 967                         pte[i] |= flags;
 968                 }
 969                 addr = 0;
 970         }
 971
 972         r = amdgpu_sync_fence(adev, &job->sync, exclusive);
 973         if (r)
 974                 goto error_free;
 975
 976         r = amdgpu_sync_resv(adev, &job->sync, vm->page_directory->tbo.resv,
 977                              owner);
 978         if (r)
 979                 goto error_free;
 980
 981         r = reservation_object_reserve_shared(vm->page_directory->tbo.resv);
 982         if (r)
 983                 goto error_free;
 984
 985         params.shadow = true;
 986         amdgpu_vm_frag_ptes(&params, vm, start, last + 1, addr, flags);
 987         params.shadow = false;
 988         amdgpu_vm_frag_ptes(&params, vm, start, last + 1, addr, flags);
 989
 990         amdgpu_ring_pad_ib(ring, params.ib);
 991         WARN_ON(params.ib->length_dw > ndw);
 992         r = amdgpu_job_submit(job, ring, &vm->entity,
 993                               AMDGPU_FENCE_OWNER_VM, &f);
 994         if (r)
 995                 goto error_free;
 996
 997         amdgpu_bo_fence(vm->page_directory, f, true);
 998         dma_fence_put(*fence);
 999         *fence = f;
1000         return 0;
1001
1002 error_free:
1003         amdgpu_job_free(job);
1004         return r;
1005 }
1006
1007 /**
1008  * amdgpu_vm_bo_split_mapping - split a mapping into smaller chunks
1009  *
1010  * @adev: amdgpu_device pointer
1011  * @exclusive: fence we need to sync to
1012  * @gtt_flags: flags as they are used for GTT
1013  * @pages_addr: DMA addresses to use for mapping
1014  * @vm: requested vm
1015  * @mapping: mapped range and flags to use for the update
1016  * @flags: HW flags for the mapping
1017  * @nodes: array of drm_mm_nodes with the MC addresses
1018  * @fence: optional resulting fence
1019  *
1020  * Split the mapping into smaller chunks so that each update fits
1021  * into a SDMA IB.
1022  * Returns 0 for success, -EINVAL for failure.
1023  */
1024 static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
1025                                       struct dma_fence *exclusive,
1026                                       uint64_t gtt_flags,
1027                                       dma_addr_t *pages_addr,
1028                                       struct amdgpu_vm *vm,
1029                                       struct amdgpu_bo_va_mapping *mapping,
1030                                       uint64_t flags,
1031                                       struct drm_mm_node *nodes,
1032                                       struct dma_fence **fence)
1033 {
1034         uint64_t pfn, src = 0, start = mapping->it.start;
1035         int r;
1036
1037         /* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here
1038          * but in case of something, we filter the flags in first place
1039          */
1040         if (!(mapping->flags & AMDGPU_PTE_READABLE))
1041                 flags &= ~AMDGPU_PTE_READABLE;
1042         if (!(mapping->flags & AMDGPU_PTE_WRITEABLE))
1043                 flags &= ~AMDGPU_PTE_WRITEABLE;
1044
1045         trace_amdgpu_vm_bo_update(mapping);
1046
1047         pfn = mapping->offset >> PAGE_SHIFT;
1048         if (nodes) {
1049                 while (pfn >= nodes->size) {
1050                         pfn -= nodes->size;
1051                         ++nodes;
1052                 }
1053         }
1054
1055         do {
1056                 uint64_t max_entries;
1057                 uint64_t addr, last;
1058
1059                 if (nodes) {
1060                         addr = nodes->start << PAGE_SHIFT;
1061                         max_entries = (nodes->size - pfn) *
1062                                 (PAGE_SIZE / AMDGPU_GPU_PAGE_SIZE);
1063                 } else {
1064                         addr = 0;
1065                         max_entries = S64_MAX;
1066                 }
1067
1068                 if (pages_addr) {
1069                         if (flags == gtt_flags)
1070                                 src = adev->gart.table_addr +
1071                                         (addr >> AMDGPU_GPU_PAGE_SHIFT) * 8;
1072                         else
1073                                 max_entries = min(max_entries, 16ull * 1024ull);
1074                         addr = 0;
1075                 } else if (flags & AMDGPU_PTE_VALID) {
1076                         addr += adev->vm_manager.vram_base_offset;
1077                 }
1078                 addr += pfn << PAGE_SHIFT;
1079
1080                 last = min((uint64_t)mapping->it.last, start + max_entries - 1);
1081                 r = amdgpu_vm_bo_update_mapping(adev, exclusive,
1082                                                 src, pages_addr, vm,
1083                                                 start, last, flags, addr,
1084                                                 fence);
1085                 if (r)
1086                         return r;
1087
1088                 pfn += last - start + 1;
1089                 if (nodes && nodes->size == pfn) {
1090                         pfn = 0;
1091                         ++nodes;
1092                 }
1093                 start = last + 1;
1094
1095         } while (unlikely(start != mapping->it.last + 1));
1096
1097         return 0;
1098 }
1099
1100 /**
1101  * amdgpu_vm_bo_update - update all BO mappings in the vm page table
1102  *
1103  * @adev: amdgpu_device pointer
1104  * @bo_va: requested BO and VM object
1105  * @clear: if true clear the entries
1106  *
1107  * Fill in the page table entries for @bo_va.
1108  * Returns 0 for success, -EINVAL for failure.
1109  */
1110 int amdgpu_vm_bo_update(struct amdgpu_device *adev,
1111                         struct amdgpu_bo_va *bo_va,
1112                         bool clear)
1113 {
1114         struct amdgpu_vm *vm = bo_va->vm;
1115         struct amdgpu_bo_va_mapping *mapping;
1116         dma_addr_t *pages_addr = NULL;
1117         uint64_t gtt_flags, flags;
1118         struct ttm_mem_reg *mem;
1119         struct drm_mm_node *nodes;
1120         struct dma_fence *exclusive;
1121         int r;
1122
1123         if (clear || !bo_va->bo) {
1124                 mem = NULL;
1125                 nodes = NULL;
1126                 exclusive = NULL;
1127         } else {
1128                 struct ttm_dma_tt *ttm;
1129
1130                 mem = &bo_va->bo->tbo.mem;
1131                 nodes = mem->mm_node;
1132                 if (mem->mem_type == TTM_PL_TT) {
1133                         ttm = container_of(bo_va->bo->tbo.ttm, struct
1134                                            ttm_dma_tt, ttm);
1135                         pages_addr = ttm->dma_address;
1136                 }
1137                 exclusive = reservation_object_get_excl(bo_va->bo->tbo.resv);
1138         }
1139
1140         if (bo_va->bo) {
1141                 flags = amdgpu_ttm_tt_pte_flags(adev, bo_va->bo->tbo.ttm, mem);
1142                 gtt_flags = (amdgpu_ttm_is_bound(bo_va->bo->tbo.ttm) &&
1143                         adev == amdgpu_ttm_adev(bo_va->bo->tbo.bdev)) ?
1144                         flags : 0;
1145         } else {
1146                 flags = 0x0;
1147                 gtt_flags = ~0x0;
1148         }
1149
1150         spin_lock(&vm->status_lock);
1151         if (!list_empty(&bo_va->vm_status))
1152                 list_splice_init(&bo_va->valids, &bo_va->invalids);
1153         spin_unlock(&vm->status_lock);
1154
1155         list_for_each_entry(mapping, &bo_va->invalids, list) {
1156                 r = amdgpu_vm_bo_split_mapping(adev, exclusive,
1157                                                gtt_flags, pages_addr, vm,
1158                                                mapping, flags, nodes,
1159                                                &bo_va->last_pt_update);
1160                 if (r)
1161                         return r;
1162         }
1163
1164         if (trace_amdgpu_vm_bo_mapping_enabled()) {
1165                 list_for_each_entry(mapping, &bo_va->valids, list)
1166                         trace_amdgpu_vm_bo_mapping(mapping);
1167
1168                 list_for_each_entry(mapping, &bo_va->invalids, list)
1169                         trace_amdgpu_vm_bo_mapping(mapping);
1170         }
1171
1172         spin_lock(&vm->status_lock);
1173         list_splice_init(&bo_va->invalids, &bo_va->valids);
1174         list_del_init(&bo_va->vm_status);
1175         if (clear)
1176                 list_add(&bo_va->vm_status, &vm->cleared);
1177         spin_unlock(&vm->status_lock);
1178
1179         return 0;
1180 }
1181
1182 /**
1183  * amdgpu_vm_update_prt_state - update the global PRT state
1184  */
1185 static void amdgpu_vm_update_prt_state(struct amdgpu_device *adev)
1186 {
1187         unsigned long flags;
1188         bool enable;
1189
1190         spin_lock_irqsave(&adev->vm_manager.prt_lock, flags);
1191         enable = !!atomic_read(&adev->vm_manager.num_prt_users);
1192         adev->gart.gart_funcs->set_prt(adev, enable);
1193         spin_unlock_irqrestore(&adev->vm_manager.prt_lock, flags);
1194 }
1195
1196 /**
1197  * amdgpu_vm_prt_get - add a PRT user
1198  */
1199 static void amdgpu_vm_prt_get(struct amdgpu_device *adev)
1200 {
1201         if (!adev->gart.gart_funcs->set_prt)
1202                 return;
1203
1204         if (atomic_inc_return(&adev->vm_manager.num_prt_users) == 1)
1205                 amdgpu_vm_update_prt_state(adev);
1206 }
1207
1208 /**
1209  * amdgpu_vm_prt_put - drop a PRT user
1210  */
1211 static void amdgpu_vm_prt_put(struct amdgpu_device *adev)
1212 {
1213         if (atomic_dec_return(&adev->vm_manager.num_prt_users) == 0)
1214                 amdgpu_vm_update_prt_state(adev);
1215 }
1216
1217 /**
1218  * amdgpu_vm_prt_cb - callback for updating the PRT status
1219  */
1220 static void amdgpu_vm_prt_cb(struct dma_fence *fence, struct dma_fence_cb *_cb)
1221 {
1222         struct amdgpu_prt_cb *cb = container_of(_cb, struct amdgpu_prt_cb, cb);
1223
1224         amdgpu_vm_prt_put(cb->adev);
1225         kfree(cb);
1226 }
1227
1228 /**
1229  * amdgpu_vm_add_prt_cb - add callback for updating the PRT status
1230  */
1231 static void amdgpu_vm_add_prt_cb(struct amdgpu_device *adev,
1232                                  struct dma_fence *fence)
1233 {
1234         struct amdgpu_prt_cb *cb;
1235
1236         if (!adev->gart.gart_funcs->set_prt)
1237                 return;
1238
1239         cb = kmalloc(sizeof(struct amdgpu_prt_cb), GFP_KERNEL);
1240         if (!cb) {
1241                 /* Last resort when we are OOM */
1242                 if (fence)
1243                         dma_fence_wait(fence, false);
1244
1245                 amdgpu_vm_prt_put(cb->adev);
1246         } else {
1247                 cb->adev = adev;
1248                 if (!fence || dma_fence_add_callback(fence, &cb->cb,
1249                                                      amdgpu_vm_prt_cb))
1250                         amdgpu_vm_prt_cb(fence, &cb->cb);
1251         }
1252 }
1253
1254 /**
1255  * amdgpu_vm_free_mapping - free a mapping
1256  *
1257  * @adev: amdgpu_device pointer
1258  * @vm: requested vm
1259  * @mapping: mapping to be freed
1260  * @fence: fence of the unmap operation
1261  *
1262  * Free a mapping and make sure we decrease the PRT usage count if applicable.
1263  */
1264 static void amdgpu_vm_free_mapping(struct amdgpu_device *adev,
1265                                    struct amdgpu_vm *vm,
1266                                    struct amdgpu_bo_va_mapping *mapping,
1267                                    struct dma_fence *fence)
1268 {
1269         if (mapping->flags & AMDGPU_PTE_PRT)
1270                 amdgpu_vm_add_prt_cb(adev, fence);
1271         kfree(mapping);
1272 }
1273
1274 /**
1275  * amdgpu_vm_prt_fini - finish all prt mappings
1276  *
1277  * @adev: amdgpu_device pointer
1278  * @vm: requested vm
1279  *
1280  * Register a cleanup callback to disable PRT support after VM dies.
1281  */
1282 static void amdgpu_vm_prt_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
1283 {
1284         struct reservation_object *resv = vm->page_directory->tbo.resv;
1285         struct dma_fence *excl, **shared;
1286         unsigned i, shared_count;
1287         int r;
1288
1289         r = reservation_object_get_fences_rcu(resv, &excl,
1290                                               &shared_count, &shared);
1291         if (r) {
1292                 /* Not enough memory to grab the fence list, as last resort
1293                  * block for all the fences to complete.
1294                  */
1295                 reservation_object_wait_timeout_rcu(resv, true, false,
1296                                                     MAX_SCHEDULE_TIMEOUT);
1297                 return;
1298         }
1299
1300         /* Add a callback for each fence in the reservation object */
1301         amdgpu_vm_prt_get(adev);
1302         amdgpu_vm_add_prt_cb(adev, excl);
1303
1304         for (i = 0; i < shared_count; ++i) {
1305                 amdgpu_vm_prt_get(adev);
1306                 amdgpu_vm_add_prt_cb(adev, shared[i]);
1307         }
1308
1309         kfree(shared);
1310 }
1311
1312 /**
1313  * amdgpu_vm_clear_freed - clear freed BOs in the PT
1314  *
1315  * @adev: amdgpu_device pointer
1316  * @vm: requested vm
1317  *
1318  * Make sure all freed BOs are cleared in the PT.
1319  * Returns 0 for success.
1320  *
1321  * PTs have to be reserved and mutex must be locked!
1322  */
1323 int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
1324                           struct amdgpu_vm *vm)
1325 {
1326         struct amdgpu_bo_va_mapping *mapping;
1327         struct dma_fence *fence = NULL;
1328         int r;
1329
1330         while (!list_empty(&vm->freed)) {
1331                 mapping = list_first_entry(&vm->freed,
1332                         struct amdgpu_bo_va_mapping, list);
1333                 list_del(&mapping->list);
1334
1335                 r = amdgpu_vm_bo_split_mapping(adev, NULL, 0, NULL, vm, mapping,
1336                                                0, 0, &fence);
1337                 amdgpu_vm_free_mapping(adev, vm, mapping, fence);
1338                 if (r) {
1339                         dma_fence_put(fence);
1340                         return r;
1341                 }
1342
1343         }
1344         dma_fence_put(fence);
1345         return 0;
1346
1347 }
1348
1349 /**
1350  * amdgpu_vm_clear_invalids - clear invalidated BOs in the PT
1351  *
1352  * @adev: amdgpu_device pointer
1353  * @vm: requested vm
1354  *
1355  * Make sure all invalidated BOs are cleared in the PT.
1356  * Returns 0 for success.
1357  *
1358  * PTs have to be reserved and mutex must be locked!
1359  */
1360 int amdgpu_vm_clear_invalids(struct amdgpu_device *adev,
1361                              struct amdgpu_vm *vm, struct amdgpu_sync *sync)
1362 {
1363         struct amdgpu_bo_va *bo_va = NULL;
1364         int r = 0;
1365
1366         spin_lock(&vm->status_lock);
1367         while (!list_empty(&vm->invalidated)) {
1368                 bo_va = list_first_entry(&vm->invalidated,
1369                         struct amdgpu_bo_va, vm_status);
1370                 spin_unlock(&vm->status_lock);
1371
1372                 r = amdgpu_vm_bo_update(adev, bo_va, true);
1373                 if (r)
1374                         return r;
1375
1376                 spin_lock(&vm->status_lock);
1377         }
1378         spin_unlock(&vm->status_lock);
1379
1380         if (bo_va)
1381                 r = amdgpu_sync_fence(adev, sync, bo_va->last_pt_update);
1382
1383         return r;
1384 }
1385
1386 /**
1387  * amdgpu_vm_bo_add - add a bo to a specific vm
1388  *
1389  * @adev: amdgpu_device pointer
1390  * @vm: requested vm
1391  * @bo: amdgpu buffer object
1392  *
1393  * Add @bo into the requested vm.
1394  * Add @bo to the list of bos associated with the vm
1395  * Returns newly added bo_va or NULL for failure
1396  *
1397  * Object has to be reserved!
1398  */
1399 struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
1400                                       struct amdgpu_vm *vm,
1401                                       struct amdgpu_bo *bo)
1402 {
1403         struct amdgpu_bo_va *bo_va;
1404
1405         bo_va = kzalloc(sizeof(struct amdgpu_bo_va), GFP_KERNEL);
1406         if (bo_va == NULL) {
1407                 return NULL;
1408         }
1409         bo_va->vm = vm;
1410         bo_va->bo = bo;
1411         bo_va->ref_count = 1;
1412         INIT_LIST_HEAD(&bo_va->bo_list);
1413         INIT_LIST_HEAD(&bo_va->valids);
1414         INIT_LIST_HEAD(&bo_va->invalids);
1415         INIT_LIST_HEAD(&bo_va->vm_status);
1416
1417         if (bo)
1418                 list_add_tail(&bo_va->bo_list, &bo->va);
1419
1420         return bo_va;
1421 }
1422
1423 /**
1424  * amdgpu_vm_bo_map - map bo inside a vm
1425  *
1426  * @adev: amdgpu_device pointer
1427  * @bo_va: bo_va to store the address
1428  * @saddr: where to map the BO
1429  * @offset: requested offset in the BO
1430  * @flags: attributes of pages (read/write/valid/etc.)
1431  *
1432  * Add a mapping of the BO at the specefied addr into the VM.
1433  * Returns 0 for success, error for failure.
1434  *
1435  * Object has to be reserved and unreserved outside!
1436  */
1437 int amdgpu_vm_bo_map(struct amdgpu_device *adev,
1438                      struct amdgpu_bo_va *bo_va,
1439                      uint64_t saddr, uint64_t offset,
1440                      uint64_t size, uint64_t flags)
1441 {
1442         struct amdgpu_bo_va_mapping *mapping;
1443         struct amdgpu_vm *vm = bo_va->vm;
1444         struct interval_tree_node *it;
1445         unsigned last_pfn, pt_idx;
1446         uint64_t eaddr;
1447         int r;
1448
1449         /* validate the parameters */
1450         if (saddr & AMDGPU_GPU_PAGE_MASK || offset & AMDGPU_GPU_PAGE_MASK ||
1451             size == 0 || size & AMDGPU_GPU_PAGE_MASK)
1452                 return -EINVAL;
1453
1454         /* make sure object fit at this offset */
1455         eaddr = saddr + size - 1;
1456         if (saddr >= eaddr ||
1457             (bo_va->bo && offset + size > amdgpu_bo_size(bo_va->bo)))
1458                 return -EINVAL;
1459
1460         last_pfn = eaddr / AMDGPU_GPU_PAGE_SIZE;
1461         if (last_pfn >= adev->vm_manager.max_pfn) {
1462                 dev_err(adev->dev, "va above limit (0x%08X >= 0x%08X)\n",
1463                         last_pfn, adev->vm_manager.max_pfn);
1464                 return -EINVAL;
1465         }
1466
1467         saddr /= AMDGPU_GPU_PAGE_SIZE;
1468         eaddr /= AMDGPU_GPU_PAGE_SIZE;
1469
1470         it = interval_tree_iter_first(&vm->va, saddr, eaddr);
1471         if (it) {
1472                 struct amdgpu_bo_va_mapping *tmp;
1473                 tmp = container_of(it, struct amdgpu_bo_va_mapping, it);
1474                 /* bo and tmp overlap, invalid addr */
1475                 dev_err(adev->dev, "bo %p va 0x%010Lx-0x%010Lx conflict with "
1476                         "0x%010lx-0x%010lx\n", bo_va->bo, saddr, eaddr,
1477                         tmp->it.start, tmp->it.last + 1);
1478                 r = -EINVAL;
1479                 goto error;
1480         }
1481
1482         mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
1483         if (!mapping) {
1484                 r = -ENOMEM;
1485                 goto error;
1486         }
1487
1488         INIT_LIST_HEAD(&mapping->list);
1489         mapping->it.start = saddr;
1490         mapping->it.last = eaddr;
1491         mapping->offset = offset;
1492         mapping->flags = flags;
1493
1494         list_add(&mapping->list, &bo_va->invalids);
1495         interval_tree_insert(&mapping->it, &vm->va);
1496
1497         /* Make sure the page tables are allocated */
1498         saddr >>= amdgpu_vm_block_size;
1499         eaddr >>= amdgpu_vm_block_size;
1500
1501         BUG_ON(eaddr >= amdgpu_vm_num_pdes(adev));
1502
1503         if (eaddr > vm->max_pde_used)
1504                 vm->max_pde_used = eaddr;
1505
1506         /* walk over the address space and allocate the page tables */
1507         for (pt_idx = saddr; pt_idx <= eaddr; ++pt_idx) {
1508                 struct reservation_object *resv = vm->page_directory->tbo.resv;
1509                 struct amdgpu_bo *pt;
1510
1511                 if (vm->page_tables[pt_idx].bo)
1512                         continue;
1513
1514                 r = amdgpu_bo_create(adev, AMDGPU_VM_PTE_COUNT * 8,
1515                                      AMDGPU_GPU_PAGE_SIZE, true,
1516                                      AMDGPU_GEM_DOMAIN_VRAM,
1517                                      AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
1518                                      AMDGPU_GEM_CREATE_SHADOW |
1519                                      AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS |
1520                                      AMDGPU_GEM_CREATE_VRAM_CLEARED,
1521                                      NULL, resv, &pt);
1522                 if (r)
1523                         goto error_free;
1524
1525                 /* Keep a reference to the page table to avoid freeing
1526                  * them up in the wrong order.
1527                  */
1528                 pt->parent = amdgpu_bo_ref(vm->page_directory);
1529
1530                 vm->page_tables[pt_idx].bo = pt;
1531                 vm->page_tables[pt_idx].addr = 0;
1532         }
1533
1534         if (flags & AMDGPU_PTE_PRT)
1535                 amdgpu_vm_prt_get(adev);
1536
1537         return 0;
1538
1539 error_free:
1540         list_del(&mapping->list);
1541         interval_tree_remove(&mapping->it, &vm->va);
1542         trace_amdgpu_vm_bo_unmap(bo_va, mapping);
1543         amdgpu_vm_free_mapping(adev, vm, mapping, NULL);
1544
1545 error:
1546         return r;
1547 }
1548
1549 /**
1550  * amdgpu_vm_bo_unmap - remove bo mapping from vm
1551  *
1552  * @adev: amdgpu_device pointer
1553  * @bo_va: bo_va to remove the address from
1554  * @saddr: where to the BO is mapped
1555  *
1556  * Remove a mapping of the BO at the specefied addr from the VM.
1557  * Returns 0 for success, error for failure.
1558  *
1559  * Object has to be reserved and unreserved outside!
1560  */
1561 int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
1562                        struct amdgpu_bo_va *bo_va,
1563                        uint64_t saddr)
1564 {
1565         struct amdgpu_bo_va_mapping *mapping;
1566         struct amdgpu_vm *vm = bo_va->vm;
1567         bool valid = true;
1568
1569         saddr /= AMDGPU_GPU_PAGE_SIZE;
1570
1571         list_for_each_entry(mapping, &bo_va->valids, list) {
1572                 if (mapping->it.start == saddr)
1573                         break;
1574         }
1575
1576         if (&mapping->list == &bo_va->valids) {
1577                 valid = false;
1578
1579                 list_for_each_entry(mapping, &bo_va->invalids, list) {
1580                         if (mapping->it.start == saddr)
1581                                 break;
1582                 }
1583
1584                 if (&mapping->list == &bo_va->invalids)
1585                         return -ENOENT;
1586         }
1587
1588         list_del(&mapping->list);
1589         interval_tree_remove(&mapping->it, &vm->va);
1590         trace_amdgpu_vm_bo_unmap(bo_va, mapping);
1591
1592         if (valid)
1593                 list_add(&mapping->list, &vm->freed);
1594         else
1595                 amdgpu_vm_free_mapping(adev, vm, mapping,
1596                                        bo_va->last_pt_update);
1597
1598         return 0;
1599 }
1600
1601 /**
1602  * amdgpu_vm_bo_rmv - remove a bo to a specific vm
1603  *
1604  * @adev: amdgpu_device pointer
1605  * @bo_va: requested bo_va
1606  *
1607  * Remove @bo_va->bo from the requested vm.
1608  *
1609  * Object have to be reserved!
1610  */
1611 void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
1612                       struct amdgpu_bo_va *bo_va)
1613 {
1614         struct amdgpu_bo_va_mapping *mapping, *next;
1615         struct amdgpu_vm *vm = bo_va->vm;
1616
1617         list_del(&bo_va->bo_list);
1618
1619         spin_lock(&vm->status_lock);
1620         list_del(&bo_va->vm_status);
1621         spin_unlock(&vm->status_lock);
1622
1623         list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
1624                 list_del(&mapping->list);
1625                 interval_tree_remove(&mapping->it, &vm->va);
1626                 trace_amdgpu_vm_bo_unmap(bo_va, mapping);
1627                 list_add(&mapping->list, &vm->freed);
1628         }
1629         list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) {
1630                 list_del(&mapping->list);
1631                 interval_tree_remove(&mapping->it, &vm->va);
1632                 amdgpu_vm_free_mapping(adev, vm, mapping,
1633                                        bo_va->last_pt_update);
1634         }
1635
1636         dma_fence_put(bo_va->last_pt_update);
1637         kfree(bo_va);
1638 }
1639
1640 /**
1641  * amdgpu_vm_bo_invalidate - mark the bo as invalid
1642  *
1643  * @adev: amdgpu_device pointer
1644  * @vm: requested vm
1645  * @bo: amdgpu buffer object
1646  *
1647  * Mark @bo as invalid.
1648  */
1649 void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,
1650                              struct amdgpu_bo *bo)
1651 {
1652         struct amdgpu_bo_va *bo_va;
1653
1654         list_for_each_entry(bo_va, &bo->va, bo_list) {
1655                 spin_lock(&bo_va->vm->status_lock);
1656                 if (list_empty(&bo_va->vm_status))
1657                         list_add(&bo_va->vm_status, &bo_va->vm->invalidated);
1658                 spin_unlock(&bo_va->vm->status_lock);
1659         }
1660 }
1661
1662 /**
1663  * amdgpu_vm_init - initialize a vm instance
1664  *
1665  * @adev: amdgpu_device pointer
1666  * @vm: requested vm
1667  *
1668  * Init @vm fields.
1669  */
1670 int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm)
1671 {
1672         const unsigned align = min(AMDGPU_VM_PTB_ALIGN_SIZE,
1673                 AMDGPU_VM_PTE_COUNT * 8);
1674         unsigned pd_size, pd_entries;
1675         unsigned ring_instance;
1676         struct amdgpu_ring *ring;
1677         struct amd_sched_rq *rq;
1678         int i, r;
1679
1680         for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
1681                 vm->ids[i] = NULL;
1682         vm->va = RB_ROOT;
1683         vm->client_id = atomic64_inc_return(&adev->vm_manager.client_counter);
1684         spin_lock_init(&vm->status_lock);
1685         INIT_LIST_HEAD(&vm->invalidated);
1686         INIT_LIST_HEAD(&vm->cleared);
1687         INIT_LIST_HEAD(&vm->freed);
1688
1689         pd_size = amdgpu_vm_directory_size(adev);
1690         pd_entries = amdgpu_vm_num_pdes(adev);
1691
1692         /* allocate page table array */
1693         vm->page_tables = drm_calloc_large(pd_entries, sizeof(struct amdgpu_vm_pt));
1694         if (vm->page_tables == NULL) {
1695                 DRM_ERROR("Cannot allocate memory for page table array\n");
1696                 return -ENOMEM;
1697         }
1698
1699         /* create scheduler entity for page table updates */
1700
1701         ring_instance = atomic_inc_return(&adev->vm_manager.vm_pte_next_ring);
1702         ring_instance %= adev->vm_manager.vm_pte_num_rings;
1703         ring = adev->vm_manager.vm_pte_rings[ring_instance];
1704         rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_KERNEL];
1705         r = amd_sched_entity_init(&ring->sched, &vm->entity,
1706                                   rq, amdgpu_sched_jobs);
1707         if (r)
1708                 goto err;
1709
1710         vm->page_directory_fence = NULL;
1711
1712         r = amdgpu_bo_create(adev, pd_size, align, true,
1713                              AMDGPU_GEM_DOMAIN_VRAM,
1714                              AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
1715                              AMDGPU_GEM_CREATE_SHADOW |
1716                              AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS |
1717                              AMDGPU_GEM_CREATE_VRAM_CLEARED,
1718                              NULL, NULL, &vm->page_directory);
1719         if (r)
1720                 goto error_free_sched_entity;
1721
1722         r = amdgpu_bo_reserve(vm->page_directory, false);
1723         if (r)
1724                 goto error_free_page_directory;
1725
1726         vm->last_eviction_counter = atomic64_read(&adev->num_evictions);
1727         amdgpu_bo_unreserve(vm->page_directory);
1728
1729         return 0;
1730
1731 error_free_page_directory:
1732         amdgpu_bo_unref(&vm->page_directory->shadow);
1733         amdgpu_bo_unref(&vm->page_directory);
1734         vm->page_directory = NULL;
1735
1736 error_free_sched_entity:
1737         amd_sched_entity_fini(&ring->sched, &vm->entity);
1738
1739 err:
1740         drm_free_large(vm->page_tables);
1741
1742         return r;
1743 }
1744
1745 /**
1746  * amdgpu_vm_fini - tear down a vm instance
1747  *
1748  * @adev: amdgpu_device pointer
1749  * @vm: requested vm
1750  *
1751  * Tear down @vm.
1752  * Unbind the VM and remove all bos from the vm bo list
1753  */
1754 void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
1755 {
1756         struct amdgpu_bo_va_mapping *mapping, *tmp;
1757         bool prt_fini_needed = !!adev->gart.gart_funcs->set_prt;
1758         int i;
1759
1760         amd_sched_entity_fini(vm->entity.sched, &vm->entity);
1761
1762         if (!RB_EMPTY_ROOT(&vm->va)) {
1763                 dev_err(adev->dev, "still active bo inside vm\n");
1764         }
1765         rbtree_postorder_for_each_entry_safe(mapping, tmp, &vm->va, it.rb) {
1766                 list_del(&mapping->list);
1767                 interval_tree_remove(&mapping->it, &vm->va);
1768                 kfree(mapping);
1769         }
1770         list_for_each_entry_safe(mapping, tmp, &vm->freed, list) {
1771                 if (mapping->flags & AMDGPU_PTE_PRT && prt_fini_needed) {
1772                         amdgpu_vm_prt_fini(adev, vm);
1773                         prt_fini_needed = false;
1774                 }
1775
1776                 list_del(&mapping->list);
1777                 amdgpu_vm_free_mapping(adev, vm, mapping, NULL);
1778         }
1779
1780         for (i = 0; i < amdgpu_vm_num_pdes(adev); i++) {
1781                 struct amdgpu_bo *pt = vm->page_tables[i].bo;
1782
1783                 if (!pt)
1784                         continue;
1785
1786                 amdgpu_bo_unref(&pt->shadow);
1787                 amdgpu_bo_unref(&pt);
1788         }
1789         drm_free_large(vm->page_tables);
1790
1791         amdgpu_bo_unref(&vm->page_directory->shadow);
1792         amdgpu_bo_unref(&vm->page_directory);
1793         dma_fence_put(vm->page_directory_fence);
1794 }
1795
1796 /**
1797  * amdgpu_vm_manager_init - init the VM manager
1798  *
1799  * @adev: amdgpu_device pointer
1800  *
1801  * Initialize the VM manager structures
1802  */
1803 void amdgpu_vm_manager_init(struct amdgpu_device *adev)
1804 {
1805         unsigned i;
1806
1807         INIT_LIST_HEAD(&adev->vm_manager.ids_lru);
1808
1809         /* skip over VMID 0, since it is the system VM */
1810         for (i = 1; i < adev->vm_manager.num_ids; ++i) {
1811                 amdgpu_vm_reset_id(adev, i);
1812                 amdgpu_sync_create(&adev->vm_manager.ids[i].active);
1813                 list_add_tail(&adev->vm_manager.ids[i].list,
1814                               &adev->vm_manager.ids_lru);
1815         }
1816
1817         adev->vm_manager.fence_context =
1818                 dma_fence_context_alloc(AMDGPU_MAX_RINGS);
1819         for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
1820                 adev->vm_manager.seqno[i] = 0;
1821
1822         atomic_set(&adev->vm_manager.vm_pte_next_ring, 0);
1823         atomic64_set(&adev->vm_manager.client_counter, 0);
1824         spin_lock_init(&adev->vm_manager.prt_lock);
1825         atomic_set(&adev->vm_manager.num_prt_users, 0);
1826 }
1827
1828 /**
1829  * amdgpu_vm_manager_fini - cleanup VM manager
1830  *
1831  * @adev: amdgpu_device pointer
1832  *
1833  * Cleanup the VM manager and free resources.
1834  */
1835 void amdgpu_vm_manager_fini(struct amdgpu_device *adev)
1836 {
1837         unsigned i;
1838
1839         for (i = 0; i < AMDGPU_NUM_VM; ++i) {
1840                 struct amdgpu_vm_id *id = &adev->vm_manager.ids[i];
1841
1842                 dma_fence_put(adev->vm_manager.ids[i].first);
1843                 amdgpu_sync_free(&adev->vm_manager.ids[i].active);
1844                 dma_fence_put(id->flushed_updates);
1845                 dma_fence_put(id->last_flush);
1846         }
1847 }