2 * Copyright © 2017 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
25 #include "../i915_selftest.h"
28 #include "huge_gem_object.h"
30 #define DW_PER_PAGE (PAGE_SIZE / sizeof(u32))
32 static struct i915_vma *
33 gpu_fill_dw(struct i915_vma *vma, u64 offset, unsigned long count, u32 value)
35 struct drm_i915_gem_object *obj;
36 const int gen = INTEL_GEN(vma->vm->i915);
37 unsigned long n, size;
41 GEM_BUG_ON(!igt_can_mi_store_dword_imm(vma->vm->i915));
43 size = (4 * count + 1) * sizeof(u32);
44 size = round_up(size, PAGE_SIZE);
45 obj = i915_gem_object_create_internal(vma->vm->i915, size);
49 cmd = i915_gem_object_pin_map(obj, I915_MAP_WB);
55 GEM_BUG_ON(offset + (count - 1) * PAGE_SIZE > vma->node.size);
56 offset += vma->node.start;
58 for (n = 0; n < count; n++) {
60 *cmd++ = MI_STORE_DWORD_IMM_GEN4;
61 *cmd++ = lower_32_bits(offset);
62 *cmd++ = upper_32_bits(offset);
64 } else if (gen >= 4) {
65 *cmd++ = MI_STORE_DWORD_IMM_GEN4 |
66 (gen < 6 ? 1 << 22 : 0);
71 *cmd++ = MI_STORE_DWORD_IMM | 1 << 22;
77 *cmd = MI_BATCH_BUFFER_END;
78 i915_gem_object_unpin_map(obj);
80 err = i915_gem_object_set_to_gtt_domain(obj, false);
84 vma = i915_vma_instance(obj, vma->vm, NULL);
90 err = i915_vma_pin(vma, 0, 0, PIN_USER);
97 i915_gem_object_put(obj);
101 static unsigned long real_page_count(struct drm_i915_gem_object *obj)
103 return huge_gem_object_phys_size(obj) >> PAGE_SHIFT;
106 static unsigned long fake_page_count(struct drm_i915_gem_object *obj)
108 return huge_gem_object_dma_size(obj) >> PAGE_SHIFT;
111 static int gpu_fill(struct drm_i915_gem_object *obj,
112 struct i915_gem_context *ctx,
113 struct intel_engine_cs *engine,
116 struct drm_i915_private *i915 = to_i915(obj->base.dev);
117 struct i915_address_space *vm =
118 ctx->ppgtt ? &ctx->ppgtt->base : &i915->ggtt.base;
119 struct drm_i915_gem_request *rq;
120 struct i915_vma *vma;
121 struct i915_vma *batch;
125 GEM_BUG_ON(obj->base.size > vm->total);
127 vma = i915_vma_instance(obj, vm, NULL);
131 err = i915_gem_object_set_to_gtt_domain(obj, false);
135 err = i915_vma_pin(vma, 0, 0, PIN_HIGH | PIN_USER);
139 /* Within the GTT the huge objects maps every page onto
140 * its 1024 real pages (using phys_pfn = dma_pfn % 1024).
141 * We set the nth dword within the page using the nth
142 * mapping via the GTT - this should exercise the GTT mapping
143 * whilst checking that each context provides a unique view
146 batch = gpu_fill_dw(vma,
147 (dw * real_page_count(obj)) << PAGE_SHIFT |
149 real_page_count(obj),
152 err = PTR_ERR(batch);
156 rq = i915_gem_request_alloc(engine, ctx);
162 err = engine->emit_flush(rq, EMIT_INVALIDATE);
166 err = i915_switch_context(rq);
171 if (INTEL_GEN(vm->i915) <= 5)
172 flags |= I915_DISPATCH_SECURE;
174 err = engine->emit_bb_start(rq,
175 batch->node.start, batch->node.size,
180 i915_vma_move_to_active(batch, rq, 0);
181 i915_gem_object_set_active_reference(batch->obj);
182 i915_vma_unpin(batch);
183 i915_vma_close(batch);
185 i915_vma_move_to_active(vma, rq, 0);
188 reservation_object_lock(obj->resv, NULL);
189 reservation_object_add_excl_fence(obj->resv, &rq->fence);
190 reservation_object_unlock(obj->resv);
192 __i915_add_request(rq, true);
197 __i915_add_request(rq, false);
199 i915_vma_unpin(batch);
205 static int cpu_fill(struct drm_i915_gem_object *obj, u32 value)
207 const bool has_llc = HAS_LLC(to_i915(obj->base.dev));
208 unsigned int n, m, need_flush;
211 err = i915_gem_obj_prepare_shmem_write(obj, &need_flush);
215 for (n = 0; n < real_page_count(obj); n++) {
218 map = kmap_atomic(i915_gem_object_get_page(obj, n));
219 for (m = 0; m < DW_PER_PAGE; m++)
222 drm_clflush_virt_range(map, PAGE_SIZE);
226 i915_gem_obj_finish_shmem_access(obj);
227 obj->base.read_domains = I915_GEM_DOMAIN_GTT | I915_GEM_DOMAIN_CPU;
228 obj->base.write_domain = 0;
232 static int cpu_check(struct drm_i915_gem_object *obj, unsigned int max)
234 unsigned int n, m, needs_flush;
237 err = i915_gem_obj_prepare_shmem_read(obj, &needs_flush);
241 for (n = 0; n < real_page_count(obj); n++) {
244 map = kmap_atomic(i915_gem_object_get_page(obj, n));
245 if (needs_flush & CLFLUSH_BEFORE)
246 drm_clflush_virt_range(map, PAGE_SIZE);
248 for (m = 0; m < max; m++) {
250 pr_err("Invalid value at page %d, offset %d: found %x expected %x\n",
257 for (; m < DW_PER_PAGE; m++) {
258 if (map[m] != 0xdeadbeef) {
259 pr_err("Invalid value at page %d, offset %d: found %x expected %x\n",
260 n, m, map[m], 0xdeadbeef);
272 i915_gem_obj_finish_shmem_access(obj);
276 static struct drm_i915_gem_object *
277 create_test_object(struct i915_gem_context *ctx,
278 struct drm_file *file,
279 struct list_head *objects)
281 struct drm_i915_gem_object *obj;
282 struct i915_address_space *vm =
283 ctx->ppgtt ? &ctx->ppgtt->base : &ctx->i915->ggtt.base;
288 size = min(vm->total / 2, 1024ull * DW_PER_PAGE * PAGE_SIZE);
289 size = round_down(size, DW_PER_PAGE * PAGE_SIZE);
291 obj = huge_gem_object(ctx->i915, DW_PER_PAGE * PAGE_SIZE, size);
295 /* tie the handle to the drm_file for easy reaping */
296 err = drm_gem_handle_create(file, &obj->base, &handle);
297 i915_gem_object_put(obj);
301 err = cpu_fill(obj, 0xdeadbeef);
303 pr_err("Failed to fill object with cpu, err=%d\n",
308 list_add_tail(&obj->st_link, objects);
312 static unsigned long max_dwords(struct drm_i915_gem_object *obj)
314 unsigned long npages = fake_page_count(obj);
316 GEM_BUG_ON(!IS_ALIGNED(npages, DW_PER_PAGE));
317 return npages / DW_PER_PAGE;
320 static int igt_ctx_exec(void *arg)
322 struct drm_i915_private *i915 = arg;
323 struct drm_i915_gem_object *obj = NULL;
324 struct drm_file *file;
325 IGT_TIMEOUT(end_time);
327 unsigned long ncontexts, ndwords, dw;
328 bool first_shared_gtt = true;
331 /* Create a few different contexts (with different mm) and write
332 * through each ctx/mm using the GPU making sure those writes end
333 * up in the expected pages of our obj.
336 file = mock_file(i915);
338 return PTR_ERR(file);
340 mutex_lock(&i915->drm.struct_mutex);
345 while (!time_after(jiffies, end_time)) {
346 struct intel_engine_cs *engine;
347 struct i915_gem_context *ctx;
350 if (first_shared_gtt) {
351 ctx = __create_hw_context(i915, file->driver_priv);
352 first_shared_gtt = false;
354 ctx = i915_gem_create_context(i915, file->driver_priv);
361 for_each_engine(engine, i915, id) {
363 obj = create_test_object(ctx, file, &objects);
370 err = gpu_fill(obj, ctx, engine, dw);
372 pr_err("Failed to fill dword %lu [%lu/%lu] with gpu (%s) in ctx %u [full-ppgtt? %s], err=%d\n",
373 ndwords, dw, max_dwords(obj),
374 engine->name, ctx->hw_id,
375 yesno(!!ctx->ppgtt), err);
379 if (++dw == max_dwords(obj)) {
387 pr_info("Submitted %lu contexts (across %u engines), filling %lu dwords\n",
388 ncontexts, INTEL_INFO(i915)->num_rings, ndwords);
391 list_for_each_entry(obj, &objects, st_link) {
393 min_t(unsigned int, ndwords - dw, max_dwords(obj));
395 err = cpu_check(obj, rem);
403 mutex_unlock(&i915->drm.struct_mutex);
405 mock_file_free(i915, file);
409 static int fake_aliasing_ppgtt_enable(struct drm_i915_private *i915)
411 struct drm_i915_gem_object *obj;
414 err = i915_gem_init_aliasing_ppgtt(i915);
418 list_for_each_entry(obj, &i915->mm.bound_list, global_link) {
419 struct i915_vma *vma;
421 vma = i915_vma_instance(obj, &i915->ggtt.base, NULL);
425 vma->flags &= ~I915_VMA_LOCAL_BIND;
431 static void fake_aliasing_ppgtt_disable(struct drm_i915_private *i915)
433 i915_gem_fini_aliasing_ppgtt(i915);
436 int i915_gem_context_live_selftests(struct drm_i915_private *dev_priv)
438 static const struct i915_subtest tests[] = {
439 SUBTEST(igt_ctx_exec),
441 bool fake_alias = false;
444 /* Install a fake aliasing gtt for exercise */
445 if (USES_PPGTT(dev_priv) && !dev_priv->mm.aliasing_ppgtt) {
446 mutex_lock(&dev_priv->drm.struct_mutex);
447 err = fake_aliasing_ppgtt_enable(dev_priv);
448 mutex_unlock(&dev_priv->drm.struct_mutex);
452 GEM_BUG_ON(!dev_priv->mm.aliasing_ppgtt);
456 err = i915_subtests(tests, dev_priv);
459 mutex_lock(&dev_priv->drm.struct_mutex);
460 fake_aliasing_ppgtt_disable(dev_priv);
461 mutex_unlock(&dev_priv->drm.struct_mutex);