drivers/gpu/drm/i915/selftests/i915_gem_coherency.c

   1 /*
   2  * Copyright © 2017 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include <linux/prime_numbers.h>
  26
  27 #include "../i915_selftest.h"
  28 #include "i915_random.h"
  29
  30 static int cpu_set(struct drm_i915_gem_object *obj,
  31                    unsigned long offset,
  32                    u32 v)
  33 {
  34         unsigned int needs_clflush;
  35         struct page *page;
  36         u32 *map;
  37         int err;
  38
  39         err = i915_gem_obj_prepare_shmem_write(obj, &needs_clflush);
  40         if (err)
  41                 return err;
  42
  43         page = i915_gem_object_get_page(obj, offset >> PAGE_SHIFT);
  44         map = kmap_atomic(page);
  45         if (needs_clflush & CLFLUSH_BEFORE)
  46                 clflush(map+offset_in_page(offset) / sizeof(*map));
  47         map[offset_in_page(offset) / sizeof(*map)] = v;
  48         if (needs_clflush & CLFLUSH_AFTER)
  49                 clflush(map+offset_in_page(offset) / sizeof(*map));
  50         kunmap_atomic(map);
  51
  52         i915_gem_obj_finish_shmem_access(obj);
  53         return 0;
  54 }
  55
  56 static int cpu_get(struct drm_i915_gem_object *obj,
  57                    unsigned long offset,
  58                    u32 *v)
  59 {
  60         unsigned int needs_clflush;
  61         struct page *page;
  62         u32 *map;
  63         int err;
  64
  65         err = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
  66         if (err)
  67                 return err;
  68
  69         page = i915_gem_object_get_page(obj, offset >> PAGE_SHIFT);
  70         map = kmap_atomic(page);
  71         if (needs_clflush & CLFLUSH_BEFORE)
  72                 clflush(map+offset_in_page(offset) / sizeof(*map));
  73         *v = map[offset_in_page(offset) / sizeof(*map)];
  74         kunmap_atomic(map);
  75
  76         i915_gem_obj_finish_shmem_access(obj);
  77         return 0;
  78 }
  79
  80 static int gtt_set(struct drm_i915_gem_object *obj,
  81                    unsigned long offset,
  82                    u32 v)
  83 {
  84         struct i915_vma *vma;
  85         u32 __iomem *map;
  86         int err;
  87
  88         err = i915_gem_object_set_to_gtt_domain(obj, true);
  89         if (err)
  90                 return err;
  91
  92         vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, PIN_MAPPABLE);
  93         if (IS_ERR(vma))
  94                 return PTR_ERR(vma);
  95
  96         map = i915_vma_pin_iomap(vma);
  97         i915_vma_unpin(vma);
  98         if (IS_ERR(map))
  99                 return PTR_ERR(map);
 100
 101         iowrite32(v, &map[offset / sizeof(*map)]);
 102         i915_vma_unpin_iomap(vma);
 103
 104         return 0;
 105 }
 106
 107 static int gtt_get(struct drm_i915_gem_object *obj,
 108                    unsigned long offset,
 109                    u32 *v)
 110 {
 111         struct i915_vma *vma;
 112         u32 __iomem *map;
 113         int err;
 114
 115         err = i915_gem_object_set_to_gtt_domain(obj, false);
 116         if (err)
 117                 return err;
 118
 119         vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, PIN_MAPPABLE);
 120         if (IS_ERR(vma))
 121                 return PTR_ERR(vma);
 122
 123         map = i915_vma_pin_iomap(vma);
 124         i915_vma_unpin(vma);
 125         if (IS_ERR(map))
 126                 return PTR_ERR(map);
 127
 128         *v = ioread32(&map[offset / sizeof(*map)]);
 129         i915_vma_unpin_iomap(vma);
 130
 131         return 0;
 132 }
 133
 134 static int wc_set(struct drm_i915_gem_object *obj,
 135                   unsigned long offset,
 136                   u32 v)
 137 {
 138         u32 *map;
 139         int err;
 140
 141         err = i915_gem_object_set_to_wc_domain(obj, true);
 142         if (err)
 143                 return err;
 144
 145         map = i915_gem_object_pin_map(obj, I915_MAP_WC);
 146         if (IS_ERR(map))
 147                 return PTR_ERR(map);
 148
 149         map[offset / sizeof(*map)] = v;
 150         i915_gem_object_unpin_map(obj);
 151
 152         return 0;
 153 }
 154
 155 static int wc_get(struct drm_i915_gem_object *obj,
 156                   unsigned long offset,
 157                   u32 *v)
 158 {
 159         u32 *map;
 160         int err;
 161
 162         err = i915_gem_object_set_to_wc_domain(obj, false);
 163         if (err)
 164                 return err;
 165
 166         map = i915_gem_object_pin_map(obj, I915_MAP_WC);
 167         if (IS_ERR(map))
 168                 return PTR_ERR(map);
 169
 170         *v = map[offset / sizeof(*map)];
 171         i915_gem_object_unpin_map(obj);
 172
 173         return 0;
 174 }
 175
 176 static int gpu_set(struct drm_i915_gem_object *obj,
 177                    unsigned long offset,
 178                    u32 v)
 179 {
 180         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 181         struct i915_request *rq;
 182         struct i915_vma *vma;
 183         u32 *cs;
 184         int err;
 185
 186         err = i915_gem_object_set_to_gtt_domain(obj, true);
 187         if (err)
 188                 return err;
 189
 190         vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 0);
 191         if (IS_ERR(vma))
 192                 return PTR_ERR(vma);
 193
 194         rq = i915_request_alloc(i915->engine[RCS], i915->kernel_context);
 195         if (IS_ERR(rq)) {
 196                 i915_vma_unpin(vma);
 197                 return PTR_ERR(rq);
 198         }
 199
 200         cs = intel_ring_begin(rq, 4);
 201         if (IS_ERR(cs)) {
 202                 i915_request_add(rq);
 203                 i915_vma_unpin(vma);
 204                 return PTR_ERR(cs);
 205         }
 206
 207         if (INTEL_GEN(i915) >= 8) {
 208                 *cs++ = MI_STORE_DWORD_IMM_GEN4 | 1 << 22;
 209                 *cs++ = lower_32_bits(i915_ggtt_offset(vma) + offset);
 210                 *cs++ = upper_32_bits(i915_ggtt_offset(vma) + offset);
 211                 *cs++ = v;
 212         } else if (INTEL_GEN(i915) >= 4) {
 213                 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
 214                 *cs++ = 0;
 215                 *cs++ = i915_ggtt_offset(vma) + offset;
 216                 *cs++ = v;
 217         } else {
 218                 *cs++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL;
 219                 *cs++ = i915_ggtt_offset(vma) + offset;
 220                 *cs++ = v;
 221                 *cs++ = MI_NOOP;
 222         }
 223         intel_ring_advance(rq, cs);
 224
 225         err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
 226         i915_vma_unpin(vma);
 227
 228         i915_request_add(rq);
 229
 230         return err;
 231 }
 232
 233 static bool always_valid(struct drm_i915_private *i915)
 234 {
 235         return true;
 236 }
 237
 238 static bool needs_fence_registers(struct drm_i915_private *i915)
 239 {
 240         return !i915_terminally_wedged(&i915->gpu_error);
 241 }
 242
 243 static bool needs_mi_store_dword(struct drm_i915_private *i915)
 244 {
 245         if (i915_terminally_wedged(&i915->gpu_error))
 246                 return false;
 247
 248         return intel_engine_can_store_dword(i915->engine[RCS]);
 249 }
 250
 251 static const struct igt_coherency_mode {
 252         const char *name;
 253         int (*set)(struct drm_i915_gem_object *, unsigned long offset, u32 v);
 254         int (*get)(struct drm_i915_gem_object *, unsigned long offset, u32 *v);
 255         bool (*valid)(struct drm_i915_private *i915);
 256 } igt_coherency_mode[] = {
 257         { "cpu", cpu_set, cpu_get, always_valid },
 258         { "gtt", gtt_set, gtt_get, needs_fence_registers },
 259         { "wc", wc_set, wc_get, always_valid },
 260         { "gpu", gpu_set, NULL, needs_mi_store_dword },
 261         { },
 262 };
 263
 264 static int igt_gem_coherency(void *arg)
 265 {
 266         const unsigned int ncachelines = PAGE_SIZE/64;
 267         I915_RND_STATE(prng);
 268         struct drm_i915_private *i915 = arg;
 269         const struct igt_coherency_mode *read, *write, *over;
 270         struct drm_i915_gem_object *obj;
 271         unsigned long count, n;
 272         u32 *offsets, *values;
 273         int err = 0;
 274
 275         /* We repeatedly write, overwrite and read from a sequence of
 276          * cachelines in order to try and detect incoherency (unflushed writes
 277          * from either the CPU or GPU). Each setter/getter uses our cache
 278          * domain API which should prevent incoherency.
 279          */
 280
 281         offsets = kmalloc_array(ncachelines, 2*sizeof(u32), GFP_KERNEL);
 282         if (!offsets)
 283                 return -ENOMEM;
 284         for (count = 0; count < ncachelines; count++)
 285                 offsets[count] = count * 64 + 4 * (count % 16);
 286
 287         values = offsets + ncachelines;
 288
 289         mutex_lock(&i915->drm.struct_mutex);
 290         for (over = igt_coherency_mode; over->name; over++) {
 291                 if (!over->set)
 292                         continue;
 293
 294                 if (!over->valid(i915))
 295                         continue;
 296
 297                 for (write = igt_coherency_mode; write->name; write++) {
 298                         if (!write->set)
 299                                 continue;
 300
 301                         if (!write->valid(i915))
 302                                 continue;
 303
 304                         for (read = igt_coherency_mode; read->name; read++) {
 305                                 if (!read->get)
 306                                         continue;
 307
 308                                 if (!read->valid(i915))
 309                                         continue;
 310
 311                                 for_each_prime_number_from(count, 1, ncachelines) {
 312                                         obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
 313                                         if (IS_ERR(obj)) {
 314                                                 err = PTR_ERR(obj);
 315                                                 goto unlock;
 316                                         }
 317
 318                                         i915_random_reorder(offsets, ncachelines, &prng);
 319                                         for (n = 0; n < count; n++)
 320                                                 values[n] = prandom_u32_state(&prng);
 321
 322                                         for (n = 0; n < count; n++) {
 323                                                 err = over->set(obj, offsets[n], ~values[n]);
 324                                                 if (err) {
 325                                                         pr_err("Failed to set stale value[%ld/%ld] in object using %s, err=%d\n",
 326                                                                n, count, over->name, err);
 327                                                         goto put_object;
 328                                                 }
 329                                         }
 330
 331                                         for (n = 0; n < count; n++) {
 332                                                 err = write->set(obj, offsets[n], values[n]);
 333                                                 if (err) {
 334                                                         pr_err("Failed to set value[%ld/%ld] in object using %s, err=%d\n",
 335                                                                n, count, write->name, err);
 336                                                         goto put_object;
 337                                                 }
 338                                         }
 339
 340                                         for (n = 0; n < count; n++) {
 341                                                 u32 found;
 342
 343                                                 err = read->get(obj, offsets[n], &found);
 344                                                 if (err) {
 345                                                         pr_err("Failed to get value[%ld/%ld] in object using %s, err=%d\n",
 346                                                                n, count, read->name, err);
 347                                                         goto put_object;
 348                                                 }
 349
 350                                                 if (found != values[n]) {
 351                                                         pr_err("Value[%ld/%ld] mismatch, (overwrite with %s) wrote [%s] %x read [%s] %x (inverse %x), at offset %x\n",
 352                                                                n, count, over->name,
 353                                                                write->name, values[n],
 354                                                                read->name, found,
 355                                                                ~values[n], offsets[n]);
 356                                                         err = -EINVAL;
 357                                                         goto put_object;
 358                                                 }
 359                                         }
 360
 361                                         __i915_gem_object_release_unless_active(obj);
 362                                 }
 363                         }
 364                 }
 365         }
 366 unlock:
 367         mutex_unlock(&i915->drm.struct_mutex);
 368         kfree(offsets);
 369         return err;
 370
 371 put_object:
 372         __i915_gem_object_release_unless_active(obj);
 373         goto unlock;
 374 }
 375
 376 int i915_gem_coherency_live_selftests(struct drm_i915_private *i915)
 377 {
 378         static const struct i915_subtest tests[] = {
 379                 SUBTEST(igt_gem_coherency),
 380         };
 381
 382         return i915_subtests(tests, i915);
 383 }