drivers/gpu/drm/i915/selftests/intel_hangcheck.c

   1 /*
   2  * Copyright © 2016 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include <linux/kthread.h>
  26
  27 #include "../i915_selftest.h"
  28 #include "i915_random.h"
  29 #include "igt_flush_test.h"
  30 #include "igt_reset.h"
  31 #include "igt_wedge_me.h"
  32
  33 #include "mock_context.h"
  34 #include "mock_drm.h"
  35
  36 #define IGT_IDLE_TIMEOUT 50 /* ms; time to wait after flushing between tests */
  37
  38 struct hang {
  39         struct drm_i915_private *i915;
  40         struct drm_i915_gem_object *hws;
  41         struct drm_i915_gem_object *obj;
  42         struct i915_gem_context *ctx;
  43         u32 *seqno;
  44         u32 *batch;
  45 };
  46
  47 static int hang_init(struct hang *h, struct drm_i915_private *i915)
  48 {
  49         void *vaddr;
  50         int err;
  51
  52         memset(h, 0, sizeof(*h));
  53         h->i915 = i915;
  54
  55         h->ctx = kernel_context(i915);
  56         if (IS_ERR(h->ctx))
  57                 return PTR_ERR(h->ctx);
  58
  59         h->hws = i915_gem_object_create_internal(i915, PAGE_SIZE);
  60         if (IS_ERR(h->hws)) {
  61                 err = PTR_ERR(h->hws);
  62                 goto err_ctx;
  63         }
  64
  65         h->obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
  66         if (IS_ERR(h->obj)) {
  67                 err = PTR_ERR(h->obj);
  68                 goto err_hws;
  69         }
  70
  71         i915_gem_object_set_cache_level(h->hws, I915_CACHE_LLC);
  72         vaddr = i915_gem_object_pin_map(h->hws, I915_MAP_WB);
  73         if (IS_ERR(vaddr)) {
  74                 err = PTR_ERR(vaddr);
  75                 goto err_obj;
  76         }
  77         h->seqno = memset(vaddr, 0xff, PAGE_SIZE);
  78
  79         vaddr = i915_gem_object_pin_map(h->obj,
  80                                         i915_coherent_map_type(i915));
  81         if (IS_ERR(vaddr)) {
  82                 err = PTR_ERR(vaddr);
  83                 goto err_unpin_hws;
  84         }
  85         h->batch = vaddr;
  86
  87         return 0;
  88
  89 err_unpin_hws:
  90         i915_gem_object_unpin_map(h->hws);
  91 err_obj:
  92         i915_gem_object_put(h->obj);
  93 err_hws:
  94         i915_gem_object_put(h->hws);
  95 err_ctx:
  96         kernel_context_close(h->ctx);
  97         return err;
  98 }
  99
 100 static u64 hws_address(const struct i915_vma *hws,
 101                        const struct i915_request *rq)
 102 {
 103         return hws->node.start + offset_in_page(sizeof(u32)*rq->fence.context);
 104 }
 105
 106 static int move_to_active(struct i915_vma *vma,
 107                           struct i915_request *rq,
 108                           unsigned int flags)
 109 {
 110         int err;
 111
 112         err = i915_vma_move_to_active(vma, rq, flags);
 113         if (err)
 114                 return err;
 115
 116         if (!i915_gem_object_has_active_reference(vma->obj)) {
 117                 i915_gem_object_get(vma->obj);
 118                 i915_gem_object_set_active_reference(vma->obj);
 119         }
 120
 121         return 0;
 122 }
 123
 124 static struct i915_request *
 125 hang_create_request(struct hang *h, struct intel_engine_cs *engine)
 126 {
 127         struct drm_i915_private *i915 = h->i915;
 128         struct i915_address_space *vm =
 129                 h->ctx->ppgtt ? &h->ctx->ppgtt->vm : &i915->ggtt.vm;
 130         struct i915_request *rq = NULL;
 131         struct i915_vma *hws, *vma;
 132         unsigned int flags;
 133         u32 *batch;
 134         int err;
 135
 136         if (i915_gem_object_is_active(h->obj)) {
 137                 struct drm_i915_gem_object *obj;
 138                 void *vaddr;
 139
 140                 obj = i915_gem_object_create_internal(h->i915, PAGE_SIZE);
 141                 if (IS_ERR(obj))
 142                         return ERR_CAST(obj);
 143
 144                 vaddr = i915_gem_object_pin_map(obj,
 145                                                 i915_coherent_map_type(h->i915));
 146                 if (IS_ERR(vaddr)) {
 147                         i915_gem_object_put(obj);
 148                         return ERR_CAST(vaddr);
 149                 }
 150
 151                 i915_gem_object_unpin_map(h->obj);
 152                 i915_gem_object_put(h->obj);
 153
 154                 h->obj = obj;
 155                 h->batch = vaddr;
 156         }
 157
 158         vma = i915_vma_instance(h->obj, vm, NULL);
 159         if (IS_ERR(vma))
 160                 return ERR_CAST(vma);
 161
 162         hws = i915_vma_instance(h->hws, vm, NULL);
 163         if (IS_ERR(hws))
 164                 return ERR_CAST(hws);
 165
 166         err = i915_vma_pin(vma, 0, 0, PIN_USER);
 167         if (err)
 168                 return ERR_PTR(err);
 169
 170         err = i915_vma_pin(hws, 0, 0, PIN_USER);
 171         if (err)
 172                 goto unpin_vma;
 173
 174         rq = i915_request_alloc(engine, h->ctx);
 175         if (IS_ERR(rq)) {
 176                 err = PTR_ERR(rq);
 177                 goto unpin_hws;
 178         }
 179
 180         err = move_to_active(vma, rq, 0);
 181         if (err)
 182                 goto cancel_rq;
 183
 184         err = move_to_active(hws, rq, 0);
 185         if (err)
 186                 goto cancel_rq;
 187
 188         batch = h->batch;
 189         if (INTEL_GEN(i915) >= 8) {
 190                 *batch++ = MI_STORE_DWORD_IMM_GEN4;
 191                 *batch++ = lower_32_bits(hws_address(hws, rq));
 192                 *batch++ = upper_32_bits(hws_address(hws, rq));
 193                 *batch++ = rq->fence.seqno;
 194                 *batch++ = MI_ARB_CHECK;
 195
 196                 memset(batch, 0, 1024);
 197                 batch += 1024 / sizeof(*batch);
 198
 199                 *batch++ = MI_ARB_CHECK;
 200                 *batch++ = MI_BATCH_BUFFER_START | 1 << 8 | 1;
 201                 *batch++ = lower_32_bits(vma->node.start);
 202                 *batch++ = upper_32_bits(vma->node.start);
 203         } else if (INTEL_GEN(i915) >= 6) {
 204                 *batch++ = MI_STORE_DWORD_IMM_GEN4;
 205                 *batch++ = 0;
 206                 *batch++ = lower_32_bits(hws_address(hws, rq));
 207                 *batch++ = rq->fence.seqno;
 208                 *batch++ = MI_ARB_CHECK;
 209
 210                 memset(batch, 0, 1024);
 211                 batch += 1024 / sizeof(*batch);
 212
 213                 *batch++ = MI_ARB_CHECK;
 214                 *batch++ = MI_BATCH_BUFFER_START | 1 << 8;
 215                 *batch++ = lower_32_bits(vma->node.start);
 216         } else if (INTEL_GEN(i915) >= 4) {
 217                 *batch++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
 218                 *batch++ = 0;
 219                 *batch++ = lower_32_bits(hws_address(hws, rq));
 220                 *batch++ = rq->fence.seqno;
 221                 *batch++ = MI_ARB_CHECK;
 222
 223                 memset(batch, 0, 1024);
 224                 batch += 1024 / sizeof(*batch);
 225
 226                 *batch++ = MI_ARB_CHECK;
 227                 *batch++ = MI_BATCH_BUFFER_START | 2 << 6;
 228                 *batch++ = lower_32_bits(vma->node.start);
 229         } else {
 230                 *batch++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL;
 231                 *batch++ = lower_32_bits(hws_address(hws, rq));
 232                 *batch++ = rq->fence.seqno;
 233                 *batch++ = MI_ARB_CHECK;
 234
 235                 memset(batch, 0, 1024);
 236                 batch += 1024 / sizeof(*batch);
 237
 238                 *batch++ = MI_ARB_CHECK;
 239                 *batch++ = MI_BATCH_BUFFER_START | 2 << 6;
 240                 *batch++ = lower_32_bits(vma->node.start);
 241         }
 242         *batch++ = MI_BATCH_BUFFER_END; /* not reached */
 243         i915_gem_chipset_flush(h->i915);
 244
 245         flags = 0;
 246         if (INTEL_GEN(vm->i915) <= 5)
 247                 flags |= I915_DISPATCH_SECURE;
 248
 249         err = rq->engine->emit_bb_start(rq, vma->node.start, PAGE_SIZE, flags);
 250
 251 cancel_rq:
 252         if (err) {
 253                 i915_request_skip(rq, err);
 254                 i915_request_add(rq);
 255         }
 256 unpin_hws:
 257         i915_vma_unpin(hws);
 258 unpin_vma:
 259         i915_vma_unpin(vma);
 260         return err ? ERR_PTR(err) : rq;
 261 }
 262
 263 static u32 hws_seqno(const struct hang *h, const struct i915_request *rq)
 264 {
 265         return READ_ONCE(h->seqno[rq->fence.context % (PAGE_SIZE/sizeof(u32))]);
 266 }
 267
 268 static void hang_fini(struct hang *h)
 269 {
 270         *h->batch = MI_BATCH_BUFFER_END;
 271         i915_gem_chipset_flush(h->i915);
 272
 273         i915_gem_object_unpin_map(h->obj);
 274         i915_gem_object_put(h->obj);
 275
 276         i915_gem_object_unpin_map(h->hws);
 277         i915_gem_object_put(h->hws);
 278
 279         kernel_context_close(h->ctx);
 280
 281         igt_flush_test(h->i915, I915_WAIT_LOCKED);
 282 }
 283
 284 static bool wait_until_running(struct hang *h, struct i915_request *rq)
 285 {
 286         return !(wait_for_us(i915_seqno_passed(hws_seqno(h, rq),
 287                                                rq->fence.seqno),
 288                              10) &&
 289                  wait_for(i915_seqno_passed(hws_seqno(h, rq),
 290                                             rq->fence.seqno),
 291                           1000));
 292 }
 293
 294 static int igt_hang_sanitycheck(void *arg)
 295 {
 296         struct drm_i915_private *i915 = arg;
 297         struct i915_request *rq;
 298         struct intel_engine_cs *engine;
 299         enum intel_engine_id id;
 300         struct hang h;
 301         int err;
 302
 303         /* Basic check that we can execute our hanging batch */
 304
 305         mutex_lock(&i915->drm.struct_mutex);
 306         err = hang_init(&h, i915);
 307         if (err)
 308                 goto unlock;
 309
 310         for_each_engine(engine, i915, id) {
 311                 struct igt_wedge_me w;
 312                 long timeout;
 313
 314                 if (!intel_engine_can_store_dword(engine))
 315                         continue;
 316
 317                 rq = hang_create_request(&h, engine);
 318                 if (IS_ERR(rq)) {
 319                         err = PTR_ERR(rq);
 320                         pr_err("Failed to create request for %s, err=%d\n",
 321                                engine->name, err);
 322                         goto fini;
 323                 }
 324
 325                 i915_request_get(rq);
 326
 327                 *h.batch = MI_BATCH_BUFFER_END;
 328                 i915_gem_chipset_flush(i915);
 329
 330                 i915_request_add(rq);
 331
 332                 timeout = 0;
 333                 igt_wedge_on_timeout(&w, i915, HZ / 10 /* 100ms timeout*/)
 334                         timeout = i915_request_wait(rq,
 335                                                     I915_WAIT_LOCKED,
 336                                                     MAX_SCHEDULE_TIMEOUT);
 337                 if (i915_terminally_wedged(&i915->gpu_error))
 338                         timeout = -EIO;
 339
 340                 i915_request_put(rq);
 341
 342                 if (timeout < 0) {
 343                         err = timeout;
 344                         pr_err("Wait for request failed on %s, err=%d\n",
 345                                engine->name, err);
 346                         goto fini;
 347                 }
 348         }
 349
 350 fini:
 351         hang_fini(&h);
 352 unlock:
 353         mutex_unlock(&i915->drm.struct_mutex);
 354         return err;
 355 }
 356
 357 static int igt_global_reset(void *arg)
 358 {
 359         struct drm_i915_private *i915 = arg;
 360         unsigned int reset_count;
 361         int err = 0;
 362
 363         /* Check that we can issue a global GPU reset */
 364
 365         igt_global_reset_lock(i915);
 366         set_bit(I915_RESET_HANDOFF, &i915->gpu_error.flags);
 367
 368         mutex_lock(&i915->drm.struct_mutex);
 369         reset_count = i915_reset_count(&i915->gpu_error);
 370
 371         i915_reset(i915, ALL_ENGINES, NULL);
 372
 373         if (i915_reset_count(&i915->gpu_error) == reset_count) {
 374                 pr_err("No GPU reset recorded!\n");
 375                 err = -EINVAL;
 376         }
 377         mutex_unlock(&i915->drm.struct_mutex);
 378
 379         GEM_BUG_ON(test_bit(I915_RESET_HANDOFF, &i915->gpu_error.flags));
 380         igt_global_reset_unlock(i915);
 381
 382         if (i915_terminally_wedged(&i915->gpu_error))
 383                 err = -EIO;
 384
 385         return err;
 386 }
 387
 388 static int igt_wedged_reset(void *arg)
 389 {
 390         struct drm_i915_private *i915 = arg;
 391
 392         /* Check that we can recover a wedged device with a GPU reset */
 393
 394         igt_global_reset_lock(i915);
 395         mutex_lock(&i915->drm.struct_mutex);
 396         intel_runtime_pm_get(i915);
 397
 398         i915_gem_set_wedged(i915);
 399         GEM_BUG_ON(!i915_terminally_wedged(&i915->gpu_error));
 400
 401         set_bit(I915_RESET_HANDOFF, &i915->gpu_error.flags);
 402         i915_reset(i915, ALL_ENGINES, NULL);
 403         GEM_BUG_ON(test_bit(I915_RESET_HANDOFF, &i915->gpu_error.flags));
 404
 405         intel_runtime_pm_put(i915);
 406         mutex_unlock(&i915->drm.struct_mutex);
 407         igt_global_reset_unlock(i915);
 408
 409         return i915_terminally_wedged(&i915->gpu_error) ? -EIO : 0;
 410 }
 411
 412 static bool wait_for_idle(struct intel_engine_cs *engine)
 413 {
 414         return wait_for(intel_engine_is_idle(engine), IGT_IDLE_TIMEOUT) == 0;
 415 }
 416
 417 static int __igt_reset_engine(struct drm_i915_private *i915, bool active)
 418 {
 419         struct intel_engine_cs *engine;
 420         enum intel_engine_id id;
 421         struct hang h;
 422         int err = 0;
 423
 424         /* Check that we can issue an engine reset on an idle engine (no-op) */
 425
 426         if (!intel_has_reset_engine(i915))
 427                 return 0;
 428
 429         if (active) {
 430                 mutex_lock(&i915->drm.struct_mutex);
 431                 err = hang_init(&h, i915);
 432                 mutex_unlock(&i915->drm.struct_mutex);
 433                 if (err)
 434                         return err;
 435         }
 436
 437         for_each_engine(engine, i915, id) {
 438                 unsigned int reset_count, reset_engine_count;
 439                 IGT_TIMEOUT(end_time);
 440
 441                 if (active && !intel_engine_can_store_dword(engine))
 442                         continue;
 443
 444                 if (!wait_for_idle(engine)) {
 445                         pr_err("%s failed to idle before reset\n",
 446                                engine->name);
 447                         err = -EIO;
 448                         break;
 449                 }
 450
 451                 reset_count = i915_reset_count(&i915->gpu_error);
 452                 reset_engine_count = i915_reset_engine_count(&i915->gpu_error,
 453                                                              engine);
 454
 455                 set_bit(I915_RESET_ENGINE + id, &i915->gpu_error.flags);
 456                 do {
 457                         u32 seqno = intel_engine_get_seqno(engine);
 458
 459                         if (active) {
 460                                 struct i915_request *rq;
 461
 462                                 mutex_lock(&i915->drm.struct_mutex);
 463                                 rq = hang_create_request(&h, engine);
 464                                 if (IS_ERR(rq)) {
 465                                         err = PTR_ERR(rq);
 466                                         mutex_unlock(&i915->drm.struct_mutex);
 467                                         break;
 468                                 }
 469
 470                                 i915_request_get(rq);
 471                                 i915_request_add(rq);
 472                                 mutex_unlock(&i915->drm.struct_mutex);
 473
 474                                 if (!wait_until_running(&h, rq)) {
 475                                         struct drm_printer p = drm_info_printer(i915->drm.dev);
 476
 477                                         pr_err("%s: Failed to start request %x, at %x\n",
 478                                                __func__, rq->fence.seqno, hws_seqno(&h, rq));
 479                                         intel_engine_dump(engine, &p,
 480                                                           "%s\n", engine->name);
 481
 482                                         i915_request_put(rq);
 483                                         err = -EIO;
 484                                         break;
 485                                 }
 486
 487                                 GEM_BUG_ON(!rq->global_seqno);
 488                                 seqno = rq->global_seqno - 1;
 489                                 i915_request_put(rq);
 490                         }
 491
 492                         err = i915_reset_engine(engine, NULL);
 493                         if (err) {
 494                                 pr_err("i915_reset_engine failed\n");
 495                                 break;
 496                         }
 497
 498                         if (i915_reset_count(&i915->gpu_error) != reset_count) {
 499                                 pr_err("Full GPU reset recorded! (engine reset expected)\n");
 500                                 err = -EINVAL;
 501                                 break;
 502                         }
 503
 504                         reset_engine_count += active;
 505                         if (i915_reset_engine_count(&i915->gpu_error, engine) !=
 506                             reset_engine_count) {
 507                                 pr_err("%s engine reset %srecorded!\n",
 508                                        engine->name, active ? "not " : "");
 509                                 err = -EINVAL;
 510                                 break;
 511                         }
 512
 513                         if (!wait_for_idle(engine)) {
 514                                 struct drm_printer p =
 515                                         drm_info_printer(i915->drm.dev);
 516
 517                                 pr_err("%s failed to idle after reset\n",
 518                                        engine->name);
 519                                 intel_engine_dump(engine, &p,
 520                                                   "%s\n", engine->name);
 521
 522                                 err = -EIO;
 523                                 break;
 524                         }
 525                 } while (time_before(jiffies, end_time));
 526                 clear_bit(I915_RESET_ENGINE + id, &i915->gpu_error.flags);
 527
 528                 if (err)
 529                         break;
 530
 531                 err = igt_flush_test(i915, 0);
 532                 if (err)
 533                         break;
 534         }
 535
 536         if (i915_terminally_wedged(&i915->gpu_error))
 537                 err = -EIO;
 538
 539         if (active) {
 540                 mutex_lock(&i915->drm.struct_mutex);
 541                 hang_fini(&h);
 542                 mutex_unlock(&i915->drm.struct_mutex);
 543         }
 544
 545         return err;
 546 }
 547
 548 static int igt_reset_idle_engine(void *arg)
 549 {
 550         return __igt_reset_engine(arg, false);
 551 }
 552
 553 static int igt_reset_active_engine(void *arg)
 554 {
 555         return __igt_reset_engine(arg, true);
 556 }
 557
 558 struct active_engine {
 559         struct task_struct *task;
 560         struct intel_engine_cs *engine;
 561         unsigned long resets;
 562         unsigned int flags;
 563 };
 564
 565 #define TEST_ACTIVE     BIT(0)
 566 #define TEST_OTHERS     BIT(1)
 567 #define TEST_SELF       BIT(2)
 568 #define TEST_PRIORITY   BIT(3)
 569
 570 static int active_request_put(struct i915_request *rq)
 571 {
 572         int err = 0;
 573
 574         if (!rq)
 575                 return 0;
 576
 577         if (i915_request_wait(rq, 0, 5 * HZ) < 0) {
 578                 GEM_TRACE("%s timed out waiting for completion of fence %llx:%d, seqno %d.\n",
 579                           rq->engine->name,
 580                           rq->fence.context,
 581                           rq->fence.seqno,
 582                           i915_request_global_seqno(rq));
 583                 GEM_TRACE_DUMP();
 584
 585                 i915_gem_set_wedged(rq->i915);
 586                 err = -EIO;
 587         }
 588
 589         i915_request_put(rq);
 590
 591         return err;
 592 }
 593
 594 static int active_engine(void *data)
 595 {
 596         I915_RND_STATE(prng);
 597         struct active_engine *arg = data;
 598         struct intel_engine_cs *engine = arg->engine;
 599         struct i915_request *rq[8] = {};
 600         struct i915_gem_context *ctx[ARRAY_SIZE(rq)];
 601         struct drm_file *file;
 602         unsigned long count = 0;
 603         int err = 0;
 604
 605         file = mock_file(engine->i915);
 606         if (IS_ERR(file))
 607                 return PTR_ERR(file);
 608
 609         for (count = 0; count < ARRAY_SIZE(ctx); count++) {
 610                 mutex_lock(&engine->i915->drm.struct_mutex);
 611                 ctx[count] = live_context(engine->i915, file);
 612                 mutex_unlock(&engine->i915->drm.struct_mutex);
 613                 if (IS_ERR(ctx[count])) {
 614                         err = PTR_ERR(ctx[count]);
 615                         while (--count)
 616                                 i915_gem_context_put(ctx[count]);
 617                         goto err_file;
 618                 }
 619         }
 620
 621         while (!kthread_should_stop()) {
 622                 unsigned int idx = count++ & (ARRAY_SIZE(rq) - 1);
 623                 struct i915_request *old = rq[idx];
 624                 struct i915_request *new;
 625
 626                 mutex_lock(&engine->i915->drm.struct_mutex);
 627                 new = i915_request_alloc(engine, ctx[idx]);
 628                 if (IS_ERR(new)) {
 629                         mutex_unlock(&engine->i915->drm.struct_mutex);
 630                         err = PTR_ERR(new);
 631                         break;
 632                 }
 633
 634                 if (arg->flags & TEST_PRIORITY)
 635                         ctx[idx]->sched.priority =
 636                                 i915_prandom_u32_max_state(512, &prng);
 637
 638                 rq[idx] = i915_request_get(new);
 639                 i915_request_add(new);
 640                 mutex_unlock(&engine->i915->drm.struct_mutex);
 641
 642                 err = active_request_put(old);
 643                 if (err)
 644                         break;
 645
 646                 cond_resched();
 647         }
 648
 649         for (count = 0; count < ARRAY_SIZE(rq); count++) {
 650                 int err__ = active_request_put(rq[count]);
 651
 652                 /* Keep the first error */
 653                 if (!err)
 654                         err = err__;
 655         }
 656
 657 err_file:
 658         mock_file_free(engine->i915, file);
 659         return err;
 660 }
 661
 662 static int __igt_reset_engines(struct drm_i915_private *i915,
 663                                const char *test_name,
 664                                unsigned int flags)
 665 {
 666         struct intel_engine_cs *engine, *other;
 667         enum intel_engine_id id, tmp;
 668         struct hang h;
 669         int err = 0;
 670
 671         /* Check that issuing a reset on one engine does not interfere
 672          * with any other engine.
 673          */
 674
 675         if (!intel_has_reset_engine(i915))
 676                 return 0;
 677
 678         if (flags & TEST_ACTIVE) {
 679                 mutex_lock(&i915->drm.struct_mutex);
 680                 err = hang_init(&h, i915);
 681                 mutex_unlock(&i915->drm.struct_mutex);
 682                 if (err)
 683                         return err;
 684
 685                 if (flags & TEST_PRIORITY)
 686                         h.ctx->sched.priority = 1024;
 687         }
 688
 689         for_each_engine(engine, i915, id) {
 690                 struct active_engine threads[I915_NUM_ENGINES] = {};
 691                 unsigned long global = i915_reset_count(&i915->gpu_error);
 692                 unsigned long count = 0, reported;
 693                 IGT_TIMEOUT(end_time);
 694
 695                 if (flags & TEST_ACTIVE &&
 696                     !intel_engine_can_store_dword(engine))
 697                         continue;
 698
 699                 if (!wait_for_idle(engine)) {
 700                         pr_err("i915_reset_engine(%s:%s): failed to idle before reset\n",
 701                                engine->name, test_name);
 702                         err = -EIO;
 703                         break;
 704                 }
 705
 706                 memset(threads, 0, sizeof(threads));
 707                 for_each_engine(other, i915, tmp) {
 708                         struct task_struct *tsk;
 709
 710                         threads[tmp].resets =
 711                                 i915_reset_engine_count(&i915->gpu_error,
 712                                                         other);
 713
 714                         if (!(flags & TEST_OTHERS))
 715                                 continue;
 716
 717                         if (other == engine && !(flags & TEST_SELF))
 718                                 continue;
 719
 720                         threads[tmp].engine = other;
 721                         threads[tmp].flags = flags;
 722
 723                         tsk = kthread_run(active_engine, &threads[tmp],
 724                                           "igt/%s", other->name);
 725                         if (IS_ERR(tsk)) {
 726                                 err = PTR_ERR(tsk);
 727                                 goto unwind;
 728                         }
 729
 730                         threads[tmp].task = tsk;
 731                         get_task_struct(tsk);
 732                 }
 733
 734                 set_bit(I915_RESET_ENGINE + id, &i915->gpu_error.flags);
 735                 do {
 736                         u32 seqno = intel_engine_get_seqno(engine);
 737                         struct i915_request *rq = NULL;
 738
 739                         if (flags & TEST_ACTIVE) {
 740                                 mutex_lock(&i915->drm.struct_mutex);
 741                                 rq = hang_create_request(&h, engine);
 742                                 if (IS_ERR(rq)) {
 743                                         err = PTR_ERR(rq);
 744                                         mutex_unlock(&i915->drm.struct_mutex);
 745                                         break;
 746                                 }
 747
 748                                 i915_request_get(rq);
 749                                 i915_request_add(rq);
 750                                 mutex_unlock(&i915->drm.struct_mutex);
 751
 752                                 if (!wait_until_running(&h, rq)) {
 753                                         struct drm_printer p = drm_info_printer(i915->drm.dev);
 754
 755                                         pr_err("%s: Failed to start request %x, at %x\n",
 756                                                __func__, rq->fence.seqno, hws_seqno(&h, rq));
 757                                         intel_engine_dump(engine, &p,
 758                                                           "%s\n", engine->name);
 759
 760                                         i915_request_put(rq);
 761                                         err = -EIO;
 762                                         break;
 763                                 }
 764
 765                                 GEM_BUG_ON(!rq->global_seqno);
 766                                 seqno = rq->global_seqno - 1;
 767                         }
 768
 769                         err = i915_reset_engine(engine, NULL);
 770                         if (err) {
 771                                 pr_err("i915_reset_engine(%s:%s): failed, err=%d\n",
 772                                        engine->name, test_name, err);
 773                                 break;
 774                         }
 775
 776                         count++;
 777
 778                         if (rq) {
 779                                 i915_request_wait(rq, 0, MAX_SCHEDULE_TIMEOUT);
 780                                 i915_request_put(rq);
 781                         }
 782
 783                         if (!(flags & TEST_SELF) && !wait_for_idle(engine)) {
 784                                 struct drm_printer p =
 785                                         drm_info_printer(i915->drm.dev);
 786
 787                                 pr_err("i915_reset_engine(%s:%s):"
 788                                        " failed to idle after reset\n",
 789                                        engine->name, test_name);
 790                                 intel_engine_dump(engine, &p,
 791                                                   "%s\n", engine->name);
 792
 793                                 err = -EIO;
 794                                 break;
 795                         }
 796                 } while (time_before(jiffies, end_time));
 797                 clear_bit(I915_RESET_ENGINE + id, &i915->gpu_error.flags);
 798                 pr_info("i915_reset_engine(%s:%s): %lu resets\n",
 799                         engine->name, test_name, count);
 800
 801                 reported = i915_reset_engine_count(&i915->gpu_error, engine);
 802                 reported -= threads[engine->id].resets;
 803                 if (reported != (flags & TEST_ACTIVE ? count : 0)) {
 804                         pr_err("i915_reset_engine(%s:%s): reset %lu times, but reported %lu, expected %lu reported\n",
 805                                engine->name, test_name, count, reported,
 806                                (flags & TEST_ACTIVE ? count : 0));
 807                         if (!err)
 808                                 err = -EINVAL;
 809                 }
 810
 811 unwind:
 812                 for_each_engine(other, i915, tmp) {
 813                         int ret;
 814
 815                         if (!threads[tmp].task)
 816                                 continue;
 817
 818                         ret = kthread_stop(threads[tmp].task);
 819                         if (ret) {
 820                                 pr_err("kthread for other engine %s failed, err=%d\n",
 821                                        other->name, ret);
 822                                 if (!err)
 823                                         err = ret;
 824                         }
 825                         put_task_struct(threads[tmp].task);
 826
 827                         if (other != engine &&
 828                             threads[tmp].resets !=
 829                             i915_reset_engine_count(&i915->gpu_error, other)) {
 830                                 pr_err("Innocent engine %s was reset (count=%ld)\n",
 831                                        other->name,
 832                                        i915_reset_engine_count(&i915->gpu_error,
 833                                                                other) -
 834                                        threads[tmp].resets);
 835                                 if (!err)
 836                                         err = -EINVAL;
 837                         }
 838                 }
 839
 840                 if (global != i915_reset_count(&i915->gpu_error)) {
 841                         pr_err("Global reset (count=%ld)!\n",
 842                                i915_reset_count(&i915->gpu_error) - global);
 843                         if (!err)
 844                                 err = -EINVAL;
 845                 }
 846
 847                 if (err)
 848                         break;
 849
 850                 err = igt_flush_test(i915, 0);
 851                 if (err)
 852                         break;
 853         }
 854
 855         if (i915_terminally_wedged(&i915->gpu_error))
 856                 err = -EIO;
 857
 858         if (flags & TEST_ACTIVE) {
 859                 mutex_lock(&i915->drm.struct_mutex);
 860                 hang_fini(&h);
 861                 mutex_unlock(&i915->drm.struct_mutex);
 862         }
 863
 864         return err;
 865 }
 866
 867 static int igt_reset_engines(void *arg)
 868 {
 869         static const struct {
 870                 const char *name;
 871                 unsigned int flags;
 872         } phases[] = {
 873                 { "idle", 0 },
 874                 { "active", TEST_ACTIVE },
 875                 { "others-idle", TEST_OTHERS },
 876                 { "others-active", TEST_OTHERS | TEST_ACTIVE },
 877                 {
 878                         "others-priority",
 879                         TEST_OTHERS | TEST_ACTIVE | TEST_PRIORITY
 880                 },
 881                 {
 882                         "self-priority",
 883                         TEST_OTHERS | TEST_ACTIVE | TEST_PRIORITY | TEST_SELF,
 884                 },
 885                 { }
 886         };
 887         struct drm_i915_private *i915 = arg;
 888         typeof(*phases) *p;
 889         int err;
 890
 891         for (p = phases; p->name; p++) {
 892                 if (p->flags & TEST_PRIORITY) {
 893                         if (!(i915->caps.scheduler & I915_SCHEDULER_CAP_PRIORITY))
 894                                 continue;
 895                 }
 896
 897                 err = __igt_reset_engines(arg, p->name, p->flags);
 898                 if (err)
 899                         return err;
 900         }
 901
 902         return 0;
 903 }
 904
 905 static u32 fake_hangcheck(struct i915_request *rq, u32 mask)
 906 {
 907         struct i915_gpu_error *error = &rq->i915->gpu_error;
 908         u32 reset_count = i915_reset_count(error);
 909
 910         error->stalled_mask = mask;
 911
 912         /* set_bit() must be after we have setup the backchannel (mask) */
 913         smp_mb__before_atomic();
 914         set_bit(I915_RESET_HANDOFF, &error->flags);
 915
 916         wake_up_all(&error->wait_queue);
 917
 918         return reset_count;
 919 }
 920
 921 static int igt_reset_wait(void *arg)
 922 {
 923         struct drm_i915_private *i915 = arg;
 924         struct i915_request *rq;
 925         unsigned int reset_count;
 926         struct hang h;
 927         long timeout;
 928         int err;
 929
 930         if (!intel_engine_can_store_dword(i915->engine[RCS]))
 931                 return 0;
 932
 933         /* Check that we detect a stuck waiter and issue a reset */
 934
 935         igt_global_reset_lock(i915);
 936
 937         mutex_lock(&i915->drm.struct_mutex);
 938         err = hang_init(&h, i915);
 939         if (err)
 940                 goto unlock;
 941
 942         rq = hang_create_request(&h, i915->engine[RCS]);
 943         if (IS_ERR(rq)) {
 944                 err = PTR_ERR(rq);
 945                 goto fini;
 946         }
 947
 948         i915_request_get(rq);
 949         i915_request_add(rq);
 950
 951         if (!wait_until_running(&h, rq)) {
 952                 struct drm_printer p = drm_info_printer(i915->drm.dev);
 953
 954                 pr_err("%s: Failed to start request %x, at %x\n",
 955                        __func__, rq->fence.seqno, hws_seqno(&h, rq));
 956                 intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name);
 957
 958                 i915_gem_set_wedged(i915);
 959
 960                 err = -EIO;
 961                 goto out_rq;
 962         }
 963
 964         reset_count = fake_hangcheck(rq, ALL_ENGINES);
 965
 966         timeout = i915_request_wait(rq, I915_WAIT_LOCKED, 10);
 967         if (timeout < 0) {
 968                 pr_err("i915_request_wait failed on a stuck request: err=%ld\n",
 969                        timeout);
 970                 err = timeout;
 971                 goto out_rq;
 972         }
 973
 974         GEM_BUG_ON(test_bit(I915_RESET_HANDOFF, &i915->gpu_error.flags));
 975         if (i915_reset_count(&i915->gpu_error) == reset_count) {
 976                 pr_err("No GPU reset recorded!\n");
 977                 err = -EINVAL;
 978                 goto out_rq;
 979         }
 980
 981 out_rq:
 982         i915_request_put(rq);
 983 fini:
 984         hang_fini(&h);
 985 unlock:
 986         mutex_unlock(&i915->drm.struct_mutex);
 987         igt_global_reset_unlock(i915);
 988
 989         if (i915_terminally_wedged(&i915->gpu_error))
 990                 return -EIO;
 991
 992         return err;
 993 }
 994
 995 struct evict_vma {
 996         struct completion completion;
 997         struct i915_vma *vma;
 998 };
 999
1000 static int evict_vma(void *data)
1001 {
1002         struct evict_vma *arg = data;
1003         struct i915_address_space *vm = arg->vma->vm;
1004         struct drm_i915_private *i915 = vm->i915;
1005         struct drm_mm_node evict = arg->vma->node;
1006         int err;
1007
1008         complete(&arg->completion);
1009
1010         mutex_lock(&i915->drm.struct_mutex);
1011         err = i915_gem_evict_for_node(vm, &evict, 0);
1012         mutex_unlock(&i915->drm.struct_mutex);
1013
1014         return err;
1015 }
1016
1017 static int evict_fence(void *data)
1018 {
1019         struct evict_vma *arg = data;
1020         struct drm_i915_private *i915 = arg->vma->vm->i915;
1021         int err;
1022
1023         complete(&arg->completion);
1024
1025         mutex_lock(&i915->drm.struct_mutex);
1026
1027         /* Mark the fence register as dirty to force the mmio update. */
1028         err = i915_gem_object_set_tiling(arg->vma->obj, I915_TILING_Y, 512);
1029         if (err) {
1030                 pr_err("Invalid Y-tiling settings; err:%d\n", err);
1031                 goto out_unlock;
1032         }
1033
1034         err = i915_vma_pin_fence(arg->vma);
1035         if (err) {
1036                 pr_err("Unable to pin Y-tiled fence; err:%d\n", err);
1037                 goto out_unlock;
1038         }
1039
1040         i915_vma_unpin_fence(arg->vma);
1041
1042 out_unlock:
1043         mutex_unlock(&i915->drm.struct_mutex);
1044
1045         return err;
1046 }
1047
1048 static int __igt_reset_evict_vma(struct drm_i915_private *i915,
1049                                  struct i915_address_space *vm,
1050                                  int (*fn)(void *),
1051                                  unsigned int flags)
1052 {
1053         struct drm_i915_gem_object *obj;
1054         struct task_struct *tsk = NULL;
1055         struct i915_request *rq;
1056         struct evict_vma arg;
1057         struct hang h;
1058         int err;
1059
1060         if (!intel_engine_can_store_dword(i915->engine[RCS]))
1061                 return 0;
1062
1063         /* Check that we can recover an unbind stuck on a hanging request */
1064
1065         igt_global_reset_lock(i915);
1066
1067         mutex_lock(&i915->drm.struct_mutex);
1068         err = hang_init(&h, i915);
1069         if (err)
1070                 goto unlock;
1071
1072         obj = i915_gem_object_create_internal(i915, SZ_1M);
1073         if (IS_ERR(obj)) {
1074                 err = PTR_ERR(obj);
1075                 goto fini;
1076         }
1077
1078         if (flags & EXEC_OBJECT_NEEDS_FENCE) {
1079                 err = i915_gem_object_set_tiling(obj, I915_TILING_X, 512);
1080                 if (err) {
1081                         pr_err("Invalid X-tiling settings; err:%d\n", err);
1082                         goto out_obj;
1083                 }
1084         }
1085
1086         arg.vma = i915_vma_instance(obj, vm, NULL);
1087         if (IS_ERR(arg.vma)) {
1088                 err = PTR_ERR(arg.vma);
1089                 goto out_obj;
1090         }
1091
1092         rq = hang_create_request(&h, i915->engine[RCS]);
1093         if (IS_ERR(rq)) {
1094                 err = PTR_ERR(rq);
1095                 goto out_obj;
1096         }
1097
1098         err = i915_vma_pin(arg.vma, 0, 0,
1099                            i915_vma_is_ggtt(arg.vma) ?
1100                            PIN_GLOBAL | PIN_MAPPABLE :
1101                            PIN_USER);
1102         if (err) {
1103                 i915_request_add(rq);
1104                 goto out_obj;
1105         }
1106
1107         if (flags & EXEC_OBJECT_NEEDS_FENCE) {
1108                 err = i915_vma_pin_fence(arg.vma);
1109                 if (err) {
1110                         pr_err("Unable to pin X-tiled fence; err:%d\n", err);
1111                         i915_vma_unpin(arg.vma);
1112                         i915_request_add(rq);
1113                         goto out_obj;
1114                 }
1115         }
1116
1117         err = i915_vma_move_to_active(arg.vma, rq, flags);
1118
1119         if (flags & EXEC_OBJECT_NEEDS_FENCE)
1120                 i915_vma_unpin_fence(arg.vma);
1121         i915_vma_unpin(arg.vma);
1122
1123         i915_request_get(rq);
1124         i915_request_add(rq);
1125         if (err)
1126                 goto out_rq;
1127
1128         mutex_unlock(&i915->drm.struct_mutex);
1129
1130         if (!wait_until_running(&h, rq)) {
1131                 struct drm_printer p = drm_info_printer(i915->drm.dev);
1132
1133                 pr_err("%s: Failed to start request %x, at %x\n",
1134                        __func__, rq->fence.seqno, hws_seqno(&h, rq));
1135                 intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name);
1136
1137                 i915_gem_set_wedged(i915);
1138                 goto out_reset;
1139         }
1140
1141         init_completion(&arg.completion);
1142
1143         tsk = kthread_run(fn, &arg, "igt/evict_vma");
1144         if (IS_ERR(tsk)) {
1145                 err = PTR_ERR(tsk);
1146                 tsk = NULL;
1147                 goto out_reset;
1148         }
1149         get_task_struct(tsk);
1150
1151         wait_for_completion(&arg.completion);
1152
1153         if (wait_for(waitqueue_active(&rq->execute), 10)) {
1154                 struct drm_printer p = drm_info_printer(i915->drm.dev);
1155
1156                 pr_err("igt/evict_vma kthread did not wait\n");
1157                 intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name);
1158
1159                 i915_gem_set_wedged(i915);
1160                 goto out_reset;
1161         }
1162
1163 out_reset:
1164         fake_hangcheck(rq, intel_engine_flag(rq->engine));
1165
1166         if (tsk) {
1167                 struct igt_wedge_me w;
1168
1169                 /* The reset, even indirectly, should take less than 10ms. */
1170                 igt_wedge_on_timeout(&w, i915, HZ / 10 /* 100ms timeout*/)
1171                         err = kthread_stop(tsk);
1172
1173                 put_task_struct(tsk);
1174         }
1175
1176         mutex_lock(&i915->drm.struct_mutex);
1177 out_rq:
1178         i915_request_put(rq);
1179 out_obj:
1180         i915_gem_object_put(obj);
1181 fini:
1182         hang_fini(&h);
1183 unlock:
1184         mutex_unlock(&i915->drm.struct_mutex);
1185         igt_global_reset_unlock(i915);
1186
1187         if (i915_terminally_wedged(&i915->gpu_error))
1188                 return -EIO;
1189
1190         return err;
1191 }
1192
1193 static int igt_reset_evict_ggtt(void *arg)
1194 {
1195         struct drm_i915_private *i915 = arg;
1196
1197         return __igt_reset_evict_vma(i915, &i915->ggtt.vm,
1198                                      evict_vma, EXEC_OBJECT_WRITE);
1199 }
1200
1201 static int igt_reset_evict_ppgtt(void *arg)
1202 {
1203         struct drm_i915_private *i915 = arg;
1204         struct i915_gem_context *ctx;
1205         struct drm_file *file;
1206         int err;
1207
1208         file = mock_file(i915);
1209         if (IS_ERR(file))
1210                 return PTR_ERR(file);
1211
1212         mutex_lock(&i915->drm.struct_mutex);
1213         ctx = live_context(i915, file);
1214         mutex_unlock(&i915->drm.struct_mutex);
1215         if (IS_ERR(ctx)) {
1216                 err = PTR_ERR(ctx);
1217                 goto out;
1218         }
1219
1220         err = 0;
1221         if (ctx->ppgtt) /* aliasing == global gtt locking, covered above */
1222                 err = __igt_reset_evict_vma(i915, &ctx->ppgtt->vm,
1223                                             evict_vma, EXEC_OBJECT_WRITE);
1224
1225 out:
1226         mock_file_free(i915, file);
1227         return err;
1228 }
1229
1230 static int igt_reset_evict_fence(void *arg)
1231 {
1232         struct drm_i915_private *i915 = arg;
1233
1234         return __igt_reset_evict_vma(i915, &i915->ggtt.vm,
1235                                      evict_fence, EXEC_OBJECT_NEEDS_FENCE);
1236 }
1237
1238 static int wait_for_others(struct drm_i915_private *i915,
1239                            struct intel_engine_cs *exclude)
1240 {
1241         struct intel_engine_cs *engine;
1242         enum intel_engine_id id;
1243
1244         for_each_engine(engine, i915, id) {
1245                 if (engine == exclude)
1246                         continue;
1247
1248                 if (!wait_for_idle(engine))
1249                         return -EIO;
1250         }
1251
1252         return 0;
1253 }
1254
1255 static int igt_reset_queue(void *arg)
1256 {
1257         struct drm_i915_private *i915 = arg;
1258         struct intel_engine_cs *engine;
1259         enum intel_engine_id id;
1260         struct hang h;
1261         int err;
1262
1263         /* Check that we replay pending requests following a hang */
1264
1265         igt_global_reset_lock(i915);
1266
1267         mutex_lock(&i915->drm.struct_mutex);
1268         err = hang_init(&h, i915);
1269         if (err)
1270                 goto unlock;
1271
1272         for_each_engine(engine, i915, id) {
1273                 struct i915_request *prev;
1274                 IGT_TIMEOUT(end_time);
1275                 unsigned int count;
1276
1277                 if (!intel_engine_can_store_dword(engine))
1278                         continue;
1279
1280                 prev = hang_create_request(&h, engine);
1281                 if (IS_ERR(prev)) {
1282                         err = PTR_ERR(prev);
1283                         goto fini;
1284                 }
1285
1286                 i915_request_get(prev);
1287                 i915_request_add(prev);
1288
1289                 count = 0;
1290                 do {
1291                         struct i915_request *rq;
1292                         unsigned int reset_count;
1293
1294                         rq = hang_create_request(&h, engine);
1295                         if (IS_ERR(rq)) {
1296                                 err = PTR_ERR(rq);
1297                                 goto fini;
1298                         }
1299
1300                         i915_request_get(rq);
1301                         i915_request_add(rq);
1302
1303                         /*
1304                          * XXX We don't handle resetting the kernel context
1305                          * very well. If we trigger a device reset twice in
1306                          * quick succession while the kernel context is
1307                          * executing, we may end up skipping the breadcrumb.
1308                          * This is really only a problem for the selftest as
1309                          * normally there is a large interlude between resets
1310                          * (hangcheck), or we focus on resetting just one
1311                          * engine and so avoid repeatedly resetting innocents.
1312                          */
1313                         err = wait_for_others(i915, engine);
1314                         if (err) {
1315                                 pr_err("%s(%s): Failed to idle other inactive engines after device reset\n",
1316                                        __func__, engine->name);
1317                                 i915_request_put(rq);
1318                                 i915_request_put(prev);
1319
1320                                 GEM_TRACE_DUMP();
1321                                 i915_gem_set_wedged(i915);
1322                                 goto fini;
1323                         }
1324
1325                         if (!wait_until_running(&h, prev)) {
1326                                 struct drm_printer p = drm_info_printer(i915->drm.dev);
1327
1328                                 pr_err("%s(%s): Failed to start request %x, at %x\n",
1329                                        __func__, engine->name,
1330                                        prev->fence.seqno, hws_seqno(&h, prev));
1331                                 intel_engine_dump(engine, &p,
1332                                                   "%s\n", engine->name);
1333
1334                                 i915_request_put(rq);
1335                                 i915_request_put(prev);
1336
1337                                 i915_gem_set_wedged(i915);
1338
1339                                 err = -EIO;
1340                                 goto fini;
1341                         }
1342
1343                         reset_count = fake_hangcheck(prev, ENGINE_MASK(id));
1344
1345                         i915_reset(i915, ENGINE_MASK(id), NULL);
1346
1347                         GEM_BUG_ON(test_bit(I915_RESET_HANDOFF,
1348                                             &i915->gpu_error.flags));
1349
1350                         if (prev->fence.error != -EIO) {
1351                                 pr_err("GPU reset not recorded on hanging request [fence.error=%d]!\n",
1352                                        prev->fence.error);
1353                                 i915_request_put(rq);
1354                                 i915_request_put(prev);
1355                                 err = -EINVAL;
1356                                 goto fini;
1357                         }
1358
1359                         if (rq->fence.error) {
1360                                 pr_err("Fence error status not zero [%d] after unrelated reset\n",
1361                                        rq->fence.error);
1362                                 i915_request_put(rq);
1363                                 i915_request_put(prev);
1364                                 err = -EINVAL;
1365                                 goto fini;
1366                         }
1367
1368                         if (i915_reset_count(&i915->gpu_error) == reset_count) {
1369                                 pr_err("No GPU reset recorded!\n");
1370                                 i915_request_put(rq);
1371                                 i915_request_put(prev);
1372                                 err = -EINVAL;
1373                                 goto fini;
1374                         }
1375
1376                         i915_request_put(prev);
1377                         prev = rq;
1378                         count++;
1379                 } while (time_before(jiffies, end_time));
1380                 pr_info("%s: Completed %d resets\n", engine->name, count);
1381
1382                 *h.batch = MI_BATCH_BUFFER_END;
1383                 i915_gem_chipset_flush(i915);
1384
1385                 i915_request_put(prev);
1386
1387                 err = igt_flush_test(i915, I915_WAIT_LOCKED);
1388                 if (err)
1389                         break;
1390         }
1391
1392 fini:
1393         hang_fini(&h);
1394 unlock:
1395         mutex_unlock(&i915->drm.struct_mutex);
1396         igt_global_reset_unlock(i915);
1397
1398         if (i915_terminally_wedged(&i915->gpu_error))
1399                 return -EIO;
1400
1401         return err;
1402 }
1403
1404 static int igt_handle_error(void *arg)
1405 {
1406         struct drm_i915_private *i915 = arg;
1407         struct intel_engine_cs *engine = i915->engine[RCS];
1408         struct hang h;
1409         struct i915_request *rq;
1410         struct i915_gpu_state *error;
1411         int err;
1412
1413         /* Check that we can issue a global GPU and engine reset */
1414
1415         if (!intel_has_reset_engine(i915))
1416                 return 0;
1417
1418         if (!engine || !intel_engine_can_store_dword(engine))
1419                 return 0;
1420
1421         mutex_lock(&i915->drm.struct_mutex);
1422
1423         err = hang_init(&h, i915);
1424         if (err)
1425                 goto err_unlock;
1426
1427         rq = hang_create_request(&h, engine);
1428         if (IS_ERR(rq)) {
1429                 err = PTR_ERR(rq);
1430                 goto err_fini;
1431         }
1432
1433         i915_request_get(rq);
1434         i915_request_add(rq);
1435
1436         if (!wait_until_running(&h, rq)) {
1437                 struct drm_printer p = drm_info_printer(i915->drm.dev);
1438
1439                 pr_err("%s: Failed to start request %x, at %x\n",
1440                        __func__, rq->fence.seqno, hws_seqno(&h, rq));
1441                 intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name);
1442
1443                 i915_gem_set_wedged(i915);
1444
1445                 err = -EIO;
1446                 goto err_request;
1447         }
1448
1449         mutex_unlock(&i915->drm.struct_mutex);
1450
1451         /* Temporarily disable error capture */
1452         error = xchg(&i915->gpu_error.first_error, (void *)-1);
1453
1454         i915_handle_error(i915, ENGINE_MASK(engine->id), 0, NULL);
1455
1456         xchg(&i915->gpu_error.first_error, error);
1457
1458         mutex_lock(&i915->drm.struct_mutex);
1459
1460         if (rq->fence.error != -EIO) {
1461                 pr_err("Guilty request not identified!\n");
1462                 err = -EINVAL;
1463                 goto err_request;
1464         }
1465
1466 err_request:
1467         i915_request_put(rq);
1468 err_fini:
1469         hang_fini(&h);
1470 err_unlock:
1471         mutex_unlock(&i915->drm.struct_mutex);
1472         return err;
1473 }
1474
1475 int intel_hangcheck_live_selftests(struct drm_i915_private *i915)
1476 {
1477         static const struct i915_subtest tests[] = {
1478                 SUBTEST(igt_global_reset), /* attempt to recover GPU first */
1479                 SUBTEST(igt_wedged_reset),
1480                 SUBTEST(igt_hang_sanitycheck),
1481                 SUBTEST(igt_reset_idle_engine),
1482                 SUBTEST(igt_reset_active_engine),
1483                 SUBTEST(igt_reset_engines),
1484                 SUBTEST(igt_reset_queue),
1485                 SUBTEST(igt_reset_wait),
1486                 SUBTEST(igt_reset_evict_ggtt),
1487                 SUBTEST(igt_reset_evict_ppgtt),
1488                 SUBTEST(igt_reset_evict_fence),
1489                 SUBTEST(igt_handle_error),
1490         };
1491         bool saved_hangcheck;
1492         int err;
1493
1494         if (!intel_has_gpu_reset(i915))
1495                 return 0;
1496
1497         if (i915_terminally_wedged(&i915->gpu_error))
1498                 return -EIO; /* we're long past hope of a successful reset */
1499
1500         intel_runtime_pm_get(i915);
1501         saved_hangcheck = fetch_and_zero(&i915_modparams.enable_hangcheck);
1502
1503         err = i915_subtests(tests, i915);
1504
1505         mutex_lock(&i915->drm.struct_mutex);
1506         igt_flush_test(i915, I915_WAIT_LOCKED);
1507         mutex_unlock(&i915->drm.struct_mutex);
1508
1509         i915_modparams.enable_hangcheck = saved_hangcheck;
1510         intel_runtime_pm_put(i915);
1511
1512         return err;
1513 }