drivers/misc/cxl/native.c

   1 /*
   2  * Copyright 2014 IBM Corp.
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU General Public License
   6  * as published by the Free Software Foundation; either version
   7  * 2 of the License, or (at your option) any later version.
   8  */
   9
  10 #include <linux/spinlock.h>
  11 #include <linux/sched.h>
  12 #include <linux/slab.h>
  13 #include <linux/sched.h>
  14 #include <linux/mutex.h>
  15 #include <linux/mm.h>
  16 #include <linux/uaccess.h>
  17 #include <linux/delay.h>
  18 #include <asm/synch.h>
  19 #include <misc/cxl-base.h>
  20
  21 #include "cxl.h"
  22 #include "trace.h"
  23
  24 static int afu_control(struct cxl_afu *afu, u64 command,
  25                        u64 result, u64 mask, bool enabled)
  26 {
  27         u64 AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An);
  28         unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT);
  29         int rc = 0;
  30
  31         spin_lock(&afu->afu_cntl_lock);
  32         pr_devel("AFU command starting: %llx\n", command);
  33
  34         trace_cxl_afu_ctrl(afu, command);
  35
  36         cxl_p2n_write(afu, CXL_AFU_Cntl_An, AFU_Cntl | command);
  37
  38         AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An);
  39         while ((AFU_Cntl & mask) != result) {
  40                 if (time_after_eq(jiffies, timeout)) {
  41                         dev_warn(&afu->dev, "WARNING: AFU control timed out!\n");
  42                         rc = -EBUSY;
  43                         goto out;
  44                 }
  45
  46                 if (!cxl_ops->link_ok(afu->adapter, afu)) {
  47                         afu->enabled = enabled;
  48                         rc = -EIO;
  49                         goto out;
  50                 }
  51
  52                 pr_devel_ratelimited("AFU control... (0x%016llx)\n",
  53                                      AFU_Cntl | command);
  54                 cpu_relax();
  55                 AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An);
  56         };
  57         pr_devel("AFU command complete: %llx\n", command);
  58         afu->enabled = enabled;
  59 out:
  60         trace_cxl_afu_ctrl_done(afu, command, rc);
  61         spin_unlock(&afu->afu_cntl_lock);
  62
  63         return rc;
  64 }
  65
  66 static int afu_enable(struct cxl_afu *afu)
  67 {
  68         pr_devel("AFU enable request\n");
  69
  70         return afu_control(afu, CXL_AFU_Cntl_An_E,
  71                            CXL_AFU_Cntl_An_ES_Enabled,
  72                            CXL_AFU_Cntl_An_ES_MASK, true);
  73 }
  74
  75 int cxl_afu_disable(struct cxl_afu *afu)
  76 {
  77         pr_devel("AFU disable request\n");
  78
  79         return afu_control(afu, 0, CXL_AFU_Cntl_An_ES_Disabled,
  80                            CXL_AFU_Cntl_An_ES_MASK, false);
  81 }
  82
  83 /* This will disable as well as reset */
  84 static int native_afu_reset(struct cxl_afu *afu)
  85 {
  86         pr_devel("AFU reset request\n");
  87
  88         return afu_control(afu, CXL_AFU_Cntl_An_RA,
  89                            CXL_AFU_Cntl_An_RS_Complete | CXL_AFU_Cntl_An_ES_Disabled,
  90                            CXL_AFU_Cntl_An_RS_MASK | CXL_AFU_Cntl_An_ES_MASK,
  91                            false);
  92 }
  93
  94 static int native_afu_check_and_enable(struct cxl_afu *afu)
  95 {
  96         if (!cxl_ops->link_ok(afu->adapter, afu)) {
  97                 WARN(1, "Refusing to enable afu while link down!\n");
  98                 return -EIO;
  99         }
 100         if (afu->enabled)
 101                 return 0;
 102         return afu_enable(afu);
 103 }
 104
 105 int cxl_psl_purge(struct cxl_afu *afu)
 106 {
 107         u64 PSL_CNTL = cxl_p1n_read(afu, CXL_PSL_SCNTL_An);
 108         u64 AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An);
 109         u64 dsisr, dar;
 110         u64 start, end;
 111         unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT);
 112         int rc = 0;
 113
 114         trace_cxl_psl_ctrl(afu, CXL_PSL_SCNTL_An_Pc);
 115
 116         pr_devel("PSL purge request\n");
 117
 118         if (!cxl_ops->link_ok(afu->adapter, afu)) {
 119                 dev_warn(&afu->dev, "PSL Purge called with link down, ignoring\n");
 120                 rc = -EIO;
 121                 goto out;
 122         }
 123
 124         if ((AFU_Cntl & CXL_AFU_Cntl_An_ES_MASK) != CXL_AFU_Cntl_An_ES_Disabled) {
 125                 WARN(1, "psl_purge request while AFU not disabled!\n");
 126                 cxl_afu_disable(afu);
 127         }
 128
 129         cxl_p1n_write(afu, CXL_PSL_SCNTL_An,
 130                        PSL_CNTL | CXL_PSL_SCNTL_An_Pc);
 131         start = local_clock();
 132         PSL_CNTL = cxl_p1n_read(afu, CXL_PSL_SCNTL_An);
 133         while ((PSL_CNTL &  CXL_PSL_SCNTL_An_Ps_MASK)
 134                         == CXL_PSL_SCNTL_An_Ps_Pending) {
 135                 if (time_after_eq(jiffies, timeout)) {
 136                         dev_warn(&afu->dev, "WARNING: PSL Purge timed out!\n");
 137                         rc = -EBUSY;
 138                         goto out;
 139                 }
 140                 if (!cxl_ops->link_ok(afu->adapter, afu)) {
 141                         rc = -EIO;
 142                         goto out;
 143                 }
 144
 145                 dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An);
 146                 pr_devel_ratelimited("PSL purging... PSL_CNTL: 0x%016llx  PSL_DSISR: 0x%016llx\n", PSL_CNTL, dsisr);
 147                 if (dsisr & CXL_PSL_DSISR_TRANS) {
 148                         dar = cxl_p2n_read(afu, CXL_PSL_DAR_An);
 149                         dev_notice(&afu->dev, "PSL purge terminating pending translation, DSISR: 0x%016llx, DAR: 0x%016llx\n", dsisr, dar);
 150                         cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_AE);
 151                 } else if (dsisr) {
 152                         dev_notice(&afu->dev, "PSL purge acknowledging pending non-translation fault, DSISR: 0x%016llx\n", dsisr);
 153                         cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_A);
 154                 } else {
 155                         cpu_relax();
 156                 }
 157                 PSL_CNTL = cxl_p1n_read(afu, CXL_PSL_SCNTL_An);
 158         };
 159         end = local_clock();
 160         pr_devel("PSL purged in %lld ns\n", end - start);
 161
 162         cxl_p1n_write(afu, CXL_PSL_SCNTL_An,
 163                        PSL_CNTL & ~CXL_PSL_SCNTL_An_Pc);
 164 out:
 165         trace_cxl_psl_ctrl_done(afu, CXL_PSL_SCNTL_An_Pc, rc);
 166         return rc;
 167 }
 168
 169 static int spa_max_procs(int spa_size)
 170 {
 171         /*
 172          * From the CAIA:
 173          *    end_of_SPA_area = SPA_Base + ((n+4) * 128) + (( ((n*8) + 127) >> 7) * 128) + 255
 174          * Most of that junk is really just an overly-complicated way of saying
 175          * the last 256 bytes are __aligned(128), so it's really:
 176          *    end_of_SPA_area = end_of_PSL_queue_area + __aligned(128) 255
 177          * and
 178          *    end_of_PSL_queue_area = SPA_Base + ((n+4) * 128) + (n*8) - 1
 179          * so
 180          *    sizeof(SPA) = ((n+4) * 128) + (n*8) + __aligned(128) 256
 181          * Ignore the alignment (which is safe in this case as long as we are
 182          * careful with our rounding) and solve for n:
 183          */
 184         return ((spa_size / 8) - 96) / 17;
 185 }
 186
 187 int cxl_alloc_spa(struct cxl_afu *afu)
 188 {
 189         /* Work out how many pages to allocate */
 190         afu->native->spa_order = 0;
 191         do {
 192                 afu->native->spa_order++;
 193                 afu->native->spa_size = (1 << afu->native->spa_order) * PAGE_SIZE;
 194                 afu->native->spa_max_procs = spa_max_procs(afu->native->spa_size);
 195         } while (afu->native->spa_max_procs < afu->num_procs);
 196
 197         WARN_ON(afu->native->spa_size > 0x100000); /* Max size supported by the hardware */
 198
 199         if (!(afu->native->spa = (struct cxl_process_element *)
 200               __get_free_pages(GFP_KERNEL | __GFP_ZERO, afu->native->spa_order))) {
 201                 pr_err("cxl_alloc_spa: Unable to allocate scheduled process area\n");
 202                 return -ENOMEM;
 203         }
 204         pr_devel("spa pages: %i afu->spa_max_procs: %i   afu->num_procs: %i\n",
 205                  1<<afu->native->spa_order, afu->native->spa_max_procs, afu->num_procs);
 206
 207         return 0;
 208 }
 209
 210 static void attach_spa(struct cxl_afu *afu)
 211 {
 212         u64 spap;
 213
 214         afu->native->sw_command_status = (__be64 *)((char *)afu->native->spa +
 215                                             ((afu->native->spa_max_procs + 3) * 128));
 216
 217         spap = virt_to_phys(afu->native->spa) & CXL_PSL_SPAP_Addr;
 218         spap |= ((afu->native->spa_size >> (12 - CXL_PSL_SPAP_Size_Shift)) - 1) & CXL_PSL_SPAP_Size;
 219         spap |= CXL_PSL_SPAP_V;
 220         pr_devel("cxl: SPA allocated at 0x%p. Max processes: %i, sw_command_status: 0x%p CXL_PSL_SPAP_An=0x%016llx\n",
 221                 afu->native->spa, afu->native->spa_max_procs,
 222                 afu->native->sw_command_status, spap);
 223         cxl_p1n_write(afu, CXL_PSL_SPAP_An, spap);
 224 }
 225
 226 static inline void detach_spa(struct cxl_afu *afu)
 227 {
 228         cxl_p1n_write(afu, CXL_PSL_SPAP_An, 0);
 229 }
 230
 231 void cxl_release_spa(struct cxl_afu *afu)
 232 {
 233         if (afu->native->spa) {
 234                 free_pages((unsigned long) afu->native->spa,
 235                         afu->native->spa_order);
 236                 afu->native->spa = NULL;
 237         }
 238 }
 239
 240 int cxl_tlb_slb_invalidate(struct cxl *adapter)
 241 {
 242         unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT);
 243
 244         pr_devel("CXL adapter wide TLBIA & SLBIA\n");
 245
 246         cxl_p1_write(adapter, CXL_PSL_AFUSEL, CXL_PSL_AFUSEL_A);
 247
 248         cxl_p1_write(adapter, CXL_PSL_TLBIA, CXL_TLB_SLB_IQ_ALL);
 249         while (cxl_p1_read(adapter, CXL_PSL_TLBIA) & CXL_TLB_SLB_P) {
 250                 if (time_after_eq(jiffies, timeout)) {
 251                         dev_warn(&adapter->dev, "WARNING: CXL adapter wide TLBIA timed out!\n");
 252                         return -EBUSY;
 253                 }
 254                 if (!cxl_ops->link_ok(adapter, NULL))
 255                         return -EIO;
 256                 cpu_relax();
 257         }
 258
 259         cxl_p1_write(adapter, CXL_PSL_SLBIA, CXL_TLB_SLB_IQ_ALL);
 260         while (cxl_p1_read(adapter, CXL_PSL_SLBIA) & CXL_TLB_SLB_P) {
 261                 if (time_after_eq(jiffies, timeout)) {
 262                         dev_warn(&adapter->dev, "WARNING: CXL adapter wide SLBIA timed out!\n");
 263                         return -EBUSY;
 264                 }
 265                 if (!cxl_ops->link_ok(adapter, NULL))
 266                         return -EIO;
 267                 cpu_relax();
 268         }
 269         return 0;
 270 }
 271
 272 static int cxl_write_sstp(struct cxl_afu *afu, u64 sstp0, u64 sstp1)
 273 {
 274         int rc;
 275
 276         /* 1. Disable SSTP by writing 0 to SSTP1[V] */
 277         cxl_p2n_write(afu, CXL_SSTP1_An, 0);
 278
 279         /* 2. Invalidate all SLB entries */
 280         if ((rc = cxl_afu_slbia(afu)))
 281                 return rc;
 282
 283         /* 3. Set SSTP0_An */
 284         cxl_p2n_write(afu, CXL_SSTP0_An, sstp0);
 285
 286         /* 4. Set SSTP1_An */
 287         cxl_p2n_write(afu, CXL_SSTP1_An, sstp1);
 288
 289         return 0;
 290 }
 291
 292 /* Using per slice version may improve performance here. (ie. SLBIA_An) */
 293 static void slb_invalid(struct cxl_context *ctx)
 294 {
 295         struct cxl *adapter = ctx->afu->adapter;
 296         u64 slbia;
 297
 298         WARN_ON(!mutex_is_locked(&ctx->afu->native->spa_mutex));
 299
 300         cxl_p1_write(adapter, CXL_PSL_LBISEL,
 301                         ((u64)be32_to_cpu(ctx->elem->common.pid) << 32) |
 302                         be32_to_cpu(ctx->elem->lpid));
 303         cxl_p1_write(adapter, CXL_PSL_SLBIA, CXL_TLB_SLB_IQ_LPIDPID);
 304
 305         while (1) {
 306                 if (!cxl_ops->link_ok(adapter, NULL))
 307                         break;
 308                 slbia = cxl_p1_read(adapter, CXL_PSL_SLBIA);
 309                 if (!(slbia & CXL_TLB_SLB_P))
 310                         break;
 311                 cpu_relax();
 312         }
 313 }
 314
 315 static int do_process_element_cmd(struct cxl_context *ctx,
 316                                   u64 cmd, u64 pe_state)
 317 {
 318         u64 state;
 319         unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT);
 320         int rc = 0;
 321
 322         trace_cxl_llcmd(ctx, cmd);
 323
 324         WARN_ON(!ctx->afu->enabled);
 325
 326         ctx->elem->software_state = cpu_to_be32(pe_state);
 327         smp_wmb();
 328         *(ctx->afu->native->sw_command_status) = cpu_to_be64(cmd | 0 | ctx->pe);
 329         smp_mb();
 330         cxl_p1n_write(ctx->afu, CXL_PSL_LLCMD_An, cmd | ctx->pe);
 331         while (1) {
 332                 if (time_after_eq(jiffies, timeout)) {
 333                         dev_warn(&ctx->afu->dev, "WARNING: Process Element Command timed out!\n");
 334                         rc = -EBUSY;
 335                         goto out;
 336                 }
 337                 if (!cxl_ops->link_ok(ctx->afu->adapter, ctx->afu)) {
 338                         dev_warn(&ctx->afu->dev, "WARNING: Device link down, aborting Process Element Command!\n");
 339                         rc = -EIO;
 340                         goto out;
 341                 }
 342                 state = be64_to_cpup(ctx->afu->native->sw_command_status);
 343                 if (state == ~0ULL) {
 344                         pr_err("cxl: Error adding process element to AFU\n");
 345                         rc = -1;
 346                         goto out;
 347                 }
 348                 if ((state & (CXL_SPA_SW_CMD_MASK | CXL_SPA_SW_STATE_MASK  | CXL_SPA_SW_LINK_MASK)) ==
 349                     (cmd | (cmd >> 16) | ctx->pe))
 350                         break;
 351                 /*
 352                  * The command won't finish in the PSL if there are
 353                  * outstanding DSIs.  Hence we need to yield here in
 354                  * case there are outstanding DSIs that we need to
 355                  * service.  Tuning possiblity: we could wait for a
 356                  * while before sched
 357                  */
 358                 schedule();
 359
 360         }
 361 out:
 362         trace_cxl_llcmd_done(ctx, cmd, rc);
 363         return rc;
 364 }
 365
 366 static int add_process_element(struct cxl_context *ctx)
 367 {
 368         int rc = 0;
 369
 370         mutex_lock(&ctx->afu->native->spa_mutex);
 371         pr_devel("%s Adding pe: %i started\n", __func__, ctx->pe);
 372         if (!(rc = do_process_element_cmd(ctx, CXL_SPA_SW_CMD_ADD, CXL_PE_SOFTWARE_STATE_V)))
 373                 ctx->pe_inserted = true;
 374         pr_devel("%s Adding pe: %i finished\n", __func__, ctx->pe);
 375         mutex_unlock(&ctx->afu->native->spa_mutex);
 376         return rc;
 377 }
 378
 379 static int terminate_process_element(struct cxl_context *ctx)
 380 {
 381         int rc = 0;
 382
 383         /* fast path terminate if it's already invalid */
 384         if (!(ctx->elem->software_state & cpu_to_be32(CXL_PE_SOFTWARE_STATE_V)))
 385                 return rc;
 386
 387         mutex_lock(&ctx->afu->native->spa_mutex);
 388         pr_devel("%s Terminate pe: %i started\n", __func__, ctx->pe);
 389         /* We could be asked to terminate when the hw is down. That
 390          * should always succeed: it's not running if the hw has gone
 391          * away and is being reset.
 392          */
 393         if (cxl_ops->link_ok(ctx->afu->adapter, ctx->afu))
 394                 rc = do_process_element_cmd(ctx, CXL_SPA_SW_CMD_TERMINATE,
 395                                             CXL_PE_SOFTWARE_STATE_V | CXL_PE_SOFTWARE_STATE_T);
 396         ctx->elem->software_state = 0;  /* Remove Valid bit */
 397         pr_devel("%s Terminate pe: %i finished\n", __func__, ctx->pe);
 398         mutex_unlock(&ctx->afu->native->spa_mutex);
 399         return rc;
 400 }
 401
 402 static int remove_process_element(struct cxl_context *ctx)
 403 {
 404         int rc = 0;
 405
 406         mutex_lock(&ctx->afu->native->spa_mutex);
 407         pr_devel("%s Remove pe: %i started\n", __func__, ctx->pe);
 408
 409         /* We could be asked to remove when the hw is down. Again, if
 410          * the hw is down, the PE is gone, so we succeed.
 411          */
 412         if (cxl_ops->link_ok(ctx->afu->adapter, ctx->afu))
 413                 rc = do_process_element_cmd(ctx, CXL_SPA_SW_CMD_REMOVE, 0);
 414
 415         if (!rc)
 416                 ctx->pe_inserted = false;
 417         slb_invalid(ctx);
 418         pr_devel("%s Remove pe: %i finished\n", __func__, ctx->pe);
 419         mutex_unlock(&ctx->afu->native->spa_mutex);
 420
 421         return rc;
 422 }
 423
 424
 425 void cxl_assign_psn_space(struct cxl_context *ctx)
 426 {
 427         if (!ctx->afu->pp_size || ctx->master) {
 428                 ctx->psn_phys = ctx->afu->psn_phys;
 429                 ctx->psn_size = ctx->afu->adapter->ps_size;
 430         } else {
 431                 ctx->psn_phys = ctx->afu->psn_phys +
 432                         (ctx->afu->native->pp_offset + ctx->afu->pp_size * ctx->pe);
 433                 ctx->psn_size = ctx->afu->pp_size;
 434         }
 435 }
 436
 437 static int activate_afu_directed(struct cxl_afu *afu)
 438 {
 439         int rc;
 440
 441         dev_info(&afu->dev, "Activating AFU directed mode\n");
 442
 443         afu->num_procs = afu->max_procs_virtualised;
 444         if (afu->native->spa == NULL) {
 445                 if (cxl_alloc_spa(afu))
 446                         return -ENOMEM;
 447         }
 448         attach_spa(afu);
 449
 450         cxl_p1n_write(afu, CXL_PSL_SCNTL_An, CXL_PSL_SCNTL_An_PM_AFU);
 451         cxl_p1n_write(afu, CXL_PSL_AMOR_An, 0xFFFFFFFFFFFFFFFFULL);
 452         cxl_p1n_write(afu, CXL_PSL_ID_An, CXL_PSL_ID_An_F | CXL_PSL_ID_An_L);
 453
 454         afu->current_mode = CXL_MODE_DIRECTED;
 455
 456         if ((rc = cxl_chardev_m_afu_add(afu)))
 457                 return rc;
 458
 459         if ((rc = cxl_sysfs_afu_m_add(afu)))
 460                 goto err;
 461
 462         if ((rc = cxl_chardev_s_afu_add(afu)))
 463                 goto err1;
 464
 465         return 0;
 466 err1:
 467         cxl_sysfs_afu_m_remove(afu);
 468 err:
 469         cxl_chardev_afu_remove(afu);
 470         return rc;
 471 }
 472
 473 #ifdef CONFIG_CPU_LITTLE_ENDIAN
 474 #define set_endian(sr) ((sr) |= CXL_PSL_SR_An_LE)
 475 #else
 476 #define set_endian(sr) ((sr) &= ~(CXL_PSL_SR_An_LE))
 477 #endif
 478
 479 static u64 calculate_sr(struct cxl_context *ctx)
 480 {
 481         u64 sr = 0;
 482
 483         set_endian(sr);
 484         if (ctx->master)
 485                 sr |= CXL_PSL_SR_An_MP;
 486         if (mfspr(SPRN_LPCR) & LPCR_TC)
 487                 sr |= CXL_PSL_SR_An_TC;
 488         if (ctx->kernel) {
 489                 sr |= CXL_PSL_SR_An_R | (mfmsr() & MSR_SF);
 490                 sr |= CXL_PSL_SR_An_HV;
 491         } else {
 492                 sr |= CXL_PSL_SR_An_PR | CXL_PSL_SR_An_R;
 493                 sr &= ~(CXL_PSL_SR_An_HV);
 494                 if (!test_tsk_thread_flag(current, TIF_32BIT))
 495                         sr |= CXL_PSL_SR_An_SF;
 496         }
 497         return sr;
 498 }
 499
 500 static int attach_afu_directed(struct cxl_context *ctx, u64 wed, u64 amr)
 501 {
 502         u32 pid;
 503         int r, result;
 504
 505         cxl_assign_psn_space(ctx);
 506
 507         ctx->elem->ctxtime = 0; /* disable */
 508         ctx->elem->lpid = cpu_to_be32(mfspr(SPRN_LPID));
 509         ctx->elem->haurp = 0; /* disable */
 510         ctx->elem->sdr = cpu_to_be64(mfspr(SPRN_SDR1));
 511
 512         pid = current->pid;
 513         if (ctx->kernel)
 514                 pid = 0;
 515         ctx->elem->common.tid = 0;
 516         ctx->elem->common.pid = cpu_to_be32(pid);
 517
 518         ctx->elem->sr = cpu_to_be64(calculate_sr(ctx));
 519
 520         ctx->elem->common.csrp = 0; /* disable */
 521         ctx->elem->common.aurp0 = 0; /* disable */
 522         ctx->elem->common.aurp1 = 0; /* disable */
 523
 524         cxl_prefault(ctx, wed);
 525
 526         ctx->elem->common.sstp0 = cpu_to_be64(ctx->sstp0);
 527         ctx->elem->common.sstp1 = cpu_to_be64(ctx->sstp1);
 528
 529         for (r = 0; r < CXL_IRQ_RANGES; r++) {
 530                 ctx->elem->ivte_offsets[r] = cpu_to_be16(ctx->irqs.offset[r]);
 531                 ctx->elem->ivte_ranges[r] = cpu_to_be16(ctx->irqs.range[r]);
 532         }
 533
 534         ctx->elem->common.amr = cpu_to_be64(amr);
 535         ctx->elem->common.wed = cpu_to_be64(wed);
 536
 537         /* first guy needs to enable */
 538         if ((result = cxl_ops->afu_check_and_enable(ctx->afu)))
 539                 return result;
 540
 541         return add_process_element(ctx);
 542 }
 543
 544 static int deactivate_afu_directed(struct cxl_afu *afu)
 545 {
 546         dev_info(&afu->dev, "Deactivating AFU directed mode\n");
 547
 548         afu->current_mode = 0;
 549         afu->num_procs = 0;
 550
 551         cxl_sysfs_afu_m_remove(afu);
 552         cxl_chardev_afu_remove(afu);
 553
 554         cxl_ops->afu_reset(afu);
 555         cxl_afu_disable(afu);
 556         cxl_psl_purge(afu);
 557
 558         return 0;
 559 }
 560
 561 static int activate_dedicated_process(struct cxl_afu *afu)
 562 {
 563         dev_info(&afu->dev, "Activating dedicated process mode\n");
 564
 565         cxl_p1n_write(afu, CXL_PSL_SCNTL_An, CXL_PSL_SCNTL_An_PM_Process);
 566
 567         cxl_p1n_write(afu, CXL_PSL_CtxTime_An, 0); /* disable */
 568         cxl_p1n_write(afu, CXL_PSL_SPAP_An, 0);    /* disable */
 569         cxl_p1n_write(afu, CXL_PSL_AMOR_An, 0xFFFFFFFFFFFFFFFFULL);
 570         cxl_p1n_write(afu, CXL_PSL_LPID_An, mfspr(SPRN_LPID));
 571         cxl_p1n_write(afu, CXL_HAURP_An, 0);       /* disable */
 572         cxl_p1n_write(afu, CXL_PSL_SDR_An, mfspr(SPRN_SDR1));
 573
 574         cxl_p2n_write(afu, CXL_CSRP_An, 0);        /* disable */
 575         cxl_p2n_write(afu, CXL_AURP0_An, 0);       /* disable */
 576         cxl_p2n_write(afu, CXL_AURP1_An, 0);       /* disable */
 577
 578         afu->current_mode = CXL_MODE_DEDICATED;
 579         afu->num_procs = 1;
 580
 581         return cxl_chardev_d_afu_add(afu);
 582 }
 583
 584 static int attach_dedicated(struct cxl_context *ctx, u64 wed, u64 amr)
 585 {
 586         struct cxl_afu *afu = ctx->afu;
 587         u64 pid;
 588         int rc;
 589
 590         pid = (u64)current->pid << 32;
 591         if (ctx->kernel)
 592                 pid = 0;
 593         cxl_p2n_write(afu, CXL_PSL_PID_TID_An, pid);
 594
 595         cxl_p1n_write(afu, CXL_PSL_SR_An, calculate_sr(ctx));
 596
 597         if ((rc = cxl_write_sstp(afu, ctx->sstp0, ctx->sstp1)))
 598                 return rc;
 599
 600         cxl_prefault(ctx, wed);
 601
 602         cxl_p1n_write(afu, CXL_PSL_IVTE_Offset_An,
 603                        (((u64)ctx->irqs.offset[0] & 0xffff) << 48) |
 604                        (((u64)ctx->irqs.offset[1] & 0xffff) << 32) |
 605                        (((u64)ctx->irqs.offset[2] & 0xffff) << 16) |
 606                         ((u64)ctx->irqs.offset[3] & 0xffff));
 607         cxl_p1n_write(afu, CXL_PSL_IVTE_Limit_An, (u64)
 608                        (((u64)ctx->irqs.range[0] & 0xffff) << 48) |
 609                        (((u64)ctx->irqs.range[1] & 0xffff) << 32) |
 610                        (((u64)ctx->irqs.range[2] & 0xffff) << 16) |
 611                         ((u64)ctx->irqs.range[3] & 0xffff));
 612
 613         cxl_p2n_write(afu, CXL_PSL_AMR_An, amr);
 614
 615         /* master only context for dedicated */
 616         cxl_assign_psn_space(ctx);
 617
 618         if ((rc = cxl_ops->afu_reset(afu)))
 619                 return rc;
 620
 621         cxl_p2n_write(afu, CXL_PSL_WED_An, wed);
 622
 623         return afu_enable(afu);
 624 }
 625
 626 static int deactivate_dedicated_process(struct cxl_afu *afu)
 627 {
 628         dev_info(&afu->dev, "Deactivating dedicated process mode\n");
 629
 630         afu->current_mode = 0;
 631         afu->num_procs = 0;
 632
 633         cxl_chardev_afu_remove(afu);
 634
 635         return 0;
 636 }
 637
 638 static int native_afu_deactivate_mode(struct cxl_afu *afu, int mode)
 639 {
 640         if (mode == CXL_MODE_DIRECTED)
 641                 return deactivate_afu_directed(afu);
 642         if (mode == CXL_MODE_DEDICATED)
 643                 return deactivate_dedicated_process(afu);
 644         return 0;
 645 }
 646
 647 static int native_afu_activate_mode(struct cxl_afu *afu, int mode)
 648 {
 649         if (!mode)
 650                 return 0;
 651         if (!(mode & afu->modes_supported))
 652                 return -EINVAL;
 653
 654         if (!cxl_ops->link_ok(afu->adapter, afu)) {
 655                 WARN(1, "Device link is down, refusing to activate!\n");
 656                 return -EIO;
 657         }
 658
 659         if (mode == CXL_MODE_DIRECTED)
 660                 return activate_afu_directed(afu);
 661         if (mode == CXL_MODE_DEDICATED)
 662                 return activate_dedicated_process(afu);
 663
 664         return -EINVAL;
 665 }
 666
 667 static int native_attach_process(struct cxl_context *ctx, bool kernel,
 668                                 u64 wed, u64 amr)
 669 {
 670         if (!cxl_ops->link_ok(ctx->afu->adapter, ctx->afu)) {
 671                 WARN(1, "Device link is down, refusing to attach process!\n");
 672                 return -EIO;
 673         }
 674
 675         ctx->kernel = kernel;
 676         if (ctx->afu->current_mode == CXL_MODE_DIRECTED)
 677                 return attach_afu_directed(ctx, wed, amr);
 678
 679         if (ctx->afu->current_mode == CXL_MODE_DEDICATED)
 680                 return attach_dedicated(ctx, wed, amr);
 681
 682         return -EINVAL;
 683 }
 684
 685 static inline int detach_process_native_dedicated(struct cxl_context *ctx)
 686 {
 687         cxl_ops->afu_reset(ctx->afu);
 688         cxl_afu_disable(ctx->afu);
 689         cxl_psl_purge(ctx->afu);
 690         return 0;
 691 }
 692
 693 static inline int detach_process_native_afu_directed(struct cxl_context *ctx)
 694 {
 695         if (!ctx->pe_inserted)
 696                 return 0;
 697         if (terminate_process_element(ctx))
 698                 return -1;
 699         if (remove_process_element(ctx))
 700                 return -1;
 701
 702         return 0;
 703 }
 704
 705 static int native_detach_process(struct cxl_context *ctx)
 706 {
 707         trace_cxl_detach(ctx);
 708
 709         if (ctx->afu->current_mode == CXL_MODE_DEDICATED)
 710                 return detach_process_native_dedicated(ctx);
 711
 712         return detach_process_native_afu_directed(ctx);
 713 }
 714
 715 static int native_get_irq_info(struct cxl_afu *afu, struct cxl_irq_info *info)
 716 {
 717         u64 pidtid;
 718
 719         /* If the adapter has gone away, we can't get any meaningful
 720          * information.
 721          */
 722         if (!cxl_ops->link_ok(afu->adapter, afu))
 723                 return -EIO;
 724
 725         info->dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An);
 726         info->dar = cxl_p2n_read(afu, CXL_PSL_DAR_An);
 727         info->dsr = cxl_p2n_read(afu, CXL_PSL_DSR_An);
 728         pidtid = cxl_p2n_read(afu, CXL_PSL_PID_TID_An);
 729         info->pid = pidtid >> 32;
 730         info->tid = pidtid & 0xffffffff;
 731         info->afu_err = cxl_p2n_read(afu, CXL_AFU_ERR_An);
 732         info->errstat = cxl_p2n_read(afu, CXL_PSL_ErrStat_An);
 733         info->proc_handle = 0;
 734
 735         return 0;
 736 }
 737
 738 static irqreturn_t native_handle_psl_slice_error(struct cxl_context *ctx,
 739                                                 u64 dsisr, u64 errstat)
 740 {
 741         u64 fir1, fir2, fir_slice, serr, afu_debug;
 742
 743         fir1 = cxl_p1_read(ctx->afu->adapter, CXL_PSL_FIR1);
 744         fir2 = cxl_p1_read(ctx->afu->adapter, CXL_PSL_FIR2);
 745         fir_slice = cxl_p1n_read(ctx->afu, CXL_PSL_FIR_SLICE_An);
 746         serr = cxl_p1n_read(ctx->afu, CXL_PSL_SERR_An);
 747         afu_debug = cxl_p1n_read(ctx->afu, CXL_AFU_DEBUG_An);
 748
 749         dev_crit(&ctx->afu->dev, "PSL ERROR STATUS: 0x%016llx\n", errstat);
 750         dev_crit(&ctx->afu->dev, "PSL_FIR1: 0x%016llx\n", fir1);
 751         dev_crit(&ctx->afu->dev, "PSL_FIR2: 0x%016llx\n", fir2);
 752         dev_crit(&ctx->afu->dev, "PSL_SERR_An: 0x%016llx\n", serr);
 753         dev_crit(&ctx->afu->dev, "PSL_FIR_SLICE_An: 0x%016llx\n", fir_slice);
 754         dev_crit(&ctx->afu->dev, "CXL_PSL_AFU_DEBUG_An: 0x%016llx\n", afu_debug);
 755
 756         dev_crit(&ctx->afu->dev, "STOPPING CXL TRACE\n");
 757         cxl_stop_trace(ctx->afu->adapter);
 758
 759         return cxl_ops->ack_irq(ctx, 0, errstat);
 760 }
 761
 762 static irqreturn_t fail_psl_irq(struct cxl_afu *afu, struct cxl_irq_info *irq_info)
 763 {
 764         if (irq_info->dsisr & CXL_PSL_DSISR_TRANS)
 765                 cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_AE);
 766         else
 767                 cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_A);
 768
 769         return IRQ_HANDLED;
 770 }
 771
 772 static irqreturn_t native_irq_multiplexed(int irq, void *data)
 773 {
 774         struct cxl_afu *afu = data;
 775         struct cxl_context *ctx;
 776         struct cxl_irq_info irq_info;
 777         int ph = cxl_p2n_read(afu, CXL_PSL_PEHandle_An) & 0xffff;
 778         int ret;
 779
 780         if ((ret = native_get_irq_info(afu, &irq_info))) {
 781                 WARN(1, "Unable to get CXL IRQ Info: %i\n", ret);
 782                 return fail_psl_irq(afu, &irq_info);
 783         }
 784
 785         rcu_read_lock();
 786         ctx = idr_find(&afu->contexts_idr, ph);
 787         if (ctx) {
 788                 ret = cxl_irq(irq, ctx, &irq_info);
 789                 rcu_read_unlock();
 790                 return ret;
 791         }
 792         rcu_read_unlock();
 793
 794         WARN(1, "Unable to demultiplex CXL PSL IRQ for PE %i DSISR %016llx DAR"
 795                 " %016llx\n(Possible AFU HW issue - was a term/remove acked"
 796                 " with outstanding transactions?)\n", ph, irq_info.dsisr,
 797                 irq_info.dar);
 798         return fail_psl_irq(afu, &irq_info);
 799 }
 800
 801 void native_irq_wait(struct cxl_context *ctx)
 802 {
 803         u64 dsisr;
 804         int timeout = 1000;
 805         int ph;
 806
 807         /*
 808          * Wait until no further interrupts are presented by the PSL
 809          * for this context.
 810          */
 811         while (timeout--) {
 812                 ph = cxl_p2n_read(ctx->afu, CXL_PSL_PEHandle_An) & 0xffff;
 813                 if (ph != ctx->pe)
 814                         return;
 815                 dsisr = cxl_p2n_read(ctx->afu, CXL_PSL_DSISR_An);
 816                 if ((dsisr & CXL_PSL_DSISR_PENDING) == 0)
 817                         return;
 818                 /*
 819                  * We are waiting for the workqueue to process our
 820                  * irq, so need to let that run here.
 821                  */
 822                 msleep(1);
 823         }
 824
 825         dev_warn(&ctx->afu->dev, "WARNING: waiting on DSI for PE %i"
 826                  " DSISR %016llx!\n", ph, dsisr);
 827         return;
 828 }
 829
 830 static irqreturn_t native_slice_irq_err(int irq, void *data)
 831 {
 832         struct cxl_afu *afu = data;
 833         u64 fir_slice, errstat, serr, afu_debug;
 834
 835         WARN(irq, "CXL SLICE ERROR interrupt %i\n", irq);
 836
 837         serr = cxl_p1n_read(afu, CXL_PSL_SERR_An);
 838         fir_slice = cxl_p1n_read(afu, CXL_PSL_FIR_SLICE_An);
 839         errstat = cxl_p2n_read(afu, CXL_PSL_ErrStat_An);
 840         afu_debug = cxl_p1n_read(afu, CXL_AFU_DEBUG_An);
 841         dev_crit(&afu->dev, "PSL_SERR_An: 0x%016llx\n", serr);
 842         dev_crit(&afu->dev, "PSL_FIR_SLICE_An: 0x%016llx\n", fir_slice);
 843         dev_crit(&afu->dev, "CXL_PSL_ErrStat_An: 0x%016llx\n", errstat);
 844         dev_crit(&afu->dev, "CXL_PSL_AFU_DEBUG_An: 0x%016llx\n", afu_debug);
 845
 846         cxl_p1n_write(afu, CXL_PSL_SERR_An, serr);
 847
 848         return IRQ_HANDLED;
 849 }
 850
 851 static irqreturn_t native_irq_err(int irq, void *data)
 852 {
 853         struct cxl *adapter = data;
 854         u64 fir1, fir2, err_ivte;
 855
 856         WARN(1, "CXL ERROR interrupt %i\n", irq);
 857
 858         err_ivte = cxl_p1_read(adapter, CXL_PSL_ErrIVTE);
 859         dev_crit(&adapter->dev, "PSL_ErrIVTE: 0x%016llx\n", err_ivte);
 860
 861         dev_crit(&adapter->dev, "STOPPING CXL TRACE\n");
 862         cxl_stop_trace(adapter);
 863
 864         fir1 = cxl_p1_read(adapter, CXL_PSL_FIR1);
 865         fir2 = cxl_p1_read(adapter, CXL_PSL_FIR2);
 866
 867         dev_crit(&adapter->dev, "PSL_FIR1: 0x%016llx\nPSL_FIR2: 0x%016llx\n", fir1, fir2);
 868
 869         return IRQ_HANDLED;
 870 }
 871
 872 int cxl_native_register_psl_err_irq(struct cxl *adapter)
 873 {
 874         int rc;
 875
 876         adapter->irq_name = kasprintf(GFP_KERNEL, "cxl-%s-err",
 877                                       dev_name(&adapter->dev));
 878         if (!adapter->irq_name)
 879                 return -ENOMEM;
 880
 881         if ((rc = cxl_register_one_irq(adapter, native_irq_err, adapter,
 882                                        &adapter->native->err_hwirq,
 883                                        &adapter->native->err_virq,
 884                                        adapter->irq_name))) {
 885                 kfree(adapter->irq_name);
 886                 adapter->irq_name = NULL;
 887                 return rc;
 888         }
 889
 890         cxl_p1_write(adapter, CXL_PSL_ErrIVTE, adapter->native->err_hwirq & 0xffff);
 891
 892         return 0;
 893 }
 894
 895 void cxl_native_release_psl_err_irq(struct cxl *adapter)
 896 {
 897         if (adapter->native->err_virq != irq_find_mapping(NULL, adapter->native->err_hwirq))
 898                 return;
 899
 900         cxl_p1_write(adapter, CXL_PSL_ErrIVTE, 0x0000000000000000);
 901         cxl_unmap_irq(adapter->native->err_virq, adapter);
 902         cxl_ops->release_one_irq(adapter, adapter->native->err_hwirq);
 903         kfree(adapter->irq_name);
 904 }
 905
 906 int cxl_native_register_serr_irq(struct cxl_afu *afu)
 907 {
 908         u64 serr;
 909         int rc;
 910
 911         afu->err_irq_name = kasprintf(GFP_KERNEL, "cxl-%s-err",
 912                                       dev_name(&afu->dev));
 913         if (!afu->err_irq_name)
 914                 return -ENOMEM;
 915
 916         if ((rc = cxl_register_one_irq(afu->adapter, native_slice_irq_err, afu,
 917                                        &afu->serr_hwirq,
 918                                        &afu->serr_virq, afu->err_irq_name))) {
 919                 kfree(afu->err_irq_name);
 920                 afu->err_irq_name = NULL;
 921                 return rc;
 922         }
 923
 924         serr = cxl_p1n_read(afu, CXL_PSL_SERR_An);
 925         serr = (serr & 0x00ffffffffff0000ULL) | (afu->serr_hwirq & 0xffff);
 926         cxl_p1n_write(afu, CXL_PSL_SERR_An, serr);
 927
 928         return 0;
 929 }
 930
 931 void cxl_native_release_serr_irq(struct cxl_afu *afu)
 932 {
 933         if (afu->serr_virq != irq_find_mapping(NULL, afu->serr_hwirq))
 934                 return;
 935
 936         cxl_p1n_write(afu, CXL_PSL_SERR_An, 0x0000000000000000);
 937         cxl_unmap_irq(afu->serr_virq, afu);
 938         cxl_ops->release_one_irq(afu->adapter, afu->serr_hwirq);
 939         kfree(afu->err_irq_name);
 940 }
 941
 942 int cxl_native_register_psl_irq(struct cxl_afu *afu)
 943 {
 944         int rc;
 945
 946         afu->psl_irq_name = kasprintf(GFP_KERNEL, "cxl-%s",
 947                                       dev_name(&afu->dev));
 948         if (!afu->psl_irq_name)
 949                 return -ENOMEM;
 950
 951         if ((rc = cxl_register_one_irq(afu->adapter, native_irq_multiplexed,
 952                                     afu, &afu->native->psl_hwirq, &afu->native->psl_virq,
 953                                     afu->psl_irq_name))) {
 954                 kfree(afu->psl_irq_name);
 955                 afu->psl_irq_name = NULL;
 956         }
 957         return rc;
 958 }
 959
 960 void cxl_native_release_psl_irq(struct cxl_afu *afu)
 961 {
 962         if (afu->native->psl_virq != irq_find_mapping(NULL, afu->native->psl_hwirq))
 963                 return;
 964
 965         cxl_unmap_irq(afu->native->psl_virq, afu);
 966         cxl_ops->release_one_irq(afu->adapter, afu->native->psl_hwirq);
 967         kfree(afu->psl_irq_name);
 968 }
 969
 970 static void recover_psl_err(struct cxl_afu *afu, u64 errstat)
 971 {
 972         u64 dsisr;
 973
 974         pr_devel("RECOVERING FROM PSL ERROR... (0x%016llx)\n", errstat);
 975
 976         /* Clear PSL_DSISR[PE] */
 977         dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An);
 978         cxl_p2n_write(afu, CXL_PSL_DSISR_An, dsisr & ~CXL_PSL_DSISR_An_PE);
 979
 980         /* Write 1s to clear error status bits */
 981         cxl_p2n_write(afu, CXL_PSL_ErrStat_An, errstat);
 982 }
 983
 984 static int native_ack_irq(struct cxl_context *ctx, u64 tfc, u64 psl_reset_mask)
 985 {
 986         trace_cxl_psl_irq_ack(ctx, tfc);
 987         if (tfc)
 988                 cxl_p2n_write(ctx->afu, CXL_PSL_TFC_An, tfc);
 989         if (psl_reset_mask)
 990                 recover_psl_err(ctx->afu, psl_reset_mask);
 991
 992         return 0;
 993 }
 994
 995 int cxl_check_error(struct cxl_afu *afu)
 996 {
 997         return (cxl_p1n_read(afu, CXL_PSL_SCNTL_An) == ~0ULL);
 998 }
 999
1000 static bool native_support_attributes(const char *attr_name,
1001                                       enum cxl_attrs type)
1002 {
1003         return true;
1004 }
1005
1006 static int native_afu_cr_read64(struct cxl_afu *afu, int cr, u64 off, u64 *out)
1007 {
1008         if (unlikely(!cxl_ops->link_ok(afu->adapter, afu)))
1009                 return -EIO;
1010         if (unlikely(off >= afu->crs_len))
1011                 return -ERANGE;
1012         *out = in_le64(afu->native->afu_desc_mmio + afu->crs_offset +
1013                 (cr * afu->crs_len) + off);
1014         return 0;
1015 }
1016
1017 static int native_afu_cr_read32(struct cxl_afu *afu, int cr, u64 off, u32 *out)
1018 {
1019         if (unlikely(!cxl_ops->link_ok(afu->adapter, afu)))
1020                 return -EIO;
1021         if (unlikely(off >= afu->crs_len))
1022                 return -ERANGE;
1023         *out = in_le32(afu->native->afu_desc_mmio + afu->crs_offset +
1024                 (cr * afu->crs_len) + off);
1025         return 0;
1026 }
1027
1028 static int native_afu_cr_read16(struct cxl_afu *afu, int cr, u64 off, u16 *out)
1029 {
1030         u64 aligned_off = off & ~0x3L;
1031         u32 val;
1032         int rc;
1033
1034         rc = native_afu_cr_read32(afu, cr, aligned_off, &val);
1035         if (!rc)
1036                 *out = (val >> ((off & 0x3) * 8)) & 0xffff;
1037         return rc;
1038 }
1039
1040 static int native_afu_cr_read8(struct cxl_afu *afu, int cr, u64 off, u8 *out)
1041 {
1042         u64 aligned_off = off & ~0x3L;
1043         u32 val;
1044         int rc;
1045
1046         rc = native_afu_cr_read32(afu, cr, aligned_off, &val);
1047         if (!rc)
1048                 *out = (val >> ((off & 0x3) * 8)) & 0xff;
1049         return rc;
1050 }
1051
1052 static int native_afu_cr_write32(struct cxl_afu *afu, int cr, u64 off, u32 in)
1053 {
1054         if (unlikely(!cxl_ops->link_ok(afu->adapter, afu)))
1055                 return -EIO;
1056         if (unlikely(off >= afu->crs_len))
1057                 return -ERANGE;
1058         out_le32(afu->native->afu_desc_mmio + afu->crs_offset +
1059                 (cr * afu->crs_len) + off, in);
1060         return 0;
1061 }
1062
1063 static int native_afu_cr_write16(struct cxl_afu *afu, int cr, u64 off, u16 in)
1064 {
1065         u64 aligned_off = off & ~0x3L;
1066         u32 val32, mask, shift;
1067         int rc;
1068
1069         rc = native_afu_cr_read32(afu, cr, aligned_off, &val32);
1070         if (rc)
1071                 return rc;
1072         shift = (off & 0x3) * 8;
1073         WARN_ON(shift == 24);
1074         mask = 0xffff << shift;
1075         val32 = (val32 & ~mask) | (in << shift);
1076
1077         rc = native_afu_cr_write32(afu, cr, aligned_off, val32);
1078         return rc;
1079 }
1080
1081 static int native_afu_cr_write8(struct cxl_afu *afu, int cr, u64 off, u8 in)
1082 {
1083         u64 aligned_off = off & ~0x3L;
1084         u32 val32, mask, shift;
1085         int rc;
1086
1087         rc = native_afu_cr_read32(afu, cr, aligned_off, &val32);
1088         if (rc)
1089                 return rc;
1090         shift = (off & 0x3) * 8;
1091         mask = 0xff << shift;
1092         val32 = (val32 & ~mask) | (in << shift);
1093
1094         rc = native_afu_cr_write32(afu, cr, aligned_off, val32);
1095         return rc;
1096 }
1097
1098 const struct cxl_backend_ops cxl_native_ops = {
1099         .module = THIS_MODULE,
1100         .adapter_reset = cxl_pci_reset,
1101         .alloc_one_irq = cxl_pci_alloc_one_irq,
1102         .release_one_irq = cxl_pci_release_one_irq,
1103         .alloc_irq_ranges = cxl_pci_alloc_irq_ranges,
1104         .release_irq_ranges = cxl_pci_release_irq_ranges,
1105         .setup_irq = cxl_pci_setup_irq,
1106         .handle_psl_slice_error = native_handle_psl_slice_error,
1107         .psl_interrupt = NULL,
1108         .ack_irq = native_ack_irq,
1109         .irq_wait = native_irq_wait,
1110         .attach_process = native_attach_process,
1111         .detach_process = native_detach_process,
1112         .support_attributes = native_support_attributes,
1113         .link_ok = cxl_adapter_link_ok,
1114         .release_afu = cxl_pci_release_afu,
1115         .afu_read_err_buffer = cxl_pci_afu_read_err_buffer,
1116         .afu_check_and_enable = native_afu_check_and_enable,
1117         .afu_activate_mode = native_afu_activate_mode,
1118         .afu_deactivate_mode = native_afu_deactivate_mode,
1119         .afu_reset = native_afu_reset,
1120         .afu_cr_read8 = native_afu_cr_read8,
1121         .afu_cr_read16 = native_afu_cr_read16,
1122         .afu_cr_read32 = native_afu_cr_read32,
1123         .afu_cr_read64 = native_afu_cr_read64,
1124         .afu_cr_write8 = native_afu_cr_write8,
1125         .afu_cr_write16 = native_afu_cr_write16,
1126         .afu_cr_write32 = native_afu_cr_write32,
1127         .read_adapter_vpd = cxl_pci_read_adapter_vpd,
1128 };