drivers/crypto/marvell/octeontx/otx_cptvf_reqmgr.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Marvell OcteonTX CPT driver
   3  *
   4  * Copyright (C) 2019 Marvell International Ltd.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 as
   8  * published by the Free Software Foundation.
   9  */
  10
  11 #include "otx_cptvf.h"
  12 #include "otx_cptvf_algs.h"
  13
  14 /* Completion code size and initial value */
  15 #define COMPLETION_CODE_SIZE    8
  16 #define COMPLETION_CODE_INIT    0
  17
  18 /* SG list header size in bytes */
  19 #define SG_LIST_HDR_SIZE        8
  20
  21 /* Default timeout when waiting for free pending entry in us */
  22 #define CPT_PENTRY_TIMEOUT      1000
  23 #define CPT_PENTRY_STEP         50
  24
  25 /* Default threshold for stopping and resuming sender requests */
  26 #define CPT_IQ_STOP_MARGIN      128
  27 #define CPT_IQ_RESUME_MARGIN    512
  28
  29 #define CPT_DMA_ALIGN           128
  30
  31 void otx_cpt_dump_sg_list(struct pci_dev *pdev, struct otx_cpt_req_info *req)
  32 {
  33         int i;
  34
  35         pr_debug("Gather list size %d\n", req->incnt);
  36         for (i = 0; i < req->incnt; i++) {
  37                 pr_debug("Buffer %d size %d, vptr 0x%p, dmaptr 0x%p\n", i,
  38                          req->in[i].size, req->in[i].vptr,
  39                          (void *) req->in[i].dma_addr);
  40                 pr_debug("Buffer hexdump (%d bytes)\n",
  41                          req->in[i].size);
  42                 print_hex_dump_debug("", DUMP_PREFIX_NONE, 16, 1,
  43                                      req->in[i].vptr, req->in[i].size, false);
  44         }
  45
  46         pr_debug("Scatter list size %d\n", req->outcnt);
  47         for (i = 0; i < req->outcnt; i++) {
  48                 pr_debug("Buffer %d size %d, vptr 0x%p, dmaptr 0x%p\n", i,
  49                          req->out[i].size, req->out[i].vptr,
  50                          (void *) req->out[i].dma_addr);
  51                 pr_debug("Buffer hexdump (%d bytes)\n", req->out[i].size);
  52                 print_hex_dump_debug("", DUMP_PREFIX_NONE, 16, 1,
  53                                      req->out[i].vptr, req->out[i].size, false);
  54         }
  55 }
  56
  57 static inline struct otx_cpt_pending_entry *get_free_pending_entry(
  58                                                 struct otx_cpt_pending_queue *q,
  59                                                 int qlen)
  60 {
  61         struct otx_cpt_pending_entry *ent = NULL;
  62
  63         ent = &q->head[q->rear];
  64         if (unlikely(ent->busy))
  65                 return NULL;
  66
  67         q->rear++;
  68         if (unlikely(q->rear == qlen))
  69                 q->rear = 0;
  70
  71         return ent;
  72 }
  73
  74 static inline u32 modulo_inc(u32 index, u32 length, u32 inc)
  75 {
  76         if (WARN_ON(inc > length))
  77                 inc = length;
  78
  79         index += inc;
  80         if (unlikely(index >= length))
  81                 index -= length;
  82
  83         return index;
  84 }
  85
  86 static inline void free_pentry(struct otx_cpt_pending_entry *pentry)
  87 {
  88         pentry->completion_addr = NULL;
  89         pentry->info = NULL;
  90         pentry->callback = NULL;
  91         pentry->areq = NULL;
  92         pentry->resume_sender = false;
  93         pentry->busy = false;
  94 }
  95
  96 static inline int setup_sgio_components(struct pci_dev *pdev,
  97                                         struct otx_cpt_buf_ptr *list,
  98                                         int buf_count, u8 *buffer)
  99 {
 100         struct otx_cpt_sglist_component *sg_ptr = NULL;
 101         int ret = 0, i, j;
 102         int components;
 103
 104         if (unlikely(!list)) {
 105                 dev_err(&pdev->dev, "Input list pointer is NULL\n");
 106                 return -EFAULT;
 107         }
 108
 109         for (i = 0; i < buf_count; i++) {
 110                 if (likely(list[i].vptr)) {
 111                         list[i].dma_addr = dma_map_single(&pdev->dev,
 112                                                           list[i].vptr,
 113                                                           list[i].size,
 114                                                           DMA_BIDIRECTIONAL);
 115                         if (unlikely(dma_mapping_error(&pdev->dev,
 116                                                        list[i].dma_addr))) {
 117                                 dev_err(&pdev->dev, "Dma mapping failed\n");
 118                                 ret = -EIO;
 119                                 goto sg_cleanup;
 120                         }
 121                 }
 122         }
 123
 124         components = buf_count / 4;
 125         sg_ptr = (struct otx_cpt_sglist_component *)buffer;
 126         for (i = 0; i < components; i++) {
 127                 sg_ptr->u.s.len0 = cpu_to_be16(list[i * 4 + 0].size);
 128                 sg_ptr->u.s.len1 = cpu_to_be16(list[i * 4 + 1].size);
 129                 sg_ptr->u.s.len2 = cpu_to_be16(list[i * 4 + 2].size);
 130                 sg_ptr->u.s.len3 = cpu_to_be16(list[i * 4 + 3].size);
 131                 sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr);
 132                 sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr);
 133                 sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr);
 134                 sg_ptr->ptr3 = cpu_to_be64(list[i * 4 + 3].dma_addr);
 135                 sg_ptr++;
 136         }
 137         components = buf_count % 4;
 138
 139         switch (components) {
 140         case 3:
 141                 sg_ptr->u.s.len2 = cpu_to_be16(list[i * 4 + 2].size);
 142                 sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr);
 143                 fallthrough;
 144         case 2:
 145                 sg_ptr->u.s.len1 = cpu_to_be16(list[i * 4 + 1].size);
 146                 sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr);
 147                 fallthrough;
 148         case 1:
 149                 sg_ptr->u.s.len0 = cpu_to_be16(list[i * 4 + 0].size);
 150                 sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr);
 151                 break;
 152         default:
 153                 break;
 154         }
 155         return ret;
 156
 157 sg_cleanup:
 158         for (j = 0; j < i; j++) {
 159                 if (list[j].dma_addr) {
 160                         dma_unmap_single(&pdev->dev, list[i].dma_addr,
 161                                          list[i].size, DMA_BIDIRECTIONAL);
 162                 }
 163
 164                 list[j].dma_addr = 0;
 165         }
 166         return ret;
 167 }
 168
 169 static inline int setup_sgio_list(struct pci_dev *pdev,
 170                                   struct otx_cpt_info_buffer **pinfo,
 171                                   struct otx_cpt_req_info *req, gfp_t gfp)
 172 {
 173         u32 dlen, align_dlen, info_len, rlen;
 174         struct otx_cpt_info_buffer *info;
 175         u16 g_sz_bytes, s_sz_bytes;
 176         int align = CPT_DMA_ALIGN;
 177         u32 total_mem_len;
 178
 179         if (unlikely(req->incnt > OTX_CPT_MAX_SG_IN_CNT ||
 180                      req->outcnt > OTX_CPT_MAX_SG_OUT_CNT)) {
 181                 dev_err(&pdev->dev, "Error too many sg components\n");
 182                 return -EINVAL;
 183         }
 184
 185         g_sz_bytes = ((req->incnt + 3) / 4) *
 186                       sizeof(struct otx_cpt_sglist_component);
 187         s_sz_bytes = ((req->outcnt + 3) / 4) *
 188                       sizeof(struct otx_cpt_sglist_component);
 189
 190         dlen = g_sz_bytes + s_sz_bytes + SG_LIST_HDR_SIZE;
 191         align_dlen = ALIGN(dlen, align);
 192         info_len = ALIGN(sizeof(*info), align);
 193         rlen = ALIGN(sizeof(union otx_cpt_res_s), align);
 194         total_mem_len = align_dlen + info_len + rlen + COMPLETION_CODE_SIZE;
 195
 196         info = kzalloc(total_mem_len, gfp);
 197         if (unlikely(!info)) {
 198                 dev_err(&pdev->dev, "Memory allocation failed\n");
 199                 return -ENOMEM;
 200         }
 201         *pinfo = info;
 202         info->dlen = dlen;
 203         info->in_buffer = (u8 *)info + info_len;
 204
 205         ((__be16 *)info->in_buffer)[0] = cpu_to_be16(req->outcnt);
 206         ((__be16 *)info->in_buffer)[1] = cpu_to_be16(req->incnt);
 207         ((u16 *)info->in_buffer)[2] = 0;
 208         ((u16 *)info->in_buffer)[3] = 0;
 209
 210         /* Setup gather (input) components */
 211         if (setup_sgio_components(pdev, req->in, req->incnt,
 212                                   &info->in_buffer[8])) {
 213                 dev_err(&pdev->dev, "Failed to setup gather list\n");
 214                 return -EFAULT;
 215         }
 216
 217         if (setup_sgio_components(pdev, req->out, req->outcnt,
 218                                   &info->in_buffer[8 + g_sz_bytes])) {
 219                 dev_err(&pdev->dev, "Failed to setup scatter list\n");
 220                 return -EFAULT;
 221         }
 222
 223         info->dma_len = total_mem_len - info_len;
 224         info->dptr_baddr = dma_map_single(&pdev->dev, (void *)info->in_buffer,
 225                                           info->dma_len, DMA_BIDIRECTIONAL);
 226         if (unlikely(dma_mapping_error(&pdev->dev, info->dptr_baddr))) {
 227                 dev_err(&pdev->dev, "DMA Mapping failed for cpt req\n");
 228                 return -EIO;
 229         }
 230         /*
 231          * Get buffer for union otx_cpt_res_s response
 232          * structure and its physical address
 233          */
 234         info->completion_addr = (u64 *)(info->in_buffer + align_dlen);
 235         info->comp_baddr = info->dptr_baddr + align_dlen;
 236
 237         /* Create and initialize RPTR */
 238         info->out_buffer = (u8 *)info->completion_addr + rlen;
 239         info->rptr_baddr = info->comp_baddr + rlen;
 240
 241         *((u64 *) info->out_buffer) = ~((u64) COMPLETION_CODE_INIT);
 242
 243         return 0;
 244 }
 245
 246
 247 static void cpt_fill_inst(union otx_cpt_inst_s *inst,
 248                           struct otx_cpt_info_buffer *info,
 249                           struct otx_cpt_iq_cmd *cmd)
 250 {
 251         inst->u[0] = 0x0;
 252         inst->s.doneint = true;
 253         inst->s.res_addr = (u64)info->comp_baddr;
 254         inst->u[2] = 0x0;
 255         inst->s.wq_ptr = 0;
 256         inst->s.ei0 = cmd->cmd.u64;
 257         inst->s.ei1 = cmd->dptr;
 258         inst->s.ei2 = cmd->rptr;
 259         inst->s.ei3 = cmd->cptr.u64;
 260 }
 261
 262 /*
 263  * On OcteonTX platform the parameter db_count is used as a count for ringing
 264  * door bell. The valid values for db_count are:
 265  * 0 - 1 CPT instruction will be enqueued however CPT will not be informed
 266  * 1 - 1 CPT instruction will be enqueued and CPT will be informed
 267  */
 268 static void cpt_send_cmd(union otx_cpt_inst_s *cptinst, struct otx_cptvf *cptvf)
 269 {
 270         struct otx_cpt_cmd_qinfo *qinfo = &cptvf->cqinfo;
 271         struct otx_cpt_cmd_queue *queue;
 272         struct otx_cpt_cmd_chunk *curr;
 273         u8 *ent;
 274
 275         queue = &qinfo->queue[0];
 276         /*
 277          * cpt_send_cmd is currently called only from critical section
 278          * therefore no locking is required for accessing instruction queue
 279          */
 280         ent = &queue->qhead->head[queue->idx * OTX_CPT_INST_SIZE];
 281         memcpy(ent, (void *) cptinst, OTX_CPT_INST_SIZE);
 282
 283         if (++queue->idx >= queue->qhead->size / 64) {
 284                 curr = queue->qhead;
 285
 286                 if (list_is_last(&curr->nextchunk, &queue->chead))
 287                         queue->qhead = queue->base;
 288                 else
 289                         queue->qhead = list_next_entry(queue->qhead, nextchunk);
 290                 queue->idx = 0;
 291         }
 292         /* make sure all memory stores are done before ringing doorbell */
 293         smp_wmb();
 294         otx_cptvf_write_vq_doorbell(cptvf, 1);
 295 }
 296
 297 static int process_request(struct pci_dev *pdev, struct otx_cpt_req_info *req,
 298                            struct otx_cpt_pending_queue *pqueue,
 299                            struct otx_cptvf *cptvf)
 300 {
 301         struct otx_cptvf_request *cpt_req = &req->req;
 302         struct otx_cpt_pending_entry *pentry = NULL;
 303         union otx_cpt_ctrl_info *ctrl = &req->ctrl;
 304         struct otx_cpt_info_buffer *info = NULL;
 305         union otx_cpt_res_s *result = NULL;
 306         struct otx_cpt_iq_cmd iq_cmd;
 307         union otx_cpt_inst_s cptinst;
 308         int retry, ret = 0;
 309         u8 resume_sender;
 310         gfp_t gfp;
 311
 312         gfp = (req->areq->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL :
 313                                                               GFP_ATOMIC;
 314         ret = setup_sgio_list(pdev, &info, req, gfp);
 315         if (unlikely(ret)) {
 316                 dev_err(&pdev->dev, "Setting up SG list failed\n");
 317                 goto request_cleanup;
 318         }
 319         cpt_req->dlen = info->dlen;
 320
 321         result = (union otx_cpt_res_s *) info->completion_addr;
 322         result->s.compcode = COMPLETION_CODE_INIT;
 323
 324         spin_lock_bh(&pqueue->lock);
 325         pentry = get_free_pending_entry(pqueue, pqueue->qlen);
 326         retry = CPT_PENTRY_TIMEOUT / CPT_PENTRY_STEP;
 327         while (unlikely(!pentry) && retry--) {
 328                 spin_unlock_bh(&pqueue->lock);
 329                 udelay(CPT_PENTRY_STEP);
 330                 spin_lock_bh(&pqueue->lock);
 331                 pentry = get_free_pending_entry(pqueue, pqueue->qlen);
 332         }
 333
 334         if (unlikely(!pentry)) {
 335                 ret = -ENOSPC;
 336                 spin_unlock_bh(&pqueue->lock);
 337                 goto request_cleanup;
 338         }
 339
 340         /*
 341          * Check if we are close to filling in entire pending queue,
 342          * if so then tell the sender to stop/sleep by returning -EBUSY
 343          * We do it only for context which can sleep (GFP_KERNEL)
 344          */
 345         if (gfp == GFP_KERNEL &&
 346             pqueue->pending_count > (pqueue->qlen - CPT_IQ_STOP_MARGIN)) {
 347                 pentry->resume_sender = true;
 348         } else
 349                 pentry->resume_sender = false;
 350         resume_sender = pentry->resume_sender;
 351         pqueue->pending_count++;
 352
 353         pentry->completion_addr = info->completion_addr;
 354         pentry->info = info;
 355         pentry->callback = req->callback;
 356         pentry->areq = req->areq;
 357         pentry->busy = true;
 358         info->pentry = pentry;
 359         info->time_in = jiffies;
 360         info->req = req;
 361
 362         /* Fill in the command */
 363         iq_cmd.cmd.u64 = 0;
 364         iq_cmd.cmd.s.opcode = cpu_to_be16(cpt_req->opcode.flags);
 365         iq_cmd.cmd.s.param1 = cpu_to_be16(cpt_req->param1);
 366         iq_cmd.cmd.s.param2 = cpu_to_be16(cpt_req->param2);
 367         iq_cmd.cmd.s.dlen   = cpu_to_be16(cpt_req->dlen);
 368
 369         iq_cmd.dptr = info->dptr_baddr;
 370         iq_cmd.rptr = info->rptr_baddr;
 371         iq_cmd.cptr.u64 = 0;
 372         iq_cmd.cptr.s.grp = ctrl->s.grp;
 373
 374         /* Fill in the CPT_INST_S type command for HW interpretation */
 375         cpt_fill_inst(&cptinst, info, &iq_cmd);
 376
 377         /* Print debug info if enabled */
 378         otx_cpt_dump_sg_list(pdev, req);
 379         pr_debug("Cpt_inst_s hexdump (%d bytes)\n", OTX_CPT_INST_SIZE);
 380         print_hex_dump_debug("", 0, 16, 1, &cptinst, OTX_CPT_INST_SIZE, false);
 381         pr_debug("Dptr hexdump (%d bytes)\n", cpt_req->dlen);
 382         print_hex_dump_debug("", 0, 16, 1, info->in_buffer,
 383                              cpt_req->dlen, false);
 384
 385         /* Send CPT command */
 386         cpt_send_cmd(&cptinst, cptvf);
 387
 388         /*
 389          * We allocate and prepare pending queue entry in critical section
 390          * together with submitting CPT instruction to CPT instruction queue
 391          * to make sure that order of CPT requests is the same in both
 392          * pending and instruction queues
 393          */
 394         spin_unlock_bh(&pqueue->lock);
 395
 396         ret = resume_sender ? -EBUSY : -EINPROGRESS;
 397         return ret;
 398
 399 request_cleanup:
 400         do_request_cleanup(pdev, info);
 401         return ret;
 402 }
 403
 404 int otx_cpt_do_request(struct pci_dev *pdev, struct otx_cpt_req_info *req,
 405                        int cpu_num)
 406 {
 407         struct otx_cptvf *cptvf = pci_get_drvdata(pdev);
 408
 409         if (!otx_cpt_device_ready(cptvf)) {
 410                 dev_err(&pdev->dev, "CPT Device is not ready\n");
 411                 return -ENODEV;
 412         }
 413
 414         if ((cptvf->vftype == OTX_CPT_SE_TYPES) && (!req->ctrl.s.se_req)) {
 415                 dev_err(&pdev->dev, "CPTVF-%d of SE TYPE got AE request\n",
 416                         cptvf->vfid);
 417                 return -EINVAL;
 418         } else if ((cptvf->vftype == OTX_CPT_AE_TYPES) &&
 419                    (req->ctrl.s.se_req)) {
 420                 dev_err(&pdev->dev, "CPTVF-%d of AE TYPE got SE request\n",
 421                         cptvf->vfid);
 422                 return -EINVAL;
 423         }
 424
 425         return process_request(pdev, req, &cptvf->pqinfo.queue[0], cptvf);
 426 }
 427
 428 static int cpt_process_ccode(struct pci_dev *pdev,
 429                              union otx_cpt_res_s *cpt_status,
 430                              struct otx_cpt_info_buffer *cpt_info,
 431                              struct otx_cpt_req_info *req, u32 *res_code)
 432 {
 433         u8 ccode = cpt_status->s.compcode;
 434         union otx_cpt_error_code ecode;
 435
 436         ecode.u = be64_to_cpup((__be64 *)cpt_info->out_buffer);
 437         switch (ccode) {
 438         case CPT_COMP_E_FAULT:
 439                 dev_err(&pdev->dev,
 440                         "Request failed with DMA fault\n");
 441                 otx_cpt_dump_sg_list(pdev, req);
 442                 break;
 443
 444         case CPT_COMP_E_SWERR:
 445                 dev_err(&pdev->dev,
 446                         "Request failed with software error code %d\n",
 447                         ecode.s.ccode);
 448                 otx_cpt_dump_sg_list(pdev, req);
 449                 break;
 450
 451         case CPT_COMP_E_HWERR:
 452                 dev_err(&pdev->dev,
 453                         "Request failed with hardware error\n");
 454                 otx_cpt_dump_sg_list(pdev, req);
 455                 break;
 456
 457         case COMPLETION_CODE_INIT:
 458                 /* check for timeout */
 459                 if (time_after_eq(jiffies, cpt_info->time_in +
 460                                   OTX_CPT_COMMAND_TIMEOUT * HZ))
 461                         dev_warn(&pdev->dev, "Request timed out 0x%p\n", req);
 462                 else if (cpt_info->extra_time < OTX_CPT_TIME_IN_RESET_COUNT) {
 463                         cpt_info->time_in = jiffies;
 464                         cpt_info->extra_time++;
 465                 }
 466                 return 1;
 467
 468         case CPT_COMP_E_GOOD:
 469                 /* Check microcode completion code */
 470                 if (ecode.s.ccode) {
 471                         /*
 472                          * If requested hmac is truncated and ucode returns
 473                          * s/g write length error then we report success
 474                          * because ucode writes as many bytes of calculated
 475                          * hmac as available in gather buffer and reports
 476                          * s/g write length error if number of bytes in gather
 477                          * buffer is less than full hmac size.
 478                          */
 479                         if (req->is_trunc_hmac &&
 480                             ecode.s.ccode == ERR_SCATTER_GATHER_WRITE_LENGTH) {
 481                                 *res_code = 0;
 482                                 break;
 483                         }
 484
 485                         dev_err(&pdev->dev,
 486                                 "Request failed with software error code 0x%x\n",
 487                                 ecode.s.ccode);
 488                         otx_cpt_dump_sg_list(pdev, req);
 489                         break;
 490                 }
 491
 492                 /* Request has been processed with success */
 493                 *res_code = 0;
 494                 break;
 495
 496         default:
 497                 dev_err(&pdev->dev, "Request returned invalid status\n");
 498                 break;
 499         }
 500
 501         return 0;
 502 }
 503
 504 static inline void process_pending_queue(struct pci_dev *pdev,
 505                                          struct otx_cpt_pending_queue *pqueue)
 506 {
 507         void (*callback)(int status, void *arg1, void *arg2);
 508         struct otx_cpt_pending_entry *resume_pentry = NULL;
 509         struct otx_cpt_pending_entry *pentry = NULL;
 510         struct otx_cpt_info_buffer *cpt_info = NULL;
 511         union otx_cpt_res_s *cpt_status = NULL;
 512         struct otx_cpt_req_info *req = NULL;
 513         struct crypto_async_request *areq;
 514         u32 res_code, resume_index;
 515
 516         while (1) {
 517                 spin_lock_bh(&pqueue->lock);
 518                 pentry = &pqueue->head[pqueue->front];
 519
 520                 if (WARN_ON(!pentry)) {
 521                         spin_unlock_bh(&pqueue->lock);
 522                         break;
 523                 }
 524
 525                 res_code = -EINVAL;
 526                 if (unlikely(!pentry->busy)) {
 527                         spin_unlock_bh(&pqueue->lock);
 528                         break;
 529                 }
 530
 531                 if (unlikely(!pentry->callback)) {
 532                         dev_err(&pdev->dev, "Callback NULL\n");
 533                         goto process_pentry;
 534                 }
 535
 536                 cpt_info = pentry->info;
 537                 if (unlikely(!cpt_info)) {
 538                         dev_err(&pdev->dev, "Pending entry post arg NULL\n");
 539                         goto process_pentry;
 540                 }
 541
 542                 req = cpt_info->req;
 543                 if (unlikely(!req)) {
 544                         dev_err(&pdev->dev, "Request NULL\n");
 545                         goto process_pentry;
 546                 }
 547
 548                 cpt_status = (union otx_cpt_res_s *) pentry->completion_addr;
 549                 if (unlikely(!cpt_status)) {
 550                         dev_err(&pdev->dev, "Completion address NULL\n");
 551                         goto process_pentry;
 552                 }
 553
 554                 if (cpt_process_ccode(pdev, cpt_status, cpt_info, req,
 555                                       &res_code)) {
 556                         spin_unlock_bh(&pqueue->lock);
 557                         return;
 558                 }
 559                 cpt_info->pdev = pdev;
 560
 561 process_pentry:
 562                 /*
 563                  * Check if we should inform sending side to resume
 564                  * We do it CPT_IQ_RESUME_MARGIN elements in advance before
 565                  * pending queue becomes empty
 566                  */
 567                 resume_index = modulo_inc(pqueue->front, pqueue->qlen,
 568                                           CPT_IQ_RESUME_MARGIN);
 569                 resume_pentry = &pqueue->head[resume_index];
 570                 if (resume_pentry &&
 571                     resume_pentry->resume_sender) {
 572                         resume_pentry->resume_sender = false;
 573                         callback = resume_pentry->callback;
 574                         areq = resume_pentry->areq;
 575
 576                         if (callback) {
 577                                 spin_unlock_bh(&pqueue->lock);
 578
 579                                 /*
 580                                  * EINPROGRESS is an indication for sending
 581                                  * side that it can resume sending requests
 582                                  */
 583                                 callback(-EINPROGRESS, areq, cpt_info);
 584                                 spin_lock_bh(&pqueue->lock);
 585                         }
 586                 }
 587
 588                 callback = pentry->callback;
 589                 areq = pentry->areq;
 590                 free_pentry(pentry);
 591
 592                 pqueue->pending_count--;
 593                 pqueue->front = modulo_inc(pqueue->front, pqueue->qlen, 1);
 594                 spin_unlock_bh(&pqueue->lock);
 595
 596                 /*
 597                  * Call callback after current pending entry has been
 598                  * processed, we don't do it if the callback pointer is
 599                  * invalid.
 600                  */
 601                 if (callback)
 602                         callback(res_code, areq, cpt_info);
 603         }
 604 }
 605
 606 void otx_cpt_post_process(struct otx_cptvf_wqe *wqe)
 607 {
 608         process_pending_queue(wqe->cptvf->pdev, &wqe->cptvf->pqinfo.queue[0]);
 609 }