drivers/crypto/ccree/cc_request_mgr.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */
   3
   4 #include <linux/kernel.h>
   5 #include <linux/nospec.h>
   6 #include "cc_driver.h"
   7 #include "cc_buffer_mgr.h"
   8 #include "cc_request_mgr.h"
   9 #include "cc_pm.h"
  10
  11 #define CC_MAX_POLL_ITER        10
  12 /* The highest descriptor count in used */
  13 #define CC_MAX_DESC_SEQ_LEN     23
  14
  15 struct cc_req_mgr_handle {
  16         /* Request manager resources */
  17         unsigned int hw_queue_size; /* HW capability */
  18         unsigned int min_free_hw_slots;
  19         unsigned int max_used_sw_slots;
  20         struct cc_crypto_req req_queue[MAX_REQUEST_QUEUE_SIZE];
  21         u32 req_queue_head;
  22         u32 req_queue_tail;
  23         u32 axi_completed;
  24         u32 q_free_slots;
  25         /* This lock protects access to HW register
  26          * that must be single request at a time
  27          */
  28         spinlock_t hw_lock;
  29         struct cc_hw_desc compl_desc;
  30         u8 *dummy_comp_buff;
  31         dma_addr_t dummy_comp_buff_dma;
  32
  33         /* backlog queue */
  34         struct list_head backlog;
  35         unsigned int bl_len;
  36         spinlock_t bl_lock; /* protect backlog queue */
  37
  38 #ifdef COMP_IN_WQ
  39         struct workqueue_struct *workq;
  40         struct delayed_work compwork;
  41 #else
  42         struct tasklet_struct comptask;
  43 #endif
  44 };
  45
  46 struct cc_bl_item {
  47         struct cc_crypto_req creq;
  48         struct cc_hw_desc desc[CC_MAX_DESC_SEQ_LEN];
  49         unsigned int len;
  50         struct list_head list;
  51         bool notif;
  52 };
  53
  54 static const u32 cc_cpp_int_masks[CC_CPP_NUM_ALGS][CC_CPP_NUM_SLOTS] = {
  55         { BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_0_INT_BIT_SHIFT),
  56           BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_1_INT_BIT_SHIFT),
  57           BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_2_INT_BIT_SHIFT),
  58           BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_3_INT_BIT_SHIFT),
  59           BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_4_INT_BIT_SHIFT),
  60           BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_5_INT_BIT_SHIFT),
  61           BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_6_INT_BIT_SHIFT),
  62           BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_7_INT_BIT_SHIFT) },
  63         { BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_0_INT_BIT_SHIFT),
  64           BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_1_INT_BIT_SHIFT),
  65           BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_2_INT_BIT_SHIFT),
  66           BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_3_INT_BIT_SHIFT),
  67           BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_4_INT_BIT_SHIFT),
  68           BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_5_INT_BIT_SHIFT),
  69           BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_6_INT_BIT_SHIFT),
  70           BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_7_INT_BIT_SHIFT) }
  71 };
  72
  73 static void comp_handler(unsigned long devarg);
  74 #ifdef COMP_IN_WQ
  75 static void comp_work_handler(struct work_struct *work);
  76 #endif
  77
  78 static inline u32 cc_cpp_int_mask(enum cc_cpp_alg alg, int slot)
  79 {
  80         alg = array_index_nospec(alg, CC_CPP_NUM_ALGS);
  81         slot = array_index_nospec(slot, CC_CPP_NUM_SLOTS);
  82
  83         return cc_cpp_int_masks[alg][slot];
  84 }
  85
  86 void cc_req_mgr_fini(struct cc_drvdata *drvdata)
  87 {
  88         struct cc_req_mgr_handle *req_mgr_h = drvdata->request_mgr_handle;
  89         struct device *dev = drvdata_to_dev(drvdata);
  90
  91         if (!req_mgr_h)
  92                 return; /* Not allocated */
  93
  94         if (req_mgr_h->dummy_comp_buff_dma) {
  95                 dma_free_coherent(dev, sizeof(u32), req_mgr_h->dummy_comp_buff,
  96                                   req_mgr_h->dummy_comp_buff_dma);
  97         }
  98
  99         dev_dbg(dev, "max_used_hw_slots=%d\n", (req_mgr_h->hw_queue_size -
 100                                                 req_mgr_h->min_free_hw_slots));
 101         dev_dbg(dev, "max_used_sw_slots=%d\n", req_mgr_h->max_used_sw_slots);
 102
 103 #ifdef COMP_IN_WQ
 104         flush_workqueue(req_mgr_h->workq);
 105         destroy_workqueue(req_mgr_h->workq);
 106 #else
 107         /* Kill tasklet */
 108         tasklet_kill(&req_mgr_h->comptask);
 109 #endif
 110         kzfree(req_mgr_h);
 111         drvdata->request_mgr_handle = NULL;
 112 }
 113
 114 int cc_req_mgr_init(struct cc_drvdata *drvdata)
 115 {
 116         struct cc_req_mgr_handle *req_mgr_h;
 117         struct device *dev = drvdata_to_dev(drvdata);
 118         int rc = 0;
 119
 120         req_mgr_h = kzalloc(sizeof(*req_mgr_h), GFP_KERNEL);
 121         if (!req_mgr_h) {
 122                 rc = -ENOMEM;
 123                 goto req_mgr_init_err;
 124         }
 125
 126         drvdata->request_mgr_handle = req_mgr_h;
 127
 128         spin_lock_init(&req_mgr_h->hw_lock);
 129         spin_lock_init(&req_mgr_h->bl_lock);
 130         INIT_LIST_HEAD(&req_mgr_h->backlog);
 131
 132 #ifdef COMP_IN_WQ
 133         dev_dbg(dev, "Initializing completion workqueue\n");
 134         req_mgr_h->workq = create_singlethread_workqueue("ccree");
 135         if (!req_mgr_h->workq) {
 136                 dev_err(dev, "Failed creating work queue\n");
 137                 rc = -ENOMEM;
 138                 goto req_mgr_init_err;
 139         }
 140         INIT_DELAYED_WORK(&req_mgr_h->compwork, comp_work_handler);
 141 #else
 142         dev_dbg(dev, "Initializing completion tasklet\n");
 143         tasklet_init(&req_mgr_h->comptask, comp_handler,
 144                      (unsigned long)drvdata);
 145 #endif
 146         req_mgr_h->hw_queue_size = cc_ioread(drvdata,
 147                                              CC_REG(DSCRPTR_QUEUE_SRAM_SIZE));
 148         dev_dbg(dev, "hw_queue_size=0x%08X\n", req_mgr_h->hw_queue_size);
 149         if (req_mgr_h->hw_queue_size < MIN_HW_QUEUE_SIZE) {
 150                 dev_err(dev, "Invalid HW queue size = %u (Min. required is %u)\n",
 151                         req_mgr_h->hw_queue_size, MIN_HW_QUEUE_SIZE);
 152                 rc = -ENOMEM;
 153                 goto req_mgr_init_err;
 154         }
 155         req_mgr_h->min_free_hw_slots = req_mgr_h->hw_queue_size;
 156         req_mgr_h->max_used_sw_slots = 0;
 157
 158         /* Allocate DMA word for "dummy" completion descriptor use */
 159         req_mgr_h->dummy_comp_buff =
 160                 dma_alloc_coherent(dev, sizeof(u32),
 161                                    &req_mgr_h->dummy_comp_buff_dma,
 162                                    GFP_KERNEL);
 163         if (!req_mgr_h->dummy_comp_buff) {
 164                 dev_err(dev, "Not enough memory to allocate DMA (%zu) dropped buffer\n",
 165                         sizeof(u32));
 166                 rc = -ENOMEM;
 167                 goto req_mgr_init_err;
 168         }
 169
 170         /* Init. "dummy" completion descriptor */
 171         hw_desc_init(&req_mgr_h->compl_desc);
 172         set_din_const(&req_mgr_h->compl_desc, 0, sizeof(u32));
 173         set_dout_dlli(&req_mgr_h->compl_desc, req_mgr_h->dummy_comp_buff_dma,
 174                       sizeof(u32), NS_BIT, 1);
 175         set_flow_mode(&req_mgr_h->compl_desc, BYPASS);
 176         set_queue_last_ind(drvdata, &req_mgr_h->compl_desc);
 177
 178         return 0;
 179
 180 req_mgr_init_err:
 181         cc_req_mgr_fini(drvdata);
 182         return rc;
 183 }
 184
 185 static void enqueue_seq(struct cc_drvdata *drvdata, struct cc_hw_desc seq[],
 186                         unsigned int seq_len)
 187 {
 188         int i, w;
 189         void __iomem *reg = drvdata->cc_base + CC_REG(DSCRPTR_QUEUE_WORD0);
 190         struct device *dev = drvdata_to_dev(drvdata);
 191
 192         /*
 193          * We do indeed write all 6 command words to the same
 194          * register. The HW supports this.
 195          */
 196
 197         for (i = 0; i < seq_len; i++) {
 198                 for (w = 0; w <= 5; w++)
 199                         writel_relaxed(seq[i].word[w], reg);
 200
 201                 if (cc_dump_desc)
 202                         dev_dbg(dev, "desc[%02d]: 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
 203                                 i, seq[i].word[0], seq[i].word[1],
 204                                 seq[i].word[2], seq[i].word[3],
 205                                 seq[i].word[4], seq[i].word[5]);
 206         }
 207 }
 208
 209 /*!
 210  * Completion will take place if and only if user requested completion
 211  * by cc_send_sync_request().
 212  *
 213  * \param dev
 214  * \param dx_compl_h The completion event to signal
 215  */
 216 static void request_mgr_complete(struct device *dev, void *dx_compl_h,
 217                                  int dummy)
 218 {
 219         struct completion *this_compl = dx_compl_h;
 220
 221         complete(this_compl);
 222 }
 223
 224 static int cc_queues_status(struct cc_drvdata *drvdata,
 225                             struct cc_req_mgr_handle *req_mgr_h,
 226                             unsigned int total_seq_len)
 227 {
 228         unsigned long poll_queue;
 229         struct device *dev = drvdata_to_dev(drvdata);
 230
 231         /* SW queue is checked only once as it will not
 232          * be changed during the poll because the spinlock_bh
 233          * is held by the thread
 234          */
 235         if (((req_mgr_h->req_queue_head + 1) & (MAX_REQUEST_QUEUE_SIZE - 1)) ==
 236             req_mgr_h->req_queue_tail) {
 237                 dev_err(dev, "SW FIFO is full. req_queue_head=%d sw_fifo_len=%d\n",
 238                         req_mgr_h->req_queue_head, MAX_REQUEST_QUEUE_SIZE);
 239                 return -ENOSPC;
 240         }
 241
 242         if (req_mgr_h->q_free_slots >= total_seq_len)
 243                 return 0;
 244
 245         /* Wait for space in HW queue. Poll constant num of iterations. */
 246         for (poll_queue = 0; poll_queue < CC_MAX_POLL_ITER ; poll_queue++) {
 247                 req_mgr_h->q_free_slots =
 248                         cc_ioread(drvdata, CC_REG(DSCRPTR_QUEUE_CONTENT));
 249                 if (req_mgr_h->q_free_slots < req_mgr_h->min_free_hw_slots)
 250                         req_mgr_h->min_free_hw_slots = req_mgr_h->q_free_slots;
 251
 252                 if (req_mgr_h->q_free_slots >= total_seq_len) {
 253                         /* If there is enough place return */
 254                         return 0;
 255                 }
 256
 257                 dev_dbg(dev, "HW FIFO is full. q_free_slots=%d total_seq_len=%d\n",
 258                         req_mgr_h->q_free_slots, total_seq_len);
 259         }
 260         /* No room in the HW queue try again later */
 261         dev_dbg(dev, "HW FIFO full, timeout. req_queue_head=%d sw_fifo_len=%d q_free_slots=%d total_seq_len=%d\n",
 262                 req_mgr_h->req_queue_head, MAX_REQUEST_QUEUE_SIZE,
 263                 req_mgr_h->q_free_slots, total_seq_len);
 264         return -ENOSPC;
 265 }
 266
 267 /*!
 268  * Enqueue caller request to crypto hardware.
 269  * Need to be called with HW lock held and PM running
 270  *
 271  * \param drvdata
 272  * \param cc_req The request to enqueue
 273  * \param desc The crypto sequence
 274  * \param len The crypto sequence length
 275  * \param add_comp If "true": add an artificial dout DMA to mark completion
 276  *
 277  */
 278 static void cc_do_send_request(struct cc_drvdata *drvdata,
 279                                struct cc_crypto_req *cc_req,
 280                                struct cc_hw_desc *desc, unsigned int len,
 281                                bool add_comp)
 282 {
 283         struct cc_req_mgr_handle *req_mgr_h = drvdata->request_mgr_handle;
 284         unsigned int used_sw_slots;
 285         unsigned int total_seq_len = len; /*initial sequence length*/
 286         struct device *dev = drvdata_to_dev(drvdata);
 287
 288         used_sw_slots = ((req_mgr_h->req_queue_head -
 289                           req_mgr_h->req_queue_tail) &
 290                          (MAX_REQUEST_QUEUE_SIZE - 1));
 291         if (used_sw_slots > req_mgr_h->max_used_sw_slots)
 292                 req_mgr_h->max_used_sw_slots = used_sw_slots;
 293
 294         /* Enqueue request - must be locked with HW lock*/
 295         req_mgr_h->req_queue[req_mgr_h->req_queue_head] = *cc_req;
 296         req_mgr_h->req_queue_head = (req_mgr_h->req_queue_head + 1) &
 297                                     (MAX_REQUEST_QUEUE_SIZE - 1);
 298         /* TODO: Use circ_buf.h ? */
 299
 300         dev_dbg(dev, "Enqueue request head=%u\n", req_mgr_h->req_queue_head);
 301
 302         /*
 303          * We are about to push command to the HW via the command registers
 304          * that may reference host memory. We need to issue a memory barrier
 305          * to make sure there are no outstanding memory writes
 306          */
 307         wmb();
 308
 309         /* STAT_PHASE_4: Push sequence */
 310
 311         enqueue_seq(drvdata, desc, len);
 312
 313         if (add_comp) {
 314                 enqueue_seq(drvdata, &req_mgr_h->compl_desc, 1);
 315                 total_seq_len++;
 316         }
 317
 318         if (req_mgr_h->q_free_slots < total_seq_len) {
 319                 /* This situation should never occur. Maybe indicating problem
 320                  * with resuming power. Set the free slot count to 0 and hope
 321                  * for the best.
 322                  */
 323                 dev_err(dev, "HW free slot count mismatch.");
 324                 req_mgr_h->q_free_slots = 0;
 325         } else {
 326                 /* Update the free slots in HW queue */
 327                 req_mgr_h->q_free_slots -= total_seq_len;
 328         }
 329 }
 330
 331 static void cc_enqueue_backlog(struct cc_drvdata *drvdata,
 332                                struct cc_bl_item *bli)
 333 {
 334         struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle;
 335         struct device *dev = drvdata_to_dev(drvdata);
 336
 337         spin_lock_bh(&mgr->bl_lock);
 338         list_add_tail(&bli->list, &mgr->backlog);
 339         ++mgr->bl_len;
 340         dev_dbg(dev, "+++bl len: %d\n", mgr->bl_len);
 341         spin_unlock_bh(&mgr->bl_lock);
 342         tasklet_schedule(&mgr->comptask);
 343 }
 344
 345 static void cc_proc_backlog(struct cc_drvdata *drvdata)
 346 {
 347         struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle;
 348         struct cc_bl_item *bli;
 349         struct cc_crypto_req *creq;
 350         void *req;
 351         struct device *dev = drvdata_to_dev(drvdata);
 352         int rc;
 353
 354         spin_lock(&mgr->bl_lock);
 355
 356         while (mgr->bl_len) {
 357                 bli = list_first_entry(&mgr->backlog, struct cc_bl_item, list);
 358                 dev_dbg(dev, "---bl len: %d\n", mgr->bl_len);
 359
 360                 spin_unlock(&mgr->bl_lock);
 361
 362
 363                 creq = &bli->creq;
 364                 req = creq->user_arg;
 365
 366                 /*
 367                  * Notify the request we're moving out of the backlog
 368                  * but only if we haven't done so already.
 369                  */
 370                 if (!bli->notif) {
 371                         creq->user_cb(dev, req, -EINPROGRESS);
 372                         bli->notif = true;
 373                 }
 374
 375                 spin_lock(&mgr->hw_lock);
 376
 377                 rc = cc_queues_status(drvdata, mgr, bli->len);
 378                 if (rc) {
 379                         /*
 380                          * There is still not room in the FIFO for
 381                          * this request. Bail out. We'll return here
 382                          * on the next completion irq.
 383                          */
 384                         spin_unlock(&mgr->hw_lock);
 385                         return;
 386                 }
 387
 388                 cc_do_send_request(drvdata, &bli->creq, bli->desc, bli->len,
 389                                    false);
 390                 spin_unlock(&mgr->hw_lock);
 391
 392                 /* Remove ourselves from the backlog list */
 393                 spin_lock(&mgr->bl_lock);
 394                 list_del(&bli->list);
 395                 --mgr->bl_len;
 396                 kfree(bli);
 397         }
 398
 399         spin_unlock(&mgr->bl_lock);
 400 }
 401
 402 int cc_send_request(struct cc_drvdata *drvdata, struct cc_crypto_req *cc_req,
 403                     struct cc_hw_desc *desc, unsigned int len,
 404                     struct crypto_async_request *req)
 405 {
 406         int rc;
 407         struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle;
 408         struct device *dev = drvdata_to_dev(drvdata);
 409         bool backlog_ok = req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG;
 410         gfp_t flags = cc_gfp_flags(req);
 411         struct cc_bl_item *bli;
 412
 413         rc = cc_pm_get(dev);
 414         if (rc) {
 415                 dev_err(dev, "cc_pm_get returned %x\n", rc);
 416                 return rc;
 417         }
 418
 419         spin_lock_bh(&mgr->hw_lock);
 420         rc = cc_queues_status(drvdata, mgr, len);
 421
 422 #ifdef CC_DEBUG_FORCE_BACKLOG
 423         if (backlog_ok)
 424                 rc = -ENOSPC;
 425 #endif /* CC_DEBUG_FORCE_BACKLOG */
 426
 427         if (rc == -ENOSPC && backlog_ok) {
 428                 spin_unlock_bh(&mgr->hw_lock);
 429
 430                 bli = kmalloc(sizeof(*bli), flags);
 431                 if (!bli) {
 432                         cc_pm_put_suspend(dev);
 433                         return -ENOMEM;
 434                 }
 435
 436                 memcpy(&bli->creq, cc_req, sizeof(*cc_req));
 437                 memcpy(&bli->desc, desc, len * sizeof(*desc));
 438                 bli->len = len;
 439                 bli->notif = false;
 440                 cc_enqueue_backlog(drvdata, bli);
 441                 return -EBUSY;
 442         }
 443
 444         if (!rc) {
 445                 cc_do_send_request(drvdata, cc_req, desc, len, false);
 446                 rc = -EINPROGRESS;
 447         }
 448
 449         spin_unlock_bh(&mgr->hw_lock);
 450         return rc;
 451 }
 452
 453 int cc_send_sync_request(struct cc_drvdata *drvdata,
 454                          struct cc_crypto_req *cc_req, struct cc_hw_desc *desc,
 455                          unsigned int len)
 456 {
 457         int rc;
 458         struct device *dev = drvdata_to_dev(drvdata);
 459         struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle;
 460
 461         init_completion(&cc_req->seq_compl);
 462         cc_req->user_cb = request_mgr_complete;
 463         cc_req->user_arg = &cc_req->seq_compl;
 464
 465         rc = cc_pm_get(dev);
 466         if (rc) {
 467                 dev_err(dev, "cc_pm_get returned %x\n", rc);
 468                 return rc;
 469         }
 470
 471         while (true) {
 472                 spin_lock_bh(&mgr->hw_lock);
 473                 rc = cc_queues_status(drvdata, mgr, len + 1);
 474
 475                 if (!rc)
 476                         break;
 477
 478                 spin_unlock_bh(&mgr->hw_lock);
 479                 if (rc != -EAGAIN) {
 480                         cc_pm_put_suspend(dev);
 481                         return rc;
 482                 }
 483                 wait_for_completion_interruptible(&drvdata->hw_queue_avail);
 484                 reinit_completion(&drvdata->hw_queue_avail);
 485         }
 486
 487         cc_do_send_request(drvdata, cc_req, desc, len, true);
 488         spin_unlock_bh(&mgr->hw_lock);
 489         wait_for_completion(&cc_req->seq_compl);
 490         return 0;
 491 }
 492
 493 /*!
 494  * Enqueue caller request to crypto hardware during init process.
 495  * assume this function is not called in middle of a flow,
 496  * since we set QUEUE_LAST_IND flag in the last descriptor.
 497  *
 498  * \param drvdata
 499  * \param desc The crypto sequence
 500  * \param len The crypto sequence length
 501  *
 502  * \return int Returns "0" upon success
 503  */
 504 int send_request_init(struct cc_drvdata *drvdata, struct cc_hw_desc *desc,
 505                       unsigned int len)
 506 {
 507         struct cc_req_mgr_handle *req_mgr_h = drvdata->request_mgr_handle;
 508         unsigned int total_seq_len = len; /*initial sequence length*/
 509         int rc = 0;
 510
 511         /* Wait for space in HW and SW FIFO. Poll for as much as FIFO_TIMEOUT.
 512          */
 513         rc = cc_queues_status(drvdata, req_mgr_h, total_seq_len);
 514         if (rc)
 515                 return rc;
 516
 517         set_queue_last_ind(drvdata, &desc[(len - 1)]);
 518
 519         /*
 520          * We are about to push command to the HW via the command registers
 521          * that may reference host memory. We need to issue a memory barrier
 522          * to make sure there are no outstanding memory writes
 523          */
 524         wmb();
 525         enqueue_seq(drvdata, desc, len);
 526
 527         /* Update the free slots in HW queue */
 528         req_mgr_h->q_free_slots =
 529                 cc_ioread(drvdata, CC_REG(DSCRPTR_QUEUE_CONTENT));
 530
 531         return 0;
 532 }
 533
 534 void complete_request(struct cc_drvdata *drvdata)
 535 {
 536         struct cc_req_mgr_handle *request_mgr_handle =
 537                                                 drvdata->request_mgr_handle;
 538
 539         complete(&drvdata->hw_queue_avail);
 540 #ifdef COMP_IN_WQ
 541         queue_delayed_work(request_mgr_handle->workq,
 542                            &request_mgr_handle->compwork, 0);
 543 #else
 544         tasklet_schedule(&request_mgr_handle->comptask);
 545 #endif
 546 }
 547
 548 #ifdef COMP_IN_WQ
 549 static void comp_work_handler(struct work_struct *work)
 550 {
 551         struct cc_drvdata *drvdata =
 552                 container_of(work, struct cc_drvdata, compwork.work);
 553
 554         comp_handler((unsigned long)drvdata);
 555 }
 556 #endif
 557
 558 static void proc_completions(struct cc_drvdata *drvdata)
 559 {
 560         struct cc_crypto_req *cc_req;
 561         struct device *dev = drvdata_to_dev(drvdata);
 562         struct cc_req_mgr_handle *request_mgr_handle =
 563                                                 drvdata->request_mgr_handle;
 564         unsigned int *tail = &request_mgr_handle->req_queue_tail;
 565         unsigned int *head = &request_mgr_handle->req_queue_head;
 566         int rc;
 567         u32 mask;
 568
 569         while (request_mgr_handle->axi_completed) {
 570                 request_mgr_handle->axi_completed--;
 571
 572                 /* Dequeue request */
 573                 if (*head == *tail) {
 574                         /* We are supposed to handle a completion but our
 575                          * queue is empty. This is not normal. Return and
 576                          * hope for the best.
 577                          */
 578                         dev_err(dev, "Request queue is empty head == tail %u\n",
 579                                 *head);
 580                         break;
 581                 }
 582
 583                 cc_req = &request_mgr_handle->req_queue[*tail];
 584
 585                 if (cc_req->cpp.is_cpp) {
 586
 587                         dev_dbg(dev, "CPP request completion slot: %d alg:%d\n",
 588                                 cc_req->cpp.slot, cc_req->cpp.alg);
 589                         mask = cc_cpp_int_mask(cc_req->cpp.alg,
 590                                                cc_req->cpp.slot);
 591                         rc = (drvdata->irq & mask ? -EPERM : 0);
 592                         dev_dbg(dev, "Got mask: %x irq: %x rc: %d\n", mask,
 593                                 drvdata->irq, rc);
 594                 } else {
 595                         dev_dbg(dev, "None CPP request completion\n");
 596                         rc = 0;
 597                 }
 598
 599                 if (cc_req->user_cb)
 600                         cc_req->user_cb(dev, cc_req->user_arg, rc);
 601                 *tail = (*tail + 1) & (MAX_REQUEST_QUEUE_SIZE - 1);
 602                 dev_dbg(dev, "Dequeue request tail=%u\n", *tail);
 603                 dev_dbg(dev, "Request completed. axi_completed=%d\n",
 604                         request_mgr_handle->axi_completed);
 605                 cc_pm_put_suspend(dev);
 606         }
 607 }
 608
 609 static inline u32 cc_axi_comp_count(struct cc_drvdata *drvdata)
 610 {
 611         return FIELD_GET(AXIM_MON_COMP_VALUE,
 612                          cc_ioread(drvdata, drvdata->axim_mon_offset));
 613 }
 614
 615 /* Deferred service handler, run as interrupt-fired tasklet */
 616 static void comp_handler(unsigned long devarg)
 617 {
 618         struct cc_drvdata *drvdata = (struct cc_drvdata *)devarg;
 619         struct cc_req_mgr_handle *request_mgr_handle =
 620                                                 drvdata->request_mgr_handle;
 621         struct device *dev = drvdata_to_dev(drvdata);
 622         u32 irq;
 623
 624         dev_dbg(dev, "Completion handler called!\n");
 625         irq = (drvdata->irq & drvdata->comp_mask);
 626
 627         /* To avoid the interrupt from firing as we unmask it,
 628          * we clear it now
 629          */
 630         cc_iowrite(drvdata, CC_REG(HOST_ICR), irq);
 631
 632         /* Avoid race with above clear: Test completion counter once more */
 633
 634         request_mgr_handle->axi_completed += cc_axi_comp_count(drvdata);
 635
 636         dev_dbg(dev, "AXI completion after updated: %d\n",
 637                 request_mgr_handle->axi_completed);
 638
 639         while (request_mgr_handle->axi_completed) {
 640                 do {
 641                         drvdata->irq |= cc_ioread(drvdata, CC_REG(HOST_IRR));
 642                         irq = (drvdata->irq & drvdata->comp_mask);
 643                         proc_completions(drvdata);
 644
 645                         /* At this point (after proc_completions()),
 646                          * request_mgr_handle->axi_completed is 0.
 647                          */
 648                         request_mgr_handle->axi_completed +=
 649                                                 cc_axi_comp_count(drvdata);
 650                 } while (request_mgr_handle->axi_completed > 0);
 651
 652                 cc_iowrite(drvdata, CC_REG(HOST_ICR), irq);
 653
 654                 request_mgr_handle->axi_completed += cc_axi_comp_count(drvdata);
 655         }
 656
 657         /* after verifying that there is nothing to do,
 658          * unmask AXI completion interrupt
 659          */
 660         cc_iowrite(drvdata, CC_REG(HOST_IMR),
 661                    cc_ioread(drvdata, CC_REG(HOST_IMR)) & ~drvdata->comp_mask);
 662
 663         cc_proc_backlog(drvdata);
 664         dev_dbg(dev, "Comp. handler done.\n");
 665 }