1 // SPDX-License-Identifier: GPL-2.0
3 * CAAM/SEC 4.x QI transport/backend driver
4 * Queue Interface backend functionality
6 * Copyright 2013-2016 Freescale Semiconductor, Inc.
7 * Copyright 2016-2017 NXP
10 #include <linux/cpumask.h>
11 #include <linux/kthread.h>
12 #include <soc/fsl/qman.h>
18 #include "desc_constr.h"
20 #define PREHDR_RSLS_SHIFT 31
23 * Use a reasonable backlog of frames (per CPU) as congestion threshold,
24 * so that resources used by the in-flight buffers do not become a memory hog.
26 #define MAX_RSP_FQ_BACKLOG_PER_CPU 256
28 #define CAAM_QI_ENQUEUE_RETRIES 10000
30 #define CAAM_NAPI_WEIGHT 63
33 * caam_napi - struct holding CAAM NAPI-related params
34 * @irqtask: IRQ task for QI backend
38 struct napi_struct irqtask;
39 struct qman_portal *p;
43 * caam_qi_pcpu_priv - percpu private data structure to main list of pending
44 * responses expected on each cpu.
45 * @caam_napi: CAAM NAPI params
46 * @net_dev: netdev used by NAPI
47 * @rsp_fq: response FQ from CAAM
49 struct caam_qi_pcpu_priv {
50 struct caam_napi caam_napi;
51 struct net_device net_dev;
52 struct qman_fq *rsp_fq;
53 } ____cacheline_aligned;
55 static DEFINE_PER_CPU(struct caam_qi_pcpu_priv, pcpu_qipriv);
56 static DEFINE_PER_CPU(int, last_cpu);
59 * caam_qi_priv - CAAM QI backend private params
60 * @cgr: QMan congestion group
61 * @qi_pdev: platform device for QI backend
65 struct platform_device *qi_pdev;
68 static struct caam_qi_priv qipriv ____cacheline_aligned;
71 * This is written by only one core - the one that initialized the CGR - and
72 * read by multiple cores (all the others).
74 bool caam_congested __read_mostly;
75 EXPORT_SYMBOL(caam_congested);
77 #ifdef CONFIG_DEBUG_FS
79 * This is a counter for the number of times the congestion group (where all
80 * the request and response queueus are) reached congestion. Incremented
81 * each time the congestion callback is called with congested == true.
83 static u64 times_congested;
87 * This is a a cache of buffers, from which the users of CAAM QI driver
88 * can allocate short (CAAM_QI_MEMCACHE_SIZE) buffers. It's faster than
89 * doing malloc on the hotpath.
90 * NOTE: A more elegant solution would be to have some headroom in the frames
91 * being processed. This could be added by the dpaa-ethernet driver.
92 * This would pose a problem for userspace application processing which
93 * cannot know of this limitation. So for now, this will work.
94 * NOTE: The memcache is SMP-safe. No need to handle spinlocks in-here
96 static struct kmem_cache *qi_cache;
98 int caam_qi_enqueue(struct device *qidev, struct caam_drv_req *req)
106 qm_fd_set_compound(&fd, qm_sg_entry_get_len(&req->fd_sgt[1]));
108 addr = dma_map_single(qidev, req->fd_sgt, sizeof(req->fd_sgt),
110 if (dma_mapping_error(qidev, addr)) {
111 dev_err(qidev, "DMA mapping error for QI enqueue request\n");
114 qm_fd_addr_set64(&fd, addr);
117 ret = qman_enqueue(req->drv_ctx->req_fq, &fd);
124 } while (num_retries < CAAM_QI_ENQUEUE_RETRIES);
126 dev_err(qidev, "qman_enqueue failed: %d\n", ret);
130 EXPORT_SYMBOL(caam_qi_enqueue);
132 static void caam_fq_ern_cb(struct qman_portal *qm, struct qman_fq *fq,
133 const union qm_mr_entry *msg)
135 const struct qm_fd *fd;
136 struct caam_drv_req *drv_req;
137 struct device *qidev = &(raw_cpu_ptr(&pcpu_qipriv)->net_dev.dev);
141 if (qm_fd_get_format(fd) != qm_fd_compound) {
142 dev_err(qidev, "Non-compound FD from CAAM\n");
146 drv_req = (struct caam_drv_req *)phys_to_virt(qm_fd_addr_get64(fd));
149 "Can't find original request for CAAM response\n");
153 dma_unmap_single(drv_req->drv_ctx->qidev, qm_fd_addr(fd),
154 sizeof(drv_req->fd_sgt), DMA_BIDIRECTIONAL);
156 drv_req->cbk(drv_req, -EIO);
159 static struct qman_fq *create_caam_req_fq(struct device *qidev,
160 struct qman_fq *rsp_fq,
165 struct qman_fq *req_fq;
166 struct qm_mcc_initfq opts;
168 req_fq = kzalloc(sizeof(*req_fq), GFP_ATOMIC);
170 return ERR_PTR(-ENOMEM);
172 req_fq->cb.ern = caam_fq_ern_cb;
173 req_fq->cb.fqs = NULL;
175 ret = qman_create_fq(0, QMAN_FQ_FLAG_DYNAMIC_FQID |
176 QMAN_FQ_FLAG_TO_DCPORTAL, req_fq);
178 dev_err(qidev, "Failed to create session req FQ\n");
179 goto create_req_fq_fail;
182 memset(&opts, 0, sizeof(opts));
183 opts.we_mask = cpu_to_be16(QM_INITFQ_WE_FQCTRL | QM_INITFQ_WE_DESTWQ |
184 QM_INITFQ_WE_CONTEXTB |
185 QM_INITFQ_WE_CONTEXTA | QM_INITFQ_WE_CGID);
186 opts.fqd.fq_ctrl = cpu_to_be16(QM_FQCTRL_CPCSTASH | QM_FQCTRL_CGE);
187 qm_fqd_set_destwq(&opts.fqd, qm_channel_caam, 2);
188 opts.fqd.context_b = cpu_to_be32(qman_fq_fqid(rsp_fq));
189 qm_fqd_context_a_set64(&opts.fqd, hwdesc);
190 opts.fqd.cgid = qipriv.cgr.cgrid;
192 ret = qman_init_fq(req_fq, fq_sched_flag, &opts);
194 dev_err(qidev, "Failed to init session req FQ\n");
195 goto init_req_fq_fail;
198 dev_dbg(qidev, "Allocated request FQ %u for CPU %u\n", req_fq->fqid,
203 qman_destroy_fq(req_fq);
209 static int empty_retired_fq(struct device *qidev, struct qman_fq *fq)
213 ret = qman_volatile_dequeue(fq, QMAN_VOLATILE_FLAG_WAIT_INT |
214 QMAN_VOLATILE_FLAG_FINISH,
215 QM_VDQCR_PRECEDENCE_VDQCR |
216 QM_VDQCR_NUMFRAMES_TILLEMPTY);
218 dev_err(qidev, "Volatile dequeue fail for FQ: %u\n", fq->fqid);
223 struct qman_portal *p;
225 p = qman_get_affine_portal(smp_processor_id());
226 qman_p_poll_dqrr(p, 16);
227 } while (fq->flags & QMAN_FQ_STATE_NE);
232 static int kill_fq(struct device *qidev, struct qman_fq *fq)
237 ret = qman_retire_fq(fq, &flags);
239 dev_err(qidev, "qman_retire_fq failed: %d\n", ret);
246 /* Async FQ retirement condition */
248 /* Retry till FQ gets in retired state */
251 } while (fq->state != qman_fq_state_retired);
253 WARN_ON(fq->flags & QMAN_FQ_STATE_BLOCKOOS);
254 WARN_ON(fq->flags & QMAN_FQ_STATE_ORL);
258 if (fq->flags & QMAN_FQ_STATE_NE) {
259 ret = empty_retired_fq(qidev, fq);
261 dev_err(qidev, "empty_retired_fq fail for FQ: %u\n",
267 ret = qman_oos_fq(fq);
269 dev_err(qidev, "OOS of FQID: %u failed\n", fq->fqid);
277 static int empty_caam_fq(struct qman_fq *fq)
280 struct qm_mcr_queryfq_np np;
282 /* Wait till the older CAAM FQ get empty */
284 ret = qman_query_fq_np(fq, &np);
288 if (!qm_mcr_np_get(&np, frm_cnt))
295 * Give extra time for pending jobs from this FQ in holding tanks
302 int caam_drv_ctx_update(struct caam_drv_ctx *drv_ctx, u32 *sh_desc)
306 struct qman_fq *new_fq, *old_fq;
307 struct device *qidev = drv_ctx->qidev;
309 num_words = desc_len(sh_desc);
310 if (num_words > MAX_SDLEN) {
311 dev_err(qidev, "Invalid descriptor len: %d words\n", num_words);
315 /* Note down older req FQ */
316 old_fq = drv_ctx->req_fq;
318 /* Create a new req FQ in parked state */
319 new_fq = create_caam_req_fq(drv_ctx->qidev, drv_ctx->rsp_fq,
320 drv_ctx->context_a, 0);
321 if (IS_ERR_OR_NULL(new_fq)) {
322 dev_err(qidev, "FQ allocation for shdesc update failed\n");
323 return PTR_ERR(new_fq);
326 /* Hook up new FQ to context so that new requests keep queuing */
327 drv_ctx->req_fq = new_fq;
329 /* Empty and remove the older FQ */
330 ret = empty_caam_fq(old_fq);
332 dev_err(qidev, "Old CAAM FQ empty failed: %d\n", ret);
334 /* We can revert to older FQ */
335 drv_ctx->req_fq = old_fq;
337 if (kill_fq(qidev, new_fq))
338 dev_warn(qidev, "New CAAM FQ kill failed\n");
344 * Re-initialise pre-header. Set RSLS and SDLEN.
345 * Update the shared descriptor for driver context.
347 drv_ctx->prehdr[0] = cpu_to_caam32((1 << PREHDR_RSLS_SHIFT) |
349 memcpy(drv_ctx->sh_desc, sh_desc, desc_bytes(sh_desc));
350 dma_sync_single_for_device(qidev, drv_ctx->context_a,
351 sizeof(drv_ctx->sh_desc) +
352 sizeof(drv_ctx->prehdr),
355 /* Put the new FQ in scheduled state */
356 ret = qman_schedule_fq(new_fq);
358 dev_err(qidev, "Fail to sched new CAAM FQ, ecode = %d\n", ret);
361 * We can kill new FQ and revert to old FQ.
362 * Since the desc is already modified, it is success case
365 drv_ctx->req_fq = old_fq;
367 if (kill_fq(qidev, new_fq))
368 dev_warn(qidev, "New CAAM FQ kill failed\n");
369 } else if (kill_fq(qidev, old_fq)) {
370 dev_warn(qidev, "Old CAAM FQ kill failed\n");
375 EXPORT_SYMBOL(caam_drv_ctx_update);
377 struct caam_drv_ctx *caam_drv_ctx_init(struct device *qidev,
384 struct caam_drv_ctx *drv_ctx;
385 const cpumask_t *cpus = qman_affine_cpus();
387 num_words = desc_len(sh_desc);
388 if (num_words > MAX_SDLEN) {
389 dev_err(qidev, "Invalid descriptor len: %d words\n",
391 return ERR_PTR(-EINVAL);
394 drv_ctx = kzalloc(sizeof(*drv_ctx), GFP_ATOMIC);
396 return ERR_PTR(-ENOMEM);
399 * Initialise pre-header - set RSLS and SDLEN - and shared descriptor
402 drv_ctx->prehdr[0] = cpu_to_caam32((1 << PREHDR_RSLS_SHIFT) |
404 memcpy(drv_ctx->sh_desc, sh_desc, desc_bytes(sh_desc));
405 size = sizeof(drv_ctx->prehdr) + sizeof(drv_ctx->sh_desc);
406 hwdesc = dma_map_single(qidev, drv_ctx->prehdr, size,
408 if (dma_mapping_error(qidev, hwdesc)) {
409 dev_err(qidev, "DMA map error for preheader + shdesc\n");
411 return ERR_PTR(-ENOMEM);
413 drv_ctx->context_a = hwdesc;
415 /* If given CPU does not own the portal, choose another one that does */
416 if (!cpumask_test_cpu(*cpu, cpus)) {
417 int *pcpu = &get_cpu_var(last_cpu);
419 *pcpu = cpumask_next(*pcpu, cpus);
420 if (*pcpu >= nr_cpu_ids)
421 *pcpu = cpumask_first(cpus);
424 put_cpu_var(last_cpu);
428 /* Find response FQ hooked with this CPU */
429 drv_ctx->rsp_fq = per_cpu(pcpu_qipriv.rsp_fq, drv_ctx->cpu);
431 /* Attach request FQ */
432 drv_ctx->req_fq = create_caam_req_fq(qidev, drv_ctx->rsp_fq, hwdesc,
433 QMAN_INITFQ_FLAG_SCHED);
434 if (IS_ERR_OR_NULL(drv_ctx->req_fq)) {
435 dev_err(qidev, "create_caam_req_fq failed\n");
436 dma_unmap_single(qidev, hwdesc, size, DMA_BIDIRECTIONAL);
438 return ERR_PTR(-ENOMEM);
441 drv_ctx->qidev = qidev;
444 EXPORT_SYMBOL(caam_drv_ctx_init);
446 void *qi_cache_alloc(gfp_t flags)
448 return kmem_cache_alloc(qi_cache, flags);
450 EXPORT_SYMBOL(qi_cache_alloc);
452 void qi_cache_free(void *obj)
454 kmem_cache_free(qi_cache, obj);
456 EXPORT_SYMBOL(qi_cache_free);
458 static int caam_qi_poll(struct napi_struct *napi, int budget)
460 struct caam_napi *np = container_of(napi, struct caam_napi, irqtask);
462 int cleaned = qman_p_poll_dqrr(np->p, budget);
464 if (cleaned < budget) {
466 qman_p_irqsource_add(np->p, QM_PIRQ_DQRI);
472 void caam_drv_ctx_rel(struct caam_drv_ctx *drv_ctx)
474 if (IS_ERR_OR_NULL(drv_ctx))
477 /* Remove request FQ */
478 if (kill_fq(drv_ctx->qidev, drv_ctx->req_fq))
479 dev_err(drv_ctx->qidev, "Crypto session req FQ kill failed\n");
481 dma_unmap_single(drv_ctx->qidev, drv_ctx->context_a,
482 sizeof(drv_ctx->sh_desc) + sizeof(drv_ctx->prehdr),
486 EXPORT_SYMBOL(caam_drv_ctx_rel);
488 void caam_qi_shutdown(struct device *qidev)
491 struct caam_qi_priv *priv = dev_get_drvdata(qidev);
492 const cpumask_t *cpus = qman_affine_cpus();
494 for_each_cpu(i, cpus) {
495 struct napi_struct *irqtask;
497 irqtask = &per_cpu_ptr(&pcpu_qipriv.caam_napi, i)->irqtask;
498 napi_disable(irqtask);
499 netif_napi_del(irqtask);
501 if (kill_fq(qidev, per_cpu(pcpu_qipriv.rsp_fq, i)))
502 dev_err(qidev, "Rsp FQ kill failed, cpu: %d\n", i);
505 qman_delete_cgr_safe(&priv->cgr);
506 qman_release_cgrid(priv->cgr.cgrid);
508 kmem_cache_destroy(qi_cache);
510 platform_device_unregister(priv->qi_pdev);
513 static void cgr_cb(struct qman_portal *qm, struct qman_cgr *cgr, int congested)
515 caam_congested = congested;
518 #ifdef CONFIG_DEBUG_FS
521 pr_debug_ratelimited("CAAM entered congestion\n");
524 pr_debug_ratelimited("CAAM exited congestion\n");
528 static int caam_qi_napi_schedule(struct qman_portal *p, struct caam_napi *np)
531 * In case of threaded ISR, for RT kernels in_irq() does not return
532 * appropriate value, so use in_serving_softirq to distinguish between
533 * softirq and irq contexts.
535 if (unlikely(in_irq() || !in_serving_softirq())) {
536 /* Disable QMan IRQ source and invoke NAPI */
537 qman_p_irqsource_remove(p, QM_PIRQ_DQRI);
539 napi_schedule(&np->irqtask);
545 static enum qman_cb_dqrr_result caam_rsp_fq_dqrr_cb(struct qman_portal *p,
546 struct qman_fq *rsp_fq,
547 const struct qm_dqrr_entry *dqrr)
549 struct caam_napi *caam_napi = raw_cpu_ptr(&pcpu_qipriv.caam_napi);
550 struct caam_drv_req *drv_req;
551 const struct qm_fd *fd;
552 struct device *qidev = &(raw_cpu_ptr(&pcpu_qipriv)->net_dev.dev);
555 if (caam_qi_napi_schedule(p, caam_napi))
556 return qman_cb_dqrr_stop;
559 status = be32_to_cpu(fd->status);
560 if (unlikely(status)) {
561 u32 ssrc = status & JRSTA_SSRC_MASK;
562 u8 err_id = status & JRSTA_CCBERR_ERRID_MASK;
564 if (ssrc != JRSTA_SSRC_CCB_ERROR ||
565 err_id != JRSTA_CCBERR_ERRID_ICVCHK)
566 dev_err(qidev, "Error: %#x in CAAM response FD\n",
570 if (unlikely(qm_fd_get_format(fd) != qm_fd_compound)) {
571 dev_err(qidev, "Non-compound FD from CAAM\n");
572 return qman_cb_dqrr_consume;
575 drv_req = (struct caam_drv_req *)phys_to_virt(qm_fd_addr_get64(fd));
576 if (unlikely(!drv_req)) {
578 "Can't find original request for caam response\n");
579 return qman_cb_dqrr_consume;
582 dma_unmap_single(drv_req->drv_ctx->qidev, qm_fd_addr(fd),
583 sizeof(drv_req->fd_sgt), DMA_BIDIRECTIONAL);
585 drv_req->cbk(drv_req, status);
586 return qman_cb_dqrr_consume;
589 static int alloc_rsp_fq_cpu(struct device *qidev, unsigned int cpu)
591 struct qm_mcc_initfq opts;
595 fq = kzalloc(sizeof(*fq), GFP_KERNEL | GFP_DMA);
599 fq->cb.dqrr = caam_rsp_fq_dqrr_cb;
601 ret = qman_create_fq(0, QMAN_FQ_FLAG_NO_ENQUEUE |
602 QMAN_FQ_FLAG_DYNAMIC_FQID, fq);
604 dev_err(qidev, "Rsp FQ create failed\n");
609 memset(&opts, 0, sizeof(opts));
610 opts.we_mask = cpu_to_be16(QM_INITFQ_WE_FQCTRL | QM_INITFQ_WE_DESTWQ |
611 QM_INITFQ_WE_CONTEXTB |
612 QM_INITFQ_WE_CONTEXTA | QM_INITFQ_WE_CGID);
613 opts.fqd.fq_ctrl = cpu_to_be16(QM_FQCTRL_CTXASTASHING |
614 QM_FQCTRL_CPCSTASH | QM_FQCTRL_CGE);
615 qm_fqd_set_destwq(&opts.fqd, qman_affine_channel(cpu), 3);
616 opts.fqd.cgid = qipriv.cgr.cgrid;
617 opts.fqd.context_a.stashing.exclusive = QM_STASHING_EXCL_CTX |
618 QM_STASHING_EXCL_DATA;
619 qm_fqd_set_stashing(&opts.fqd, 0, 1, 1);
621 ret = qman_init_fq(fq, QMAN_INITFQ_FLAG_SCHED, &opts);
623 dev_err(qidev, "Rsp FQ init failed\n");
628 per_cpu(pcpu_qipriv.rsp_fq, cpu) = fq;
630 dev_dbg(qidev, "Allocated response FQ %u for CPU %u", fq->fqid, cpu);
634 static int init_cgr(struct device *qidev)
637 struct qm_mcc_initcgr opts;
638 const u64 val = (u64)cpumask_weight(qman_affine_cpus()) *
639 MAX_RSP_FQ_BACKLOG_PER_CPU;
641 ret = qman_alloc_cgrid(&qipriv.cgr.cgrid);
643 dev_err(qidev, "CGR alloc failed for rsp FQs: %d\n", ret);
647 qipriv.cgr.cb = cgr_cb;
648 memset(&opts, 0, sizeof(opts));
649 opts.we_mask = cpu_to_be16(QM_CGR_WE_CSCN_EN | QM_CGR_WE_CS_THRES |
651 opts.cgr.cscn_en = QM_CGR_EN;
652 opts.cgr.mode = QMAN_CGR_MODE_FRAME;
653 qm_cgr_cs_thres_set64(&opts.cgr.cs_thres, val, 1);
655 ret = qman_create_cgr(&qipriv.cgr, QMAN_CGR_FLAG_USE_INIT, &opts);
657 dev_err(qidev, "Error %d creating CAAM CGRID: %u\n", ret,
662 dev_dbg(qidev, "Congestion threshold set to %llu\n", val);
666 static int alloc_rsp_fqs(struct device *qidev)
669 const cpumask_t *cpus = qman_affine_cpus();
671 /*Now create response FQs*/
672 for_each_cpu(i, cpus) {
673 ret = alloc_rsp_fq_cpu(qidev, i);
675 dev_err(qidev, "CAAM rsp FQ alloc failed, cpu: %u", i);
683 static void free_rsp_fqs(void)
686 const cpumask_t *cpus = qman_affine_cpus();
688 for_each_cpu(i, cpus)
689 kfree(per_cpu(pcpu_qipriv.rsp_fq, i));
692 int caam_qi_init(struct platform_device *caam_pdev)
695 struct platform_device *qi_pdev;
696 struct device *ctrldev = &caam_pdev->dev, *qidev;
697 struct caam_drv_private *ctrlpriv;
698 const cpumask_t *cpus = qman_affine_cpus();
699 static struct platform_device_info qi_pdev_info = {
701 .id = PLATFORM_DEVID_NONE
704 qi_pdev_info.parent = ctrldev;
705 qi_pdev_info.dma_mask = dma_get_mask(ctrldev);
706 qi_pdev = platform_device_register_full(&qi_pdev_info);
708 return PTR_ERR(qi_pdev);
709 set_dma_ops(&qi_pdev->dev, get_dma_ops(ctrldev));
711 ctrlpriv = dev_get_drvdata(ctrldev);
712 qidev = &qi_pdev->dev;
714 qipriv.qi_pdev = qi_pdev;
715 dev_set_drvdata(qidev, &qipriv);
717 /* Initialize the congestion detection */
718 err = init_cgr(qidev);
720 dev_err(qidev, "CGR initialization failed: %d\n", err);
721 platform_device_unregister(qi_pdev);
725 /* Initialise response FQs */
726 err = alloc_rsp_fqs(qidev);
728 dev_err(qidev, "Can't allocate CAAM response FQs: %d\n", err);
730 platform_device_unregister(qi_pdev);
735 * Enable the NAPI contexts on each of the core which has an affine
738 for_each_cpu(i, cpus) {
739 struct caam_qi_pcpu_priv *priv = per_cpu_ptr(&pcpu_qipriv, i);
740 struct caam_napi *caam_napi = &priv->caam_napi;
741 struct napi_struct *irqtask = &caam_napi->irqtask;
742 struct net_device *net_dev = &priv->net_dev;
744 net_dev->dev = *qidev;
745 INIT_LIST_HEAD(&net_dev->napi_list);
747 netif_napi_add(net_dev, irqtask, caam_qi_poll,
750 napi_enable(irqtask);
753 /* Hook up QI device to parent controlling caam device */
754 ctrlpriv->qidev = qidev;
756 qi_cache = kmem_cache_create("caamqicache", CAAM_QI_MEMCACHE_SIZE, 0,
757 SLAB_CACHE_DMA, NULL);
759 dev_err(qidev, "Can't allocate CAAM cache\n");
761 platform_device_unregister(qi_pdev);
765 #ifdef CONFIG_DEBUG_FS
766 debugfs_create_file("qi_congested", 0444, ctrlpriv->ctl,
767 ×_congested, &caam_fops_u64_ro);
769 dev_info(qidev, "Linux CAAM Queue I/F driver initialised\n");