1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2019 HiSilicon Limited. */
3 #include <crypto/internal/acompress.h>
4 #include <linux/bitfield.h>
5 #include <linux/dma-mapping.h>
6 #include <linux/scatterlist.h>
9 #define HZIP_ZLIB_HEAD_SIZE 2
10 #define HZIP_GZIP_HEAD_SIZE 10
12 #define GZIP_HEAD_FHCRC_BIT BIT(1)
13 #define GZIP_HEAD_FEXTRA_BIT BIT(2)
14 #define GZIP_HEAD_FNAME_BIT BIT(3)
15 #define GZIP_HEAD_FCOMMENT_BIT BIT(4)
17 #define GZIP_HEAD_FLG_SHIFT 3
18 #define GZIP_HEAD_FEXTRA_SHIFT 10
19 #define GZIP_HEAD_FEXTRA_XLEN 2
20 #define GZIP_HEAD_FHCRC_SIZE 2
22 #define HZIP_CTX_Q_NUM 2
23 #define HZIP_GZIP_HEAD_BUF 256
24 #define HZIP_ALG_PRIORITY 300
25 #define HZIP_SGL_SGE_NR 10
27 static const u8 zlib_head[HZIP_ZLIB_HEAD_SIZE] = {0x78, 0x9c};
28 static const u8 gzip_head[HZIP_GZIP_HEAD_SIZE] = {0x1f, 0x8b, 0x08, 0x0, 0x0,
29 0x0, 0x0, 0x0, 0x0, 0x03};
30 enum hisi_zip_alg_type {
31 HZIP_ALG_TYPE_COMP = 0,
32 HZIP_ALG_TYPE_DECOMP = 1,
35 #define COMP_NAME_TO_TYPE(alg_name) \
36 (!strcmp((alg_name), "zlib-deflate") ? HZIP_ALG_TYPE_ZLIB : \
37 !strcmp((alg_name), "gzip") ? HZIP_ALG_TYPE_GZIP : 0) \
39 #define TO_HEAD_SIZE(req_type) \
40 (((req_type) == HZIP_ALG_TYPE_ZLIB) ? sizeof(zlib_head) : \
41 ((req_type) == HZIP_ALG_TYPE_GZIP) ? sizeof(gzip_head) : 0) \
43 #define TO_HEAD(req_type) \
44 (((req_type) == HZIP_ALG_TYPE_ZLIB) ? zlib_head : \
45 ((req_type) == HZIP_ALG_TYPE_GZIP) ? gzip_head : NULL) \
48 struct acomp_req *req;
49 struct scatterlist *src;
50 struct scatterlist *dst;
53 struct hisi_acc_hw_sgl *hw_src;
54 struct hisi_acc_hw_sgl *hw_dst;
60 struct hisi_zip_req_q {
61 struct hisi_zip_req *q;
62 unsigned long *req_bitmap;
67 struct hisi_zip_qp_ctx {
69 struct hisi_zip_sqe zip_sqe;
70 struct hisi_zip_req_q req_q;
71 struct hisi_acc_sgl_pool *sgl_pool;
72 struct hisi_zip *zip_dev;
73 struct hisi_zip_ctx *ctx;
79 struct hisi_zip_qp_ctx qp_ctx[HZIP_CTX_Q_NUM];
82 static int sgl_sge_nr_set(const char *val, const struct kernel_param *kp)
90 ret = kstrtou16(val, 10, &n);
91 if (ret || n == 0 || n > HISI_ACC_SGL_SGE_NR_MAX)
94 return param_set_int(val, kp);
97 static const struct kernel_param_ops sgl_sge_nr_ops = {
98 .set = sgl_sge_nr_set,
102 static u16 sgl_sge_nr = HZIP_SGL_SGE_NR;
103 module_param_cb(sgl_sge_nr, &sgl_sge_nr_ops, &sgl_sge_nr, 0444);
104 MODULE_PARM_DESC(sgl_sge_nr, "Number of sge in sgl(1-255)");
106 static void hisi_zip_config_buf_type(struct hisi_zip_sqe *sqe, u8 buf_type)
110 val = (sqe->dw9) & ~HZIP_BUF_TYPE_M;
111 val |= FIELD_PREP(HZIP_BUF_TYPE_M, buf_type);
115 static void hisi_zip_config_tag(struct hisi_zip_sqe *sqe, u32 tag)
120 static void hisi_zip_fill_sqe(struct hisi_zip_sqe *sqe, u8 req_type,
121 dma_addr_t s_addr, dma_addr_t d_addr, u32 slen,
124 memset(sqe, 0, sizeof(struct hisi_zip_sqe));
126 sqe->input_data_length = slen;
127 sqe->dw9 = FIELD_PREP(HZIP_REQ_TYPE_M, req_type);
128 sqe->dest_avail_out = dlen;
129 sqe->source_addr_l = lower_32_bits(s_addr);
130 sqe->source_addr_h = upper_32_bits(s_addr);
131 sqe->dest_addr_l = lower_32_bits(d_addr);
132 sqe->dest_addr_h = upper_32_bits(d_addr);
135 static int hisi_zip_create_qp(struct hisi_qm *qm, struct hisi_zip_qp_ctx *ctx,
136 int alg_type, int req_type)
141 qp = hisi_qm_create_qp(qm, alg_type);
145 qp->req_type = req_type;
149 ret = hisi_qm_start_qp(qp, 0);
156 hisi_qm_release_qp(qp);
160 static void hisi_zip_release_qp(struct hisi_zip_qp_ctx *ctx)
162 hisi_qm_stop_qp(ctx->qp);
163 hisi_qm_release_qp(ctx->qp);
166 static int hisi_zip_ctx_init(struct hisi_zip_ctx *hisi_zip_ctx, u8 req_type)
168 struct hisi_zip *hisi_zip;
172 /* find the proper zip device */
173 hisi_zip = find_zip_device(cpu_to_node(smp_processor_id()));
175 pr_err("Failed to find a proper ZIP device!\n");
180 for (i = 0; i < HZIP_CTX_Q_NUM; i++) {
181 /* alg_type = 0 for compress, 1 for decompress in hw sqe */
182 ret = hisi_zip_create_qp(qm, &hisi_zip_ctx->qp_ctx[i], i,
187 hisi_zip_ctx->qp_ctx[i].zip_dev = hisi_zip;
192 for (j = i - 1; j >= 0; j--)
193 hisi_zip_release_qp(&hisi_zip_ctx->qp_ctx[j]);
198 static void hisi_zip_ctx_exit(struct hisi_zip_ctx *hisi_zip_ctx)
202 for (i = 1; i >= 0; i--)
203 hisi_zip_release_qp(&hisi_zip_ctx->qp_ctx[i]);
206 static u16 get_extra_field_size(const u8 *start)
208 return *((u16 *)start) + GZIP_HEAD_FEXTRA_XLEN;
211 static u32 get_name_field_size(const u8 *start)
213 return strlen(start) + 1;
216 static u32 get_comment_field_size(const u8 *start)
218 return strlen(start) + 1;
221 static u32 __get_gzip_head_size(const u8 *src)
223 u8 head_flg = *(src + GZIP_HEAD_FLG_SHIFT);
224 u32 size = GZIP_HEAD_FEXTRA_SHIFT;
226 if (head_flg & GZIP_HEAD_FEXTRA_BIT)
227 size += get_extra_field_size(src + size);
228 if (head_flg & GZIP_HEAD_FNAME_BIT)
229 size += get_name_field_size(src + size);
230 if (head_flg & GZIP_HEAD_FCOMMENT_BIT)
231 size += get_comment_field_size(src + size);
232 if (head_flg & GZIP_HEAD_FHCRC_BIT)
233 size += GZIP_HEAD_FHCRC_SIZE;
238 static int hisi_zip_create_req_q(struct hisi_zip_ctx *ctx)
240 struct hisi_zip_req_q *req_q;
243 for (i = 0; i < HZIP_CTX_Q_NUM; i++) {
244 req_q = &ctx->qp_ctx[i].req_q;
245 req_q->size = QM_Q_DEPTH;
247 req_q->req_bitmap = kcalloc(BITS_TO_LONGS(req_q->size),
248 sizeof(long), GFP_KERNEL);
249 if (!req_q->req_bitmap) {
256 rwlock_init(&req_q->req_lock);
258 req_q->q = kcalloc(req_q->size, sizeof(struct hisi_zip_req),
263 goto err_free_bitmap;
272 kfree(ctx->qp_ctx[QPC_DECOMP].req_q.req_bitmap);
274 kfree(ctx->qp_ctx[QPC_COMP].req_q.q);
276 kfree(ctx->qp_ctx[QPC_COMP].req_q.req_bitmap);
280 static void hisi_zip_release_req_q(struct hisi_zip_ctx *ctx)
284 for (i = 0; i < HZIP_CTX_Q_NUM; i++) {
285 kfree(ctx->qp_ctx[i].req_q.q);
286 kfree(ctx->qp_ctx[i].req_q.req_bitmap);
290 static int hisi_zip_create_sgl_pool(struct hisi_zip_ctx *ctx)
292 struct hisi_zip_qp_ctx *tmp;
296 for (i = 0; i < HZIP_CTX_Q_NUM; i++) {
297 tmp = &ctx->qp_ctx[i];
298 dev = &tmp->qp->qm->pdev->dev;
299 tmp->sgl_pool = hisi_acc_create_sgl_pool(dev, QM_Q_DEPTH << 1,
301 if (IS_ERR(tmp->sgl_pool)) {
303 goto err_free_sgl_pool0;
311 hisi_acc_free_sgl_pool(&ctx->qp_ctx[QPC_COMP].qp->qm->pdev->dev,
312 ctx->qp_ctx[QPC_COMP].sgl_pool);
316 static void hisi_zip_release_sgl_pool(struct hisi_zip_ctx *ctx)
320 for (i = 0; i < HZIP_CTX_Q_NUM; i++)
321 hisi_acc_free_sgl_pool(&ctx->qp_ctx[i].qp->qm->pdev->dev,
322 ctx->qp_ctx[i].sgl_pool);
325 static void hisi_zip_remove_req(struct hisi_zip_qp_ctx *qp_ctx,
326 struct hisi_zip_req *req)
328 struct hisi_zip_req_q *req_q = &qp_ctx->req_q;
330 if (qp_ctx->qp->alg_type == HZIP_ALG_TYPE_COMP)
335 write_lock(&req_q->req_lock);
336 clear_bit(req->req_id, req_q->req_bitmap);
337 memset(req, 0, sizeof(struct hisi_zip_req));
338 write_unlock(&req_q->req_lock);
341 static void hisi_zip_acomp_cb(struct hisi_qp *qp, void *data)
343 struct hisi_zip_sqe *sqe = data;
344 struct hisi_zip_qp_ctx *qp_ctx = qp->qp_ctx;
345 struct hisi_zip_req_q *req_q = &qp_ctx->req_q;
346 struct hisi_zip_req *req = req_q->q + sqe->tag;
347 struct acomp_req *acomp_req = req->req;
348 struct device *dev = &qp->qm->pdev->dev;
349 u32 status, dlen, head_size;
352 status = sqe->dw3 & HZIP_BD_STATUS_M;
354 if (status != 0 && status != HZIP_NC_ERR) {
355 dev_err(dev, "%scompress fail in qp%u: %u, output: %u\n",
356 (qp->alg_type == 0) ? "" : "de", qp->qp_id, status,
360 dlen = sqe->produced;
362 hisi_acc_sg_buf_unmap(dev, req->src, req->hw_src);
363 hisi_acc_sg_buf_unmap(dev, req->dst, req->hw_dst);
365 head_size = (qp->alg_type == 0) ? TO_HEAD_SIZE(qp->req_type) : 0;
366 acomp_req->dlen = dlen + head_size;
368 if (acomp_req->base.complete)
369 acomp_request_complete(acomp_req, err);
371 hisi_zip_remove_req(qp_ctx, req);
374 static void hisi_zip_set_acomp_cb(struct hisi_zip_ctx *ctx,
375 void (*fn)(struct hisi_qp *, void *))
379 for (i = 0; i < HZIP_CTX_Q_NUM; i++)
380 ctx->qp_ctx[i].qp->req_cb = fn;
383 static int hisi_zip_acomp_init(struct crypto_acomp *tfm)
385 const char *alg_name = crypto_tfm_alg_name(&tfm->base);
386 struct hisi_zip_ctx *ctx = crypto_tfm_ctx(&tfm->base);
389 ret = hisi_zip_ctx_init(ctx, COMP_NAME_TO_TYPE(alg_name));
393 ret = hisi_zip_create_req_q(ctx);
397 ret = hisi_zip_create_sgl_pool(ctx);
399 goto err_release_req_q;
401 hisi_zip_set_acomp_cb(ctx, hisi_zip_acomp_cb);
406 hisi_zip_release_req_q(ctx);
408 hisi_zip_ctx_exit(ctx);
412 static void hisi_zip_acomp_exit(struct crypto_acomp *tfm)
414 struct hisi_zip_ctx *ctx = crypto_tfm_ctx(&tfm->base);
416 hisi_zip_set_acomp_cb(ctx, NULL);
417 hisi_zip_release_sgl_pool(ctx);
418 hisi_zip_release_req_q(ctx);
419 hisi_zip_ctx_exit(ctx);
422 static int add_comp_head(struct scatterlist *dst, u8 req_type)
424 int head_size = TO_HEAD_SIZE(req_type);
425 const u8 *head = TO_HEAD(req_type);
428 ret = sg_copy_from_buffer(dst, sg_nents(dst), head, head_size);
429 if (ret != head_size)
435 static size_t get_gzip_head_size(struct scatterlist *sgl)
437 char buf[HZIP_GZIP_HEAD_BUF];
439 sg_copy_to_buffer(sgl, sg_nents(sgl), buf, sizeof(buf));
441 return __get_gzip_head_size(buf);
444 static size_t get_comp_head_size(struct scatterlist *src, u8 req_type)
447 case HZIP_ALG_TYPE_ZLIB:
448 return TO_HEAD_SIZE(HZIP_ALG_TYPE_ZLIB);
449 case HZIP_ALG_TYPE_GZIP:
450 return get_gzip_head_size(src);
452 pr_err("request type does not support!\n");
457 static int get_sg_skip_bytes(struct scatterlist *sgl, size_t bytes,
458 size_t remains, struct scatterlist **out)
461 size_t split_sizes[SPLIT_NUM];
462 int out_mapped_nents[SPLIT_NUM];
464 split_sizes[0] = bytes;
465 split_sizes[1] = remains;
467 return sg_split(sgl, 0, 0, SPLIT_NUM, split_sizes, out,
468 out_mapped_nents, GFP_KERNEL);
471 static struct hisi_zip_req *hisi_zip_create_req(struct acomp_req *req,
472 struct hisi_zip_qp_ctx *qp_ctx,
473 size_t head_size, bool is_comp)
475 struct hisi_zip_req_q *req_q = &qp_ctx->req_q;
476 struct hisi_zip_req *q = req_q->q;
477 struct hisi_zip_req *req_cache;
478 struct scatterlist *out[2];
479 struct scatterlist *sgl;
484 * remove/add zlib/gzip head, as hardware operations do not include
485 * comp head. so split req->src to get sgl without heads in acomp, or
486 * add comp head to req->dst ahead of that hardware output compressed
487 * data in sgl splited from req->dst without comp head.
491 len = req->dlen - head_size;
494 len = req->slen - head_size;
497 ret = get_sg_skip_bytes(sgl, head_size, len, out);
501 /* sgl for comp head is useless, so free it now */
504 write_lock(&req_q->req_lock);
506 req_id = find_first_zero_bit(req_q->req_bitmap, req_q->size);
507 if (req_id >= req_q->size) {
508 write_unlock(&req_q->req_lock);
509 dev_dbg(&qp_ctx->qp->qm->pdev->dev, "req cache is full!\n");
511 return ERR_PTR(-EBUSY);
513 set_bit(req_id, req_q->req_bitmap);
515 req_cache = q + req_id;
516 req_cache->req_id = req_id;
517 req_cache->req = req;
519 req_cache->src = req->src;
520 req_cache->dst = out[1];
521 req_cache->slen = req->slen;
522 req_cache->dlen = req->dlen - head_size;
524 req_cache->src = out[1];
525 req_cache->dst = req->dst;
526 req_cache->slen = req->slen - head_size;
527 req_cache->dlen = req->dlen;
530 write_unlock(&req_q->req_lock);
535 static int hisi_zip_do_work(struct hisi_zip_req *req,
536 struct hisi_zip_qp_ctx *qp_ctx)
538 struct hisi_zip_sqe *zip_sqe = &qp_ctx->zip_sqe;
539 struct hisi_qp *qp = qp_ctx->qp;
540 struct device *dev = &qp->qm->pdev->dev;
541 struct hisi_acc_sgl_pool *pool = qp_ctx->sgl_pool;
546 if (!req->src || !req->slen || !req->dst || !req->dlen)
549 req->hw_src = hisi_acc_sg_buf_map_to_hw_sgl(dev, req->src, pool,
550 req->req_id << 1, &input);
551 if (IS_ERR(req->hw_src))
552 return PTR_ERR(req->hw_src);
553 req->dma_src = input;
555 req->hw_dst = hisi_acc_sg_buf_map_to_hw_sgl(dev, req->dst, pool,
556 (req->req_id << 1) + 1,
558 if (IS_ERR(req->hw_dst)) {
559 ret = PTR_ERR(req->hw_dst);
560 goto err_unmap_input;
562 req->dma_dst = output;
564 hisi_zip_fill_sqe(zip_sqe, qp->req_type, input, output, req->slen,
566 hisi_zip_config_buf_type(zip_sqe, HZIP_SGL);
567 hisi_zip_config_tag(zip_sqe, req->req_id);
569 /* send command to start a task */
570 ret = hisi_qp_send(qp, zip_sqe);
572 goto err_unmap_output;
577 hisi_acc_sg_buf_unmap(dev, req->dst, req->hw_dst);
579 hisi_acc_sg_buf_unmap(dev, req->src, req->hw_src);
583 static int hisi_zip_acompress(struct acomp_req *acomp_req)
585 struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm);
586 struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[QPC_COMP];
587 struct hisi_zip_req *req;
591 /* let's output compression head now */
592 head_size = add_comp_head(acomp_req->dst, qp_ctx->qp->req_type);
596 req = hisi_zip_create_req(acomp_req, qp_ctx, (size_t)head_size, true);
600 ret = hisi_zip_do_work(req, qp_ctx);
601 if (ret != -EINPROGRESS)
602 hisi_zip_remove_req(qp_ctx, req);
607 static int hisi_zip_adecompress(struct acomp_req *acomp_req)
609 struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm);
610 struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[QPC_DECOMP];
611 struct hisi_zip_req *req;
615 head_size = get_comp_head_size(acomp_req->src, qp_ctx->qp->req_type);
617 req = hisi_zip_create_req(acomp_req, qp_ctx, head_size, false);
621 ret = hisi_zip_do_work(req, qp_ctx);
622 if (ret != -EINPROGRESS)
623 hisi_zip_remove_req(qp_ctx, req);
628 static struct acomp_alg hisi_zip_acomp_zlib = {
629 .init = hisi_zip_acomp_init,
630 .exit = hisi_zip_acomp_exit,
631 .compress = hisi_zip_acompress,
632 .decompress = hisi_zip_adecompress,
634 .cra_name = "zlib-deflate",
635 .cra_driver_name = "hisi-zlib-acomp",
636 .cra_module = THIS_MODULE,
637 .cra_priority = HZIP_ALG_PRIORITY,
638 .cra_ctxsize = sizeof(struct hisi_zip_ctx),
642 static struct acomp_alg hisi_zip_acomp_gzip = {
643 .init = hisi_zip_acomp_init,
644 .exit = hisi_zip_acomp_exit,
645 .compress = hisi_zip_acompress,
646 .decompress = hisi_zip_adecompress,
649 .cra_driver_name = "hisi-gzip-acomp",
650 .cra_module = THIS_MODULE,
651 .cra_priority = HZIP_ALG_PRIORITY,
652 .cra_ctxsize = sizeof(struct hisi_zip_ctx),
656 int hisi_zip_register_to_crypto(void)
660 ret = crypto_register_acomp(&hisi_zip_acomp_zlib);
662 pr_err("Zlib acomp algorithm registration failed\n");
666 ret = crypto_register_acomp(&hisi_zip_acomp_gzip);
668 pr_err("Gzip acomp algorithm registration failed\n");
669 crypto_unregister_acomp(&hisi_zip_acomp_zlib);
675 void hisi_zip_unregister_from_crypto(void)
677 crypto_unregister_acomp(&hisi_zip_acomp_gzip);
678 crypto_unregister_acomp(&hisi_zip_acomp_zlib);