Commit | Line | Data |
---|---|---|
5b2efa2b TZ |
1 | /* SPDX-License-Identifier: GPL-2.0-or-later */ |
2 | /* | |
3 | * SM4 Cipher Algorithm, AES-NI/AVX2 optimized. | |
4 | * as specified in | |
5 | * https://tools.ietf.org/id/draft-ribose-cfrg-sm4-10.html | |
6 | * | |
7 | * Copyright (c) 2021, Alibaba Group. | |
8 | * Copyright (c) 2021 Tianjia Zhang <tianjia.zhang@linux.alibaba.com> | |
9 | */ | |
10 | ||
11 | #include <linux/module.h> | |
12 | #include <linux/crypto.h> | |
13 | #include <linux/kernel.h> | |
14 | #include <asm/simd.h> | |
15 | #include <crypto/internal/simd.h> | |
16 | #include <crypto/internal/skcipher.h> | |
17 | #include <crypto/sm4.h> | |
18 | #include "sm4-avx.h" | |
19 | ||
20 | #define SM4_CRYPT16_BLOCK_SIZE (SM4_BLOCK_SIZE * 16) | |
21 | ||
22 | asmlinkage void sm4_aesni_avx2_ctr_enc_blk16(const u32 *rk, u8 *dst, | |
23 | const u8 *src, u8 *iv); | |
24 | asmlinkage void sm4_aesni_avx2_cbc_dec_blk16(const u32 *rk, u8 *dst, | |
25 | const u8 *src, u8 *iv); | |
5b2efa2b TZ |
26 | |
27 | static int sm4_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, | |
28 | unsigned int key_len) | |
29 | { | |
30 | struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm); | |
31 | ||
32 | return sm4_expandkey(ctx, key, key_len); | |
33 | } | |
34 | ||
35 | static int cbc_decrypt(struct skcipher_request *req) | |
36 | { | |
37 | return sm4_avx_cbc_decrypt(req, SM4_CRYPT16_BLOCK_SIZE, | |
38 | sm4_aesni_avx2_cbc_dec_blk16); | |
39 | } | |
40 | ||
41 | ||
5b2efa2b TZ |
42 | static int ctr_crypt(struct skcipher_request *req) |
43 | { | |
44 | return sm4_avx_ctr_crypt(req, SM4_CRYPT16_BLOCK_SIZE, | |
45 | sm4_aesni_avx2_ctr_enc_blk16); | |
46 | } | |
47 | ||
48 | static struct skcipher_alg sm4_aesni_avx2_skciphers[] = { | |
49 | { | |
50 | .base = { | |
51 | .cra_name = "__ecb(sm4)", | |
52 | .cra_driver_name = "__ecb-sm4-aesni-avx2", | |
53 | .cra_priority = 500, | |
54 | .cra_flags = CRYPTO_ALG_INTERNAL, | |
55 | .cra_blocksize = SM4_BLOCK_SIZE, | |
56 | .cra_ctxsize = sizeof(struct sm4_ctx), | |
57 | .cra_module = THIS_MODULE, | |
58 | }, | |
59 | .min_keysize = SM4_KEY_SIZE, | |
60 | .max_keysize = SM4_KEY_SIZE, | |
61 | .walksize = 16 * SM4_BLOCK_SIZE, | |
62 | .setkey = sm4_skcipher_setkey, | |
63 | .encrypt = sm4_avx_ecb_encrypt, | |
64 | .decrypt = sm4_avx_ecb_decrypt, | |
65 | }, { | |
66 | .base = { | |
67 | .cra_name = "__cbc(sm4)", | |
68 | .cra_driver_name = "__cbc-sm4-aesni-avx2", | |
69 | .cra_priority = 500, | |
70 | .cra_flags = CRYPTO_ALG_INTERNAL, | |
71 | .cra_blocksize = SM4_BLOCK_SIZE, | |
72 | .cra_ctxsize = sizeof(struct sm4_ctx), | |
73 | .cra_module = THIS_MODULE, | |
74 | }, | |
75 | .min_keysize = SM4_KEY_SIZE, | |
76 | .max_keysize = SM4_KEY_SIZE, | |
77 | .ivsize = SM4_BLOCK_SIZE, | |
78 | .walksize = 16 * SM4_BLOCK_SIZE, | |
79 | .setkey = sm4_skcipher_setkey, | |
80 | .encrypt = sm4_cbc_encrypt, | |
81 | .decrypt = cbc_decrypt, | |
5b2efa2b TZ |
82 | }, { |
83 | .base = { | |
84 | .cra_name = "__ctr(sm4)", | |
85 | .cra_driver_name = "__ctr-sm4-aesni-avx2", | |
86 | .cra_priority = 500, | |
87 | .cra_flags = CRYPTO_ALG_INTERNAL, | |
88 | .cra_blocksize = 1, | |
89 | .cra_ctxsize = sizeof(struct sm4_ctx), | |
90 | .cra_module = THIS_MODULE, | |
91 | }, | |
92 | .min_keysize = SM4_KEY_SIZE, | |
93 | .max_keysize = SM4_KEY_SIZE, | |
94 | .ivsize = SM4_BLOCK_SIZE, | |
95 | .chunksize = SM4_BLOCK_SIZE, | |
96 | .walksize = 16 * SM4_BLOCK_SIZE, | |
97 | .setkey = sm4_skcipher_setkey, | |
98 | .encrypt = ctr_crypt, | |
99 | .decrypt = ctr_crypt, | |
100 | } | |
101 | }; | |
102 | ||
103 | static struct simd_skcipher_alg * | |
104 | simd_sm4_aesni_avx2_skciphers[ARRAY_SIZE(sm4_aesni_avx2_skciphers)]; | |
105 | ||
106 | static int __init sm4_init(void) | |
107 | { | |
108 | const char *feature_name; | |
109 | ||
110 | if (!boot_cpu_has(X86_FEATURE_AVX) || | |
111 | !boot_cpu_has(X86_FEATURE_AVX2) || | |
112 | !boot_cpu_has(X86_FEATURE_AES) || | |
113 | !boot_cpu_has(X86_FEATURE_OSXSAVE)) { | |
114 | pr_info("AVX2 or AES-NI instructions are not detected.\n"); | |
115 | return -ENODEV; | |
116 | } | |
117 | ||
118 | if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, | |
119 | &feature_name)) { | |
120 | pr_info("CPU feature '%s' is not supported.\n", feature_name); | |
121 | return -ENODEV; | |
122 | } | |
123 | ||
124 | return simd_register_skciphers_compat(sm4_aesni_avx2_skciphers, | |
125 | ARRAY_SIZE(sm4_aesni_avx2_skciphers), | |
126 | simd_sm4_aesni_avx2_skciphers); | |
127 | } | |
128 | ||
129 | static void __exit sm4_exit(void) | |
130 | { | |
131 | simd_unregister_skciphers(sm4_aesni_avx2_skciphers, | |
132 | ARRAY_SIZE(sm4_aesni_avx2_skciphers), | |
133 | simd_sm4_aesni_avx2_skciphers); | |
134 | } | |
135 | ||
136 | module_init(sm4_init); | |
137 | module_exit(sm4_exit); | |
138 | ||
139 | MODULE_LICENSE("GPL v2"); | |
140 | MODULE_AUTHOR("Tianjia Zhang <tianjia.zhang@linux.alibaba.com>"); | |
141 | MODULE_DESCRIPTION("SM4 Cipher Algorithm, AES-NI/AVX2 optimized"); | |
142 | MODULE_ALIAS_CRYPTO("sm4"); | |
143 | MODULE_ALIAS_CRYPTO("sm4-aesni-avx2"); |