1 /* Glue code for DES encryption optimized for sparc64 crypto opcodes.
3 * Copyright (C) 2012 David S. Miller <davem@davemloft.net>
6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
8 #include <linux/crypto.h>
9 #include <linux/init.h>
10 #include <linux/module.h>
12 #include <linux/types.h>
13 #include <crypto/algapi.h>
14 #include <crypto/des.h>
16 #include <asm/fpumacro.h>
17 #include <asm/pstate.h>
22 struct des_sparc64_ctx {
23 u64 encrypt_expkey[DES_EXPKEY_WORDS / 2];
24 u64 decrypt_expkey[DES_EXPKEY_WORDS / 2];
27 struct des3_ede_sparc64_ctx {
28 u64 encrypt_expkey[DES3_EDE_EXPKEY_WORDS / 2];
29 u64 decrypt_expkey[DES3_EDE_EXPKEY_WORDS / 2];
32 static void encrypt_to_decrypt(u64 *d, const u64 *e)
34 const u64 *s = e + (DES_EXPKEY_WORDS / 2) - 1;
37 for (i = 0; i < DES_EXPKEY_WORDS / 2; i++)
41 extern void des_sparc64_key_expand(const u32 *input_key, u64 *key);
43 static int des_set_key(struct crypto_tfm *tfm, const u8 *key,
46 struct des_sparc64_ctx *dctx = crypto_tfm_ctx(tfm);
47 u32 *flags = &tfm->crt_flags;
48 u32 tmp[DES_EXPKEY_WORDS];
51 /* Even though we have special instructions for key expansion,
52 * we call des_ekey() so that we don't have to write our own
53 * weak key detection code.
55 ret = des_ekey(tmp, key);
56 if (unlikely(ret == 0) && (*flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) {
57 *flags |= CRYPTO_TFM_RES_WEAK_KEY;
61 des_sparc64_key_expand((const u32 *) key, &dctx->encrypt_expkey[0]);
62 encrypt_to_decrypt(&dctx->decrypt_expkey[0], &dctx->encrypt_expkey[0]);
67 extern void des_sparc64_crypt(const u64 *key, const u64 *input,
70 static void des_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
72 struct des_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
73 const u64 *K = ctx->encrypt_expkey;
75 des_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
78 static void des_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
80 struct des_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
81 const u64 *K = ctx->decrypt_expkey;
83 des_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
86 extern void des_sparc64_load_keys(const u64 *key);
88 extern void des_sparc64_ecb_crypt(const u64 *input, u64 *output,
91 #define DES_BLOCK_MASK (~(DES_BLOCK_SIZE - 1))
93 static int __ecb_crypt(struct blkcipher_desc *desc,
94 struct scatterlist *dst, struct scatterlist *src,
95 unsigned int nbytes, bool encrypt)
97 struct des_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
98 struct blkcipher_walk walk;
101 blkcipher_walk_init(&walk, dst, src, nbytes);
102 err = blkcipher_walk_virt(desc, &walk);
103 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
106 des_sparc64_load_keys(&ctx->encrypt_expkey[0]);
108 des_sparc64_load_keys(&ctx->decrypt_expkey[0]);
109 while ((nbytes = walk.nbytes)) {
110 unsigned int block_len = nbytes & DES_BLOCK_MASK;
112 if (likely(block_len)) {
113 des_sparc64_ecb_crypt((const u64 *)walk.src.virt.addr,
114 (u64 *) walk.dst.virt.addr,
117 nbytes &= DES_BLOCK_SIZE - 1;
118 err = blkcipher_walk_done(desc, &walk, nbytes);
124 static int ecb_encrypt(struct blkcipher_desc *desc,
125 struct scatterlist *dst, struct scatterlist *src,
128 return __ecb_crypt(desc, dst, src, nbytes, true);
131 static int ecb_decrypt(struct blkcipher_desc *desc,
132 struct scatterlist *dst, struct scatterlist *src,
135 return __ecb_crypt(desc, dst, src, nbytes, false);
138 extern void des_sparc64_cbc_encrypt(const u64 *input, u64 *output,
139 unsigned int len, u64 *iv);
141 static int cbc_encrypt(struct blkcipher_desc *desc,
142 struct scatterlist *dst, struct scatterlist *src,
145 struct des_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
146 struct blkcipher_walk walk;
149 blkcipher_walk_init(&walk, dst, src, nbytes);
150 err = blkcipher_walk_virt(desc, &walk);
151 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
153 des_sparc64_load_keys(&ctx->encrypt_expkey[0]);
154 while ((nbytes = walk.nbytes)) {
155 unsigned int block_len = nbytes & DES_BLOCK_MASK;
157 if (likely(block_len)) {
158 des_sparc64_cbc_encrypt((const u64 *)walk.src.virt.addr,
159 (u64 *) walk.dst.virt.addr,
160 block_len, (u64 *) walk.iv);
162 nbytes &= DES_BLOCK_SIZE - 1;
163 err = blkcipher_walk_done(desc, &walk, nbytes);
169 extern void des_sparc64_cbc_decrypt(const u64 *input, u64 *output,
170 unsigned int len, u64 *iv);
172 static int cbc_decrypt(struct blkcipher_desc *desc,
173 struct scatterlist *dst, struct scatterlist *src,
176 struct des_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
177 struct blkcipher_walk walk;
180 blkcipher_walk_init(&walk, dst, src, nbytes);
181 err = blkcipher_walk_virt(desc, &walk);
182 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
184 des_sparc64_load_keys(&ctx->decrypt_expkey[0]);
185 while ((nbytes = walk.nbytes)) {
186 unsigned int block_len = nbytes & DES_BLOCK_MASK;
188 if (likely(block_len)) {
189 des_sparc64_cbc_decrypt((const u64 *)walk.src.virt.addr,
190 (u64 *) walk.dst.virt.addr,
191 block_len, (u64 *) walk.iv);
193 nbytes &= DES_BLOCK_SIZE - 1;
194 err = blkcipher_walk_done(desc, &walk, nbytes);
200 static int des3_ede_set_key(struct crypto_tfm *tfm, const u8 *key,
203 struct des3_ede_sparc64_ctx *dctx = crypto_tfm_ctx(tfm);
204 u32 *flags = &tfm->crt_flags;
205 u64 k1[DES_EXPKEY_WORDS / 2];
206 u64 k2[DES_EXPKEY_WORDS / 2];
207 u64 k3[DES_EXPKEY_WORDS / 2];
210 err = __des3_verify_key(flags, key);
214 des_sparc64_key_expand((const u32 *)key, k1);
216 des_sparc64_key_expand((const u32 *)key, k2);
218 des_sparc64_key_expand((const u32 *)key, k3);
220 memcpy(&dctx->encrypt_expkey[0], &k1[0], sizeof(k1));
221 encrypt_to_decrypt(&dctx->encrypt_expkey[DES_EXPKEY_WORDS / 2], &k2[0]);
222 memcpy(&dctx->encrypt_expkey[(DES_EXPKEY_WORDS / 2) * 2],
225 encrypt_to_decrypt(&dctx->decrypt_expkey[0], &k3[0]);
226 memcpy(&dctx->decrypt_expkey[DES_EXPKEY_WORDS / 2],
228 encrypt_to_decrypt(&dctx->decrypt_expkey[(DES_EXPKEY_WORDS / 2) * 2],
234 extern void des3_ede_sparc64_crypt(const u64 *key, const u64 *input,
237 static void des3_ede_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
239 struct des3_ede_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
240 const u64 *K = ctx->encrypt_expkey;
242 des3_ede_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
245 static void des3_ede_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
247 struct des3_ede_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
248 const u64 *K = ctx->decrypt_expkey;
250 des3_ede_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
253 extern void des3_ede_sparc64_load_keys(const u64 *key);
255 extern void des3_ede_sparc64_ecb_crypt(const u64 *expkey, const u64 *input,
256 u64 *output, unsigned int len);
258 static int __ecb3_crypt(struct blkcipher_desc *desc,
259 struct scatterlist *dst, struct scatterlist *src,
260 unsigned int nbytes, bool encrypt)
262 struct des3_ede_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
263 struct blkcipher_walk walk;
267 blkcipher_walk_init(&walk, dst, src, nbytes);
268 err = blkcipher_walk_virt(desc, &walk);
269 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
272 K = &ctx->encrypt_expkey[0];
274 K = &ctx->decrypt_expkey[0];
275 des3_ede_sparc64_load_keys(K);
276 while ((nbytes = walk.nbytes)) {
277 unsigned int block_len = nbytes & DES_BLOCK_MASK;
279 if (likely(block_len)) {
280 const u64 *src64 = (const u64 *)walk.src.virt.addr;
281 des3_ede_sparc64_ecb_crypt(K, src64,
282 (u64 *) walk.dst.virt.addr,
285 nbytes &= DES_BLOCK_SIZE - 1;
286 err = blkcipher_walk_done(desc, &walk, nbytes);
292 static int ecb3_encrypt(struct blkcipher_desc *desc,
293 struct scatterlist *dst, struct scatterlist *src,
296 return __ecb3_crypt(desc, dst, src, nbytes, true);
299 static int ecb3_decrypt(struct blkcipher_desc *desc,
300 struct scatterlist *dst, struct scatterlist *src,
303 return __ecb3_crypt(desc, dst, src, nbytes, false);
306 extern void des3_ede_sparc64_cbc_encrypt(const u64 *expkey, const u64 *input,
307 u64 *output, unsigned int len,
310 static int cbc3_encrypt(struct blkcipher_desc *desc,
311 struct scatterlist *dst, struct scatterlist *src,
314 struct des3_ede_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
315 struct blkcipher_walk walk;
319 blkcipher_walk_init(&walk, dst, src, nbytes);
320 err = blkcipher_walk_virt(desc, &walk);
321 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
323 K = &ctx->encrypt_expkey[0];
324 des3_ede_sparc64_load_keys(K);
325 while ((nbytes = walk.nbytes)) {
326 unsigned int block_len = nbytes & DES_BLOCK_MASK;
328 if (likely(block_len)) {
329 const u64 *src64 = (const u64 *)walk.src.virt.addr;
330 des3_ede_sparc64_cbc_encrypt(K, src64,
331 (u64 *) walk.dst.virt.addr,
335 nbytes &= DES_BLOCK_SIZE - 1;
336 err = blkcipher_walk_done(desc, &walk, nbytes);
342 extern void des3_ede_sparc64_cbc_decrypt(const u64 *expkey, const u64 *input,
343 u64 *output, unsigned int len,
346 static int cbc3_decrypt(struct blkcipher_desc *desc,
347 struct scatterlist *dst, struct scatterlist *src,
350 struct des3_ede_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
351 struct blkcipher_walk walk;
355 blkcipher_walk_init(&walk, dst, src, nbytes);
356 err = blkcipher_walk_virt(desc, &walk);
357 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
359 K = &ctx->decrypt_expkey[0];
360 des3_ede_sparc64_load_keys(K);
361 while ((nbytes = walk.nbytes)) {
362 unsigned int block_len = nbytes & DES_BLOCK_MASK;
364 if (likely(block_len)) {
365 const u64 *src64 = (const u64 *)walk.src.virt.addr;
366 des3_ede_sparc64_cbc_decrypt(K, src64,
367 (u64 *) walk.dst.virt.addr,
371 nbytes &= DES_BLOCK_SIZE - 1;
372 err = blkcipher_walk_done(desc, &walk, nbytes);
378 static struct crypto_alg algs[] = { {
380 .cra_driver_name = "des-sparc64",
381 .cra_priority = SPARC_CR_OPCODE_PRIORITY,
382 .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
383 .cra_blocksize = DES_BLOCK_SIZE,
384 .cra_ctxsize = sizeof(struct des_sparc64_ctx),
386 .cra_module = THIS_MODULE,
389 .cia_min_keysize = DES_KEY_SIZE,
390 .cia_max_keysize = DES_KEY_SIZE,
391 .cia_setkey = des_set_key,
392 .cia_encrypt = des_encrypt,
393 .cia_decrypt = des_decrypt
397 .cra_name = "ecb(des)",
398 .cra_driver_name = "ecb-des-sparc64",
399 .cra_priority = SPARC_CR_OPCODE_PRIORITY,
400 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
401 .cra_blocksize = DES_BLOCK_SIZE,
402 .cra_ctxsize = sizeof(struct des_sparc64_ctx),
404 .cra_type = &crypto_blkcipher_type,
405 .cra_module = THIS_MODULE,
408 .min_keysize = DES_KEY_SIZE,
409 .max_keysize = DES_KEY_SIZE,
410 .setkey = des_set_key,
411 .encrypt = ecb_encrypt,
412 .decrypt = ecb_decrypt,
416 .cra_name = "cbc(des)",
417 .cra_driver_name = "cbc-des-sparc64",
418 .cra_priority = SPARC_CR_OPCODE_PRIORITY,
419 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
420 .cra_blocksize = DES_BLOCK_SIZE,
421 .cra_ctxsize = sizeof(struct des_sparc64_ctx),
423 .cra_type = &crypto_blkcipher_type,
424 .cra_module = THIS_MODULE,
427 .min_keysize = DES_KEY_SIZE,
428 .max_keysize = DES_KEY_SIZE,
429 .ivsize = DES_BLOCK_SIZE,
430 .setkey = des_set_key,
431 .encrypt = cbc_encrypt,
432 .decrypt = cbc_decrypt,
436 .cra_name = "des3_ede",
437 .cra_driver_name = "des3_ede-sparc64",
438 .cra_priority = SPARC_CR_OPCODE_PRIORITY,
439 .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
440 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
441 .cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx),
443 .cra_module = THIS_MODULE,
446 .cia_min_keysize = DES3_EDE_KEY_SIZE,
447 .cia_max_keysize = DES3_EDE_KEY_SIZE,
448 .cia_setkey = des3_ede_set_key,
449 .cia_encrypt = des3_ede_encrypt,
450 .cia_decrypt = des3_ede_decrypt
454 .cra_name = "ecb(des3_ede)",
455 .cra_driver_name = "ecb-des3_ede-sparc64",
456 .cra_priority = SPARC_CR_OPCODE_PRIORITY,
457 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
458 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
459 .cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx),
461 .cra_type = &crypto_blkcipher_type,
462 .cra_module = THIS_MODULE,
465 .min_keysize = DES3_EDE_KEY_SIZE,
466 .max_keysize = DES3_EDE_KEY_SIZE,
467 .setkey = des3_ede_set_key,
468 .encrypt = ecb3_encrypt,
469 .decrypt = ecb3_decrypt,
473 .cra_name = "cbc(des3_ede)",
474 .cra_driver_name = "cbc-des3_ede-sparc64",
475 .cra_priority = SPARC_CR_OPCODE_PRIORITY,
476 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
477 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
478 .cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx),
480 .cra_type = &crypto_blkcipher_type,
481 .cra_module = THIS_MODULE,
484 .min_keysize = DES3_EDE_KEY_SIZE,
485 .max_keysize = DES3_EDE_KEY_SIZE,
486 .ivsize = DES3_EDE_BLOCK_SIZE,
487 .setkey = des3_ede_set_key,
488 .encrypt = cbc3_encrypt,
489 .decrypt = cbc3_decrypt,
494 static bool __init sparc64_has_des_opcode(void)
498 if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO))
501 __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr));
502 if (!(cfr & CFR_DES))
508 static int __init des_sparc64_mod_init(void)
510 if (sparc64_has_des_opcode()) {
511 pr_info("Using sparc64 des opcodes optimized DES implementation\n");
512 return crypto_register_algs(algs, ARRAY_SIZE(algs));
514 pr_info("sparc64 des opcodes not available.\n");
518 static void __exit des_sparc64_mod_fini(void)
520 crypto_unregister_algs(algs, ARRAY_SIZE(algs));
523 module_init(des_sparc64_mod_init);
524 module_exit(des_sparc64_mod_fini);
526 MODULE_LICENSE("GPL");
527 MODULE_DESCRIPTION("DES & Triple DES EDE Cipher Algorithms, sparc64 des opcode accelerated");
529 MODULE_ALIAS_CRYPTO("des");
530 MODULE_ALIAS_CRYPTO("des3_ede");
532 #include "crop_devid.c"