projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
mm: replace get_user_pages() write/force parameters with gup_flags
[linux-2.6-block.git] / arch / cris / arch-v32 / drivers / cryptocop.c
CommitLineData
635c45c1 1/*
51533b61
MS		 2 * Stream co-processor driver for the ETRAX FS
3 *
635c45c1 4 * Copyright (C) 2003-2007 Axis Communications AB
51533b61
MS		 5 */
6
7#include <linux/init.h>
8#include <linux/sched.h>
9#include <linux/module.h>
10#include <linux/slab.h>
11#include <linux/string.h>
12#include <linux/fs.h>
13#include <linux/mm.h>
14#include <linux/spinlock.h>
15#include <linux/stddef.h>
16
17#include <asm/uaccess.h>
18#include <asm/io.h>
60063497 19#include <linux/atomic.h>
51533b61
MS		 20
21#include <linux/list.h>
22#include <linux/interrupt.h>
23
24#include <asm/signal.h>
25#include <asm/irq.h>
26
635c45c1
JN		 27#include <dma.h>
28#include <hwregs/dma.h>
29#include <hwregs/reg_map.h>
30#include <hwregs/reg_rdwr.h>
31#include <hwregs/intr_vect_defs.h>
32
33#include <hwregs/strcop.h>
34#include <hwregs/strcop_defs.h>
35#include <cryptocop.h>
36
37#ifdef CONFIG_ETRAXFS
38#define IN_DMA 9
39#define OUT_DMA 8
40#define IN_DMA_INST regi_dma9
41#define OUT_DMA_INST regi_dma8
42#define DMA_IRQ DMA9_INTR_VECT
43#else
44#define IN_DMA 3
45#define OUT_DMA 2
46#define IN_DMA_INST regi_dma3
47#define OUT_DMA_INST regi_dma2
48#define DMA_IRQ DMA3_INTR_VECT
49#endif
51533b61
MS		 50
51#define DESCR_ALLOC_PAD (31)
52
53struct cryptocop_dma_desc {
54 char *free_buf; /* If non-null will be kfreed in free_cdesc() */
55 dma_descr_data *dma_descr;
56
57 unsigned char dma_descr_buf[sizeof(dma_descr_data) + DESCR_ALLOC_PAD];
58
59 unsigned int from_pool:1; /* If 1 'allocated' from the descriptor pool. */
60 struct cryptocop_dma_desc *next;
61};
62
63
64struct cryptocop_int_operation{
65 void *alloc_ptr;
66 cryptocop_session_id sid;
67
68 dma_descr_context ctx_out;
69 dma_descr_context ctx_in;
70
71 /* DMA descriptors allocated by driver. */
72 struct cryptocop_dma_desc *cdesc_out;
73 struct cryptocop_dma_desc *cdesc_in;
74
75 /* Strcop config to use. */
76 cryptocop_3des_mode tdes_mode;
77 cryptocop_csum_type csum_mode;
78
79 /* DMA descrs provided by consumer. */
80 dma_descr_data *ddesc_out;
81 dma_descr_data *ddesc_in;
82};
83
84
85struct cryptocop_tfrm_ctx {
86 cryptocop_tfrm_id tid;
87 unsigned int blocklength;
88
89 unsigned int start_ix;
90
91 struct cryptocop_tfrm_cfg *tcfg;
92 struct cryptocop_transform_ctx *tctx;
93
94 unsigned char previous_src;
95 unsigned char current_src;
96
97 /* Values to use in metadata out. */
98 unsigned char hash_conf;
99 unsigned char hash_mode;
100 unsigned char ciph_conf;
101 unsigned char cbcmode;
102 unsigned char decrypt;
103
104 unsigned int requires_padding:1;
105 unsigned int strict_block_length:1;
106 unsigned int active:1;
107 unsigned int done:1;
108 size_t consumed;
109 size_t produced;
110
111 /* Pad (input) descriptors to put in the DMA out list when the transform
112 * output is put on the DMA in list. */
113 struct cryptocop_dma_desc *pad_descs;
114
115 struct cryptocop_tfrm_ctx *prev_src;
116 struct cryptocop_tfrm_ctx *curr_src;
117
118 /* Mapping to HW. */
119 unsigned char unit_no;
120};
121
122
123struct cryptocop_private{
124 cryptocop_session_id sid;
125 struct cryptocop_private *next;
126};
127
128/* Session list. */
129
130struct cryptocop_transform_ctx{
131 struct cryptocop_transform_init init;
132 unsigned char dec_key[CRYPTOCOP_MAX_KEY_LENGTH];
133 unsigned int dec_key_set:1;
134
135 struct cryptocop_transform_ctx *next;
136};
137
138
139struct cryptocop_session{
140 cryptocop_session_id sid;
141
142 struct cryptocop_transform_ctx *tfrm_ctx;
143
144 struct cryptocop_session *next;
145};
146
147/* Priority levels for jobs sent to the cryptocop. Checksum operations from
148 kernel have highest priority since TCPIP stack processing must not
149 be a bottleneck. */
150typedef enum {
151 cryptocop_prio_kernel_csum = 0,
152 cryptocop_prio_kernel = 1,
153 cryptocop_prio_user = 2,
154 cryptocop_prio_no_prios = 3
155} cryptocop_queue_priority;
156
157struct cryptocop_prio_queue{
158 struct list_head jobs;
159 cryptocop_queue_priority prio;
160};
161
162struct cryptocop_prio_job{
163 struct list_head node;
164 cryptocop_queue_priority prio;
165
166 struct cryptocop_operation *oper;
167 struct cryptocop_int_operation *iop;
168};
169
170struct ioctl_job_cb_ctx {
171 unsigned int processed:1;
172};
173
174
175static struct cryptocop_session *cryptocop_sessions = NULL;
176spinlock_t cryptocop_sessions_lock;
177
178/* Next Session ID to assign. */
179static cryptocop_session_id next_sid = 1;
180
181/* Pad for checksum. */
182static const char csum_zero_pad[1] = {0x00};
183
184/* Trash buffer for mem2mem operations. */
185#define MEM2MEM_DISCARD_BUF_LENGTH (512)
186static unsigned char mem2mem_discard_buf[MEM2MEM_DISCARD_BUF_LENGTH];
187
188/* Descriptor pool. */
189/* FIXME Tweak this value. */
190#define CRYPTOCOP_DESCRIPTOR_POOL_SIZE (100)
191static struct cryptocop_dma_desc descr_pool[CRYPTOCOP_DESCRIPTOR_POOL_SIZE];
192static struct cryptocop_dma_desc *descr_pool_free_list;
193static int descr_pool_no_free;
194static spinlock_t descr_pool_lock;
195
196/* Lock to stop cryptocop to start processing of a new operation. The holder
197 of this lock MUST call cryptocop_start_job() after it is unlocked. */
198spinlock_t cryptocop_process_lock;
199
200static struct cryptocop_prio_queue cryptocop_job_queues[cryptocop_prio_no_prios];
201static spinlock_t cryptocop_job_queue_lock;
202static struct cryptocop_prio_job *cryptocop_running_job = NULL;
203static spinlock_t running_job_lock;
204
205/* The interrupt handler appends completed jobs to this list. The scehduled
206 * tasklet removes them upon sending the response to the crypto consumer. */
207static struct list_head cryptocop_completed_jobs;
208static spinlock_t cryptocop_completed_jobs_lock;
209
210DECLARE_WAIT_QUEUE_HEAD(cryptocop_ioc_process_wq);
211
212
213/** Local functions. **/
214
215static int cryptocop_open(struct inode *, struct file *);
216
217static int cryptocop_release(struct inode *, struct file *);
218
90276a1a 219static long cryptocop_ioctl(struct file *file,
51533b61
MS		 220 unsigned int cmd, unsigned long arg);
221
222static void cryptocop_start_job(void);
223
224static int cryptocop_job_queue_insert(cryptocop_queue_priority prio, struct cryptocop_operation *operation);
225static int cryptocop_job_setup(struct cryptocop_prio_job **pj, struct cryptocop_operation *operation);
226
227static int cryptocop_job_queue_init(void);
228static void cryptocop_job_queue_close(void);
229
230static int create_md5_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length);
231
232static int create_sha1_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length);
233
234static int transform_ok(struct cryptocop_transform_init *tinit);
235
236static struct cryptocop_session *get_session(cryptocop_session_id sid);
237
238static struct cryptocop_transform_ctx *get_transform_ctx(struct cryptocop_session *sess, cryptocop_tfrm_id tid);
239
240static void delete_internal_operation(struct cryptocop_int_operation *iop);
241
242static void get_aes_decrypt_key(unsigned char *dec_key, const unsigned char *key, unsigned int keylength);
243
244static int init_stream_coprocessor(void);
245
246static void __exit exit_stream_coprocessor(void);
247
248/*#define LDEBUG*/
249#ifdef LDEBUG
250#define DEBUG(s) s
251#define DEBUG_API(s) s
252static void print_cryptocop_operation(struct cryptocop_operation *cop);
253static void print_dma_descriptors(struct cryptocop_int_operation *iop);
254static void print_strcop_crypto_op(struct strcop_crypto_op *cop);
255static void print_lock_status(void);
256static void print_user_dma_lists(struct cryptocop_dma_list_operation *dma_op);
257#define assert(s) do{if (!(s)) panic(#s);} while(0);
258#else
259#define DEBUG(s)
260#define DEBUG_API(s)
261#define assert(s)
262#endif
263
264
265/* Transform constants. */
266#define DES_BLOCK_LENGTH (8)
267#define AES_BLOCK_LENGTH (16)
268#define MD5_BLOCK_LENGTH (64)
269#define SHA1_BLOCK_LENGTH (64)
270#define CSUM_BLOCK_LENGTH (2)
271#define MD5_STATE_LENGTH (16)
272#define SHA1_STATE_LENGTH (20)
273
274/* The device number. */
275#define CRYPTOCOP_MAJOR (254)
276#define CRYPTOCOP_MINOR (0)
277
278
279
5dfe4c96 280const struct file_operations cryptocop_fops = {
90276a1a
JN		 281 .owner = THIS_MODULE,
282 .open = cryptocop_open,
283 .release = cryptocop_release,
6038f373
AB		 284 .unlocked_ioctl = cryptocop_ioctl,
285 .llseek = noop_llseek,
51533b61
MS		 286};
287
288
289static void free_cdesc(struct cryptocop_dma_desc *cdesc)
290{
291 DEBUG(printk("free_cdesc: cdesc 0x%p, from_pool=%d\n", cdesc, cdesc->from_pool));
b2325fe1 292 kfree(cdesc->free_buf);
51533b61
MS		 293
294 if (cdesc->from_pool) {
295 unsigned long int flags;
296 spin_lock_irqsave(&descr_pool_lock, flags);
297 cdesc->next = descr_pool_free_list;
298 descr_pool_free_list = cdesc;
299 ++descr_pool_no_free;
300 spin_unlock_irqrestore(&descr_pool_lock, flags);
301 } else {
302 kfree(cdesc);
303 }
304}
305
306
307static struct cryptocop_dma_desc *alloc_cdesc(int alloc_flag)
308{
309 int use_pool = (alloc_flag & GFP_ATOMIC) ? 1 : 0;
310 struct cryptocop_dma_desc *cdesc;
311
312 if (use_pool) {
313 unsigned long int flags;
314 spin_lock_irqsave(&descr_pool_lock, flags);
315 if (!descr_pool_free_list) {
316 spin_unlock_irqrestore(&descr_pool_lock, flags);
317 DEBUG_API(printk("alloc_cdesc: pool is empty\n"));
318 return NULL;
319 }
320 cdesc = descr_pool_free_list;
321 descr_pool_free_list = descr_pool_free_list->next;
322 --descr_pool_no_free;
323 spin_unlock_irqrestore(&descr_pool_lock, flags);
324 cdesc->from_pool = 1;
325 } else {
326 cdesc = kmalloc(sizeof(struct cryptocop_dma_desc), alloc_flag);
327 if (!cdesc) {
328 DEBUG_API(printk("alloc_cdesc: kmalloc\n"));
329 return NULL;
330 }
331 cdesc->from_pool = 0;
332 }
333 cdesc->dma_descr = (dma_descr_data*)(((unsigned long int)cdesc + offsetof(struct cryptocop_dma_desc, dma_descr_buf) + DESCR_ALLOC_PAD) & ~0x0000001F);
334
335 cdesc->next = NULL;
336
337 cdesc->free_buf = NULL;
338 cdesc->dma_descr->out_eop = 0;
339 cdesc->dma_descr->in_eop = 0;
340 cdesc->dma_descr->intr = 0;
341 cdesc->dma_descr->eol = 0;
342 cdesc->dma_descr->wait = 0;
343 cdesc->dma_descr->buf = NULL;
344 cdesc->dma_descr->after = NULL;
345
346 DEBUG_API(printk("alloc_cdesc: return 0x%p, cdesc->dma_descr=0x%p, from_pool=%d\n", cdesc, cdesc->dma_descr, cdesc->from_pool));
347 return cdesc;
348}
349
350
351static void setup_descr_chain(struct cryptocop_dma_desc *cd)
352{
353 DEBUG(printk("setup_descr_chain: entering\n"));
354 while (cd) {
355 if (cd->next) {
356 cd->dma_descr->next = (dma_descr_data*)virt_to_phys(cd->next->dma_descr);
357 } else {
358 cd->dma_descr->next = NULL;
359 }
360 cd = cd->next;
361 }
362 DEBUG(printk("setup_descr_chain: exit\n"));
363}
364
365
366/* Create a pad descriptor for the transform.
367 * Return -1 for error, 0 if pad created. */
368static int create_pad_descriptor(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **pad_desc, int alloc_flag)
369{
370 struct cryptocop_dma_desc *cdesc = NULL;
371 int error = 0;
372 struct strcop_meta_out mo = {
373 .ciphsel = src_none,
374 .hashsel = src_none,
375 .csumsel = src_none
376 };
377 char *pad;
378 size_t plen;
379
380 DEBUG(printk("create_pad_descriptor: start.\n"));
381 /* Setup pad descriptor. */
382
383 DEBUG(printk("create_pad_descriptor: setting up padding.\n"));
384 cdesc = alloc_cdesc(alloc_flag);
385 if (!cdesc){
386 DEBUG_API(printk("create_pad_descriptor: alloc pad desc\n"));
387 goto error_cleanup;
388 }
389 switch (tc->unit_no) {
390 case src_md5:
391 error = create_md5_pad(alloc_flag, tc->consumed, &pad, &plen);
392 if (error){
393 DEBUG_API(printk("create_pad_descriptor: create_md5_pad_failed\n"));
394 goto error_cleanup;
395 }
396 cdesc->free_buf = pad;
397 mo.hashsel = src_dma;
398 mo.hashconf = tc->hash_conf;
399 mo.hashmode = tc->hash_mode;
400 break;
401 case src_sha1:
402 error = create_sha1_pad(alloc_flag, tc->consumed, &pad, &plen);
403 if (error){
404 DEBUG_API(printk("create_pad_descriptor: create_sha1_pad_failed\n"));
405 goto error_cleanup;
406 }
407 cdesc->free_buf = pad;
408 mo.hashsel = src_dma;
409 mo.hashconf = tc->hash_conf;
410 mo.hashmode = tc->hash_mode;
411 break;
412 case src_csum:
413 if (tc->consumed % tc->blocklength){
414 pad = (char*)csum_zero_pad;
415 plen = 1;
416 } else {
417 pad = (char*)cdesc; /* Use any pointer. */
418 plen = 0;
419 }
420 mo.csumsel = src_dma;
421 break;
422 }
423 cdesc->dma_descr->wait = 1;
424 cdesc->dma_descr->out_eop = 1; /* Since this is a pad output is pushed. EOP is ok here since the padded unit is the only one active. */
425 cdesc->dma_descr->buf = (char*)virt_to_phys((char*)pad);
426 cdesc->dma_descr->after = cdesc->dma_descr->buf + plen;
427
428 cdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
429 *pad_desc = cdesc;
430
431 return 0;
432
433 error_cleanup:
434 if (cdesc) free_cdesc(cdesc);
435 return -1;
436}
437
438
439static int setup_key_dl_desc(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **kd, int alloc_flag)
440{
441 struct cryptocop_dma_desc *key_desc = alloc_cdesc(alloc_flag);
442 struct strcop_meta_out mo = {0};
443
444 DEBUG(printk("setup_key_dl_desc\n"));
445
446 if (!key_desc) {
447 DEBUG_API(printk("setup_key_dl_desc: failed descriptor allocation.\n"));
448 return -ENOMEM;
449 }
450
451 /* Download key. */
452 if ((tc->tctx->init.alg == cryptocop_alg_aes) && (tc->tcfg->flags & CRYPTOCOP_DECRYPT)) {
453 /* Precook the AES decrypt key. */
454 if (!tc->tctx->dec_key_set){
455 get_aes_decrypt_key(tc->tctx->dec_key, tc->tctx->init.key, tc->tctx->init.keylen);
456 tc->tctx->dec_key_set = 1;
457 }
458 key_desc->dma_descr->buf = (char*)virt_to_phys(tc->tctx->dec_key);
459 key_desc->dma_descr->after = key_desc->dma_descr->buf + tc->tctx->init.keylen/8;
460 } else {
461 key_desc->dma_descr->buf = (char*)virt_to_phys(tc->tctx->init.key);
462 key_desc->dma_descr->after = key_desc->dma_descr->buf + tc->tctx->init.keylen/8;
463 }
464 /* Setup metadata. */
465 mo.dlkey = 1;
466 switch (tc->tctx->init.keylen) {
467 case 64:
468 mo.decrypt = 0;
469 mo.hashmode = 0;
470 break;
471 case 128:
472 mo.decrypt = 0;
473 mo.hashmode = 1;
474 break;
475 case 192:
476 mo.decrypt = 1;
477 mo.hashmode = 0;
478 break;
479 case 256:
480 mo.decrypt = 1;
481 mo.hashmode = 1;
482 break;
483 default:
484 break;
485 }
486 mo.ciphsel = mo.hashsel = mo.csumsel = src_none;
487 key_desc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
488
489 key_desc->dma_descr->out_eop = 1;
490 key_desc->dma_descr->wait = 1;
491 key_desc->dma_descr->intr = 0;
492
493 *kd = key_desc;
494 return 0;
495}
496
497static int setup_cipher_iv_desc(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **id, int alloc_flag)
498{
499 struct cryptocop_dma_desc *iv_desc = alloc_cdesc(alloc_flag);
500 struct strcop_meta_out mo = {0};
501
502 DEBUG(printk("setup_cipher_iv_desc\n"));
503
504 if (!iv_desc) {
505 DEBUG_API(printk("setup_cipher_iv_desc: failed CBC IV descriptor allocation.\n"));
506 return -ENOMEM;
507 }
508 /* Download IV. */
509 iv_desc->dma_descr->buf = (char*)virt_to_phys(tc->tcfg->iv);
510 iv_desc->dma_descr->after = iv_desc->dma_descr->buf + tc->blocklength;
511
512 /* Setup metadata. */
513 mo.hashsel = mo.csumsel = src_none;
514 mo.ciphsel = src_dma;
515 mo.ciphconf = tc->ciph_conf;
516 mo.cbcmode = tc->cbcmode;
517
518 iv_desc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
519
520 iv_desc->dma_descr->out_eop = 0;
521 iv_desc->dma_descr->wait = 1;
522 iv_desc->dma_descr->intr = 0;
523
524 *id = iv_desc;
525 return 0;
526}
527
014b38ec 528/* Map the output length of the transform to operation output starting on the inject index. */
51533b61
MS		 529static int create_input_descriptors(struct cryptocop_operation *operation, struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **id, int alloc_flag)
530{
531 int err = 0;
532 struct cryptocop_dma_desc head = {0};
533 struct cryptocop_dma_desc *outdesc = &head;
534 size_t iov_offset = 0;
535 size_t out_ix = 0;
536 int outiov_ix = 0;
537 struct strcop_meta_in mi = {0};
538
539 size_t out_length = tc->produced;
540 int rem_length;
541 int dlength;
542
543 assert(out_length != 0);
544 if (((tc->produced + tc->tcfg->inject_ix) > operation->tfrm_op.outlen) || (tc->produced && (operation->tfrm_op.outlen == 0))) {
545 DEBUG_API(printk("create_input_descriptors: operation outdata too small\n"));
546 return -EINVAL;
547 }
548 /* Traverse the out iovec until the result inject index is reached. */
549 while ((outiov_ix < operation->tfrm_op.outcount) && ((out_ix + operation->tfrm_op.outdata[outiov_ix].iov_len) <= tc->tcfg->inject_ix)){
550 out_ix += operation->tfrm_op.outdata[outiov_ix].iov_len;
551 outiov_ix++;
552 }
553 if (outiov_ix >= operation->tfrm_op.outcount){
554 DEBUG_API(printk("create_input_descriptors: operation outdata too small\n"));
555 return -EINVAL;
556 }
557 iov_offset = tc->tcfg->inject_ix - out_ix;
558 mi.dmasel = tc->unit_no;
559
560 /* Setup the output descriptors. */
561 while ((out_length > 0) && (outiov_ix < operation->tfrm_op.outcount)) {
562 outdesc->next = alloc_cdesc(alloc_flag);
563 if (!outdesc->next) {
564 DEBUG_API(printk("create_input_descriptors: alloc_cdesc\n"));
565 err = -ENOMEM;
566 goto error_cleanup;
567 }
568 outdesc = outdesc->next;
569 rem_length = operation->tfrm_op.outdata[outiov_ix].iov_len - iov_offset;
570 dlength = (out_length < rem_length) ? out_length : rem_length;
571
572 DEBUG(printk("create_input_descriptors:\n"
573 "outiov_ix=%d, rem_length=%d, dlength=%d\n"
574 "iov_offset=%d, outdata[outiov_ix].iov_len=%d\n"
575 "outcount=%d, outiov_ix=%d\n",
576 outiov_ix, rem_length, dlength, iov_offset, operation->tfrm_op.outdata[outiov_ix].iov_len, operation->tfrm_op.outcount, outiov_ix));
577
578 outdesc->dma_descr->buf = (char*)virt_to_phys(operation->tfrm_op.outdata[outiov_ix].iov_base + iov_offset);
579 outdesc->dma_descr->after = outdesc->dma_descr->buf + dlength;
580 outdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
581
582 out_length -= dlength;
583 iov_offset += dlength;
584 if (iov_offset >= operation->tfrm_op.outdata[outiov_ix].iov_len) {
585 iov_offset = 0;
586 ++outiov_ix;
587 }
588 }
589 if (out_length > 0){
590 DEBUG_API(printk("create_input_descriptors: not enough room for output, %d remained\n", out_length));
591 err = -EINVAL;
592 goto error_cleanup;
593 }
594 /* Set sync in last descriptor. */
595 mi.sync = 1;
596 outdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
597
598 *id = head.next;
599 return 0;
600
601 error_cleanup:
602 while (head.next) {
603 outdesc = head.next->next;
604 free_cdesc(head.next);
605 head.next = outdesc;
606 }
607 return err;
608}
609
610
611static int create_output_descriptors(struct cryptocop_operation *operation, int *iniov_ix, int *iniov_offset, size_t desc_len, struct cryptocop_dma_desc **current_out_cdesc, struct strcop_meta_out *meta_out, int alloc_flag)
612{
613 while (desc_len != 0) {
614 struct cryptocop_dma_desc *cdesc;
615 int rem_length = operation->tfrm_op.indata[*iniov_ix].iov_len - *iniov_offset;
616 int dlength = (desc_len < rem_length) ? desc_len : rem_length;
617
618 cdesc = alloc_cdesc(alloc_flag);
619 if (!cdesc) {
620 DEBUG_API(printk("create_output_descriptors: alloc_cdesc\n"));
621 return -ENOMEM;
622 }
623 (*current_out_cdesc)->next = cdesc;
624 (*current_out_cdesc) = cdesc;
625
626 cdesc->free_buf = NULL;
627
628 cdesc->dma_descr->buf = (char*)virt_to_phys(operation->tfrm_op.indata[*iniov_ix].iov_base + *iniov_offset);
629 cdesc->dma_descr->after = cdesc->dma_descr->buf + dlength;
630
cda1c5af 631 assert(desc_len >= dlength);
51533b61
MS		 632 desc_len -= dlength;
633 *iniov_offset += dlength;
51533b61
MS		 634 if (*iniov_offset >= operation->tfrm_op.indata[*iniov_ix].iov_len) {
635 *iniov_offset = 0;
636 ++(*iniov_ix);
637 if (*iniov_ix > operation->tfrm_op.incount) {
638 DEBUG_API(printk("create_output_descriptors: not enough indata in operation."));
639 return -EINVAL;
640 }
641 }
642 cdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, (*meta_out));
643 } /* while (desc_len != 0) */
644 /* Last DMA descriptor gets a 'wait' bit to signal expected change in metadata. */
645 (*current_out_cdesc)->dma_descr->wait = 1; /* This will set extraneous WAIT in some situations, e.g. when padding hashes and checksums. */
646
647 return 0;
648}
649
650
651static int append_input_descriptors(struct cryptocop_operation *operation, struct cryptocop_dma_desc **current_in_cdesc, struct cryptocop_dma_desc **current_out_cdesc, struct cryptocop_tfrm_ctx *tc, int alloc_flag)
652{
653 DEBUG(printk("append_input_descriptors, tc=0x%p, unit_no=%d\n", tc, tc->unit_no));
654 if (tc->tcfg) {
655 int failed = 0;
656 struct cryptocop_dma_desc *idescs = NULL;
657 DEBUG(printk("append_input_descriptors: pushing output, consumed %d produced %d bytes.\n", tc->consumed, tc->produced));
658 if (tc->pad_descs) {
659 DEBUG(printk("append_input_descriptors: append pad descriptors to DMA out list.\n"));
660 while (tc->pad_descs) {
661 DEBUG(printk("append descriptor 0x%p\n", tc->pad_descs));
662 (*current_out_cdesc)->next = tc->pad_descs;
663 tc->pad_descs = tc->pad_descs->next;
664 (*current_out_cdesc) = (*current_out_cdesc)->next;
665 }
666 }
667
668 /* Setup and append output descriptors to DMA in list. */
669 if (tc->unit_no == src_dma){
670 /* mem2mem. Setup DMA in descriptors to discard all input prior to the requested mem2mem data. */
671 struct strcop_meta_in mi = {.sync = 0, .dmasel = src_dma};
672 unsigned int start_ix = tc->start_ix;
673 while (start_ix){
674 unsigned int desclen = start_ix < MEM2MEM_DISCARD_BUF_LENGTH ? start_ix : MEM2MEM_DISCARD_BUF_LENGTH;
675 (*current_in_cdesc)->next = alloc_cdesc(alloc_flag);
676 if (!(*current_in_cdesc)->next){
677 DEBUG_API(printk("append_input_descriptors: alloc_cdesc mem2mem discard failed\n"));
678 return -ENOMEM;
679 }
680 (*current_in_cdesc) = (*current_in_cdesc)->next;
681 (*current_in_cdesc)->dma_descr->buf = (char*)virt_to_phys(mem2mem_discard_buf);
682 (*current_in_cdesc)->dma_descr->after = (*current_in_cdesc)->dma_descr->buf + desclen;
683 (*current_in_cdesc)->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
684 start_ix -= desclen;
685 }
686 mi.sync = 1;
687 (*current_in_cdesc)->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
688 }
689
690 failed = create_input_descriptors(operation, tc, &idescs, alloc_flag);
691 if (failed){
692 DEBUG_API(printk("append_input_descriptors: output descriptor setup failed\n"));
693 return failed;
694 }
695 DEBUG(printk("append_input_descriptors: append output descriptors to DMA in list.\n"));
696 while (idescs) {
697 DEBUG(printk("append descriptor 0x%p\n", idescs));
698 (*current_in_cdesc)->next = idescs;
699 idescs = idescs->next;
700 (*current_in_cdesc) = (*current_in_cdesc)->next;
701 }
702 }
703 return 0;
704}
705
706
707
708static int cryptocop_setup_dma_list(struct cryptocop_operation *operation, struct cryptocop_int_operation **int_op, int alloc_flag)
709{
710 struct cryptocop_session *sess;
711 struct cryptocop_transform_ctx *tctx;
712
713 struct cryptocop_tfrm_ctx digest_ctx = {
714 .previous_src = src_none,
715 .current_src = src_none,
716 .start_ix = 0,
717 .requires_padding = 1,
718 .strict_block_length = 0,
719 .hash_conf = 0,
720 .hash_mode = 0,
721 .ciph_conf = 0,
722 .cbcmode = 0,
723 .decrypt = 0,
724 .consumed = 0,
725 .produced = 0,
726 .pad_descs = NULL,
727 .active = 0,
728 .done = 0,
729 .prev_src = NULL,
730 .curr_src = NULL,
731 .tcfg = NULL};
732 struct cryptocop_tfrm_ctx cipher_ctx = {
733 .previous_src = src_none,
734 .current_src = src_none,
735 .start_ix = 0,
736 .requires_padding = 0,
737 .strict_block_length = 1,
738 .hash_conf = 0,
739 .hash_mode = 0,
740 .ciph_conf = 0,
741 .cbcmode = 0,
742 .decrypt = 0,
743 .consumed = 0,
744 .produced = 0,
745 .pad_descs = NULL,
746 .active = 0,
747 .done = 0,
748 .prev_src = NULL,
749 .curr_src = NULL,
750 .tcfg = NULL};
751 struct cryptocop_tfrm_ctx csum_ctx = {
752 .previous_src = src_none,
753 .current_src = src_none,
754 .start_ix = 0,
755 .blocklength = 2,
756 .requires_padding = 1,
757 .strict_block_length = 0,
758 .hash_conf = 0,
759 .hash_mode = 0,
760 .ciph_conf = 0,
761 .cbcmode = 0,
762 .decrypt = 0,
763 .consumed = 0,
764 .produced = 0,
765 .pad_descs = NULL,
766 .active = 0,
767 .done = 0,
768 .tcfg = NULL,
769 .prev_src = NULL,
770 .curr_src = NULL,
771 .unit_no = src_csum};
772 struct cryptocop_tfrm_cfg *tcfg = operation->tfrm_op.tfrm_cfg;
773
774 unsigned int indata_ix = 0;
775
776 /* iovec accounting. */
777 int iniov_ix = 0;
778 int iniov_offset = 0;
779
780 /* Operation descriptor cfg traversal pointer. */
781 struct cryptocop_desc *odsc;
782
783 int failed = 0;
784 /* List heads for allocated descriptors. */
785 struct cryptocop_dma_desc out_cdesc_head = {0};
786 struct cryptocop_dma_desc in_cdesc_head = {0};
787
788 struct cryptocop_dma_desc *current_out_cdesc = &out_cdesc_head;
789 struct cryptocop_dma_desc *current_in_cdesc = &in_cdesc_head;
790
791 struct cryptocop_tfrm_ctx *output_tc = NULL;
792 void *iop_alloc_ptr;
793
794 assert(operation != NULL);
795 assert(int_op != NULL);
796
797 DEBUG(printk("cryptocop_setup_dma_list: start\n"));
798 DEBUG(print_cryptocop_operation(operation));
799
800 sess = get_session(operation->sid);
801 if (!sess) {
802 DEBUG_API(printk("cryptocop_setup_dma_list: no session found for operation.\n"));
803 failed = -EINVAL;
804 goto error_cleanup;
805 }
806 iop_alloc_ptr = kmalloc(DESCR_ALLOC_PAD + sizeof(struct cryptocop_int_operation), alloc_flag);
807 if (!iop_alloc_ptr) {
808 DEBUG_API(printk("cryptocop_setup_dma_list: kmalloc cryptocop_int_operation\n"));
809 failed = -ENOMEM;
810 goto error_cleanup;
811 }
812 (*int_op) = (struct cryptocop_int_operation*)(((unsigned long int)(iop_alloc_ptr + DESCR_ALLOC_PAD + offsetof(struct cryptocop_int_operation, ctx_out)) & ~0x0000001F) - offsetof(struct cryptocop_int_operation, ctx_out));
813 DEBUG(memset((*int_op), 0xff, sizeof(struct cryptocop_int_operation)));
814 (*int_op)->alloc_ptr = iop_alloc_ptr;
815 DEBUG(printk("cryptocop_setup_dma_list: *int_op=0x%p, alloc_ptr=0x%p\n", *int_op, (*int_op)->alloc_ptr));
816
817 (*int_op)->sid = operation->sid;
818 (*int_op)->cdesc_out = NULL;
819 (*int_op)->cdesc_in = NULL;
820 (*int_op)->tdes_mode = cryptocop_3des_ede;
821 (*int_op)->csum_mode = cryptocop_csum_le;
822 (*int_op)->ddesc_out = NULL;
823 (*int_op)->ddesc_in = NULL;
824
825 /* Scan operation->tfrm_op.tfrm_cfg for bad configuration and set up the local contexts. */
826 if (!tcfg) {
827 DEBUG_API(printk("cryptocop_setup_dma_list: no configured transforms in operation.\n"));
828 failed = -EINVAL;
829 goto error_cleanup;
830 }
831 while (tcfg) {
832 tctx = get_transform_ctx(sess, tcfg->tid);
833 if (!tctx) {
834 DEBUG_API(printk("cryptocop_setup_dma_list: no transform id %d in session.\n", tcfg->tid));
835 failed = -EINVAL;
836 goto error_cleanup;
837 }
838 if (tcfg->inject_ix > operation->tfrm_op.outlen){
839 DEBUG_API(printk("cryptocop_setup_dma_list: transform id %d inject_ix (%d) > operation->tfrm_op.outlen(%d)", tcfg->tid, tcfg->inject_ix, operation->tfrm_op.outlen));
840 failed = -EINVAL;
841 goto error_cleanup;
842 }
843 switch (tctx->init.alg){
844 case cryptocop_alg_mem2mem:
845 if (cipher_ctx.tcfg != NULL){
846 DEBUG_API(printk("cryptocop_setup_dma_list: multiple ciphers in operation.\n"));
847 failed = -EINVAL;
848 goto error_cleanup;
849 }
850 /* mem2mem is handled as a NULL cipher. */
851 cipher_ctx.cbcmode = 0;
852 cipher_ctx.decrypt = 0;
853 cipher_ctx.blocklength = 1;
854 cipher_ctx.ciph_conf = 0;
855 cipher_ctx.unit_no = src_dma;
856 cipher_ctx.tcfg = tcfg;
857 cipher_ctx.tctx = tctx;
858 break;
859 case cryptocop_alg_des:
860 case cryptocop_alg_3des:
861 case cryptocop_alg_aes:
862 /* cipher */
863 if (cipher_ctx.tcfg != NULL){
864 DEBUG_API(printk("cryptocop_setup_dma_list: multiple ciphers in operation.\n"));
865 failed = -EINVAL;
866 goto error_cleanup;
867 }
868 cipher_ctx.tcfg = tcfg;
869 cipher_ctx.tctx = tctx;
870 if (cipher_ctx.tcfg->flags & CRYPTOCOP_DECRYPT){
871 cipher_ctx.decrypt = 1;
872 }
873 switch (tctx->init.cipher_mode) {
874 case cryptocop_cipher_mode_ecb:
875 cipher_ctx.cbcmode = 0;
876 break;
877 case cryptocop_cipher_mode_cbc:
878 cipher_ctx.cbcmode = 1;
879 break;
880 default:
881 DEBUG_API(printk("cryptocop_setup_dma_list: cipher_ctx, bad cipher mode==%d\n", tctx->init.cipher_mode));
882 failed = -EINVAL;
883 goto error_cleanup;
884 }
885 DEBUG(printk("cryptocop_setup_dma_list: cipher_ctx, set CBC mode==%d\n", cipher_ctx.cbcmode));
886 switch (tctx->init.alg){
887 case cryptocop_alg_des:
888 cipher_ctx.ciph_conf = 0;
889 cipher_ctx.unit_no = src_des;
890 cipher_ctx.blocklength = DES_BLOCK_LENGTH;
891 break;
892 case cryptocop_alg_3des:
893 cipher_ctx.ciph_conf = 1;
894 cipher_ctx.unit_no = src_des;
895 cipher_ctx.blocklength = DES_BLOCK_LENGTH;
896 break;
897 case cryptocop_alg_aes:
898 cipher_ctx.ciph_conf = 2;
899 cipher_ctx.unit_no = src_aes;
900 cipher_ctx.blocklength = AES_BLOCK_LENGTH;
901 break;
902 default:
903 panic("cryptocop_setup_dma_list: impossible algorithm %d\n", tctx->init.alg);
904 }
905 (*int_op)->tdes_mode = tctx->init.tdes_mode;
906 break;
907 case cryptocop_alg_md5:
908 case cryptocop_alg_sha1:
909 /* digest */
910 if (digest_ctx.tcfg != NULL){
911 DEBUG_API(printk("cryptocop_setup_dma_list: multiple digests in operation.\n"));
912 failed = -EINVAL;
913 goto error_cleanup;
914 }
915 digest_ctx.tcfg = tcfg;
916 digest_ctx.tctx = tctx;
917 digest_ctx.hash_mode = 0; /* Don't use explicit IV in this API. */
918 switch (tctx->init.alg){
919 case cryptocop_alg_md5:
920 digest_ctx.blocklength = MD5_BLOCK_LENGTH;
921 digest_ctx.unit_no = src_md5;
922 digest_ctx.hash_conf = 1; /* 1 => MD-5 */
923 break;
924 case cryptocop_alg_sha1:
925 digest_ctx.blocklength = SHA1_BLOCK_LENGTH;
926 digest_ctx.unit_no = src_sha1;
927 digest_ctx.hash_conf = 0; /* 0 => SHA-1 */
928 break;
929 default:
930 panic("cryptocop_setup_dma_list: impossible digest algorithm\n");
931 }
932 break;
933 case cryptocop_alg_csum:
934 /* digest */
935 if (csum_ctx.tcfg != NULL){
936 DEBUG_API(printk("cryptocop_setup_dma_list: multiple checksums in operation.\n"));
937 failed = -EINVAL;
938 goto error_cleanup;
939 }
940 (*int_op)->csum_mode = tctx->init.csum_mode;
941 csum_ctx.tcfg = tcfg;
942 csum_ctx.tctx = tctx;
943 break;
944 default:
945 /* no algorithm. */
946 DEBUG_API(printk("cryptocop_setup_dma_list: invalid algorithm %d specified in tfrm %d.\n", tctx->init.alg, tcfg->tid));
947 failed = -EINVAL;
948 goto error_cleanup;
949 }
950 tcfg = tcfg->next;
951 }
952 /* Download key if a cipher is used. */
953 if (cipher_ctx.tcfg && (cipher_ctx.tctx->init.alg != cryptocop_alg_mem2mem)){
954 struct cryptocop_dma_desc *key_desc = NULL;
955
956 failed = setup_key_dl_desc(&cipher_ctx, &key_desc, alloc_flag);
957 if (failed) {
958 DEBUG_API(printk("cryptocop_setup_dma_list: setup key dl\n"));
959 goto error_cleanup;
960 }
961 current_out_cdesc->next = key_desc;
962 current_out_cdesc = key_desc;
963 indata_ix += (unsigned int)(key_desc->dma_descr->after - key_desc->dma_descr->buf);
964
965 /* Download explicit IV if a cipher is used and CBC mode and explicit IV selected. */
966 if ((cipher_ctx.tctx->init.cipher_mode == cryptocop_cipher_mode_cbc) && (cipher_ctx.tcfg->flags & CRYPTOCOP_EXPLICIT_IV)) {
967 struct cryptocop_dma_desc *iv_desc = NULL;
968
969 DEBUG(printk("cryptocop_setup_dma_list: setup cipher CBC IV descriptor.\n"));
970
971 failed = setup_cipher_iv_desc(&cipher_ctx, &iv_desc, alloc_flag);
972 if (failed) {
973 DEBUG_API(printk("cryptocop_setup_dma_list: CBC IV descriptor.\n"));
974 goto error_cleanup;
975 }
976 current_out_cdesc->next = iv_desc;
977 current_out_cdesc = iv_desc;
978 indata_ix += (unsigned int)(iv_desc->dma_descr->after - iv_desc->dma_descr->buf);
979 }
980 }
981
982 /* Process descriptors. */
983 odsc = operation->tfrm_op.desc;
984 while (odsc) {
985 struct cryptocop_desc_cfg *dcfg = odsc->cfg;
986 struct strcop_meta_out meta_out = {0};
987 size_t desc_len = odsc->length;
988 int active_count, eop_needed_count;
989
990 output_tc = NULL;
991
992 DEBUG(printk("cryptocop_setup_dma_list: parsing an operation descriptor\n"));
993
994 while (dcfg) {
995 struct cryptocop_tfrm_ctx *tc = NULL;
996
997 DEBUG(printk("cryptocop_setup_dma_list: parsing an operation descriptor configuration.\n"));
998 /* Get the local context for the transform and mark it as the output unit if it produces output. */
999 if (digest_ctx.tcfg && (digest_ctx.tcfg->tid == dcfg->tid)){
1000 tc = &digest_ctx;
1001 } else if (cipher_ctx.tcfg && (cipher_ctx.tcfg->tid == dcfg->tid)){
1002 tc = &cipher_ctx;
1003 } else if (csum_ctx.tcfg && (csum_ctx.tcfg->tid == dcfg->tid)){
1004 tc = &csum_ctx;
1005 }
1006 if (!tc) {
1007 DEBUG_API(printk("cryptocop_setup_dma_list: invalid transform %d specified in descriptor.\n", dcfg->tid));
1008 failed = -EINVAL;
1009 goto error_cleanup;
1010 }
1011 if (tc->done) {
1012 DEBUG_API(printk("cryptocop_setup_dma_list: completed transform %d reused.\n", dcfg->tid));
1013 failed = -EINVAL;
1014 goto error_cleanup;
1015 }
1016 if (!tc->active) {
1017 tc->start_ix = indata_ix;
1018 tc->active = 1;
1019 }
1020
1021 tc->previous_src = tc->current_src;
1022 tc->prev_src = tc->curr_src;
1023 /* Map source unit id to DMA source config. */
1024 switch (dcfg->src){
1025 case cryptocop_source_dma:
1026 tc->current_src = src_dma;
1027 break;
1028 case cryptocop_source_des:
1029 tc->current_src = src_des;
1030 break;
1031 case cryptocop_source_3des:
1032 tc->current_src = src_des;
1033 break;
1034 case cryptocop_source_aes:
1035 tc->current_src = src_aes;
1036 break;
1037 case cryptocop_source_md5:
1038 case cryptocop_source_sha1:
1039 case cryptocop_source_csum:
1040 case cryptocop_source_none:
1041 default:
1042 /* We do not allow using accumulating style units (SHA-1, MD5, checksum) as sources to other units.
1043 */
1044 DEBUG_API(printk("cryptocop_setup_dma_list: bad unit source configured %d.\n", dcfg->src));
1045 failed = -EINVAL;
1046 goto error_cleanup;
1047 }
1048 if (tc->current_src != src_dma) {
1049 /* Find the unit we are sourcing from. */
1050 if (digest_ctx.unit_no == tc->current_src){
1051 tc->curr_src = &digest_ctx;
1052 } else if (cipher_ctx.unit_no == tc->current_src){
1053 tc->curr_src = &cipher_ctx;
1054 } else if (csum_ctx.unit_no == tc->current_src){
1055 tc->curr_src = &csum_ctx;
1056 }
1057 if ((tc->curr_src == tc) && (tc->unit_no != src_dma)){
1058 DEBUG_API(printk("cryptocop_setup_dma_list: unit %d configured to source from itself.\n", tc->unit_no));
1059 failed = -EINVAL;
1060 goto error_cleanup;
1061 }
1062 } else {
1063 tc->curr_src = NULL;
1064 }
1065
1066 /* Detect source switch. */
1067 DEBUG(printk("cryptocop_setup_dma_list: tc->active=%d tc->unit_no=%d tc->current_src=%d tc->previous_src=%d, tc->curr_src=0x%p, tc->prev_srv=0x%p\n", tc->active, tc->unit_no, tc->current_src, tc->previous_src, tc->curr_src, tc->prev_src));
1068 if (tc->active && (tc->current_src != tc->previous_src)) {
1069 /* Only allow source switch when both the old source unit and the new one have
1070 * no pending data to process (i.e. the consumed length must be a multiple of the
1071 * transform blocklength). */
1072 /* Note: if the src == NULL we are actually sourcing from DMA out. */
1073 if (((tc->prev_src != NULL) && (tc->prev_src->consumed % tc->prev_src->blocklength)) ||
1074 ((tc->curr_src != NULL) && (tc->curr_src->consumed % tc->curr_src->blocklength)))
1075 {
1076 DEBUG_API(printk("cryptocop_setup_dma_list: can only disconnect from or connect to a unit on a multiple of the blocklength, old: cons=%d, prod=%d, block=%d, new: cons=%d prod=%d, block=%d.\n", tc->prev_src ? tc->prev_src->consumed : INT_MIN, tc->prev_src ? tc->prev_src->produced : INT_MIN, tc->prev_src ? tc->prev_src->blocklength : INT_MIN, tc->curr_src ? tc->curr_src->consumed : INT_MIN, tc->curr_src ? tc->curr_src->produced : INT_MIN, tc->curr_src ? tc->curr_src->blocklength : INT_MIN));
1077 failed = -EINVAL;
1078 goto error_cleanup;
1079 }
1080 }
1081 /* Detect unit deactivation. */
1082 if (dcfg->last) {
1083 /* Length check of this is handled below. */
1084 tc->done = 1;
1085 }
1086 dcfg = dcfg->next;
1087 } /* while (dcfg) */
1088 DEBUG(printk("cryptocop_setup_dma_list: parsing operation descriptor configuration complete.\n"));
1089
1090 if (cipher_ctx.active && (cipher_ctx.curr_src != NULL) && !cipher_ctx.curr_src->active){
1091 DEBUG_API(printk("cryptocop_setup_dma_list: cipher source from inactive unit %d\n", cipher_ctx.curr_src->unit_no));
1092 failed = -EINVAL;
1093 goto error_cleanup;
1094 }
1095 if (digest_ctx.active && (digest_ctx.curr_src != NULL) && !digest_ctx.curr_src->active){
1096 DEBUG_API(printk("cryptocop_setup_dma_list: digest source from inactive unit %d\n", digest_ctx.curr_src->unit_no));
1097 failed = -EINVAL;
1098 goto error_cleanup;
1099 }
1100 if (csum_ctx.active && (csum_ctx.curr_src != NULL) && !csum_ctx.curr_src->active){
1101 DEBUG_API(printk("cryptocop_setup_dma_list: cipher source from inactive unit %d\n", csum_ctx.curr_src->unit_no));
1102 failed = -EINVAL;
1103 goto error_cleanup;
1104 }
1105
1106 /* Update consumed and produced lengths.
1107
1108 The consumed length accounting here is actually cheating. If a unit source from DMA (or any
1109 other unit that process data in blocks of one octet) it is correct, but if it source from a
1110 block processing unit, i.e. a cipher, it will be temporarily incorrect at some times. However
1111 since it is only allowed--by the HW--to change source to or from a block processing unit at times where that
1112 unit has processed an exact multiple of its block length the end result will be correct.
1113 Beware that if the source change restriction change this code will need to be (much) reworked.
1114 */
1115 DEBUG(printk("cryptocop_setup_dma_list: desc->length=%d, desc_len=%d.\n", odsc->length, desc_len));
1116
1117 if (csum_ctx.active) {
1118 csum_ctx.consumed += desc_len;
1119 if (csum_ctx.done) {
1120 csum_ctx.produced = 2;
1121 }
1122 DEBUG(printk("cryptocop_setup_dma_list: csum_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", csum_ctx.consumed, csum_ctx.produced, csum_ctx.blocklength));
1123 }
1124 if (digest_ctx.active) {
1125 digest_ctx.consumed += desc_len;
1126 if (digest_ctx.done) {
1127 if (digest_ctx.unit_no == src_md5) {
1128 digest_ctx.produced = MD5_STATE_LENGTH;
1129 } else {
1130 digest_ctx.produced = SHA1_STATE_LENGTH;
1131 }
1132 }
1133 DEBUG(printk("cryptocop_setup_dma_list: digest_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", digest_ctx.consumed, digest_ctx.produced, digest_ctx.blocklength));
1134 }
1135 if (cipher_ctx.active) {
1136 /* Ciphers are allowed only to source from DMA out. That is filtered above. */
1137 assert(cipher_ctx.current_src == src_dma);
1138 cipher_ctx.consumed += desc_len;
1139 cipher_ctx.produced = cipher_ctx.blocklength * (cipher_ctx.consumed / cipher_ctx.blocklength);
1140 if (cipher_ctx.cbcmode && !(cipher_ctx.tcfg->flags & CRYPTOCOP_EXPLICIT_IV) && cipher_ctx.produced){
1141 cipher_ctx.produced -= cipher_ctx.blocklength; /* Compensate for CBC iv. */
1142 }
1143 DEBUG(printk("cryptocop_setup_dma_list: cipher_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", cipher_ctx.consumed, cipher_ctx.produced, cipher_ctx.blocklength));
1144 }
1145
1146 /* Setup the DMA out descriptors. */
1147 /* Configure the metadata. */
1148 active_count = 0;
1149 eop_needed_count = 0;
1150 if (cipher_ctx.active) {
1151 ++active_count;
1152 if (cipher_ctx.unit_no == src_dma){
1153 /* mem2mem */
1154 meta_out.ciphsel = src_none;
1155 } else {
1156 meta_out.ciphsel = cipher_ctx.current_src;
1157 }
1158 meta_out.ciphconf = cipher_ctx.ciph_conf;
1159 meta_out.cbcmode = cipher_ctx.cbcmode;
1160 meta_out.decrypt = cipher_ctx.decrypt;
1161 DEBUG(printk("set ciphsel=%d ciphconf=%d cbcmode=%d decrypt=%d\n", meta_out.ciphsel, meta_out.ciphconf, meta_out.cbcmode, meta_out.decrypt));
1162 if (cipher_ctx.done) ++eop_needed_count;
1163 } else {
1164 meta_out.ciphsel = src_none;
1165 }
1166
1167 if (digest_ctx.active) {
1168 ++active_count;
1169 meta_out.hashsel = digest_ctx.current_src;
1170 meta_out.hashconf = digest_ctx.hash_conf;
1171 meta_out.hashmode = 0; /* Explicit mode is not used here. */
1172 DEBUG(printk("set hashsel=%d hashconf=%d hashmode=%d\n", meta_out.hashsel, meta_out.hashconf, meta_out.hashmode));
1173 if (digest_ctx.done) {
1174 assert(digest_ctx.pad_descs == NULL);
1175 failed = create_pad_descriptor(&digest_ctx, &digest_ctx.pad_descs, alloc_flag);
1176 if (failed) {
1177 DEBUG_API(printk("cryptocop_setup_dma_list: failed digest pad creation.\n"));
1178 goto error_cleanup;
1179 }
1180 }
1181 } else {
1182 meta_out.hashsel = src_none;
1183 }
1184
1185 if (csum_ctx.active) {
1186 ++active_count;
1187 meta_out.csumsel = csum_ctx.current_src;
1188 if (csum_ctx.done) {
1189 assert(csum_ctx.pad_descs == NULL);
1190 failed = create_pad_descriptor(&csum_ctx, &csum_ctx.pad_descs, alloc_flag);
1191 if (failed) {
1192 DEBUG_API(printk("cryptocop_setup_dma_list: failed csum pad creation.\n"));
1193 goto error_cleanup;
1194 }
1195 }
1196 } else {
1197 meta_out.csumsel = src_none;
1198 }
1199 DEBUG(printk("cryptocop_setup_dma_list: %d eop needed, %d active units\n", eop_needed_count, active_count));
1200 /* Setup DMA out descriptors for the indata. */
1201 failed = create_output_descriptors(operation, &iniov_ix, &iniov_offset, desc_len, &current_out_cdesc, &meta_out, alloc_flag);
1202 if (failed) {
1203 DEBUG_API(printk("cryptocop_setup_dma_list: create_output_descriptors %d\n", failed));
1204 goto error_cleanup;
1205 }
1206 /* Setup out EOP. If there are active units that are not done here they cannot get an EOP
1207 * so we ust setup a zero length descriptor to DMA to signal EOP only to done units.
1208 * If there is a pad descriptor EOP for the padded unit will be EOPed by it.
1209 */
1210 assert(active_count >= eop_needed_count);
1211 assert((eop_needed_count == 0) || (eop_needed_count == 1));
1212 if (eop_needed_count) {
520c1748 1213 /* This means that the bulk operation (cipher/m2m) is terminated. */
51533b61
MS		 1214 if (active_count > 1) {
1215 /* Use zero length EOP descriptor. */
1216 struct cryptocop_dma_desc *ed = alloc_cdesc(alloc_flag);
1217 struct strcop_meta_out ed_mo = {0};
1218 if (!ed) {
1219 DEBUG_API(printk("cryptocop_setup_dma_list: alloc EOP descriptor for cipher\n"));
1220 failed = -ENOMEM;
1221 goto error_cleanup;
1222 }
1223
1224 assert(cipher_ctx.active && cipher_ctx.done);
1225
1226 if (cipher_ctx.unit_no == src_dma){
1227 /* mem2mem */
1228 ed_mo.ciphsel = src_none;
1229 } else {
1230 ed_mo.ciphsel = cipher_ctx.current_src;
1231 }
1232 ed_mo.ciphconf = cipher_ctx.ciph_conf;
1233 ed_mo.cbcmode = cipher_ctx.cbcmode;
1234 ed_mo.decrypt = cipher_ctx.decrypt;
1235
1236 ed->free_buf = NULL;
1237 ed->dma_descr->wait = 1;
1238 ed->dma_descr->out_eop = 1;
1239
1240 ed->dma_descr->buf = (char*)virt_to_phys(&ed); /* Use any valid physical address for zero length descriptor. */
1241 ed->dma_descr->after = ed->dma_descr->buf;
1242 ed->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, ed_mo);
1243 current_out_cdesc->next = ed;
1244 current_out_cdesc = ed;
1245 } else {
1246 /* Set EOP in the current out descriptor since the only active module is
1247 * the one needing the EOP. */
1248
1249 current_out_cdesc->dma_descr->out_eop = 1;
1250 }
1251 }
1252
1253 if (cipher_ctx.done && cipher_ctx.active) cipher_ctx.active = 0;
1254 if (digest_ctx.done && digest_ctx.active) digest_ctx.active = 0;
1255 if (csum_ctx.done && csum_ctx.active) csum_ctx.active = 0;
1256 indata_ix += odsc->length;
1257 odsc = odsc->next;
1258 } /* while (odsc) */ /* Process descriptors. */
1259 DEBUG(printk("cryptocop_setup_dma_list: done parsing operation descriptors\n"));
1260 if (cipher_ctx.tcfg && (cipher_ctx.active || !cipher_ctx.done)){
1261 DEBUG_API(printk("cryptocop_setup_dma_list: cipher operation not terminated.\n"));
1262 failed = -EINVAL;
1263 goto error_cleanup;
1264 }
1265 if (digest_ctx.tcfg && (digest_ctx.active || !digest_ctx.done)){
1266 DEBUG_API(printk("cryptocop_setup_dma_list: digest operation not terminated.\n"));
1267 failed = -EINVAL;
1268 goto error_cleanup;
1269 }
1270 if (csum_ctx.tcfg && (csum_ctx.active || !csum_ctx.done)){
1271 DEBUG_API(printk("cryptocop_setup_dma_list: csum operation not terminated.\n"));
1272 failed = -EINVAL;
1273 goto error_cleanup;
1274 }
1275
1276 failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &cipher_ctx, alloc_flag);
1277 if (failed){
1278 DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1279 goto error_cleanup;
1280 }
1281 failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &digest_ctx, alloc_flag);
1282 if (failed){
1283 DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1284 goto error_cleanup;
1285 }
1286 failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &csum_ctx, alloc_flag);
1287 if (failed){
1288 DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1289 goto error_cleanup;
1290 }
1291
1292 DEBUG(printk("cryptocop_setup_dma_list: int_op=0x%p, *int_op=0x%p\n", int_op, *int_op));
1293 (*int_op)->cdesc_out = out_cdesc_head.next;
1294 (*int_op)->cdesc_in = in_cdesc_head.next;
1295 DEBUG(printk("cryptocop_setup_dma_list: out_cdesc_head=0x%p in_cdesc_head=0x%p\n", (*int_op)->cdesc_out, (*int_op)->cdesc_in));
1296
1297 setup_descr_chain(out_cdesc_head.next);
1298 setup_descr_chain(in_cdesc_head.next);
1299
1300 /* Last but not least: mark the last DMA in descriptor for a INTR and EOL and the the
1301 * last DMA out descriptor for EOL.
1302 */
1303 current_in_cdesc->dma_descr->intr = 1;
1304 current_in_cdesc->dma_descr->eol = 1;
1305 current_out_cdesc->dma_descr->eol = 1;
1306
1307 /* Setup DMA contexts. */
1308 (*int_op)->ctx_out.next = NULL;
1309 (*int_op)->ctx_out.eol = 1;
1310 (*int_op)->ctx_out.intr = 0;
1311 (*int_op)->ctx_out.store_mode = 0;
1312 (*int_op)->ctx_out.en = 0;
1313 (*int_op)->ctx_out.dis = 0;
1314 (*int_op)->ctx_out.md0 = 0;
1315 (*int_op)->ctx_out.md1 = 0;
1316 (*int_op)->ctx_out.md2 = 0;
1317 (*int_op)->ctx_out.md3 = 0;
1318 (*int_op)->ctx_out.md4 = 0;
1319 (*int_op)->ctx_out.saved_data = (dma_descr_data*)virt_to_phys((*int_op)->cdesc_out->dma_descr);
1320 (*int_op)->ctx_out.saved_data_buf = (*int_op)->cdesc_out->dma_descr->buf; /* Already physical address. */
1321
1322 (*int_op)->ctx_in.next = NULL;
1323 (*int_op)->ctx_in.eol = 1;
1324 (*int_op)->ctx_in.intr = 0;
1325 (*int_op)->ctx_in.store_mode = 0;
1326 (*int_op)->ctx_in.en = 0;
1327 (*int_op)->ctx_in.dis = 0;
1328 (*int_op)->ctx_in.md0 = 0;
1329 (*int_op)->ctx_in.md1 = 0;
1330 (*int_op)->ctx_in.md2 = 0;
1331 (*int_op)->ctx_in.md3 = 0;
1332 (*int_op)->ctx_in.md4 = 0;
1333
1334 (*int_op)->ctx_in.saved_data = (dma_descr_data*)virt_to_phys((*int_op)->cdesc_in->dma_descr);
1335 (*int_op)->ctx_in.saved_data_buf = (*int_op)->cdesc_in->dma_descr->buf; /* Already physical address. */
1336
1337 DEBUG(printk("cryptocop_setup_dma_list: done\n"));
1338 return 0;
1339
1340error_cleanup:
1341 {
1342 /* Free all allocated resources. */
1343 struct cryptocop_dma_desc *tmp_cdesc;
1344 while (digest_ctx.pad_descs){
1345 tmp_cdesc = digest_ctx.pad_descs->next;
1346 free_cdesc(digest_ctx.pad_descs);
1347 digest_ctx.pad_descs = tmp_cdesc;
1348 }
1349 while (csum_ctx.pad_descs){
1350 tmp_cdesc = csum_ctx.pad_descs->next;
1351 free_cdesc(csum_ctx.pad_descs);
1352 csum_ctx.pad_descs = tmp_cdesc;
1353 }
1354 assert(cipher_ctx.pad_descs == NULL); /* The ciphers are never padded. */
1355
1356 if (*int_op != NULL) delete_internal_operation(*int_op);
1357 }
1358 DEBUG_API(printk("cryptocop_setup_dma_list: done with error %d\n", failed));
1359 return failed;
1360}
1361
1362
1363static void delete_internal_operation(struct cryptocop_int_operation *iop)
1364{
1365 void *ptr = iop->alloc_ptr;
1366 struct cryptocop_dma_desc *cd = iop->cdesc_out;
1367 struct cryptocop_dma_desc *next;
1368
1369 DEBUG(printk("delete_internal_operation: iop=0x%p, alloc_ptr=0x%p\n", iop, ptr));
1370
1371 while (cd) {
1372 next = cd->next;
1373 free_cdesc(cd);
1374 cd = next;
1375 }
1376 cd = iop->cdesc_in;
1377 while (cd) {
1378 next = cd->next;
1379 free_cdesc(cd);
1380 cd = next;
1381 }
1382 kfree(ptr);
1383}
1384
1385#define MD5_MIN_PAD_LENGTH (9)
1386#define MD5_PAD_LENGTH_FIELD_LENGTH (8)
1387
1388static int create_md5_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length)
1389{
1390 size_t padlen = MD5_BLOCK_LENGTH - (hashed_length % MD5_BLOCK_LENGTH);
1391 unsigned char *p;
1392 int i;
1393 unsigned long long int bit_length = hashed_length << 3;
1394
1395 if (padlen < MD5_MIN_PAD_LENGTH) padlen += MD5_BLOCK_LENGTH;
1396
a52bdec3 1397 p = kzalloc(padlen, alloc_flag);
91a120d0 1398 if (!p) return -ENOMEM;
51533b61
MS		 1399
1400 *p = 0x80;
51533b61
MS		 1401
1402 DEBUG(printk("create_md5_pad: hashed_length=%lld bits == %lld bytes\n", bit_length, hashed_length));
1403
1404 i = padlen - MD5_PAD_LENGTH_FIELD_LENGTH;
1405 while (bit_length != 0){
1406 p[i++] = bit_length % 0x100;
1407 bit_length >>= 8;
1408 }
1409
1410 *pad = (char*)p;
1411 *pad_length = padlen;
1412
1413 return 0;
1414}
1415
1416#define SHA1_MIN_PAD_LENGTH (9)
1417#define SHA1_PAD_LENGTH_FIELD_LENGTH (8)
1418
1419static int create_sha1_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length)
1420{
1421 size_t padlen = SHA1_BLOCK_LENGTH - (hashed_length % SHA1_BLOCK_LENGTH);
1422 unsigned char *p;
1423 int i;
1424 unsigned long long int bit_length = hashed_length << 3;
1425
1426 if (padlen < SHA1_MIN_PAD_LENGTH) padlen += SHA1_BLOCK_LENGTH;
1427
a52bdec3 1428 p = kzalloc(padlen, alloc_flag);
91a120d0 1429 if (!p) return -ENOMEM;
51533b61
MS		 1430
1431 *p = 0x80;
51533b61
MS		 1432
1433 DEBUG(printk("create_sha1_pad: hashed_length=%lld bits == %lld bytes\n", bit_length, hashed_length));
1434
1435 i = padlen - 1;
1436 while (bit_length != 0){
1437 p[i--] = bit_length % 0x100;
1438 bit_length >>= 8;
1439 }
1440
1441 *pad = (char*)p;
1442 *pad_length = padlen;
1443
1444 return 0;
1445}
1446
1447
1448static int transform_ok(struct cryptocop_transform_init *tinit)
1449{
1450 switch (tinit->alg){
1451 case cryptocop_alg_csum:
1452 switch (tinit->csum_mode){
1453 case cryptocop_csum_le:
1454 case cryptocop_csum_be:
1455 break;
1456 default:
1457 DEBUG_API(printk("transform_ok: Bad mode set for csum transform\n"));
1458 return -EINVAL;
1459 }
1460 case cryptocop_alg_mem2mem:
1461 case cryptocop_alg_md5:
1462 case cryptocop_alg_sha1:
1463 if (tinit->keylen != 0) {
1464 DEBUG_API(printk("transform_ok: non-zero keylength, %d, for a digest/csum algorithm\n", tinit->keylen));
1465 return -EINVAL; /* This check is a bit strict. */
1466 }
1467 break;
1468 case cryptocop_alg_des:
1469 if (tinit->keylen != 64) {
1470 DEBUG_API(printk("transform_ok: keylen %d invalid for DES\n", tinit->keylen));
1471 return -EINVAL;
1472 }
1473 break;
1474 case cryptocop_alg_3des:
1475 if (tinit->keylen != 192) {
1476 DEBUG_API(printk("transform_ok: keylen %d invalid for 3DES\n", tinit->keylen));
1477 return -EINVAL;
1478 }
1479 break;
1480 case cryptocop_alg_aes:
1481 if (tinit->keylen != 128 && tinit->keylen != 192 && tinit->keylen != 256) {
1482 DEBUG_API(printk("transform_ok: keylen %d invalid for AES\n", tinit->keylen));
1483 return -EINVAL;
1484 }
1485 break;
1486 case cryptocop_no_alg:
1487 default:
1488 DEBUG_API(printk("transform_ok: no such algorithm %d\n", tinit->alg));
1489 return -EINVAL;
1490 }
1491
1492 switch (tinit->alg){
1493 case cryptocop_alg_des:
1494 case cryptocop_alg_3des:
1495 case cryptocop_alg_aes:
1496 if (tinit->cipher_mode != cryptocop_cipher_mode_ecb && tinit->cipher_mode != cryptocop_cipher_mode_cbc) return -EINVAL;
1497 default:
1498 break;
1499 }
1500 return 0;
1501}
1502
1503
1504int cryptocop_new_session(cryptocop_session_id *sid, struct cryptocop_transform_init *tinit, int alloc_flag)
1505{
1506 struct cryptocop_session *sess;
1507 struct cryptocop_transform_init *tfrm_in = tinit;
1508 struct cryptocop_transform_init *tmp_in;
1509 int no_tfrms = 0;
1510 int i;
1511 unsigned long int flags;
1512
1513 init_stream_coprocessor(); /* For safety if we are called early */
1514
1515 while (tfrm_in){
1516 int err;
1517 ++no_tfrms;
1518 if ((err = transform_ok(tfrm_in))) {
1519 DEBUG_API(printk("cryptocop_new_session, bad transform\n"));
1520 return err;
1521 }
1522 tfrm_in = tfrm_in->next;
1523 }
1524 if (0 == no_tfrms) {
1525 DEBUG_API(printk("cryptocop_new_session, no transforms specified\n"));
1526 return -EINVAL;
1527 }
1528
1529 sess = kmalloc(sizeof(struct cryptocop_session), alloc_flag);
1530 if (!sess){
1531 DEBUG_API(printk("cryptocop_new_session, kmalloc cryptocop_session\n"));
1532 return -ENOMEM;
1533 }
1534
1535 sess->tfrm_ctx = kmalloc(no_tfrms * sizeof(struct cryptocop_transform_ctx), alloc_flag);
1536 if (!sess->tfrm_ctx) {
1537 DEBUG_API(printk("cryptocop_new_session, kmalloc cryptocop_transform_ctx\n"));
1538 kfree(sess);
1539 return -ENOMEM;
1540 }
1541
1542 tfrm_in = tinit;
1543 for (i = 0; i < no_tfrms; i++){
1544 tmp_in = tfrm_in->next;
1545 while (tmp_in){
1546 if (tmp_in->tid == tfrm_in->tid) {
1547 DEBUG_API(printk("cryptocop_new_session, duplicate transform ids\n"));
1548 kfree(sess->tfrm_ctx);
1549 kfree(sess);
1550 return -EINVAL;
1551 }
1552 tmp_in = tmp_in->next;
1553 }
1554 memcpy(&sess->tfrm_ctx[i].init, tfrm_in, sizeof(struct cryptocop_transform_init));
1555 sess->tfrm_ctx[i].dec_key_set = 0;
1556 sess->tfrm_ctx[i].next = &sess->tfrm_ctx[i] + 1;
1557
1558 tfrm_in = tfrm_in->next;
1559 }
1560 sess->tfrm_ctx[i-1].next = NULL;
1561
1562 spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1563 sess->sid = next_sid;
1564 next_sid++;
1565 /* TODO If we are really paranoid we should do duplicate check to handle sid wraparound.
1566 * OTOH 2^64 is a really large number of session. */
1567 if (next_sid == 0) next_sid = 1;
1568
1569 /* Prepend to session list. */
1570 sess->next = cryptocop_sessions;
1571 cryptocop_sessions = sess;
1572 spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1573 *sid = sess->sid;
1574 return 0;
1575}
1576
1577
1578int cryptocop_free_session(cryptocop_session_id sid)
1579{
1580 struct cryptocop_transform_ctx *tc;
1581 struct cryptocop_session *sess = NULL;
1582 struct cryptocop_session *psess = NULL;
1583 unsigned long int flags;
1584 int i;
1585 LIST_HEAD(remove_list);
1586 struct list_head *node, *tmp;
1587 struct cryptocop_prio_job *pj;
1588
1589 DEBUG(printk("cryptocop_free_session: sid=%lld\n", sid));
1590
1591 spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1592 sess = cryptocop_sessions;
1593 while (sess && sess->sid != sid){
1594 psess = sess;
1595 sess = sess->next;
1596 }
1597 if (sess){
1598 if (psess){
1599 psess->next = sess->next;
1600 } else {
1601 cryptocop_sessions = sess->next;
1602 }
1603 }
1604 spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1605
1606 if (!sess) return -EINVAL;
1607
1608 /* Remove queued jobs. */
1609 spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
1610
1611 for (i = 0; i < cryptocop_prio_no_prios; i++){
1612 if (!list_empty(&(cryptocop_job_queues[i].jobs))){
1613 list_for_each_safe(node, tmp, &(cryptocop_job_queues[i].jobs)) {
1614 pj = list_entry(node, struct cryptocop_prio_job, node);
1615 if (pj->oper->sid == sid) {
1616 list_move_tail(node, &remove_list);
1617 }
1618 }
1619 }
1620 }
1621 spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
1622
1623 list_for_each_safe(node, tmp, &remove_list) {
1624 list_del(node);
1625 pj = list_entry(node, struct cryptocop_prio_job, node);
1626 pj->oper->operation_status = -EAGAIN; /* EAGAIN is not ideal for job/session terminated but it's the best choice I know of. */
1627 DEBUG(printk("cryptocop_free_session: pj=0x%p, pj->oper=0x%p, pj->iop=0x%p\n", pj, pj->oper, pj->iop));
1628 pj->oper->cb(pj->oper, pj->oper->cb_data);
1629 delete_internal_operation(pj->iop);
1630 kfree(pj);
1631 }
1632
1633 tc = sess->tfrm_ctx;
1634 /* Erase keying data. */
1635 while (tc){
1636 DEBUG(printk("cryptocop_free_session: memset keys, tfrm id=%d\n", tc->init.tid));
1637 memset(tc->init.key, 0xff, CRYPTOCOP_MAX_KEY_LENGTH);
1638 memset(tc->dec_key, 0xff, CRYPTOCOP_MAX_KEY_LENGTH);
1639 tc = tc->next;
1640 }
1641 kfree(sess->tfrm_ctx);
1642 kfree(sess);
1643
1644 return 0;
1645}
1646
1647static struct cryptocop_session *get_session(cryptocop_session_id sid)
1648{
1649 struct cryptocop_session *sess;
1650 unsigned long int flags;
1651
1652 spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1653 sess = cryptocop_sessions;
1654 while (sess && (sess->sid != sid)){
1655 sess = sess->next;
1656 }
1657 spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1658
1659 return sess;
1660}
1661
1662static struct cryptocop_transform_ctx *get_transform_ctx(struct cryptocop_session *sess, cryptocop_tfrm_id tid)
1663{
1664 struct cryptocop_transform_ctx *tc = sess->tfrm_ctx;
1665
1666 DEBUG(printk("get_transform_ctx, sess=0x%p, tid=%d\n", sess, tid));
1667 assert(sess != NULL);
1668 while (tc && tc->init.tid != tid){
1669 DEBUG(printk("tc=0x%p, tc->next=0x%p\n", tc, tc->next));
1670 tc = tc->next;
1671 }
1672 DEBUG(printk("get_transform_ctx, returning tc=0x%p\n", tc));
1673 return tc;
1674}
1675
1676
1677
1678/* The AES s-transform matrix (s-box). */
1679static const u8 aes_sbox[256] = {
1680 99, 124, 119, 123, 242, 107, 111, 197, 48, 1, 103, 43, 254, 215, 171, 118,
1681 202, 130, 201, 125, 250, 89, 71, 240, 173, 212, 162, 175, 156, 164, 114, 192,
1682 183, 253, 147, 38, 54, 63, 247, 204, 52, 165, 229, 241, 113, 216, 49, 21,
1683 4, 199, 35, 195, 24, 150, 5, 154, 7, 18, 128, 226, 235, 39, 178, 117,
1684 9, 131, 44, 26, 27, 110, 90, 160, 82, 59, 214, 179, 41, 227, 47, 132,
1685 83, 209, 0, 237, 32, 252, 177, 91, 106, 203, 190, 57, 74, 76, 88, 207,
1686 208, 239, 170, 251, 67, 77, 51, 133, 69, 249, 2, 127, 80, 60, 159, 168,
1687 81, 163, 64, 143, 146, 157, 56, 245, 188, 182, 218, 33, 16, 255, 243, 210,
1688 205, 12, 19, 236, 95, 151, 68, 23, 196, 167, 126, 61, 100, 93, 25, 115,
1689 96, 129, 79, 220, 34, 42, 144, 136, 70, 238, 184, 20, 222, 94, 11, 219,
1690 224, 50, 58, 10, 73, 6, 36, 92, 194, 211, 172, 98, 145, 149, 228, 121,
1691 231, 200, 55, 109, 141, 213, 78, 169, 108, 86, 244, 234, 101, 122, 174, 8,
1692 186, 120, 37, 46, 28, 166, 180, 198, 232, 221, 116, 31, 75, 189, 139, 138,
1693 112, 62, 181, 102, 72, 3, 246, 14, 97, 53, 87, 185, 134, 193, 29, 158,
1694 225, 248, 152, 17, 105, 217, 142, 148, 155, 30, 135, 233, 206, 85, 40, 223,
1695 140, 161, 137, 13, 191, 230, 66, 104, 65, 153, 45, 15, 176, 84, 187, 22
1696};
1697
1698/* AES has a 32 bit word round constants for each round in the
1699 * key schedule. round_constant[i] is really Rcon[i+1] in FIPS187.
1700 */
1701static u32 round_constant[11] = {
1702 0x01000000, 0x02000000, 0x04000000, 0x08000000,
1703 0x10000000, 0x20000000, 0x40000000, 0x80000000,
1704 0x1B000000, 0x36000000, 0x6C000000
1705};
1706
1707/* Apply the s-box to each of the four occtets in w. */
1708static u32 aes_ks_subword(const u32 w)
1709{
1710 u8 bytes[4];
1711
1712 *(u32*)(&bytes[0]) = w;
1713 bytes[0] = aes_sbox[bytes[0]];
1714 bytes[1] = aes_sbox[bytes[1]];
1715 bytes[2] = aes_sbox[bytes[2]];
1716 bytes[3] = aes_sbox[bytes[3]];
1717 return *(u32*)(&bytes[0]);
1718}
1719
1720/* The encrypt (forward) Rijndael key schedule algorithm pseudo code:
1721 * (Note that AES words are 32 bit long)
1722 *
1723 * KeyExpansion(byte key[4*Nk], word w[Nb*(Nr+1)], Nk){
1724 * word temp
1725 * i = 0
1726 * while (i < Nk) {
1727 * w[i] = word(key[4*i, 4*i + 1, 4*i + 2, 4*i + 3])
1728 * i = i + 1
1729 * }
1730 * i = Nk
1731 *
1732 * while (i < (Nb * (Nr + 1))) {
1733 * temp = w[i - 1]
1734 * if ((i mod Nk) == 0) {
1735 * temp = SubWord(RotWord(temp)) xor Rcon[i/Nk]
1736 * }
1737 * else if ((Nk > 6) && ((i mod Nk) == 4)) {
1738 * temp = SubWord(temp)
1739 * }
1740 * w[i] = w[i - Nk] xor temp
1741 * }
1742 * RotWord(t) does a 8 bit cyclic shift left on a 32 bit word.
1743 * SubWord(t) applies the AES s-box individually to each octet
1744 * in a 32 bit word.
1745 *
1746 * For AES Nk can have the values 4, 6, and 8 (corresponding to
1747 * values for Nr of 10, 12, and 14). Nb is always 4.
1748 *
1749 * To construct w[i], w[i - 1] and w[i - Nk] must be
1750 * available. Consequently we must keep a state of the last Nk words
1751 * to be able to create the last round keys.
1752 */
1753static void get_aes_decrypt_key(unsigned char *dec_key, const unsigned char *key, unsigned int keylength)
1754{
1755 u32 temp;
1756 u32 w_ring[8]; /* nk is max 8, use elements 0..(nk - 1) as a ringbuffer */
1757 u8 w_last_ix;
1758 int i;
1759 u8 nr, nk;
1760
1761 switch (keylength){
1762 case 128:
1763 nk = 4;
1764 nr = 10;
1765 break;
1766 case 192:
1767 nk = 6;
1768 nr = 12;
1769 break;
1770 case 256:
1771 nk = 8;
1772 nr = 14;
1773 break;
1774 default:
1775 panic("stream co-processor: bad aes key length in get_aes_decrypt_key\n");
1776 };
1777
1778 /* Need to do host byte order correction here since key is byte oriented and the
1779 * kx algorithm is word (u32) oriented. */
1780 for (i = 0; i < nk; i+=1) {
1781 w_ring[i] = be32_to_cpu(*(u32*)&key[4*i]);
1782 }
1783
1784 i = (int)nk;
1785 w_last_ix = i - 1;
1786 while (i < (4 * (nr + 2))) {
1787 temp = w_ring[w_last_ix];
1788 if (!(i % nk)) {
1789 /* RotWord(temp) */
1790 temp = (temp << 8) | (temp >> 24);
1791 temp = aes_ks_subword(temp);
1792 temp ^= round_constant[i/nk - 1];
1793 } else if ((nk > 6) && ((i % nk) == 4)) {
1794 temp = aes_ks_subword(temp);
1795 }
1796 w_last_ix = (w_last_ix + 1) % nk; /* This is the same as (i-Nk) mod Nk */
1797 temp ^= w_ring[w_last_ix];
1798 w_ring[w_last_ix] = temp;
1799
1800 /* We need the round keys for round Nr+1 and Nr+2 (round key
1801 * Nr+2 is the round key beyond the last one used when
1802 * encrypting). Rounds are numbered starting from 0, Nr=10
1803 * implies 11 rounds are used in encryption/decryption.
1804 */
1805 if (i >= (4 * nr)) {
1806 /* Need to do host byte order correction here, the key
1807 * is byte oriented. */
1808 *(u32*)dec_key = cpu_to_be32(temp);
1809 dec_key += 4;
1810 }
1811 ++i;
1812 }
1813}
1814
1815
1816/**** Job/operation management. ****/
1817
1818int cryptocop_job_queue_insert_csum(struct cryptocop_operation *operation)
1819{
1820 return cryptocop_job_queue_insert(cryptocop_prio_kernel_csum, operation);
1821}
1822
1823int cryptocop_job_queue_insert_crypto(struct cryptocop_operation *operation)
1824{
1825 return cryptocop_job_queue_insert(cryptocop_prio_kernel, operation);
1826}
1827
1828int cryptocop_job_queue_insert_user_job(struct cryptocop_operation *operation)
1829{
1830 return cryptocop_job_queue_insert(cryptocop_prio_user, operation);
1831}
1832
1833static int cryptocop_job_queue_insert(cryptocop_queue_priority prio, struct cryptocop_operation *operation)
1834{
1835 int ret;
1836 struct cryptocop_prio_job *pj = NULL;
1837 unsigned long int flags;
1838
1839 DEBUG(printk("cryptocop_job_queue_insert(%d, 0x%p)\n", prio, operation));
1840
1841 if (!operation || !operation->cb){
1842 DEBUG_API(printk("cryptocop_job_queue_insert oper=0x%p, NULL operation or callback\n", operation));
1843 return -EINVAL;
1844 }
1845
1846 if ((ret = cryptocop_job_setup(&pj, operation)) != 0){
1847 DEBUG_API(printk("cryptocop_job_queue_insert: job setup failed\n"));
1848 return ret;
1849 }
1850 assert(pj != NULL);
1851
1852 spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
1853 list_add_tail(&pj->node, &cryptocop_job_queues[prio].jobs);
1854 spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
1855
1856 /* Make sure a job is running */
1857 cryptocop_start_job();
1858 return 0;
1859}
1860
1861static void cryptocop_do_tasklet(unsigned long unused);
1862DECLARE_TASKLET (cryptocop_tasklet, cryptocop_do_tasklet, 0);
1863
1864static void cryptocop_do_tasklet(unsigned long unused)
1865{
1866 struct list_head *node;
1867 struct cryptocop_prio_job *pj = NULL;
1868 unsigned long flags;
1869
1870 DEBUG(printk("cryptocop_do_tasklet: entering\n"));
1871
1872 do {
1873 spin_lock_irqsave(&cryptocop_completed_jobs_lock, flags);
1874 if (!list_empty(&cryptocop_completed_jobs)){
1875 node = cryptocop_completed_jobs.next;
1876 list_del(node);
1877 pj = list_entry(node, struct cryptocop_prio_job, node);
1878 } else {
1879 pj = NULL;
1880 }
1881 spin_unlock_irqrestore(&cryptocop_completed_jobs_lock, flags);
1882 if (pj) {
1883 assert(pj->oper != NULL);
1884
1885 /* Notify consumer of operation completeness. */
1886 DEBUG(printk("cryptocop_do_tasklet: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
1887
1888 pj->oper->operation_status = 0; /* Job is completed. */
1889 pj->oper->cb(pj->oper, pj->oper->cb_data);
1890 delete_internal_operation(pj->iop);
1891 kfree(pj);
1892 }
1893 } while (pj != NULL);
1894
1895 DEBUG(printk("cryptocop_do_tasklet: exiting\n"));
1896}
1897
1898static irqreturn_t
635c45c1 1899dma_done_interrupt(int irq, void *dev_id)
51533b61
MS		 1900{
1901 struct cryptocop_prio_job *done_job;
1902 reg_dma_rw_ack_intr ack_intr = {
1903 .data = 1,
1904 };
1905
635c45c1 1906 REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
51533b61
MS		 1907
1908 DEBUG(printk("cryptocop DMA done\n"));
1909
1910 spin_lock(&running_job_lock);
1911 if (cryptocop_running_job == NULL){
1912 printk("stream co-processor got interrupt when not busy\n");
1913 spin_unlock(&running_job_lock);
1914 return IRQ_HANDLED;
1915 }
1916 done_job = cryptocop_running_job;
1917 cryptocop_running_job = NULL;
1918 spin_unlock(&running_job_lock);
1919
1920 /* Start processing a job. */
1921 if (!spin_trylock(&cryptocop_process_lock)){
1922 DEBUG(printk("cryptocop irq handler, not starting a job\n"));
1923 } else {
1924 cryptocop_start_job();
1925 spin_unlock(&cryptocop_process_lock);
1926 }
1927
1928 done_job->oper->operation_status = 0; /* Job is completed. */
1929 if (done_job->oper->fast_callback){
1930 /* This operation wants callback from interrupt. */
1931 done_job->oper->cb(done_job->oper, done_job->oper->cb_data);
1932 delete_internal_operation(done_job->iop);
1933 kfree(done_job);
1934 } else {
1935 spin_lock(&cryptocop_completed_jobs_lock);
1936 list_add_tail(&(done_job->node), &cryptocop_completed_jobs);
1937 spin_unlock(&cryptocop_completed_jobs_lock);
1938 tasklet_schedule(&cryptocop_tasklet);
1939 }
1940
1941 DEBUG(printk("cryptocop leave irq handler\n"));
1942 return IRQ_HANDLED;
1943}
1944
1945
1946/* Setup interrupts and DMA channels. */
1947static int init_cryptocop(void)
1948{
1949 unsigned long flags;
51533b61
MS		 1950 reg_dma_rw_cfg dma_cfg = {.en = 1};
1951 reg_dma_rw_intr_mask intr_mask_in = {.data = regk_dma_yes}; /* Only want descriptor interrupts from the DMA in channel. */
1952 reg_dma_rw_ack_intr ack_intr = {.data = 1,.in_eop = 1 };
1953 reg_strcop_rw_cfg strcop_cfg = {
1954 .ipend = regk_strcop_little,
1955 .td1 = regk_strcop_e,
1956 .td2 = regk_strcop_d,
1957 .td3 = regk_strcop_e,
1958 .ignore_sync = 0,
1959 .en = 1
1960 };
1961
635c45c1
JN		 1962 if (request_irq(DMA_IRQ, dma_done_interrupt, 0,
1963 "stream co-processor DMA", NULL))
1964 panic("request_irq stream co-processor irq dma9");
51533b61 1965
635c45c1
JN		 1966 (void)crisv32_request_dma(OUT_DMA, "strcop", DMA_PANIC_ON_ERROR,
1967 0, dma_strp);
1968 (void)crisv32_request_dma(IN_DMA, "strcop", DMA_PANIC_ON_ERROR,
1969 0, dma_strp);
51533b61
MS		 1970
1971 local_irq_save(flags);
1972
1973 /* Reset and enable the cryptocop. */
1974 strcop_cfg.en = 0;
1975 REG_WR(strcop, regi_strcop, rw_cfg, strcop_cfg);
1976 strcop_cfg.en = 1;
1977 REG_WR(strcop, regi_strcop, rw_cfg, strcop_cfg);
1978
51533b61 1979 /* Enable DMAs. */
635c45c1
JN		 1980 REG_WR(dma, IN_DMA_INST, rw_cfg, dma_cfg); /* input DMA */
1981 REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_cfg); /* output DMA */
51533b61
MS		 1982
1983 /* Set up wordsize = 4 for DMAs. */
635c45c1
JN		 1984 DMA_WR_CMD(OUT_DMA_INST, regk_dma_set_w_size4);
1985 DMA_WR_CMD(IN_DMA_INST, regk_dma_set_w_size4);
51533b61
MS		 1986
1987 /* Enable interrupts. */
635c45c1 1988 REG_WR(dma, IN_DMA_INST, rw_intr_mask, intr_mask_in);
51533b61
MS		 1989
1990 /* Clear intr ack. */
635c45c1 1991 REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
51533b61
MS		 1992
1993 local_irq_restore(flags);
1994
1995 return 0;
1996}
1997
1998/* Free used cryptocop hw resources (interrupt and DMA channels). */
1999static void release_cryptocop(void)
2000{
2001 unsigned long flags;
51533b61
MS		 2002 reg_dma_rw_cfg dma_cfg = {.en = 0};
2003 reg_dma_rw_intr_mask intr_mask_in = {0};
2004 reg_dma_rw_ack_intr ack_intr = {.data = 1,.in_eop = 1 };
2005
2006 local_irq_save(flags);
2007
2008 /* Clear intr ack. */
635c45c1 2009 REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
51533b61
MS		 2010
2011 /* Disable DMAs. */
635c45c1
JN		 2012 REG_WR(dma, IN_DMA_INST, rw_cfg, dma_cfg); /* input DMA */
2013 REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_cfg); /* output DMA */
51533b61
MS		 2014
2015 /* Disable interrupts. */
635c45c1 2016 REG_WR(dma, IN_DMA_INST, rw_intr_mask, intr_mask_in);
51533b61
MS		 2017
2018 local_irq_restore(flags);
2019
635c45c1 2020 free_irq(DMA_IRQ, NULL);
51533b61 2021
635c45c1
JN		 2022 (void)crisv32_free_dma(OUT_DMA);
2023 (void)crisv32_free_dma(IN_DMA);
51533b61
MS		 2024}
2025
2026
2027/* Init job queue. */
2028static int cryptocop_job_queue_init(void)
2029{
2030 int i;
2031
2032 INIT_LIST_HEAD(&cryptocop_completed_jobs);
2033
2034 for (i = 0; i < cryptocop_prio_no_prios; i++){
2035 cryptocop_job_queues[i].prio = (cryptocop_queue_priority)i;
2036 INIT_LIST_HEAD(&cryptocop_job_queues[i].jobs);
2037 }
2038 return 0;
2039}
2040
2041
2042static void cryptocop_job_queue_close(void)
2043{
2044 struct list_head *node, *tmp;
2045 struct cryptocop_prio_job *pj = NULL;
2046 unsigned long int process_flags, flags;
2047 int i;
2048
2049 /* FIXME: This is as yet untested code. */
2050
2051 /* Stop strcop from getting an operation to process while we are closing the
2052 module. */
2053 spin_lock_irqsave(&cryptocop_process_lock, process_flags);
2054
2055 /* Empty the job queue. */
51533b61
MS		 2056 for (i = 0; i < cryptocop_prio_no_prios; i++){
2057 if (!list_empty(&(cryptocop_job_queues[i].jobs))){
2058 list_for_each_safe(node, tmp, &(cryptocop_job_queues[i].jobs)) {
2059 pj = list_entry(node, struct cryptocop_prio_job, node);
2060 list_del(node);
2061
2062 /* Call callback to notify consumer of job removal. */
2063 DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2064 pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2065 pj->oper->cb(pj->oper, pj->oper->cb_data);
2066
2067 delete_internal_operation(pj->iop);
2068 kfree(pj);
2069 }
2070 }
2071 }
2072 spin_unlock_irqrestore(&cryptocop_process_lock, process_flags);
2073
2074 /* Remove the running job, if any. */
2075 spin_lock_irqsave(&running_job_lock, flags);
2076 if (cryptocop_running_job){
2077 reg_strcop_rw_cfg rw_cfg;
2078 reg_dma_rw_cfg dma_out_cfg, dma_in_cfg;
2079
2080 /* Stop DMA. */
635c45c1 2081 dma_out_cfg = REG_RD(dma, OUT_DMA_INST, rw_cfg);
51533b61 2082 dma_out_cfg.en = regk_dma_no;
635c45c1 2083 REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_out_cfg);
51533b61 2084
635c45c1 2085 dma_in_cfg = REG_RD(dma, IN_DMA_INST, rw_cfg);
51533b61 2086 dma_in_cfg.en = regk_dma_no;
635c45c1 2087 REG_WR(dma, IN_DMA_INST, rw_cfg, dma_in_cfg);
51533b61
MS		 2088
2089 /* Disble the cryptocop. */
2090 rw_cfg = REG_RD(strcop, regi_strcop, rw_cfg);
2091 rw_cfg.en = 0;
2092 REG_WR(strcop, regi_strcop, rw_cfg, rw_cfg);
2093
2094 pj = cryptocop_running_job;
2095 cryptocop_running_job = NULL;
2096
2097 /* Call callback to notify consumer of job removal. */
2098 DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2099 pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2100 pj->oper->cb(pj->oper, pj->oper->cb_data);
2101
2102 delete_internal_operation(pj->iop);
2103 kfree(pj);
2104 }
2105 spin_unlock_irqrestore(&running_job_lock, flags);
2106
2107 /* Remove completed jobs, if any. */
2108 spin_lock_irqsave(&cryptocop_completed_jobs_lock, flags);
2109
2110 list_for_each_safe(node, tmp, &cryptocop_completed_jobs) {
2111 pj = list_entry(node, struct cryptocop_prio_job, node);
2112 list_del(node);
2113 /* Call callback to notify consumer of job removal. */
2114 DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2115 pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2116 pj->oper->cb(pj->oper, pj->oper->cb_data);
2117
2118 delete_internal_operation(pj->iop);
2119 kfree(pj);
2120 }
2121 spin_unlock_irqrestore(&cryptocop_completed_jobs_lock, flags);
2122}
2123
2124
2125static void cryptocop_start_job(void)
2126{
2127 int i;
2128 struct cryptocop_prio_job *pj;
2129 unsigned long int flags;
2130 unsigned long int running_job_flags;
2131 reg_strcop_rw_cfg rw_cfg = {.en = 1, .ignore_sync = 0};
2132
2133 DEBUG(printk("cryptocop_start_job: entering\n"));
2134
2135 spin_lock_irqsave(&running_job_lock, running_job_flags);
2136 if (cryptocop_running_job != NULL){
2137 /* Already running. */
2138 DEBUG(printk("cryptocop_start_job: already running, exit\n"));
2139 spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2140 return;
2141 }
2142 spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
2143
2144 /* Check the queues in priority order. */
2145 for (i = cryptocop_prio_kernel_csum; (i < cryptocop_prio_no_prios) && list_empty(&cryptocop_job_queues[i].jobs); i++);
2146 if (i == cryptocop_prio_no_prios) {
2147 spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
2148 spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2149 DEBUG(printk("cryptocop_start_job: no jobs to run\n"));
2150 return; /* No jobs to run */
2151 }
2152 DEBUG(printk("starting job for prio %d\n", i));
2153
2154 /* TODO: Do not starve lower priority jobs. Let in a lower
2155 * prio job for every N-th processed higher prio job or some
2156 * other scheduling policy. This could reasonably be
2157 * tweakable since the optimal balance would depend on the
2158 * type of load on the system. */
2159
2160 /* Pull the DMA lists from the job and start the DMA client. */
2161 pj = list_entry(cryptocop_job_queues[i].jobs.next, struct cryptocop_prio_job, node);
2162 list_del(&pj->node);
2163 spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
2164 cryptocop_running_job = pj;
2165
2166 /* Set config register (3DES and CSUM modes). */
2167 switch (pj->iop->tdes_mode){
2168 case cryptocop_3des_eee:
2169 rw_cfg.td1 = regk_strcop_e;
2170 rw_cfg.td2 = regk_strcop_e;
2171 rw_cfg.td3 = regk_strcop_e;
2172 break;
2173 case cryptocop_3des_eed:
2174 rw_cfg.td1 = regk_strcop_e;
2175 rw_cfg.td2 = regk_strcop_e;
2176 rw_cfg.td3 = regk_strcop_d;
2177 break;
2178 case cryptocop_3des_ede:
2179 rw_cfg.td1 = regk_strcop_e;
2180 rw_cfg.td2 = regk_strcop_d;
2181 rw_cfg.td3 = regk_strcop_e;
2182 break;
2183 case cryptocop_3des_edd:
2184 rw_cfg.td1 = regk_strcop_e;
2185 rw_cfg.td2 = regk_strcop_d;
2186 rw_cfg.td3 = regk_strcop_d;
2187 break;
2188 case cryptocop_3des_dee:
2189 rw_cfg.td1 = regk_strcop_d;
2190 rw_cfg.td2 = regk_strcop_e;
2191 rw_cfg.td3 = regk_strcop_e;
2192 break;
2193 case cryptocop_3des_ded:
2194 rw_cfg.td1 = regk_strcop_d;
2195 rw_cfg.td2 = regk_strcop_e;
2196 rw_cfg.td3 = regk_strcop_d;
2197 break;
2198 case cryptocop_3des_dde:
2199 rw_cfg.td1 = regk_strcop_d;
2200 rw_cfg.td2 = regk_strcop_d;
2201 rw_cfg.td3 = regk_strcop_e;
2202 break;
2203 case cryptocop_3des_ddd:
2204 rw_cfg.td1 = regk_strcop_d;
2205 rw_cfg.td2 = regk_strcop_d;
2206 rw_cfg.td3 = regk_strcop_d;
2207 break;
2208 default:
2209 DEBUG(printk("cryptocop_setup_dma_list: bad 3DES mode\n"));
2210 }
2211 switch (pj->iop->csum_mode){
2212 case cryptocop_csum_le:
2213 rw_cfg.ipend = regk_strcop_little;
2214 break;
2215 case cryptocop_csum_be:
2216 rw_cfg.ipend = regk_strcop_big;
2217 break;
2218 default:
2219 DEBUG(printk("cryptocop_setup_dma_list: bad checksum mode\n"));
2220 }
2221 REG_WR(strcop, regi_strcop, rw_cfg, rw_cfg);
2222
2223 DEBUG(printk("cryptocop_start_job: starting DMA, new cryptocop_running_job=0x%p\n"
2224 "ctx_in: 0x%p, phys: 0x%p\n"
2225 "ctx_out: 0x%p, phys: 0x%p\n",
2226 pj,
2227 &pj->iop->ctx_in, (char*)virt_to_phys(&pj->iop->ctx_in),
2228 &pj->iop->ctx_out, (char*)virt_to_phys(&pj->iop->ctx_out)));
2229
2230 /* Start input DMA. */
635c45c1
JN		 2231 flush_dma_context(&pj->iop->ctx_in);
2232 DMA_START_CONTEXT(IN_DMA_INST, virt_to_phys(&pj->iop->ctx_in));
51533b61
MS		 2233
2234 /* Start output DMA. */
635c45c1 2235 DMA_START_CONTEXT(OUT_DMA_INST, virt_to_phys(&pj->iop->ctx_out));
51533b61
MS		 2236
2237 spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2238 DEBUG(printk("cryptocop_start_job: exiting\n"));
2239}
2240
2241
2242static int cryptocop_job_setup(struct cryptocop_prio_job **pj, struct cryptocop_operation *operation)
2243{
2244 int err;
2245 int alloc_flag = operation->in_interrupt ? GFP_ATOMIC : GFP_KERNEL;
2246 void *iop_alloc_ptr = NULL;
2247
2248 *pj = kmalloc(sizeof (struct cryptocop_prio_job), alloc_flag);
2249 if (!*pj) return -ENOMEM;
2250
2251 DEBUG(printk("cryptocop_job_setup: operation=0x%p\n", operation));
2252
2253 (*pj)->oper = operation;
2254 DEBUG(printk("cryptocop_job_setup, cb=0x%p cb_data=0x%p\n", (*pj)->oper->cb, (*pj)->oper->cb_data));
2255
2256 if (operation->use_dmalists) {
2257 DEBUG(print_user_dma_lists(&operation->list_op));
2258 if (!operation->list_op.inlist || !operation->list_op.outlist || !operation->list_op.out_data_buf || !operation->list_op.in_data_buf){
2259 DEBUG_API(printk("cryptocop_job_setup: bad indata (use_dmalists)\n"));
2260 kfree(*pj);
2261 return -EINVAL;
2262 }
2263 iop_alloc_ptr = kmalloc(DESCR_ALLOC_PAD + sizeof(struct cryptocop_int_operation), alloc_flag);
2264 if (!iop_alloc_ptr) {
2265 DEBUG_API(printk("cryptocop_job_setup: kmalloc cryptocop_int_operation\n"));
2266 kfree(*pj);
2267 return -ENOMEM;
2268 }
2269 (*pj)->iop = (struct cryptocop_int_operation*)(((unsigned long int)(iop_alloc_ptr + DESCR_ALLOC_PAD + offsetof(struct cryptocop_int_operation, ctx_out)) & ~0x0000001F) - offsetof(struct cryptocop_int_operation, ctx_out));
2270 DEBUG(memset((*pj)->iop, 0xff, sizeof(struct cryptocop_int_operation)));
2271 (*pj)->iop->alloc_ptr = iop_alloc_ptr;
2272 (*pj)->iop->sid = operation->sid;
2273 (*pj)->iop->cdesc_out = NULL;
2274 (*pj)->iop->cdesc_in = NULL;
2275 (*pj)->iop->tdes_mode = operation->list_op.tdes_mode;
2276 (*pj)->iop->csum_mode = operation->list_op.csum_mode;
2277 (*pj)->iop->ddesc_out = operation->list_op.outlist;
2278 (*pj)->iop->ddesc_in = operation->list_op.inlist;
2279
2280 /* Setup DMA contexts. */
2281 (*pj)->iop->ctx_out.next = NULL;
2282 (*pj)->iop->ctx_out.eol = 1;
2283 (*pj)->iop->ctx_out.saved_data = operation->list_op.outlist;
2284 (*pj)->iop->ctx_out.saved_data_buf = operation->list_op.out_data_buf;
2285
2286 (*pj)->iop->ctx_in.next = NULL;
2287 (*pj)->iop->ctx_in.eol = 1;
2288 (*pj)->iop->ctx_in.saved_data = operation->list_op.inlist;
2289 (*pj)->iop->ctx_in.saved_data_buf = operation->list_op.in_data_buf;
2290 } else {
2291 if ((err = cryptocop_setup_dma_list(operation, &(*pj)->iop, alloc_flag))) {
2292 DEBUG_API(printk("cryptocop_job_setup: cryptocop_setup_dma_list failed %d\n", err));
2293 kfree(*pj);
2294 return err;
2295 }
2296 }
2297 DEBUG(print_dma_descriptors((*pj)->iop));
2298
2299 DEBUG(printk("cryptocop_job_setup, DMA list setup successful\n"));
2300
2301 return 0;
2302}
2303
51533b61
MS		 2304static int cryptocop_open(struct inode *inode, struct file *filp)
2305{
32ea086b 2306 int p = iminor(inode);
51533b61
MS		 2307
2308 if (p != CRYPTOCOP_MINOR) return -EINVAL;
2309
2310 filp->private_data = NULL;
2311 return 0;
2312}
2313
2314
2315static int cryptocop_release(struct inode *inode, struct file *filp)
2316{
2317 struct cryptocop_private *dev = filp->private_data;
2318 struct cryptocop_private *dev_next;
2319
2320 while (dev){
2321 dev_next = dev->next;
2322 if (dev->sid != CRYPTOCOP_SESSION_ID_NONE) {
2323 (void)cryptocop_free_session(dev->sid);
2324 }
2325 kfree(dev);
2326 dev = dev_next;
2327 }
2328
2329 return 0;
2330}
2331
2332
2333static int cryptocop_ioctl_close_session(struct inode *inode, struct file *filp,
2334 unsigned int cmd, unsigned long arg)
2335{
2336 struct cryptocop_private *dev = filp->private_data;
2337 struct cryptocop_private *prev_dev = NULL;
2338 struct strcop_session_op *sess_op = (struct strcop_session_op *)arg;
2339 struct strcop_session_op sop;
2340 int err;
2341
2342 DEBUG(printk("cryptocop_ioctl_close_session\n"));
2343
2344 if (!access_ok(VERIFY_READ, sess_op, sizeof(struct strcop_session_op)))
2345 return -EFAULT;
2346 err = copy_from_user(&sop, sess_op, sizeof(struct strcop_session_op));
2347 if (err) return -EFAULT;
2348
2349 while (dev && (dev->sid != sop.ses_id)) {
2350 prev_dev = dev;
2351 dev = dev->next;
2352 }
2353 if (dev){
2354 if (prev_dev){
2355 prev_dev->next = dev->next;
2356 } else {
2357 filp->private_data = dev->next;
2358 }
2359 err = cryptocop_free_session(dev->sid);
2360 if (err) return -EFAULT;
2361 } else {
2362 DEBUG_API(printk("cryptocop_ioctl_close_session: session %lld not found\n", sop.ses_id));
2363 return -EINVAL;
2364 }
2365 return 0;
2366}
2367
2368
2369static void ioctl_process_job_callback(struct cryptocop_operation *op, void*cb_data)
2370{
2371 struct ioctl_job_cb_ctx *jc = (struct ioctl_job_cb_ctx *)cb_data;
2372
2373 DEBUG(printk("ioctl_process_job_callback: op=0x%p, cb_data=0x%p\n", op, cb_data));
2374
2375 jc->processed = 1;
2376 wake_up(&cryptocop_ioc_process_wq);
2377}
2378
2379
2380#define CRYPTOCOP_IOCTL_CIPHER_TID (1)
2381#define CRYPTOCOP_IOCTL_DIGEST_TID (2)
2382#define CRYPTOCOP_IOCTL_CSUM_TID (3)
2383
2384static size_t first_cfg_change_ix(struct strcop_crypto_op *crp_op)
2385{
2386 size_t ch_ix = 0;
2387
2388 if (crp_op->do_cipher) ch_ix = crp_op->cipher_start;
2389 if (crp_op->do_digest && (crp_op->digest_start < ch_ix)) ch_ix = crp_op->digest_start;
2390 if (crp_op->do_csum && (crp_op->csum_start < ch_ix)) ch_ix = crp_op->csum_start;
2391
2392 DEBUG(printk("first_cfg_change_ix: ix=%d\n", ch_ix));
2393 return ch_ix;
2394}
2395
2396
2397static size_t next_cfg_change_ix(struct strcop_crypto_op *crp_op, size_t ix)
2398{
2399 size_t ch_ix = INT_MAX;
2400 size_t tmp_ix = 0;
2401
2402 if (crp_op->do_cipher && ((crp_op->cipher_start + crp_op->cipher_len) > ix)){
2403 if (crp_op->cipher_start > ix) {
2404 ch_ix = crp_op->cipher_start;
2405 } else {
2406 ch_ix = crp_op->cipher_start + crp_op->cipher_len;
2407 }
2408 }
2409 if (crp_op->do_digest && ((crp_op->digest_start + crp_op->digest_len) > ix)){
2410 if (crp_op->digest_start > ix) {
2411 tmp_ix = crp_op->digest_start;
2412 } else {
2413 tmp_ix = crp_op->digest_start + crp_op->digest_len;
2414 }
2415 if (tmp_ix < ch_ix) ch_ix = tmp_ix;
2416 }
2417 if (crp_op->do_csum && ((crp_op->csum_start + crp_op->csum_len) > ix)){
2418 if (crp_op->csum_start > ix) {
2419 tmp_ix = crp_op->csum_start;
2420 } else {
2421 tmp_ix = crp_op->csum_start + crp_op->csum_len;
2422 }
2423 if (tmp_ix < ch_ix) ch_ix = tmp_ix;
2424 }
2425 if (ch_ix == INT_MAX) ch_ix = ix;
2426 DEBUG(printk("next_cfg_change_ix prev ix=%d, next ix=%d\n", ix, ch_ix));
2427 return ch_ix;
2428}
2429
2430
2431/* Map map_length bytes from the pages starting on *pageix and *pageoffset to iovecs starting on *iovix.
2432 * Return -1 for ok, 0 for fail. */
2433static int map_pages_to_iovec(struct iovec *iov, int iovlen, int *iovix, struct page **pages, int nopages, int *pageix, int *pageoffset, int map_length )
2434{
2435 int tmplen;
2436
2437 assert(iov != NULL);
2438 assert(iovix != NULL);
2439 assert(pages != NULL);
2440 assert(pageix != NULL);
2441 assert(pageoffset != NULL);
2442
2443 DEBUG(printk("map_pages_to_iovec, map_length=%d, iovlen=%d, *iovix=%d, nopages=%d, *pageix=%d, *pageoffset=%d\n", map_length, iovlen, *iovix, nopages, *pageix, *pageoffset));
2444
2445 while (map_length > 0){
2446 DEBUG(printk("map_pages_to_iovec, map_length=%d, iovlen=%d, *iovix=%d, nopages=%d, *pageix=%d, *pageoffset=%d\n", map_length, iovlen, *iovix, nopages, *pageix, *pageoffset));
2447 if (*iovix >= iovlen){
2448 DEBUG_API(printk("map_page_to_iovec: *iovix=%d >= iovlen=%d\n", *iovix, iovlen));
2449 return 0;
2450 }
2451 if (*pageix >= nopages){
2452 DEBUG_API(printk("map_page_to_iovec: *pageix=%d >= nopages=%d\n", *pageix, nopages));
2453 return 0;
2454 }
2455 iov[*iovix].iov_base = (unsigned char*)page_address(pages[*pageix]) + *pageoffset;
2456 tmplen = PAGE_SIZE - *pageoffset;
2457 if (tmplen < map_length){
2458 (*pageoffset) = 0;
2459 (*pageix)++;
2460 } else {
2461 tmplen = map_length;
2462 (*pageoffset) += map_length;
2463 }
2464 DEBUG(printk("mapping %d bytes from page %d (or %d) to iovec %d\n", tmplen, *pageix, *pageix-1, *iovix));
2465 iov[*iovix].iov_len = tmplen;
2466 map_length -= tmplen;
2467 (*iovix)++;
2468 }
2469 DEBUG(printk("map_page_to_iovec, exit, *iovix=%d\n", *iovix));
2470 return -1;
2471}
2472
2473
2474
2475static int cryptocop_ioctl_process(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
2476{
2477 int i;
2478 struct cryptocop_private *dev = filp->private_data;
2479 struct strcop_crypto_op *crp_oper = (struct strcop_crypto_op *)arg;
2480 struct strcop_crypto_op oper = {0};
2481 int err = 0;
2482 struct cryptocop_operation *cop = NULL;
2483
2484 struct ioctl_job_cb_ctx *jc = NULL;
2485
2486 struct page **inpages = NULL;
2487 struct page **outpages = NULL;
2488 int noinpages = 0;
2489 int nooutpages = 0;
2490
2491 struct cryptocop_desc descs[5]; /* Max 5 descriptors are needed, there are three transforms that
2492 * can get connected/disconnected on different places in the indata. */
2493 struct cryptocop_desc_cfg dcfgs[5*3];
2494 int desc_ix = 0;
2495 int dcfg_ix = 0;
2496 struct cryptocop_tfrm_cfg ciph_tcfg = {0};
2497 struct cryptocop_tfrm_cfg digest_tcfg = {0};
2498 struct cryptocop_tfrm_cfg csum_tcfg = {0};
2499
2500 unsigned char *digest_result = NULL;
2501 int digest_length = 0;
2502 int cblocklen = 0;
2503 unsigned char csum_result[CSUM_BLOCK_LENGTH];
2504 struct cryptocop_session *sess;
2505
2506 int iovlen = 0;
2507 int iovix = 0;
2508 int pageix = 0;
2509 int pageoffset = 0;
2510
2511 size_t prev_ix = 0;
2512 size_t next_ix;
2513
2514 int cipher_active, digest_active, csum_active;
2515 int end_digest, end_csum;
2516 int digest_done = 0;
2517 int cipher_done = 0;
2518 int csum_done = 0;
2519
2520 DEBUG(printk("cryptocop_ioctl_process\n"));
2521
2522 if (!access_ok(VERIFY_WRITE, crp_oper, sizeof(struct strcop_crypto_op))){
2523 DEBUG_API(printk("cryptocop_ioctl_process: !access_ok crp_oper!\n"));
2524 return -EFAULT;
2525 }
2526 if (copy_from_user(&oper, crp_oper, sizeof(struct strcop_crypto_op))) {
2527 DEBUG_API(printk("cryptocop_ioctl_process: copy_from_user\n"));
2528 return -EFAULT;
2529 }
2530 DEBUG(print_strcop_crypto_op(&oper));
2531
2532 while (dev && dev->sid != oper.ses_id) dev = dev->next;
2533 if (!dev){
2534 DEBUG_API(printk("cryptocop_ioctl_process: session %lld not found\n", oper.ses_id));
2535 return -EINVAL;
2536 }
2537
2538 /* Check buffers. */
2539 if (((oper.indata + oper.inlen) < oper.indata) || ((oper.cipher_outdata + oper.cipher_outlen) < oper.cipher_outdata)){
2540 DEBUG_API(printk("cryptocop_ioctl_process: user buffers wrapped around, bad user!\n"));
2541 return -EINVAL;
2542 }
2543
2544 if (!access_ok(VERIFY_WRITE, oper.cipher_outdata, oper.cipher_outlen)){
2545 DEBUG_API(printk("cryptocop_ioctl_process: !access_ok out data!\n"));
2546 return -EFAULT;
2547 }
2548 if (!access_ok(VERIFY_READ, oper.indata, oper.inlen)){
2549 DEBUG_API(printk("cryptocop_ioctl_process: !access_ok in data!\n"));
2550 return -EFAULT;
2551 }
2552
2553 cop = kmalloc(sizeof(struct cryptocop_operation), GFP_KERNEL);
2554 if (!cop) {
2555 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc\n"));
2556 return -ENOMEM;
2557 }
2558 jc = kmalloc(sizeof(struct ioctl_job_cb_ctx), GFP_KERNEL);
2559 if (!jc) {
2560 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc\n"));
2561 err = -ENOMEM;
2562 goto error_cleanup;
2563 }
2564 jc->processed = 0;
2565
2566 cop->cb_data = jc;
2567 cop->cb = ioctl_process_job_callback;
2568 cop->operation_status = 0;
2569 cop->use_dmalists = 0;
2570 cop->in_interrupt = 0;
2571 cop->fast_callback = 0;
2572 cop->tfrm_op.tfrm_cfg = NULL;
2573 cop->tfrm_op.desc = NULL;
2574 cop->tfrm_op.indata = NULL;
2575 cop->tfrm_op.incount = 0;
2576 cop->tfrm_op.inlen = 0;
2577 cop->tfrm_op.outdata = NULL;
2578 cop->tfrm_op.outcount = 0;
2579 cop->tfrm_op.outlen = 0;
2580
2581 sess = get_session(oper.ses_id);
2582 if (!sess){
2583 DEBUG_API(printk("cryptocop_ioctl_process: bad session id.\n"));
2584 kfree(cop);
2585 kfree(jc);
2586 return -EINVAL;
2587 }
2588
2589 if (oper.do_cipher) {
2590 unsigned int cipher_outlen = 0;
2591 struct cryptocop_transform_ctx *tc = get_transform_ctx(sess, CRYPTOCOP_IOCTL_CIPHER_TID);
2592 if (!tc) {
2593 DEBUG_API(printk("cryptocop_ioctl_process: no cipher transform in session.\n"));
2594 err = -EINVAL;
2595 goto error_cleanup;
2596 }
2597 ciph_tcfg.tid = CRYPTOCOP_IOCTL_CIPHER_TID;
2598 ciph_tcfg.inject_ix = 0;
2599 ciph_tcfg.flags = 0;
2600 if ((oper.cipher_start < 0) || (oper.cipher_len <= 0) || (oper.cipher_start > oper.inlen) || ((oper.cipher_start + oper.cipher_len) > oper.inlen)){
2601 DEBUG_API(printk("cryptocop_ioctl_process: bad cipher length\n"));
2602 kfree(cop);
2603 kfree(jc);
2604 return -EINVAL;
2605 }
2606 cblocklen = tc->init.alg == cryptocop_alg_aes ? AES_BLOCK_LENGTH : DES_BLOCK_LENGTH;
2607 if (oper.cipher_len % cblocklen) {
2608 kfree(cop);
2609 kfree(jc);
2610 DEBUG_API(printk("cryptocop_ioctl_process: cipher inlength not multiple of block length.\n"));
2611 return -EINVAL;
2612 }
2613 cipher_outlen = oper.cipher_len;
2614 if (tc->init.cipher_mode == cryptocop_cipher_mode_cbc){
2615 if (oper.cipher_explicit) {
2616 ciph_tcfg.flags |= CRYPTOCOP_EXPLICIT_IV;
2617 memcpy(ciph_tcfg.iv, oper.cipher_iv, cblocklen);
2618 } else {
2619 cipher_outlen = oper.cipher_len - cblocklen;
2620 }
2621 } else {
2622 if (oper.cipher_explicit){
2623 kfree(cop);
2624 kfree(jc);
2625 DEBUG_API(printk("cryptocop_ioctl_process: explicit_iv when not CBC mode\n"));
2626 return -EINVAL;
2627 }
2628 }
2629 if (oper.cipher_outlen != cipher_outlen) {
2630 kfree(cop);
2631 kfree(jc);
2632 DEBUG_API(printk("cryptocop_ioctl_process: cipher_outlen incorrect, should be %d not %d.\n", cipher_outlen, oper.cipher_outlen));
2633 return -EINVAL;
2634 }
2635
2636 if (oper.decrypt){
2637 ciph_tcfg.flags |= CRYPTOCOP_DECRYPT;
2638 } else {
2639 ciph_tcfg.flags |= CRYPTOCOP_ENCRYPT;
2640 }
2641 ciph_tcfg.next = cop->tfrm_op.tfrm_cfg;
2642 cop->tfrm_op.tfrm_cfg = &ciph_tcfg;
2643 }
2644 if (oper.do_digest){
2645 struct cryptocop_transform_ctx *tc = get_transform_ctx(sess, CRYPTOCOP_IOCTL_DIGEST_TID);
2646 if (!tc) {
2647 DEBUG_API(printk("cryptocop_ioctl_process: no digest transform in session.\n"));
2648 err = -EINVAL;
2649 goto error_cleanup;
2650 }
2651 digest_length = tc->init.alg == cryptocop_alg_md5 ? 16 : 20;
2652 digest_result = kmalloc(digest_length, GFP_KERNEL);
2653 if (!digest_result) {
2654 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc digest_result\n"));
2655 err = -EINVAL;
2656 goto error_cleanup;
2657 }
2658 DEBUG(memset(digest_result, 0xff, digest_length));
2659
2660 digest_tcfg.tid = CRYPTOCOP_IOCTL_DIGEST_TID;
2661 digest_tcfg.inject_ix = 0;
2662 ciph_tcfg.inject_ix += digest_length;
2663 if ((oper.digest_start < 0) || (oper.digest_len <= 0) || (oper.digest_start > oper.inlen) || ((oper.digest_start + oper.digest_len) > oper.inlen)){
2664 DEBUG_API(printk("cryptocop_ioctl_process: bad digest length\n"));
2665 err = -EINVAL;
2666 goto error_cleanup;
2667 }
2668
2669 digest_tcfg.next = cop->tfrm_op.tfrm_cfg;
2670 cop->tfrm_op.tfrm_cfg = &digest_tcfg;
2671 }
2672 if (oper.do_csum){
2673 csum_tcfg.tid = CRYPTOCOP_IOCTL_CSUM_TID;
2674 csum_tcfg.inject_ix = digest_length;
2675 ciph_tcfg.inject_ix += 2;
2676
2677 if ((oper.csum_start < 0) || (oper.csum_len <= 0) || (oper.csum_start > oper.inlen) || ((oper.csum_start + oper.csum_len) > oper.inlen)){
2678 DEBUG_API(printk("cryptocop_ioctl_process: bad csum length\n"));
2679 kfree(cop);
2680 kfree(jc);
2681 return -EINVAL;
2682 }
2683
2684 csum_tcfg.next = cop->tfrm_op.tfrm_cfg;
2685 cop->tfrm_op.tfrm_cfg = &csum_tcfg;
2686 }
2687
2688 prev_ix = first_cfg_change_ix(&oper);
2689 if (prev_ix > oper.inlen) {
2690 DEBUG_API(printk("cryptocop_ioctl_process: length mismatch\n"));
2691 nooutpages = noinpages = 0;
2692 err = -EINVAL;
2693 goto error_cleanup;
2694 }
2695 DEBUG(printk("cryptocop_ioctl_process: inlen=%d, cipher_outlen=%d\n", oper.inlen, oper.cipher_outlen));
2696
2697 /* Map user pages for in and out data of the operation. */
2698 noinpages = (((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK) + oper.inlen - 1 - prev_ix + ~PAGE_MASK) >> PAGE_SHIFT;
2699 DEBUG(printk("cryptocop_ioctl_process: noinpages=%d\n", noinpages));
2700 inpages = kmalloc(noinpages * sizeof(struct page*), GFP_KERNEL);
2701 if (!inpages){
2702 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc inpages\n"));
2703 nooutpages = noinpages = 0;
2704 err = -ENOMEM;
2705 goto error_cleanup;
2706 }
2707 if (oper.do_cipher){
2708 nooutpages = (((unsigned long int)oper.cipher_outdata & ~PAGE_MASK) + oper.cipher_outlen - 1 + ~PAGE_MASK) >> PAGE_SHIFT;
2709 DEBUG(printk("cryptocop_ioctl_process: nooutpages=%d\n", nooutpages));
2710 outpages = kmalloc(nooutpages * sizeof(struct page*), GFP_KERNEL);
2711 if (!outpages){
2712 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc outpages\n"));
2713 nooutpages = noinpages = 0;
2714 err = -ENOMEM;
2715 goto error_cleanup;
2716 }
2717 }
2718
2719 /* Acquire the mm page semaphore. */
2720 down_read(&current->mm->mmap_sem);
2721
d4edcf0d 2722 err = get_user_pages((unsigned long int)(oper.indata + prev_ix),
51533b61
MS		 2723 noinpages,
2724 0, /* read access only for in data */
51533b61
MS		 2725 inpages,
2726 NULL);
2727
2728 if (err < 0) {
2729 up_read(&current->mm->mmap_sem);
2730 nooutpages = noinpages = 0;
2731 DEBUG_API(printk("cryptocop_ioctl_process: get_user_pages indata\n"));
2732 goto error_cleanup;
2733 }
2734 noinpages = err;
2735 if (oper.do_cipher){
d4edcf0d 2736 err = get_user_pages((unsigned long int)oper.cipher_outdata,
51533b61 2737 nooutpages,
768ae309 2738 FOLL_WRITE, /* write access for out data */
51533b61
MS		 2739 outpages,
2740 NULL);
2741 up_read(&current->mm->mmap_sem);
2742 if (err < 0) {
2743 nooutpages = 0;
2744 DEBUG_API(printk("cryptocop_ioctl_process: get_user_pages outdata\n"));
2745 goto error_cleanup;
2746 }
2747 nooutpages = err;
2748 } else {
2749 up_read(&current->mm->mmap_sem);
2750 }
2751
2752 /* Add 6 to nooutpages to make room for possibly inserted buffers for storing digest and
2753 * csum output and splits when units are (dis-)connected. */
2754 cop->tfrm_op.indata = kmalloc((noinpages) * sizeof(struct iovec), GFP_KERNEL);
2755 cop->tfrm_op.outdata = kmalloc((6 + nooutpages) * sizeof(struct iovec), GFP_KERNEL);
2756 if (!cop->tfrm_op.indata || !cop->tfrm_op.outdata) {
2757 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc iovecs\n"));
2758 err = -ENOMEM;
2759 goto error_cleanup;
2760 }
2761
2762 cop->tfrm_op.inlen = oper.inlen - prev_ix;
2763 cop->tfrm_op.outlen = 0;
2764 if (oper.do_cipher) cop->tfrm_op.outlen += oper.cipher_outlen;
2765 if (oper.do_digest) cop->tfrm_op.outlen += digest_length;
2766 if (oper.do_csum) cop->tfrm_op.outlen += 2;
2767
2768 /* Setup the in iovecs. */
2769 cop->tfrm_op.incount = noinpages;
2770 if (noinpages > 1){
2771 size_t tmplen = cop->tfrm_op.inlen;
2772
2773 cop->tfrm_op.indata[0].iov_len = PAGE_SIZE - ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2774 cop->tfrm_op.indata[0].iov_base = (unsigned char*)page_address(inpages[0]) + ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2775 tmplen -= cop->tfrm_op.indata[0].iov_len;
2776 for (i = 1; i<noinpages; i++){
2777 cop->tfrm_op.indata[i].iov_len = tmplen < PAGE_SIZE ? tmplen : PAGE_SIZE;
2778 cop->tfrm_op.indata[i].iov_base = (unsigned char*)page_address(inpages[i]);
2779 tmplen -= PAGE_SIZE;
2780 }
2781 } else {
2782 cop->tfrm_op.indata[0].iov_len = oper.inlen - prev_ix;
2783 cop->tfrm_op.indata[0].iov_base = (unsigned char*)page_address(inpages[0]) + ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2784 }
2785
2786 iovlen = nooutpages + 6;
2787 pageoffset = oper.do_cipher ? ((unsigned long int)oper.cipher_outdata & ~PAGE_MASK) : 0;
2788
2789 next_ix = next_cfg_change_ix(&oper, prev_ix);
2790 if (prev_ix == next_ix){
2791 DEBUG_API(printk("cryptocop_ioctl_process: length configuration broken.\n"));
2792 err = -EINVAL; /* This should be impossible barring bugs. */
2793 goto error_cleanup;
2794 }
2795 while (prev_ix != next_ix){
2796 end_digest = end_csum = cipher_active = digest_active = csum_active = 0;
2797 descs[desc_ix].cfg = NULL;
2798 descs[desc_ix].length = next_ix - prev_ix;
2799
2800 if (oper.do_cipher && (oper.cipher_start < next_ix) && (prev_ix < (oper.cipher_start + oper.cipher_len))) {
2801 dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_CIPHER_TID;
2802 dcfgs[dcfg_ix].src = cryptocop_source_dma;
2803 cipher_active = 1;
2804
2805 if (next_ix == (oper.cipher_start + oper.cipher_len)){
2806 cipher_done = 1;
2807 dcfgs[dcfg_ix].last = 1;
2808 } else {
2809 dcfgs[dcfg_ix].last = 0;
2810 }
2811 dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2812 descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2813 ++dcfg_ix;
2814 }
2815 if (oper.do_digest && (oper.digest_start < next_ix) && (prev_ix < (oper.digest_start + oper.digest_len))) {
2816 digest_active = 1;
2817 dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_DIGEST_TID;
2818 dcfgs[dcfg_ix].src = cryptocop_source_dma;
2819 if (next_ix == (oper.digest_start + oper.digest_len)){
2820 assert(!digest_done);
2821 digest_done = 1;
2822 dcfgs[dcfg_ix].last = 1;
2823 } else {
2824 dcfgs[dcfg_ix].last = 0;
2825 }
2826 dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2827 descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2828 ++dcfg_ix;
2829 }
2830 if (oper.do_csum && (oper.csum_start < next_ix) && (prev_ix < (oper.csum_start + oper.csum_len))){
2831 csum_active = 1;
2832 dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_CSUM_TID;
2833 dcfgs[dcfg_ix].src = cryptocop_source_dma;
2834 if (next_ix == (oper.csum_start + oper.csum_len)){
2835 csum_done = 1;
2836 dcfgs[dcfg_ix].last = 1;
2837 } else {
2838 dcfgs[dcfg_ix].last = 0;
2839 }
2840 dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2841 descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2842 ++dcfg_ix;
2843 }
2844 if (!descs[desc_ix].cfg){
2845 DEBUG_API(printk("cryptocop_ioctl_process: data segment %d (%d to %d) had no active transforms\n", desc_ix, prev_ix, next_ix));
2846 err = -EINVAL;
2847 goto error_cleanup;
2848 }
2849 descs[desc_ix].next = &(descs[desc_ix]) + 1;
2850 ++desc_ix;
2851 prev_ix = next_ix;
2852 next_ix = next_cfg_change_ix(&oper, prev_ix);
2853 }
2854 if (desc_ix > 0){
2855 descs[desc_ix-1].next = NULL;
2856 } else {
2857 descs[0].next = NULL;
2858 }
2859 if (oper.do_digest) {
2860 DEBUG(printk("cryptocop_ioctl_process: mapping %d byte digest output to iovec %d\n", digest_length, iovix));
2861 /* Add outdata iovec, length == <length of type of digest> */
2862 cop->tfrm_op.outdata[iovix].iov_base = digest_result;
2863 cop->tfrm_op.outdata[iovix].iov_len = digest_length;
2864 ++iovix;
2865 }
2866 if (oper.do_csum) {
2867 /* Add outdata iovec, length == 2, the length of csum. */
2868 DEBUG(printk("cryptocop_ioctl_process: mapping 2 byte csum output to iovec %d\n", iovix));
2869 /* Add outdata iovec, length == <length of type of digest> */
2870 cop->tfrm_op.outdata[iovix].iov_base = csum_result;
2871 cop->tfrm_op.outdata[iovix].iov_len = 2;
2872 ++iovix;
2873 }
2874 if (oper.do_cipher) {
2875 if (!map_pages_to_iovec(cop->tfrm_op.outdata, iovlen, &iovix, outpages, nooutpages, &pageix, &pageoffset, oper.cipher_outlen)){
2876 DEBUG_API(printk("cryptocop_ioctl_process: failed to map pages to iovec.\n"));
2877 err = -ENOSYS; /* This should be impossible barring bugs. */
2878 goto error_cleanup;
2879 }
2880 }
2881 DEBUG(printk("cryptocop_ioctl_process: setting cop->tfrm_op.outcount %d\n", iovix));
2882 cop->tfrm_op.outcount = iovix;
2883 assert(iovix <= (nooutpages + 6));
2884
2885 cop->sid = oper.ses_id;
2886 cop->tfrm_op.desc = &descs[0];
2887
2888 DEBUG(printk("cryptocop_ioctl_process: inserting job, cb_data=0x%p\n", cop->cb_data));
2889
2890 if ((err = cryptocop_job_queue_insert_user_job(cop)) != 0) {
2891 DEBUG_API(printk("cryptocop_ioctl_process: insert job %d\n", err));
2892 err = -EINVAL;
2893 goto error_cleanup;
2894 }
2895
2896 DEBUG(printk("cryptocop_ioctl_process: begin wait for result\n"));
2897
2898 wait_event(cryptocop_ioc_process_wq, (jc->processed != 0));
2899 DEBUG(printk("cryptocop_ioctl_process: end wait for result\n"));
2900 if (!jc->processed){
2901 printk(KERN_WARNING "cryptocop_ioctl_process: job not processed at completion\n");
2902 err = -EIO;
2903 goto error_cleanup;
2904 }
2905
2906 /* Job process done. Cipher output should already be correct in job so no post processing of outdata. */
2907 DEBUG(printk("cryptocop_ioctl_process: operation_status = %d\n", cop->operation_status));
2908 if (cop->operation_status == 0){
2909 if (oper.do_digest){
2910 DEBUG(printk("cryptocop_ioctl_process: copy %d bytes digest to user\n", digest_length));
2911 err = copy_to_user((unsigned char*)crp_oper + offsetof(struct strcop_crypto_op, digest), digest_result, digest_length);
2912 if (0 != err){
2913 DEBUG_API(printk("cryptocop_ioctl_process: copy_to_user, digest length %d, err %d\n", digest_length, err));
2914 err = -EFAULT;
2915 goto error_cleanup;
2916 }
2917 }
2918 if (oper.do_csum){
2919 DEBUG(printk("cryptocop_ioctl_process: copy 2 bytes checksum to user\n"));
2920 err = copy_to_user((unsigned char*)crp_oper + offsetof(struct strcop_crypto_op, csum), csum_result, 2);
2921 if (0 != err){
2922 DEBUG_API(printk("cryptocop_ioctl_process: copy_to_user, csum, err %d\n", err));
2923 err = -EFAULT;
2924 goto error_cleanup;
2925 }
2926 }
2927 err = 0;
2928 } else {
2929 DEBUG(printk("cryptocop_ioctl_process: returning err = operation_status = %d\n", cop->operation_status));
2930 err = cop->operation_status;
2931 }
2932
2933 error_cleanup:
2934 /* Release page caches. */
2935 for (i = 0; i < noinpages; i++){
2936 put_page(inpages[i]);
2937 }
2938 for (i = 0; i < nooutpages; i++){
2939 int spdl_err;
2940 /* Mark output pages dirty. */
2941 spdl_err = set_page_dirty_lock(outpages[i]);
4741c9fd 2942 DEBUG(if (spdl_err < 0)printk("cryptocop_ioctl_process: set_page_dirty_lock returned %d\n", spdl_err));
51533b61
MS		 2943 }
2944 for (i = 0; i < nooutpages; i++){
2945 put_page(outpages[i]);
2946 }
2947
b2325fe1
JJ		 2948 kfree(digest_result);
2949 kfree(inpages);
2950 kfree(outpages);
51533b61 2951 if (cop){
b2325fe1
JJ		 2952 kfree(cop->tfrm_op.indata);
2953 kfree(cop->tfrm_op.outdata);
51533b61
MS		 2954 kfree(cop);
2955 }
b2325fe1 2956 kfree(jc);
51533b61
MS		 2957
2958 DEBUG(print_lock_status());
2959
2960 return err;
2961}
2962
2963
2964static int cryptocop_ioctl_create_session(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
2965{
2966 cryptocop_session_id sid;
2967 int err;
2968 struct cryptocop_private *dev;
2969 struct strcop_session_op *sess_op = (struct strcop_session_op *)arg;
2970 struct strcop_session_op sop;
2971 struct cryptocop_transform_init *tis = NULL;
2972 struct cryptocop_transform_init ti_cipher = {0};
2973 struct cryptocop_transform_init ti_digest = {0};
2974 struct cryptocop_transform_init ti_csum = {0};
2975
2976 if (!access_ok(VERIFY_WRITE, sess_op, sizeof(struct strcop_session_op)))
2977 return -EFAULT;
2978 err = copy_from_user(&sop, sess_op, sizeof(struct strcop_session_op));
2979 if (err) return -EFAULT;
2980 if (sop.cipher != cryptocop_cipher_none) {
2981 if (!access_ok(VERIFY_READ, sop.key, sop.keylen)) return -EFAULT;
2982 }
2983 DEBUG(printk("cryptocop_ioctl_create_session, sess_op:\n"));
2984
2985 DEBUG(printk("\tcipher:%d\n"
2986 "\tcipher_mode:%d\n"
2987 "\tdigest:%d\n"
2988 "\tcsum:%d\n",
2989 (int)sop.cipher,
2990 (int)sop.cmode,
2991 (int)sop.digest,
2992 (int)sop.csum));
2993
2994 if (sop.cipher != cryptocop_cipher_none){
2995 /* Init the cipher. */
2996 switch (sop.cipher){
2997 case cryptocop_cipher_des:
2998 ti_cipher.alg = cryptocop_alg_des;
2999 break;
3000 case cryptocop_cipher_3des:
3001 ti_cipher.alg = cryptocop_alg_3des;
3002 break;
3003 case cryptocop_cipher_aes:
3004 ti_cipher.alg = cryptocop_alg_aes;
3005 break;
3006 default:
3007 DEBUG_API(printk("create session, bad cipher algorithm %d\n", sop.cipher));
3008 return -EINVAL;
3009 };
3010 DEBUG(printk("setting cipher transform %d\n", ti_cipher.alg));
3011 copy_from_user(ti_cipher.key, sop.key, sop.keylen/8);
3012 ti_cipher.keylen = sop.keylen;
3013 switch (sop.cmode){
3014 case cryptocop_cipher_mode_cbc:
3015 case cryptocop_cipher_mode_ecb:
3016 ti_cipher.cipher_mode = sop.cmode;
3017 break;
3018 default:
3019 DEBUG_API(printk("create session, bad cipher mode %d\n", sop.cmode));
3020 return -EINVAL;
3021 }
3022 DEBUG(printk("cryptocop_ioctl_create_session: setting CBC mode %d\n", ti_cipher.cipher_mode));
3023 switch (sop.des3_mode){
3024 case cryptocop_3des_eee:
3025 case cryptocop_3des_eed:
3026 case cryptocop_3des_ede:
3027 case cryptocop_3des_edd:
3028 case cryptocop_3des_dee:
3029 case cryptocop_3des_ded:
3030 case cryptocop_3des_dde:
3031 case cryptocop_3des_ddd:
3032 ti_cipher.tdes_mode = sop.des3_mode;
3033 break;
3034 default:
3035 DEBUG_API(printk("create session, bad 3DES mode %d\n", sop.des3_mode));
3036 return -EINVAL;
3037 }
3038 ti_cipher.tid = CRYPTOCOP_IOCTL_CIPHER_TID;
3039 ti_cipher.next = tis;
3040 tis = &ti_cipher;
3041 } /* if (sop.cipher != cryptocop_cipher_none) */
3042 if (sop.digest != cryptocop_digest_none){
3043 DEBUG(printk("setting digest transform\n"));
3044 switch (sop.digest){
3045 case cryptocop_digest_md5:
3046 ti_digest.alg = cryptocop_alg_md5;
3047 break;
3048 case cryptocop_digest_sha1:
3049 ti_digest.alg = cryptocop_alg_sha1;
3050 break;
3051 default:
3052 DEBUG_API(printk("create session, bad digest algorithm %d\n", sop.digest));
3053 return -EINVAL;
3054 }
3055 ti_digest.tid = CRYPTOCOP_IOCTL_DIGEST_TID;
3056 ti_digest.next = tis;
3057 tis = &ti_digest;
3058 } /* if (sop.digest != cryptocop_digest_none) */
3059 if (sop.csum != cryptocop_csum_none){
3060 DEBUG(printk("setting csum transform\n"));
3061 switch (sop.csum){
3062 case cryptocop_csum_le:
3063 case cryptocop_csum_be:
3064 ti_csum.csum_mode = sop.csum;
3065 break;
3066 default:
3067 DEBUG_API(printk("create session, bad checksum algorithm %d\n", sop.csum));
3068 return -EINVAL;
3069 }
3070 ti_csum.alg = cryptocop_alg_csum;
3071 ti_csum.tid = CRYPTOCOP_IOCTL_CSUM_TID;
3072 ti_csum.next = tis;
3073 tis = &ti_csum;
3074 } /* (sop.csum != cryptocop_csum_none) */
3075 dev = kmalloc(sizeof(struct cryptocop_private), GFP_KERNEL);
3076 if (!dev){
3077 DEBUG_API(printk("create session, alloc dev\n"));
3078 return -ENOMEM;
3079 }
3080
3081 err = cryptocop_new_session(&sid, tis, GFP_KERNEL);
3082 DEBUG({ if (err) printk("create session, cryptocop_new_session %d\n", err);});
3083
3084 if (err) {
3085 kfree(dev);
3086 return err;
3087 }
3088 sess_op->ses_id = sid;
3089 dev->sid = sid;
3090 dev->next = filp->private_data;
3091 filp->private_data = dev;
3092
3093 return 0;
3094}
3095
90276a1a
JN		 3096static long cryptocop_ioctl_unlocked(struct inode *inode,
3097 struct file *filp, unsigned int cmd, unsigned long arg)
51533b61
MS		 3098{
3099 int err = 0;
3100 if (_IOC_TYPE(cmd) != ETRAXCRYPTOCOP_IOCTYPE) {
3101 DEBUG_API(printk("cryptocop_ioctl: wrong type\n"));
3102 return -ENOTTY;
3103 }
3104 if (_IOC_NR(cmd) > CRYPTOCOP_IO_MAXNR){
3105 return -ENOTTY;
3106 }
3107 /* Access check of the argument. Some commands, e.g. create session and process op,
3108 needs additional checks. Those are handled in the command handling functions. */
3109 if (_IOC_DIR(cmd) & _IOC_READ)
3110 err = !access_ok(VERIFY_WRITE, (void *)arg, _IOC_SIZE(cmd));
3111 else if (_IOC_DIR(cmd) & _IOC_WRITE)
3112 err = !access_ok(VERIFY_READ, (void *)arg, _IOC_SIZE(cmd));
3113 if (err) return -EFAULT;
3114
3115 switch (cmd) {
3116 case CRYPTOCOP_IO_CREATE_SESSION:
3117 return cryptocop_ioctl_create_session(inode, filp, cmd, arg);
3118 case CRYPTOCOP_IO_CLOSE_SESSION:
3119 return cryptocop_ioctl_close_session(inode, filp, cmd, arg);
3120 case CRYPTOCOP_IO_PROCESS_OP:
3121 return cryptocop_ioctl_process(inode, filp, cmd, arg);
3122 default:
3123 DEBUG_API(printk("cryptocop_ioctl: unknown command\n"));
3124 return -ENOTTY;
3125 }
3126 return 0;
3127}
3128
90276a1a
JN		 3129static long
3130cryptocop_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
3131{
90276a1a
JN		 3132 long ret;
3133
0890b588 3134 mutex_lock(&cryptocop_mutex);
496ad9aa 3135 ret = cryptocop_ioctl_unlocked(file_inode(filp), filp, cmd, arg);
0890b588 3136 mutex_unlock(&cryptocop_mutex);
90276a1a
JN		 3137
3138 return ret;
3139}
3140
51533b61
MS		 3141
3142#ifdef LDEBUG
3143static void print_dma_descriptors(struct cryptocop_int_operation *iop)
3144{
3145 struct cryptocop_dma_desc *cdesc_out = iop->cdesc_out;
3146 struct cryptocop_dma_desc *cdesc_in = iop->cdesc_in;
3147 int i;
3148
3149 printk("print_dma_descriptors start\n");
3150
3151 printk("iop:\n");
3152 printk("\tsid: 0x%lld\n", iop->sid);
3153
3154 printk("\tcdesc_out: 0x%p\n", iop->cdesc_out);
3155 printk("\tcdesc_in: 0x%p\n", iop->cdesc_in);
3156 printk("\tddesc_out: 0x%p\n", iop->ddesc_out);
3157 printk("\tddesc_in: 0x%p\n", iop->ddesc_in);
3158
3159 printk("\niop->ctx_out: 0x%p phys: 0x%p\n", &iop->ctx_out, (char*)virt_to_phys(&iop->ctx_out));
3160 printk("\tnext: 0x%p\n"
3161 "\tsaved_data: 0x%p\n"
3162 "\tsaved_data_buf: 0x%p\n",
3163 iop->ctx_out.next,
3164 iop->ctx_out.saved_data,
3165 iop->ctx_out.saved_data_buf);
3166
3167 printk("\niop->ctx_in: 0x%p phys: 0x%p\n", &iop->ctx_in, (char*)virt_to_phys(&iop->ctx_in));
3168 printk("\tnext: 0x%p\n"
3169 "\tsaved_data: 0x%p\n"
3170 "\tsaved_data_buf: 0x%p\n",
3171 iop->ctx_in.next,
3172 iop->ctx_in.saved_data,
3173 iop->ctx_in.saved_data_buf);
3174
3175 i = 0;
3176 while (cdesc_out) {
3177 dma_descr_data *td;
3178 printk("cdesc_out %d, desc=0x%p\n", i, cdesc_out->dma_descr);
3179 printk("\n\tvirt_to_phys(desc): 0x%p\n", (char*)virt_to_phys(cdesc_out->dma_descr));
3180 td = cdesc_out->dma_descr;
3181 printk("\n\tbuf: 0x%p\n"
3182 "\tafter: 0x%p\n"
3183 "\tmd: 0x%04x\n"
3184 "\tnext: 0x%p\n",
3185 td->buf,
3186 td->after,
3187 td->md,
3188 td->next);
3189 printk("flags:\n"
3190 "\twait:\t%d\n"
3191 "\teol:\t%d\n"
3192 "\touteop:\t%d\n"
3193 "\tineop:\t%d\n"
3194 "\tintr:\t%d\n",
3195 td->wait,
3196 td->eol,
3197 td->out_eop,
3198 td->in_eop,
3199 td->intr);
3200 cdesc_out = cdesc_out->next;
3201 i++;
3202 }
3203 i = 0;
3204 while (cdesc_in) {
3205 dma_descr_data *td;
3206 printk("cdesc_in %d, desc=0x%p\n", i, cdesc_in->dma_descr);
3207 printk("\n\tvirt_to_phys(desc): 0x%p\n", (char*)virt_to_phys(cdesc_in->dma_descr));
3208 td = cdesc_in->dma_descr;
3209 printk("\n\tbuf: 0x%p\n"
3210 "\tafter: 0x%p\n"
3211 "\tmd: 0x%04x\n"
3212 "\tnext: 0x%p\n",
3213 td->buf,
3214 td->after,
3215 td->md,
3216 td->next);
3217 printk("flags:\n"
3218 "\twait:\t%d\n"
3219 "\teol:\t%d\n"
3220 "\touteop:\t%d\n"
3221 "\tineop:\t%d\n"
3222 "\tintr:\t%d\n",
3223 td->wait,
3224 td->eol,
3225 td->out_eop,
3226 td->in_eop,
3227 td->intr);
3228 cdesc_in = cdesc_in->next;
3229 i++;
3230 }
3231
3232 printk("print_dma_descriptors end\n");
3233}
3234
3235
3236static void print_strcop_crypto_op(struct strcop_crypto_op *cop)
3237{
3238 printk("print_strcop_crypto_op, 0x%p\n", cop);
3239
3240 /* Indata. */
3241 printk("indata=0x%p\n"
3242 "inlen=%d\n"
3243 "do_cipher=%d\n"
3244 "decrypt=%d\n"
3245 "cipher_explicit=%d\n"
3246 "cipher_start=%d\n"
3247 "cipher_len=%d\n"
3248 "outdata=0x%p\n"
3249 "outlen=%d\n",
3250 cop->indata,
3251 cop->inlen,
3252 cop->do_cipher,
3253 cop->decrypt,
3254 cop->cipher_explicit,
3255 cop->cipher_start,
3256 cop->cipher_len,
3257 cop->cipher_outdata,
3258 cop->cipher_outlen);
3259
3260 printk("do_digest=%d\n"
3261 "digest_start=%d\n"
3262 "digest_len=%d\n",
3263 cop->do_digest,
3264 cop->digest_start,
3265 cop->digest_len);
3266
3267 printk("do_csum=%d\n"
3268 "csum_start=%d\n"
3269 "csum_len=%d\n",
3270 cop->do_csum,
3271 cop->csum_start,
3272 cop->csum_len);
3273}
3274
3275static void print_cryptocop_operation(struct cryptocop_operation *cop)
3276{
3277 struct cryptocop_desc *d;
3278 struct cryptocop_tfrm_cfg *tc;
3279 struct cryptocop_desc_cfg *dc;
3280 int i;
3281
3282 printk("print_cryptocop_operation, cop=0x%p\n\n", cop);
3283 printk("sid: %lld\n", cop->sid);
3284 printk("operation_status=%d\n"
3285 "use_dmalists=%d\n"
3286 "in_interrupt=%d\n"
3287 "fast_callback=%d\n",
3288 cop->operation_status,
3289 cop->use_dmalists,
3290 cop->in_interrupt,
3291 cop->fast_callback);
3292
3293 if (cop->use_dmalists){
3294 print_user_dma_lists(&cop->list_op);
3295 } else {
3296 printk("cop->tfrm_op\n"
3297 "tfrm_cfg=0x%p\n"
3298 "desc=0x%p\n"
3299 "indata=0x%p\n"
3300 "incount=%d\n"
3301 "inlen=%d\n"
3302 "outdata=0x%p\n"
3303 "outcount=%d\n"
3304 "outlen=%d\n\n",
3305 cop->tfrm_op.tfrm_cfg,
3306 cop->tfrm_op.desc,
3307 cop->tfrm_op.indata,
3308 cop->tfrm_op.incount,
3309 cop->tfrm_op.inlen,
3310 cop->tfrm_op.outdata,
3311 cop->tfrm_op.outcount,
3312 cop->tfrm_op.outlen);
3313
3314 tc = cop->tfrm_op.tfrm_cfg;
3315 while (tc){
3316 printk("tfrm_cfg, 0x%p\n"
3317 "tid=%d\n"
3318 "flags=%d\n"
3319 "inject_ix=%d\n"
3320 "next=0x%p\n",
3321 tc,
3322 tc->tid,
3323 tc->flags,
3324 tc->inject_ix,
3325 tc->next);
3326 tc = tc->next;
3327 }
3328 d = cop->tfrm_op.desc;
3329 while (d){
3330 printk("\n======================desc, 0x%p\n"
3331 "length=%d\n"
3332 "cfg=0x%p\n"
3333 "next=0x%p\n",
3334 d,
3335 d->length,
3336 d->cfg,
3337 d->next);
3338 dc = d->cfg;
3339 while (dc){
3340 printk("=========desc_cfg, 0x%p\n"
3341 "tid=%d\n"
3342 "src=%d\n"
3343 "last=%d\n"
3344 "next=0x%p\n",
3345 dc,
3346 dc->tid,
3347 dc->src,
3348 dc->last,
3349 dc->next);
3350 dc = dc->next;
3351 }
3352 d = d->next;
3353 }
3354 printk("\n====iniov\n");
3355 for (i = 0; i < cop->tfrm_op.incount; i++){
3356 printk("indata[%d]\n"
3357 "base=0x%p\n"
3358 "len=%d\n",
3359 i,
3360 cop->tfrm_op.indata[i].iov_base,
3361 cop->tfrm_op.indata[i].iov_len);
3362 }
3363 printk("\n====outiov\n");
3364 for (i = 0; i < cop->tfrm_op.outcount; i++){
3365 printk("outdata[%d]\n"
3366 "base=0x%p\n"
3367 "len=%d\n",
3368 i,
3369 cop->tfrm_op.outdata[i].iov_base,
3370 cop->tfrm_op.outdata[i].iov_len);
3371 }
3372 }
3373 printk("------------end print_cryptocop_operation\n");
3374}
3375
3376
3377static void print_user_dma_lists(struct cryptocop_dma_list_operation *dma_op)
3378{
3379 dma_descr_data *dd;
3380 int i;
3381
3382 printk("print_user_dma_lists, dma_op=0x%p\n", dma_op);
3383
3384 printk("out_data_buf = 0x%p, phys_to_virt(out_data_buf) = 0x%p\n", dma_op->out_data_buf, phys_to_virt((unsigned long int)dma_op->out_data_buf));
3385 printk("in_data_buf = 0x%p, phys_to_virt(in_data_buf) = 0x%p\n", dma_op->in_data_buf, phys_to_virt((unsigned long int)dma_op->in_data_buf));
3386
3387 printk("##############outlist\n");
3388 dd = phys_to_virt((unsigned long int)dma_op->outlist);
3389 i = 0;
3390 while (dd != NULL) {
3391 printk("#%d phys_to_virt(desc) 0x%p\n", i, dd);
3392 printk("\n\tbuf: 0x%p\n"
3393 "\tafter: 0x%p\n"
3394 "\tmd: 0x%04x\n"
3395 "\tnext: 0x%p\n",
3396 dd->buf,
3397 dd->after,
3398 dd->md,
3399 dd->next);
3400 printk("flags:\n"
3401 "\twait:\t%d\n"
3402 "\teol:\t%d\n"
3403 "\touteop:\t%d\n"
3404 "\tineop:\t%d\n"
3405 "\tintr:\t%d\n",
3406 dd->wait,
3407 dd->eol,
3408 dd->out_eop,
3409 dd->in_eop,
3410 dd->intr);
3411 if (dd->eol)
3412 dd = NULL;
3413 else
3414 dd = phys_to_virt((unsigned long int)dd->next);
3415 ++i;
3416 }
3417
3418 printk("##############inlist\n");
3419 dd = phys_to_virt((unsigned long int)dma_op->inlist);
3420 i = 0;
3421 while (dd != NULL) {
3422 printk("#%d phys_to_virt(desc) 0x%p\n", i, dd);
3423 printk("\n\tbuf: 0x%p\n"
3424 "\tafter: 0x%p\n"
3425 "\tmd: 0x%04x\n"
3426 "\tnext: 0x%p\n",
3427 dd->buf,
3428 dd->after,
3429 dd->md,
3430 dd->next);
3431 printk("flags:\n"
3432 "\twait:\t%d\n"
3433 "\teol:\t%d\n"
3434 "\touteop:\t%d\n"
3435 "\tineop:\t%d\n"
3436 "\tintr:\t%d\n",
3437 dd->wait,
3438 dd->eol,
3439 dd->out_eop,
3440 dd->in_eop,
3441 dd->intr);
3442 if (dd->eol)
3443 dd = NULL;
3444 else
3445 dd = phys_to_virt((unsigned long int)dd->next);
3446 ++i;
3447 }
3448}
3449
3450
3451static void print_lock_status(void)
3452{
3453 printk("**********************print_lock_status\n");
3454 printk("cryptocop_completed_jobs_lock %d\n", spin_is_locked(&cryptocop_completed_jobs_lock));
3455 printk("cryptocop_job_queue_lock %d\n", spin_is_locked(&cryptocop_job_queue_lock));
3456 printk("descr_pool_lock %d\n", spin_is_locked(&descr_pool_lock));
3457 printk("cryptocop_sessions_lock %d\n", spin_is_locked(cryptocop_sessions_lock));
3458 printk("running_job_lock %d\n", spin_is_locked(running_job_lock));
3459 printk("cryptocop_process_lock %d\n", spin_is_locked(cryptocop_process_lock));
3460}
3461#endif /* LDEBUG */
3462
3463
3464static const char cryptocop_name[] = "ETRAX FS stream co-processor";
3465
3466static int init_stream_coprocessor(void)
3467{
3468 int err;
3469 int i;
3470 static int initialized = 0;
3471
3472 if (initialized)
3473 return 0;
3474
3475 initialized = 1;
3476
3477 printk("ETRAX FS stream co-processor driver v0.01, (c) 2003 Axis Communications AB\n");
3478
3479 err = register_chrdev(CRYPTOCOP_MAJOR, cryptocop_name, &cryptocop_fops);
3480 if (err < 0) {
3481 printk(KERN_ERR "stream co-processor: could not get major number.\n");
3482 return err;
3483 }
3484
3485 err = init_cryptocop();
3486 if (err) {
3487 (void)unregister_chrdev(CRYPTOCOP_MAJOR, cryptocop_name);
3488 return err;
3489 }
3490 err = cryptocop_job_queue_init();
3491 if (err) {
3492 release_cryptocop();
3493 (void)unregister_chrdev(CRYPTOCOP_MAJOR, cryptocop_name);
3494 return err;
3495 }
3496 /* Init the descriptor pool. */
3497 for (i = 0; i < CRYPTOCOP_DESCRIPTOR_POOL_SIZE - 1; i++) {
3498 descr_pool[i].from_pool = 1;
3499 descr_pool[i].next = &descr_pool[i + 1];
3500 }
3501 descr_pool[i].from_pool = 1;
3502 descr_pool[i].next = NULL;
3503 descr_pool_free_list = &descr_pool[0];
3504 descr_pool_no_free = CRYPTOCOP_DESCRIPTOR_POOL_SIZE;
3505
3506 spin_lock_init(&cryptocop_completed_jobs_lock);
3507 spin_lock_init(&cryptocop_job_queue_lock);
3508 spin_lock_init(&descr_pool_lock);
3509 spin_lock_init(&cryptocop_sessions_lock);
3510 spin_lock_init(&running_job_lock);
3511 spin_lock_init(&cryptocop_process_lock);
3512
3513 cryptocop_sessions = NULL;
3514 next_sid = 1;
3515
3516 cryptocop_running_job = NULL;
3517
3518 printk("stream co-processor: init done.\n");
3519 return 0;
3520}
3521
3522static void __exit exit_stream_coprocessor(void)
3523{
3524 release_cryptocop();
3525 cryptocop_job_queue_close();
3526}
3527
3528module_init(init_stream_coprocessor);
3529module_exit(exit_stream_coprocessor);
3530
Linux 6.x block layer and io_uring tree(s)