projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
llseek: automatically add .llseek fop
[linux-block.git] / arch / cris / arch-v32 / drivers / cryptocop.c
CommitLineData
635c45c1 1/*
51533b61
MS		 2 * Stream co-processor driver for the ETRAX FS
3 *
635c45c1 4 * Copyright (C) 2003-2007 Axis Communications AB
51533b61
MS		 5 */
6
7#include <linux/init.h>
8#include <linux/sched.h>
9#include <linux/module.h>
10#include <linux/slab.h>
11#include <linux/string.h>
12#include <linux/fs.h>
13#include <linux/mm.h>
14#include <linux/spinlock.h>
15#include <linux/stddef.h>
16
17#include <asm/uaccess.h>
18#include <asm/io.h>
19#include <asm/atomic.h>
20
21#include <linux/list.h>
22#include <linux/interrupt.h>
23
24#include <asm/signal.h>
25#include <asm/irq.h>
26
635c45c1
JN		 27#include <dma.h>
28#include <hwregs/dma.h>
29#include <hwregs/reg_map.h>
30#include <hwregs/reg_rdwr.h>
31#include <hwregs/intr_vect_defs.h>
32
33#include <hwregs/strcop.h>
34#include <hwregs/strcop_defs.h>
35#include <cryptocop.h>
36
37#ifdef CONFIG_ETRAXFS
38#define IN_DMA 9
39#define OUT_DMA 8
40#define IN_DMA_INST regi_dma9
41#define OUT_DMA_INST regi_dma8
42#define DMA_IRQ DMA9_INTR_VECT
43#else
44#define IN_DMA 3
45#define OUT_DMA 2
46#define IN_DMA_INST regi_dma3
47#define OUT_DMA_INST regi_dma2
48#define DMA_IRQ DMA3_INTR_VECT
49#endif
51533b61
MS		 50
51#define DESCR_ALLOC_PAD (31)
52
53struct cryptocop_dma_desc {
54 char *free_buf; /* If non-null will be kfreed in free_cdesc() */
55 dma_descr_data *dma_descr;
56
57 unsigned char dma_descr_buf[sizeof(dma_descr_data) + DESCR_ALLOC_PAD];
58
59 unsigned int from_pool:1; /* If 1 'allocated' from the descriptor pool. */
60 struct cryptocop_dma_desc *next;
61};
62
63
64struct cryptocop_int_operation{
65 void *alloc_ptr;
66 cryptocop_session_id sid;
67
68 dma_descr_context ctx_out;
69 dma_descr_context ctx_in;
70
71 /* DMA descriptors allocated by driver. */
72 struct cryptocop_dma_desc *cdesc_out;
73 struct cryptocop_dma_desc *cdesc_in;
74
75 /* Strcop config to use. */
76 cryptocop_3des_mode tdes_mode;
77 cryptocop_csum_type csum_mode;
78
79 /* DMA descrs provided by consumer. */
80 dma_descr_data *ddesc_out;
81 dma_descr_data *ddesc_in;
82};
83
84
85struct cryptocop_tfrm_ctx {
86 cryptocop_tfrm_id tid;
87 unsigned int blocklength;
88
89 unsigned int start_ix;
90
91 struct cryptocop_tfrm_cfg *tcfg;
92 struct cryptocop_transform_ctx *tctx;
93
94 unsigned char previous_src;
95 unsigned char current_src;
96
97 /* Values to use in metadata out. */
98 unsigned char hash_conf;
99 unsigned char hash_mode;
100 unsigned char ciph_conf;
101 unsigned char cbcmode;
102 unsigned char decrypt;
103
104 unsigned int requires_padding:1;
105 unsigned int strict_block_length:1;
106 unsigned int active:1;
107 unsigned int done:1;
108 size_t consumed;
109 size_t produced;
110
111 /* Pad (input) descriptors to put in the DMA out list when the transform
112 * output is put on the DMA in list. */
113 struct cryptocop_dma_desc *pad_descs;
114
115 struct cryptocop_tfrm_ctx *prev_src;
116 struct cryptocop_tfrm_ctx *curr_src;
117
118 /* Mapping to HW. */
119 unsigned char unit_no;
120};
121
122
123struct cryptocop_private{
124 cryptocop_session_id sid;
125 struct cryptocop_private *next;
126};
127
128/* Session list. */
129
130struct cryptocop_transform_ctx{
131 struct cryptocop_transform_init init;
132 unsigned char dec_key[CRYPTOCOP_MAX_KEY_LENGTH];
133 unsigned int dec_key_set:1;
134
135 struct cryptocop_transform_ctx *next;
136};
137
138
139struct cryptocop_session{
140 cryptocop_session_id sid;
141
142 struct cryptocop_transform_ctx *tfrm_ctx;
143
144 struct cryptocop_session *next;
145};
146
147/* Priority levels for jobs sent to the cryptocop. Checksum operations from
148 kernel have highest priority since TCPIP stack processing must not
149 be a bottleneck. */
150typedef enum {
151 cryptocop_prio_kernel_csum = 0,
152 cryptocop_prio_kernel = 1,
153 cryptocop_prio_user = 2,
154 cryptocop_prio_no_prios = 3
155} cryptocop_queue_priority;
156
157struct cryptocop_prio_queue{
158 struct list_head jobs;
159 cryptocop_queue_priority prio;
160};
161
162struct cryptocop_prio_job{
163 struct list_head node;
164 cryptocop_queue_priority prio;
165
166 struct cryptocop_operation *oper;
167 struct cryptocop_int_operation *iop;
168};
169
170struct ioctl_job_cb_ctx {
171 unsigned int processed:1;
172};
173
174
175static struct cryptocop_session *cryptocop_sessions = NULL;
176spinlock_t cryptocop_sessions_lock;
177
178/* Next Session ID to assign. */
179static cryptocop_session_id next_sid = 1;
180
181/* Pad for checksum. */
182static const char csum_zero_pad[1] = {0x00};
183
184/* Trash buffer for mem2mem operations. */
185#define MEM2MEM_DISCARD_BUF_LENGTH (512)
186static unsigned char mem2mem_discard_buf[MEM2MEM_DISCARD_BUF_LENGTH];
187
188/* Descriptor pool. */
189/* FIXME Tweak this value. */
190#define CRYPTOCOP_DESCRIPTOR_POOL_SIZE (100)
191static struct cryptocop_dma_desc descr_pool[CRYPTOCOP_DESCRIPTOR_POOL_SIZE];
192static struct cryptocop_dma_desc *descr_pool_free_list;
193static int descr_pool_no_free;
194static spinlock_t descr_pool_lock;
195
196/* Lock to stop cryptocop to start processing of a new operation. The holder
197 of this lock MUST call cryptocop_start_job() after it is unlocked. */
198spinlock_t cryptocop_process_lock;
199
200static struct cryptocop_prio_queue cryptocop_job_queues[cryptocop_prio_no_prios];
201static spinlock_t cryptocop_job_queue_lock;
202static struct cryptocop_prio_job *cryptocop_running_job = NULL;
203static spinlock_t running_job_lock;
204
205/* The interrupt handler appends completed jobs to this list. The scehduled
206 * tasklet removes them upon sending the response to the crypto consumer. */
207static struct list_head cryptocop_completed_jobs;
208static spinlock_t cryptocop_completed_jobs_lock;
209
210DECLARE_WAIT_QUEUE_HEAD(cryptocop_ioc_process_wq);
211
212
213/** Local functions. **/
214
215static int cryptocop_open(struct inode *, struct file *);
216
217static int cryptocop_release(struct inode *, struct file *);
218
90276a1a 219static long cryptocop_ioctl(struct file *file,
51533b61
MS		 220 unsigned int cmd, unsigned long arg);
221
222static void cryptocop_start_job(void);
223
224static int cryptocop_job_queue_insert(cryptocop_queue_priority prio, struct cryptocop_operation *operation);
225static int cryptocop_job_setup(struct cryptocop_prio_job **pj, struct cryptocop_operation *operation);
226
227static int cryptocop_job_queue_init(void);
228static void cryptocop_job_queue_close(void);
229
230static int create_md5_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length);
231
232static int create_sha1_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length);
233
234static int transform_ok(struct cryptocop_transform_init *tinit);
235
236static struct cryptocop_session *get_session(cryptocop_session_id sid);
237
238static struct cryptocop_transform_ctx *get_transform_ctx(struct cryptocop_session *sess, cryptocop_tfrm_id tid);
239
240static void delete_internal_operation(struct cryptocop_int_operation *iop);
241
242static void get_aes_decrypt_key(unsigned char *dec_key, const unsigned char *key, unsigned int keylength);
243
244static int init_stream_coprocessor(void);
245
246static void __exit exit_stream_coprocessor(void);
247
248/*#define LDEBUG*/
249#ifdef LDEBUG
250#define DEBUG(s) s
251#define DEBUG_API(s) s
252static void print_cryptocop_operation(struct cryptocop_operation *cop);
253static void print_dma_descriptors(struct cryptocop_int_operation *iop);
254static void print_strcop_crypto_op(struct strcop_crypto_op *cop);
255static void print_lock_status(void);
256static void print_user_dma_lists(struct cryptocop_dma_list_operation *dma_op);
257#define assert(s) do{if (!(s)) panic(#s);} while(0);
258#else
259#define DEBUG(s)
260#define DEBUG_API(s)
261#define assert(s)
262#endif
263
264
265/* Transform constants. */
266#define DES_BLOCK_LENGTH (8)
267#define AES_BLOCK_LENGTH (16)
268#define MD5_BLOCK_LENGTH (64)
269#define SHA1_BLOCK_LENGTH (64)
270#define CSUM_BLOCK_LENGTH (2)
271#define MD5_STATE_LENGTH (16)
272#define SHA1_STATE_LENGTH (20)
273
274/* The device number. */
275#define CRYPTOCOP_MAJOR (254)
276#define CRYPTOCOP_MINOR (0)
277
278
279
5dfe4c96 280const struct file_operations cryptocop_fops = {
90276a1a
JN		 281 .owner = THIS_MODULE,
282 .open = cryptocop_open,
283 .release = cryptocop_release,
6038f373
AB		 284 .unlocked_ioctl = cryptocop_ioctl,
285 .llseek = noop_llseek,
51533b61
MS		 286};
287
288
289static void free_cdesc(struct cryptocop_dma_desc *cdesc)
290{
291 DEBUG(printk("free_cdesc: cdesc 0x%p, from_pool=%d\n", cdesc, cdesc->from_pool));
b2325fe1 292 kfree(cdesc->free_buf);
51533b61
MS		 293
294 if (cdesc->from_pool) {
295 unsigned long int flags;
296 spin_lock_irqsave(&descr_pool_lock, flags);
297 cdesc->next = descr_pool_free_list;
298 descr_pool_free_list = cdesc;
299 ++descr_pool_no_free;
300 spin_unlock_irqrestore(&descr_pool_lock, flags);
301 } else {
302 kfree(cdesc);
303 }
304}
305
306
307static struct cryptocop_dma_desc *alloc_cdesc(int alloc_flag)
308{
309 int use_pool = (alloc_flag & GFP_ATOMIC) ? 1 : 0;
310 struct cryptocop_dma_desc *cdesc;
311
312 if (use_pool) {
313 unsigned long int flags;
314 spin_lock_irqsave(&descr_pool_lock, flags);
315 if (!descr_pool_free_list) {
316 spin_unlock_irqrestore(&descr_pool_lock, flags);
317 DEBUG_API(printk("alloc_cdesc: pool is empty\n"));
318 return NULL;
319 }
320 cdesc = descr_pool_free_list;
321 descr_pool_free_list = descr_pool_free_list->next;
322 --descr_pool_no_free;
323 spin_unlock_irqrestore(&descr_pool_lock, flags);
324 cdesc->from_pool = 1;
325 } else {
326 cdesc = kmalloc(sizeof(struct cryptocop_dma_desc), alloc_flag);
327 if (!cdesc) {
328 DEBUG_API(printk("alloc_cdesc: kmalloc\n"));
329 return NULL;
330 }
331 cdesc->from_pool = 0;
332 }
333 cdesc->dma_descr = (dma_descr_data*)(((unsigned long int)cdesc + offsetof(struct cryptocop_dma_desc, dma_descr_buf) + DESCR_ALLOC_PAD) & ~0x0000001F);
334
335 cdesc->next = NULL;
336
337 cdesc->free_buf = NULL;
338 cdesc->dma_descr->out_eop = 0;
339 cdesc->dma_descr->in_eop = 0;
340 cdesc->dma_descr->intr = 0;
341 cdesc->dma_descr->eol = 0;
342 cdesc->dma_descr->wait = 0;
343 cdesc->dma_descr->buf = NULL;
344 cdesc->dma_descr->after = NULL;
345
346 DEBUG_API(printk("alloc_cdesc: return 0x%p, cdesc->dma_descr=0x%p, from_pool=%d\n", cdesc, cdesc->dma_descr, cdesc->from_pool));
347 return cdesc;
348}
349
350
351static void setup_descr_chain(struct cryptocop_dma_desc *cd)
352{
353 DEBUG(printk("setup_descr_chain: entering\n"));
354 while (cd) {
355 if (cd->next) {
356 cd->dma_descr->next = (dma_descr_data*)virt_to_phys(cd->next->dma_descr);
357 } else {
358 cd->dma_descr->next = NULL;
359 }
360 cd = cd->next;
361 }
362 DEBUG(printk("setup_descr_chain: exit\n"));
363}
364
365
366/* Create a pad descriptor for the transform.
367 * Return -1 for error, 0 if pad created. */
368static int create_pad_descriptor(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **pad_desc, int alloc_flag)
369{
370 struct cryptocop_dma_desc *cdesc = NULL;
371 int error = 0;
372 struct strcop_meta_out mo = {
373 .ciphsel = src_none,
374 .hashsel = src_none,
375 .csumsel = src_none
376 };
377 char *pad;
378 size_t plen;
379
380 DEBUG(printk("create_pad_descriptor: start.\n"));
381 /* Setup pad descriptor. */
382
383 DEBUG(printk("create_pad_descriptor: setting up padding.\n"));
384 cdesc = alloc_cdesc(alloc_flag);
385 if (!cdesc){
386 DEBUG_API(printk("create_pad_descriptor: alloc pad desc\n"));
387 goto error_cleanup;
388 }
389 switch (tc->unit_no) {
390 case src_md5:
391 error = create_md5_pad(alloc_flag, tc->consumed, &pad, &plen);
392 if (error){
393 DEBUG_API(printk("create_pad_descriptor: create_md5_pad_failed\n"));
394 goto error_cleanup;
395 }
396 cdesc->free_buf = pad;
397 mo.hashsel = src_dma;
398 mo.hashconf = tc->hash_conf;
399 mo.hashmode = tc->hash_mode;
400 break;
401 case src_sha1:
402 error = create_sha1_pad(alloc_flag, tc->consumed, &pad, &plen);
403 if (error){
404 DEBUG_API(printk("create_pad_descriptor: create_sha1_pad_failed\n"));
405 goto error_cleanup;
406 }
407 cdesc->free_buf = pad;
408 mo.hashsel = src_dma;
409 mo.hashconf = tc->hash_conf;
410 mo.hashmode = tc->hash_mode;
411 break;
412 case src_csum:
413 if (tc->consumed % tc->blocklength){
414 pad = (char*)csum_zero_pad;
415 plen = 1;
416 } else {
417 pad = (char*)cdesc; /* Use any pointer. */
418 plen = 0;
419 }
420 mo.csumsel = src_dma;
421 break;
422 }
423 cdesc->dma_descr->wait = 1;
424 cdesc->dma_descr->out_eop = 1; /* Since this is a pad output is pushed. EOP is ok here since the padded unit is the only one active. */
425 cdesc->dma_descr->buf = (char*)virt_to_phys((char*)pad);
426 cdesc->dma_descr->after = cdesc->dma_descr->buf + plen;
427
428 cdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
429 *pad_desc = cdesc;
430
431 return 0;
432
433 error_cleanup:
434 if (cdesc) free_cdesc(cdesc);
435 return -1;
436}
437
438
439static int setup_key_dl_desc(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **kd, int alloc_flag)
440{
441 struct cryptocop_dma_desc *key_desc = alloc_cdesc(alloc_flag);
442 struct strcop_meta_out mo = {0};
443
444 DEBUG(printk("setup_key_dl_desc\n"));
445
446 if (!key_desc) {
447 DEBUG_API(printk("setup_key_dl_desc: failed descriptor allocation.\n"));
448 return -ENOMEM;
449 }
450
451 /* Download key. */
452 if ((tc->tctx->init.alg == cryptocop_alg_aes) && (tc->tcfg->flags & CRYPTOCOP_DECRYPT)) {
453 /* Precook the AES decrypt key. */
454 if (!tc->tctx->dec_key_set){
455 get_aes_decrypt_key(tc->tctx->dec_key, tc->tctx->init.key, tc->tctx->init.keylen);
456 tc->tctx->dec_key_set = 1;
457 }
458 key_desc->dma_descr->buf = (char*)virt_to_phys(tc->tctx->dec_key);
459 key_desc->dma_descr->after = key_desc->dma_descr->buf + tc->tctx->init.keylen/8;
460 } else {
461 key_desc->dma_descr->buf = (char*)virt_to_phys(tc->tctx->init.key);
462 key_desc->dma_descr->after = key_desc->dma_descr->buf + tc->tctx->init.keylen/8;
463 }
464 /* Setup metadata. */
465 mo.dlkey = 1;
466 switch (tc->tctx->init.keylen) {
467 case 64:
468 mo.decrypt = 0;
469 mo.hashmode = 0;
470 break;
471 case 128:
472 mo.decrypt = 0;
473 mo.hashmode = 1;
474 break;
475 case 192:
476 mo.decrypt = 1;
477 mo.hashmode = 0;
478 break;
479 case 256:
480 mo.decrypt = 1;
481 mo.hashmode = 1;
482 break;
483 default:
484 break;
485 }
486 mo.ciphsel = mo.hashsel = mo.csumsel = src_none;
487 key_desc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
488
489 key_desc->dma_descr->out_eop = 1;
490 key_desc->dma_descr->wait = 1;
491 key_desc->dma_descr->intr = 0;
492
493 *kd = key_desc;
494 return 0;
495}
496
497static int setup_cipher_iv_desc(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **id, int alloc_flag)
498{
499 struct cryptocop_dma_desc *iv_desc = alloc_cdesc(alloc_flag);
500 struct strcop_meta_out mo = {0};
501
502 DEBUG(printk("setup_cipher_iv_desc\n"));
503
504 if (!iv_desc) {
505 DEBUG_API(printk("setup_cipher_iv_desc: failed CBC IV descriptor allocation.\n"));
506 return -ENOMEM;
507 }
508 /* Download IV. */
509 iv_desc->dma_descr->buf = (char*)virt_to_phys(tc->tcfg->iv);
510 iv_desc->dma_descr->after = iv_desc->dma_descr->buf + tc->blocklength;
511
512 /* Setup metadata. */
513 mo.hashsel = mo.csumsel = src_none;
514 mo.ciphsel = src_dma;
515 mo.ciphconf = tc->ciph_conf;
516 mo.cbcmode = tc->cbcmode;
517
518 iv_desc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
519
520 iv_desc->dma_descr->out_eop = 0;
521 iv_desc->dma_descr->wait = 1;
522 iv_desc->dma_descr->intr = 0;
523
524 *id = iv_desc;
525 return 0;
526}
527
528/* Map the ouput length of the transform to operation output starting on the inject index. */
529static int create_input_descriptors(struct cryptocop_operation *operation, struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **id, int alloc_flag)
530{
531 int err = 0;
532 struct cryptocop_dma_desc head = {0};
533 struct cryptocop_dma_desc *outdesc = &head;
534 size_t iov_offset = 0;
535 size_t out_ix = 0;
536 int outiov_ix = 0;
537 struct strcop_meta_in mi = {0};
538
539 size_t out_length = tc->produced;
540 int rem_length;
541 int dlength;
542
543 assert(out_length != 0);
544 if (((tc->produced + tc->tcfg->inject_ix) > operation->tfrm_op.outlen) || (tc->produced && (operation->tfrm_op.outlen == 0))) {
545 DEBUG_API(printk("create_input_descriptors: operation outdata too small\n"));
546 return -EINVAL;
547 }
548 /* Traverse the out iovec until the result inject index is reached. */
549 while ((outiov_ix < operation->tfrm_op.outcount) && ((out_ix + operation->tfrm_op.outdata[outiov_ix].iov_len) <= tc->tcfg->inject_ix)){
550 out_ix += operation->tfrm_op.outdata[outiov_ix].iov_len;
551 outiov_ix++;
552 }
553 if (outiov_ix >= operation->tfrm_op.outcount){
554 DEBUG_API(printk("create_input_descriptors: operation outdata too small\n"));
555 return -EINVAL;
556 }
557 iov_offset = tc->tcfg->inject_ix - out_ix;
558 mi.dmasel = tc->unit_no;
559
560 /* Setup the output descriptors. */
561 while ((out_length > 0) && (outiov_ix < operation->tfrm_op.outcount)) {
562 outdesc->next = alloc_cdesc(alloc_flag);
563 if (!outdesc->next) {
564 DEBUG_API(printk("create_input_descriptors: alloc_cdesc\n"));
565 err = -ENOMEM;
566 goto error_cleanup;
567 }
568 outdesc = outdesc->next;
569 rem_length = operation->tfrm_op.outdata[outiov_ix].iov_len - iov_offset;
570 dlength = (out_length < rem_length) ? out_length : rem_length;
571
572 DEBUG(printk("create_input_descriptors:\n"
573 "outiov_ix=%d, rem_length=%d, dlength=%d\n"
574 "iov_offset=%d, outdata[outiov_ix].iov_len=%d\n"
575 "outcount=%d, outiov_ix=%d\n",
576 outiov_ix, rem_length, dlength, iov_offset, operation->tfrm_op.outdata[outiov_ix].iov_len, operation->tfrm_op.outcount, outiov_ix));
577
578 outdesc->dma_descr->buf = (char*)virt_to_phys(operation->tfrm_op.outdata[outiov_ix].iov_base + iov_offset);
579 outdesc->dma_descr->after = outdesc->dma_descr->buf + dlength;
580 outdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
581
582 out_length -= dlength;
583 iov_offset += dlength;
584 if (iov_offset >= operation->tfrm_op.outdata[outiov_ix].iov_len) {
585 iov_offset = 0;
586 ++outiov_ix;
587 }
588 }
589 if (out_length > 0){
590 DEBUG_API(printk("create_input_descriptors: not enough room for output, %d remained\n", out_length));
591 err = -EINVAL;
592 goto error_cleanup;
593 }
594 /* Set sync in last descriptor. */
595 mi.sync = 1;
596 outdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
597
598 *id = head.next;
599 return 0;
600
601 error_cleanup:
602 while (head.next) {
603 outdesc = head.next->next;
604 free_cdesc(head.next);
605 head.next = outdesc;
606 }
607 return err;
608}
609
610
611static int create_output_descriptors(struct cryptocop_operation *operation, int *iniov_ix, int *iniov_offset, size_t desc_len, struct cryptocop_dma_desc **current_out_cdesc, struct strcop_meta_out *meta_out, int alloc_flag)
612{
613 while (desc_len != 0) {
614 struct cryptocop_dma_desc *cdesc;
615 int rem_length = operation->tfrm_op.indata[*iniov_ix].iov_len - *iniov_offset;
616 int dlength = (desc_len < rem_length) ? desc_len : rem_length;
617
618 cdesc = alloc_cdesc(alloc_flag);
619 if (!cdesc) {
620 DEBUG_API(printk("create_output_descriptors: alloc_cdesc\n"));
621 return -ENOMEM;
622 }
623 (*current_out_cdesc)->next = cdesc;
624 (*current_out_cdesc) = cdesc;
625
626 cdesc->free_buf = NULL;
627
628 cdesc->dma_descr->buf = (char*)virt_to_phys(operation->tfrm_op.indata[*iniov_ix].iov_base + *iniov_offset);
629 cdesc->dma_descr->after = cdesc->dma_descr->buf + dlength;
630
cda1c5af 631 assert(desc_len >= dlength);
51533b61
MS		 632 desc_len -= dlength;
633 *iniov_offset += dlength;
51533b61
MS		 634 if (*iniov_offset >= operation->tfrm_op.indata[*iniov_ix].iov_len) {
635 *iniov_offset = 0;
636 ++(*iniov_ix);
637 if (*iniov_ix > operation->tfrm_op.incount) {
638 DEBUG_API(printk("create_output_descriptors: not enough indata in operation."));
639 return -EINVAL;
640 }
641 }
642 cdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, (*meta_out));
643 } /* while (desc_len != 0) */
644 /* Last DMA descriptor gets a 'wait' bit to signal expected change in metadata. */
645 (*current_out_cdesc)->dma_descr->wait = 1; /* This will set extraneous WAIT in some situations, e.g. when padding hashes and checksums. */
646
647 return 0;
648}
649
650
651static int append_input_descriptors(struct cryptocop_operation *operation, struct cryptocop_dma_desc **current_in_cdesc, struct cryptocop_dma_desc **current_out_cdesc, struct cryptocop_tfrm_ctx *tc, int alloc_flag)
652{
653 DEBUG(printk("append_input_descriptors, tc=0x%p, unit_no=%d\n", tc, tc->unit_no));
654 if (tc->tcfg) {
655 int failed = 0;
656 struct cryptocop_dma_desc *idescs = NULL;
657 DEBUG(printk("append_input_descriptors: pushing output, consumed %d produced %d bytes.\n", tc->consumed, tc->produced));
658 if (tc->pad_descs) {
659 DEBUG(printk("append_input_descriptors: append pad descriptors to DMA out list.\n"));
660 while (tc->pad_descs) {
661 DEBUG(printk("append descriptor 0x%p\n", tc->pad_descs));
662 (*current_out_cdesc)->next = tc->pad_descs;
663 tc->pad_descs = tc->pad_descs->next;
664 (*current_out_cdesc) = (*current_out_cdesc)->next;
665 }
666 }
667
668 /* Setup and append output descriptors to DMA in list. */
669 if (tc->unit_no == src_dma){
670 /* mem2mem. Setup DMA in descriptors to discard all input prior to the requested mem2mem data. */
671 struct strcop_meta_in mi = {.sync = 0, .dmasel = src_dma};
672 unsigned int start_ix = tc->start_ix;
673 while (start_ix){
674 unsigned int desclen = start_ix < MEM2MEM_DISCARD_BUF_LENGTH ? start_ix : MEM2MEM_DISCARD_BUF_LENGTH;
675 (*current_in_cdesc)->next = alloc_cdesc(alloc_flag);
676 if (!(*current_in_cdesc)->next){
677 DEBUG_API(printk("append_input_descriptors: alloc_cdesc mem2mem discard failed\n"));
678 return -ENOMEM;
679 }
680 (*current_in_cdesc) = (*current_in_cdesc)->next;
681 (*current_in_cdesc)->dma_descr->buf = (char*)virt_to_phys(mem2mem_discard_buf);
682 (*current_in_cdesc)->dma_descr->after = (*current_in_cdesc)->dma_descr->buf + desclen;
683 (*current_in_cdesc)->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
684 start_ix -= desclen;
685 }
686 mi.sync = 1;
687 (*current_in_cdesc)->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
688 }
689
690 failed = create_input_descriptors(operation, tc, &idescs, alloc_flag);
691 if (failed){
692 DEBUG_API(printk("append_input_descriptors: output descriptor setup failed\n"));
693 return failed;
694 }
695 DEBUG(printk("append_input_descriptors: append output descriptors to DMA in list.\n"));
696 while (idescs) {
697 DEBUG(printk("append descriptor 0x%p\n", idescs));
698 (*current_in_cdesc)->next = idescs;
699 idescs = idescs->next;
700 (*current_in_cdesc) = (*current_in_cdesc)->next;
701 }
702 }
703 return 0;
704}
705
706
707
708static int cryptocop_setup_dma_list(struct cryptocop_operation *operation, struct cryptocop_int_operation **int_op, int alloc_flag)
709{
710 struct cryptocop_session *sess;
711 struct cryptocop_transform_ctx *tctx;
712
713 struct cryptocop_tfrm_ctx digest_ctx = {
714 .previous_src = src_none,
715 .current_src = src_none,
716 .start_ix = 0,
717 .requires_padding = 1,
718 .strict_block_length = 0,
719 .hash_conf = 0,
720 .hash_mode = 0,
721 .ciph_conf = 0,
722 .cbcmode = 0,
723 .decrypt = 0,
724 .consumed = 0,
725 .produced = 0,
726 .pad_descs = NULL,
727 .active = 0,
728 .done = 0,
729 .prev_src = NULL,
730 .curr_src = NULL,
731 .tcfg = NULL};
732 struct cryptocop_tfrm_ctx cipher_ctx = {
733 .previous_src = src_none,
734 .current_src = src_none,
735 .start_ix = 0,
736 .requires_padding = 0,
737 .strict_block_length = 1,
738 .hash_conf = 0,
739 .hash_mode = 0,
740 .ciph_conf = 0,
741 .cbcmode = 0,
742 .decrypt = 0,
743 .consumed = 0,
744 .produced = 0,
745 .pad_descs = NULL,
746 .active = 0,
747 .done = 0,
748 .prev_src = NULL,
749 .curr_src = NULL,
750 .tcfg = NULL};
751 struct cryptocop_tfrm_ctx csum_ctx = {
752 .previous_src = src_none,
753 .current_src = src_none,
754 .start_ix = 0,
755 .blocklength = 2,
756 .requires_padding = 1,
757 .strict_block_length = 0,
758 .hash_conf = 0,
759 .hash_mode = 0,
760 .ciph_conf = 0,
761 .cbcmode = 0,
762 .decrypt = 0,
763 .consumed = 0,
764 .produced = 0,
765 .pad_descs = NULL,
766 .active = 0,
767 .done = 0,
768 .tcfg = NULL,
769 .prev_src = NULL,
770 .curr_src = NULL,
771 .unit_no = src_csum};
772 struct cryptocop_tfrm_cfg *tcfg = operation->tfrm_op.tfrm_cfg;
773
774 unsigned int indata_ix = 0;
775
776 /* iovec accounting. */
777 int iniov_ix = 0;
778 int iniov_offset = 0;
779
780 /* Operation descriptor cfg traversal pointer. */
781 struct cryptocop_desc *odsc;
782
783 int failed = 0;
784 /* List heads for allocated descriptors. */
785 struct cryptocop_dma_desc out_cdesc_head = {0};
786 struct cryptocop_dma_desc in_cdesc_head = {0};
787
788 struct cryptocop_dma_desc *current_out_cdesc = &out_cdesc_head;
789 struct cryptocop_dma_desc *current_in_cdesc = &in_cdesc_head;
790
791 struct cryptocop_tfrm_ctx *output_tc = NULL;
792 void *iop_alloc_ptr;
793
794 assert(operation != NULL);
795 assert(int_op != NULL);
796
797 DEBUG(printk("cryptocop_setup_dma_list: start\n"));
798 DEBUG(print_cryptocop_operation(operation));
799
800 sess = get_session(operation->sid);
801 if (!sess) {
802 DEBUG_API(printk("cryptocop_setup_dma_list: no session found for operation.\n"));
803 failed = -EINVAL;
804 goto error_cleanup;
805 }
806 iop_alloc_ptr = kmalloc(DESCR_ALLOC_PAD + sizeof(struct cryptocop_int_operation), alloc_flag);
807 if (!iop_alloc_ptr) {
808 DEBUG_API(printk("cryptocop_setup_dma_list: kmalloc cryptocop_int_operation\n"));
809 failed = -ENOMEM;
810 goto error_cleanup;
811 }
812 (*int_op) = (struct cryptocop_int_operation*)(((unsigned long int)(iop_alloc_ptr + DESCR_ALLOC_PAD + offsetof(struct cryptocop_int_operation, ctx_out)) & ~0x0000001F) - offsetof(struct cryptocop_int_operation, ctx_out));
813 DEBUG(memset((*int_op), 0xff, sizeof(struct cryptocop_int_operation)));
814 (*int_op)->alloc_ptr = iop_alloc_ptr;
815 DEBUG(printk("cryptocop_setup_dma_list: *int_op=0x%p, alloc_ptr=0x%p\n", *int_op, (*int_op)->alloc_ptr));
816
817 (*int_op)->sid = operation->sid;
818 (*int_op)->cdesc_out = NULL;
819 (*int_op)->cdesc_in = NULL;
820 (*int_op)->tdes_mode = cryptocop_3des_ede;
821 (*int_op)->csum_mode = cryptocop_csum_le;
822 (*int_op)->ddesc_out = NULL;
823 (*int_op)->ddesc_in = NULL;
824
825 /* Scan operation->tfrm_op.tfrm_cfg for bad configuration and set up the local contexts. */
826 if (!tcfg) {
827 DEBUG_API(printk("cryptocop_setup_dma_list: no configured transforms in operation.\n"));
828 failed = -EINVAL;
829 goto error_cleanup;
830 }
831 while (tcfg) {
832 tctx = get_transform_ctx(sess, tcfg->tid);
833 if (!tctx) {
834 DEBUG_API(printk("cryptocop_setup_dma_list: no transform id %d in session.\n", tcfg->tid));
835 failed = -EINVAL;
836 goto error_cleanup;
837 }
838 if (tcfg->inject_ix > operation->tfrm_op.outlen){
839 DEBUG_API(printk("cryptocop_setup_dma_list: transform id %d inject_ix (%d) > operation->tfrm_op.outlen(%d)", tcfg->tid, tcfg->inject_ix, operation->tfrm_op.outlen));
840 failed = -EINVAL;
841 goto error_cleanup;
842 }
843 switch (tctx->init.alg){
844 case cryptocop_alg_mem2mem:
845 if (cipher_ctx.tcfg != NULL){
846 DEBUG_API(printk("cryptocop_setup_dma_list: multiple ciphers in operation.\n"));
847 failed = -EINVAL;
848 goto error_cleanup;
849 }
850 /* mem2mem is handled as a NULL cipher. */
851 cipher_ctx.cbcmode = 0;
852 cipher_ctx.decrypt = 0;
853 cipher_ctx.blocklength = 1;
854 cipher_ctx.ciph_conf = 0;
855 cipher_ctx.unit_no = src_dma;
856 cipher_ctx.tcfg = tcfg;
857 cipher_ctx.tctx = tctx;
858 break;
859 case cryptocop_alg_des:
860 case cryptocop_alg_3des:
861 case cryptocop_alg_aes:
862 /* cipher */
863 if (cipher_ctx.tcfg != NULL){
864 DEBUG_API(printk("cryptocop_setup_dma_list: multiple ciphers in operation.\n"));
865 failed = -EINVAL;
866 goto error_cleanup;
867 }
868 cipher_ctx.tcfg = tcfg;
869 cipher_ctx.tctx = tctx;
870 if (cipher_ctx.tcfg->flags & CRYPTOCOP_DECRYPT){
871 cipher_ctx.decrypt = 1;
872 }
873 switch (tctx->init.cipher_mode) {
874 case cryptocop_cipher_mode_ecb:
875 cipher_ctx.cbcmode = 0;
876 break;
877 case cryptocop_cipher_mode_cbc:
878 cipher_ctx.cbcmode = 1;
879 break;
880 default:
881 DEBUG_API(printk("cryptocop_setup_dma_list: cipher_ctx, bad cipher mode==%d\n", tctx->init.cipher_mode));
882 failed = -EINVAL;
883 goto error_cleanup;
884 }
885 DEBUG(printk("cryptocop_setup_dma_list: cipher_ctx, set CBC mode==%d\n", cipher_ctx.cbcmode));
886 switch (tctx->init.alg){
887 case cryptocop_alg_des:
888 cipher_ctx.ciph_conf = 0;
889 cipher_ctx.unit_no = src_des;
890 cipher_ctx.blocklength = DES_BLOCK_LENGTH;
891 break;
892 case cryptocop_alg_3des:
893 cipher_ctx.ciph_conf = 1;
894 cipher_ctx.unit_no = src_des;
895 cipher_ctx.blocklength = DES_BLOCK_LENGTH;
896 break;
897 case cryptocop_alg_aes:
898 cipher_ctx.ciph_conf = 2;
899 cipher_ctx.unit_no = src_aes;
900 cipher_ctx.blocklength = AES_BLOCK_LENGTH;
901 break;
902 default:
903 panic("cryptocop_setup_dma_list: impossible algorithm %d\n", tctx->init.alg);
904 }
905 (*int_op)->tdes_mode = tctx->init.tdes_mode;
906 break;
907 case cryptocop_alg_md5:
908 case cryptocop_alg_sha1:
909 /* digest */
910 if (digest_ctx.tcfg != NULL){
911 DEBUG_API(printk("cryptocop_setup_dma_list: multiple digests in operation.\n"));
912 failed = -EINVAL;
913 goto error_cleanup;
914 }
915 digest_ctx.tcfg = tcfg;
916 digest_ctx.tctx = tctx;
917 digest_ctx.hash_mode = 0; /* Don't use explicit IV in this API. */
918 switch (tctx->init.alg){
919 case cryptocop_alg_md5:
920 digest_ctx.blocklength = MD5_BLOCK_LENGTH;
921 digest_ctx.unit_no = src_md5;
922 digest_ctx.hash_conf = 1; /* 1 => MD-5 */
923 break;
924 case cryptocop_alg_sha1:
925 digest_ctx.blocklength = SHA1_BLOCK_LENGTH;
926 digest_ctx.unit_no = src_sha1;
927 digest_ctx.hash_conf = 0; /* 0 => SHA-1 */
928 break;
929 default:
930 panic("cryptocop_setup_dma_list: impossible digest algorithm\n");
931 }
932 break;
933 case cryptocop_alg_csum:
934 /* digest */
935 if (csum_ctx.tcfg != NULL){
936 DEBUG_API(printk("cryptocop_setup_dma_list: multiple checksums in operation.\n"));
937 failed = -EINVAL;
938 goto error_cleanup;
939 }
940 (*int_op)->csum_mode = tctx->init.csum_mode;
941 csum_ctx.tcfg = tcfg;
942 csum_ctx.tctx = tctx;
943 break;
944 default:
945 /* no algorithm. */
946 DEBUG_API(printk("cryptocop_setup_dma_list: invalid algorithm %d specified in tfrm %d.\n", tctx->init.alg, tcfg->tid));
947 failed = -EINVAL;
948 goto error_cleanup;
949 }
950 tcfg = tcfg->next;
951 }
952 /* Download key if a cipher is used. */
953 if (cipher_ctx.tcfg && (cipher_ctx.tctx->init.alg != cryptocop_alg_mem2mem)){
954 struct cryptocop_dma_desc *key_desc = NULL;
955
956 failed = setup_key_dl_desc(&cipher_ctx, &key_desc, alloc_flag);
957 if (failed) {
958 DEBUG_API(printk("cryptocop_setup_dma_list: setup key dl\n"));
959 goto error_cleanup;
960 }
961 current_out_cdesc->next = key_desc;
962 current_out_cdesc = key_desc;
963 indata_ix += (unsigned int)(key_desc->dma_descr->after - key_desc->dma_descr->buf);
964
965 /* Download explicit IV if a cipher is used and CBC mode and explicit IV selected. */
966 if ((cipher_ctx.tctx->init.cipher_mode == cryptocop_cipher_mode_cbc) && (cipher_ctx.tcfg->flags & CRYPTOCOP_EXPLICIT_IV)) {
967 struct cryptocop_dma_desc *iv_desc = NULL;
968
969 DEBUG(printk("cryptocop_setup_dma_list: setup cipher CBC IV descriptor.\n"));
970
971 failed = setup_cipher_iv_desc(&cipher_ctx, &iv_desc, alloc_flag);
972 if (failed) {
973 DEBUG_API(printk("cryptocop_setup_dma_list: CBC IV descriptor.\n"));
974 goto error_cleanup;
975 }
976 current_out_cdesc->next = iv_desc;
977 current_out_cdesc = iv_desc;
978 indata_ix += (unsigned int)(iv_desc->dma_descr->after - iv_desc->dma_descr->buf);
979 }
980 }
981
982 /* Process descriptors. */
983 odsc = operation->tfrm_op.desc;
984 while (odsc) {
985 struct cryptocop_desc_cfg *dcfg = odsc->cfg;
986 struct strcop_meta_out meta_out = {0};
987 size_t desc_len = odsc->length;
988 int active_count, eop_needed_count;
989
990 output_tc = NULL;
991
992 DEBUG(printk("cryptocop_setup_dma_list: parsing an operation descriptor\n"));
993
994 while (dcfg) {
995 struct cryptocop_tfrm_ctx *tc = NULL;
996
997 DEBUG(printk("cryptocop_setup_dma_list: parsing an operation descriptor configuration.\n"));
998 /* Get the local context for the transform and mark it as the output unit if it produces output. */
999 if (digest_ctx.tcfg && (digest_ctx.tcfg->tid == dcfg->tid)){
1000 tc = &digest_ctx;
1001 } else if (cipher_ctx.tcfg && (cipher_ctx.tcfg->tid == dcfg->tid)){
1002 tc = &cipher_ctx;
1003 } else if (csum_ctx.tcfg && (csum_ctx.tcfg->tid == dcfg->tid)){
1004 tc = &csum_ctx;
1005 }
1006 if (!tc) {
1007 DEBUG_API(printk("cryptocop_setup_dma_list: invalid transform %d specified in descriptor.\n", dcfg->tid));
1008 failed = -EINVAL;
1009 goto error_cleanup;
1010 }
1011 if (tc->done) {
1012 DEBUG_API(printk("cryptocop_setup_dma_list: completed transform %d reused.\n", dcfg->tid));
1013 failed = -EINVAL;
1014 goto error_cleanup;
1015 }
1016 if (!tc->active) {
1017 tc->start_ix = indata_ix;
1018 tc->active = 1;
1019 }
1020
1021 tc->previous_src = tc->current_src;
1022 tc->prev_src = tc->curr_src;
1023 /* Map source unit id to DMA source config. */
1024 switch (dcfg->src){
1025 case cryptocop_source_dma:
1026 tc->current_src = src_dma;
1027 break;
1028 case cryptocop_source_des:
1029 tc->current_src = src_des;
1030 break;
1031 case cryptocop_source_3des:
1032 tc->current_src = src_des;
1033 break;
1034 case cryptocop_source_aes:
1035 tc->current_src = src_aes;
1036 break;
1037 case cryptocop_source_md5:
1038 case cryptocop_source_sha1:
1039 case cryptocop_source_csum:
1040 case cryptocop_source_none:
1041 default:
1042 /* We do not allow using accumulating style units (SHA-1, MD5, checksum) as sources to other units.
1043 */
1044 DEBUG_API(printk("cryptocop_setup_dma_list: bad unit source configured %d.\n", dcfg->src));
1045 failed = -EINVAL;
1046 goto error_cleanup;
1047 }
1048 if (tc->current_src != src_dma) {
1049 /* Find the unit we are sourcing from. */
1050 if (digest_ctx.unit_no == tc->current_src){
1051 tc->curr_src = &digest_ctx;
1052 } else if (cipher_ctx.unit_no == tc->current_src){
1053 tc->curr_src = &cipher_ctx;
1054 } else if (csum_ctx.unit_no == tc->current_src){
1055 tc->curr_src = &csum_ctx;
1056 }
1057 if ((tc->curr_src == tc) && (tc->unit_no != src_dma)){
1058 DEBUG_API(printk("cryptocop_setup_dma_list: unit %d configured to source from itself.\n", tc->unit_no));
1059 failed = -EINVAL;
1060 goto error_cleanup;
1061 }
1062 } else {
1063 tc->curr_src = NULL;
1064 }
1065
1066 /* Detect source switch. */
1067 DEBUG(printk("cryptocop_setup_dma_list: tc->active=%d tc->unit_no=%d tc->current_src=%d tc->previous_src=%d, tc->curr_src=0x%p, tc->prev_srv=0x%p\n", tc->active, tc->unit_no, tc->current_src, tc->previous_src, tc->curr_src, tc->prev_src));
1068 if (tc->active && (tc->current_src != tc->previous_src)) {
1069 /* Only allow source switch when both the old source unit and the new one have
1070 * no pending data to process (i.e. the consumed length must be a multiple of the
1071 * transform blocklength). */
1072 /* Note: if the src == NULL we are actually sourcing from DMA out. */
1073 if (((tc->prev_src != NULL) && (tc->prev_src->consumed % tc->prev_src->blocklength)) ||
1074 ((tc->curr_src != NULL) && (tc->curr_src->consumed % tc->curr_src->blocklength)))
1075 {
1076 DEBUG_API(printk("cryptocop_setup_dma_list: can only disconnect from or connect to a unit on a multiple of the blocklength, old: cons=%d, prod=%d, block=%d, new: cons=%d prod=%d, block=%d.\n", tc->prev_src ? tc->prev_src->consumed : INT_MIN, tc->prev_src ? tc->prev_src->produced : INT_MIN, tc->prev_src ? tc->prev_src->blocklength : INT_MIN, tc->curr_src ? tc->curr_src->consumed : INT_MIN, tc->curr_src ? tc->curr_src->produced : INT_MIN, tc->curr_src ? tc->curr_src->blocklength : INT_MIN));
1077 failed = -EINVAL;
1078 goto error_cleanup;
1079 }
1080 }
1081 /* Detect unit deactivation. */
1082 if (dcfg->last) {
1083 /* Length check of this is handled below. */
1084 tc->done = 1;
1085 }
1086 dcfg = dcfg->next;
1087 } /* while (dcfg) */
1088 DEBUG(printk("cryptocop_setup_dma_list: parsing operation descriptor configuration complete.\n"));
1089
1090 if (cipher_ctx.active && (cipher_ctx.curr_src != NULL) && !cipher_ctx.curr_src->active){
1091 DEBUG_API(printk("cryptocop_setup_dma_list: cipher source from inactive unit %d\n", cipher_ctx.curr_src->unit_no));
1092 failed = -EINVAL;
1093 goto error_cleanup;
1094 }
1095 if (digest_ctx.active && (digest_ctx.curr_src != NULL) && !digest_ctx.curr_src->active){
1096 DEBUG_API(printk("cryptocop_setup_dma_list: digest source from inactive unit %d\n", digest_ctx.curr_src->unit_no));
1097 failed = -EINVAL;
1098 goto error_cleanup;
1099 }
1100 if (csum_ctx.active && (csum_ctx.curr_src != NULL) && !csum_ctx.curr_src->active){
1101 DEBUG_API(printk("cryptocop_setup_dma_list: cipher source from inactive unit %d\n", csum_ctx.curr_src->unit_no));
1102 failed = -EINVAL;
1103 goto error_cleanup;
1104 }
1105
1106 /* Update consumed and produced lengths.
1107
1108 The consumed length accounting here is actually cheating. If a unit source from DMA (or any
1109 other unit that process data in blocks of one octet) it is correct, but if it source from a
1110 block processing unit, i.e. a cipher, it will be temporarily incorrect at some times. However
1111 since it is only allowed--by the HW--to change source to or from a block processing unit at times where that
1112 unit has processed an exact multiple of its block length the end result will be correct.
1113 Beware that if the source change restriction change this code will need to be (much) reworked.
1114 */
1115 DEBUG(printk("cryptocop_setup_dma_list: desc->length=%d, desc_len=%d.\n", odsc->length, desc_len));
1116
1117 if (csum_ctx.active) {
1118 csum_ctx.consumed += desc_len;
1119 if (csum_ctx.done) {
1120 csum_ctx.produced = 2;
1121 }
1122 DEBUG(printk("cryptocop_setup_dma_list: csum_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", csum_ctx.consumed, csum_ctx.produced, csum_ctx.blocklength));
1123 }
1124 if (digest_ctx.active) {
1125 digest_ctx.consumed += desc_len;
1126 if (digest_ctx.done) {
1127 if (digest_ctx.unit_no == src_md5) {
1128 digest_ctx.produced = MD5_STATE_LENGTH;
1129 } else {
1130 digest_ctx.produced = SHA1_STATE_LENGTH;
1131 }
1132 }
1133 DEBUG(printk("cryptocop_setup_dma_list: digest_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", digest_ctx.consumed, digest_ctx.produced, digest_ctx.blocklength));
1134 }
1135 if (cipher_ctx.active) {
1136 /* Ciphers are allowed only to source from DMA out. That is filtered above. */
1137 assert(cipher_ctx.current_src == src_dma);
1138 cipher_ctx.consumed += desc_len;
1139 cipher_ctx.produced = cipher_ctx.blocklength * (cipher_ctx.consumed / cipher_ctx.blocklength);
1140 if (cipher_ctx.cbcmode && !(cipher_ctx.tcfg->flags & CRYPTOCOP_EXPLICIT_IV) && cipher_ctx.produced){
1141 cipher_ctx.produced -= cipher_ctx.blocklength; /* Compensate for CBC iv. */
1142 }
1143 DEBUG(printk("cryptocop_setup_dma_list: cipher_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", cipher_ctx.consumed, cipher_ctx.produced, cipher_ctx.blocklength));
1144 }
1145
1146 /* Setup the DMA out descriptors. */
1147 /* Configure the metadata. */
1148 active_count = 0;
1149 eop_needed_count = 0;
1150 if (cipher_ctx.active) {
1151 ++active_count;
1152 if (cipher_ctx.unit_no == src_dma){
1153 /* mem2mem */
1154 meta_out.ciphsel = src_none;
1155 } else {
1156 meta_out.ciphsel = cipher_ctx.current_src;
1157 }
1158 meta_out.ciphconf = cipher_ctx.ciph_conf;
1159 meta_out.cbcmode = cipher_ctx.cbcmode;
1160 meta_out.decrypt = cipher_ctx.decrypt;
1161 DEBUG(printk("set ciphsel=%d ciphconf=%d cbcmode=%d decrypt=%d\n", meta_out.ciphsel, meta_out.ciphconf, meta_out.cbcmode, meta_out.decrypt));
1162 if (cipher_ctx.done) ++eop_needed_count;
1163 } else {
1164 meta_out.ciphsel = src_none;
1165 }
1166
1167 if (digest_ctx.active) {
1168 ++active_count;
1169 meta_out.hashsel = digest_ctx.current_src;
1170 meta_out.hashconf = digest_ctx.hash_conf;
1171 meta_out.hashmode = 0; /* Explicit mode is not used here. */
1172 DEBUG(printk("set hashsel=%d hashconf=%d hashmode=%d\n", meta_out.hashsel, meta_out.hashconf, meta_out.hashmode));
1173 if (digest_ctx.done) {
1174 assert(digest_ctx.pad_descs == NULL);
1175 failed = create_pad_descriptor(&digest_ctx, &digest_ctx.pad_descs, alloc_flag);
1176 if (failed) {
1177 DEBUG_API(printk("cryptocop_setup_dma_list: failed digest pad creation.\n"));
1178 goto error_cleanup;
1179 }
1180 }
1181 } else {
1182 meta_out.hashsel = src_none;
1183 }
1184
1185 if (csum_ctx.active) {
1186 ++active_count;
1187 meta_out.csumsel = csum_ctx.current_src;
1188 if (csum_ctx.done) {
1189 assert(csum_ctx.pad_descs == NULL);
1190 failed = create_pad_descriptor(&csum_ctx, &csum_ctx.pad_descs, alloc_flag);
1191 if (failed) {
1192 DEBUG_API(printk("cryptocop_setup_dma_list: failed csum pad creation.\n"));
1193 goto error_cleanup;
1194 }
1195 }
1196 } else {
1197 meta_out.csumsel = src_none;
1198 }
1199 DEBUG(printk("cryptocop_setup_dma_list: %d eop needed, %d active units\n", eop_needed_count, active_count));
1200 /* Setup DMA out descriptors for the indata. */
1201 failed = create_output_descriptors(operation, &iniov_ix, &iniov_offset, desc_len, &current_out_cdesc, &meta_out, alloc_flag);
1202 if (failed) {
1203 DEBUG_API(printk("cryptocop_setup_dma_list: create_output_descriptors %d\n", failed));
1204 goto error_cleanup;
1205 }
1206 /* Setup out EOP. If there are active units that are not done here they cannot get an EOP
1207 * so we ust setup a zero length descriptor to DMA to signal EOP only to done units.
1208 * If there is a pad descriptor EOP for the padded unit will be EOPed by it.
1209 */
1210 assert(active_count >= eop_needed_count);
1211 assert((eop_needed_count == 0) || (eop_needed_count == 1));
1212 if (eop_needed_count) {
1213 /* This means that the bulk operation (cipeher/m2m) is terminated. */
1214 if (active_count > 1) {
1215 /* Use zero length EOP descriptor. */
1216 struct cryptocop_dma_desc *ed = alloc_cdesc(alloc_flag);
1217 struct strcop_meta_out ed_mo = {0};
1218 if (!ed) {
1219 DEBUG_API(printk("cryptocop_setup_dma_list: alloc EOP descriptor for cipher\n"));
1220 failed = -ENOMEM;
1221 goto error_cleanup;
1222 }
1223
1224 assert(cipher_ctx.active && cipher_ctx.done);
1225
1226 if (cipher_ctx.unit_no == src_dma){
1227 /* mem2mem */
1228 ed_mo.ciphsel = src_none;
1229 } else {
1230 ed_mo.ciphsel = cipher_ctx.current_src;
1231 }
1232 ed_mo.ciphconf = cipher_ctx.ciph_conf;
1233 ed_mo.cbcmode = cipher_ctx.cbcmode;
1234 ed_mo.decrypt = cipher_ctx.decrypt;
1235
1236 ed->free_buf = NULL;
1237 ed->dma_descr->wait = 1;
1238 ed->dma_descr->out_eop = 1;
1239
1240 ed->dma_descr->buf = (char*)virt_to_phys(&ed); /* Use any valid physical address for zero length descriptor. */
1241 ed->dma_descr->after = ed->dma_descr->buf;
1242 ed->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, ed_mo);
1243 current_out_cdesc->next = ed;
1244 current_out_cdesc = ed;
1245 } else {
1246 /* Set EOP in the current out descriptor since the only active module is
1247 * the one needing the EOP. */
1248
1249 current_out_cdesc->dma_descr->out_eop = 1;
1250 }
1251 }
1252
1253 if (cipher_ctx.done && cipher_ctx.active) cipher_ctx.active = 0;
1254 if (digest_ctx.done && digest_ctx.active) digest_ctx.active = 0;
1255 if (csum_ctx.done && csum_ctx.active) csum_ctx.active = 0;
1256 indata_ix += odsc->length;
1257 odsc = odsc->next;
1258 } /* while (odsc) */ /* Process descriptors. */
1259 DEBUG(printk("cryptocop_setup_dma_list: done parsing operation descriptors\n"));
1260 if (cipher_ctx.tcfg && (cipher_ctx.active || !cipher_ctx.done)){
1261 DEBUG_API(printk("cryptocop_setup_dma_list: cipher operation not terminated.\n"));
1262 failed = -EINVAL;
1263 goto error_cleanup;
1264 }
1265 if (digest_ctx.tcfg && (digest_ctx.active || !digest_ctx.done)){
1266 DEBUG_API(printk("cryptocop_setup_dma_list: digest operation not terminated.\n"));
1267 failed = -EINVAL;
1268 goto error_cleanup;
1269 }
1270 if (csum_ctx.tcfg && (csum_ctx.active || !csum_ctx.done)){
1271 DEBUG_API(printk("cryptocop_setup_dma_list: csum operation not terminated.\n"));
1272 failed = -EINVAL;
1273 goto error_cleanup;
1274 }
1275
1276 failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &cipher_ctx, alloc_flag);
1277 if (failed){
1278 DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1279 goto error_cleanup;
1280 }
1281 failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &digest_ctx, alloc_flag);
1282 if (failed){
1283 DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1284 goto error_cleanup;
1285 }
1286 failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &csum_ctx, alloc_flag);
1287 if (failed){
1288 DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1289 goto error_cleanup;
1290 }
1291
1292 DEBUG(printk("cryptocop_setup_dma_list: int_op=0x%p, *int_op=0x%p\n", int_op, *int_op));
1293 (*int_op)->cdesc_out = out_cdesc_head.next;
1294 (*int_op)->cdesc_in = in_cdesc_head.next;
1295 DEBUG(printk("cryptocop_setup_dma_list: out_cdesc_head=0x%p in_cdesc_head=0x%p\n", (*int_op)->cdesc_out, (*int_op)->cdesc_in));
1296
1297 setup_descr_chain(out_cdesc_head.next);
1298 setup_descr_chain(in_cdesc_head.next);
1299
1300 /* Last but not least: mark the last DMA in descriptor for a INTR and EOL and the the
1301 * last DMA out descriptor for EOL.
1302 */
1303 current_in_cdesc->dma_descr->intr = 1;
1304 current_in_cdesc->dma_descr->eol = 1;
1305 current_out_cdesc->dma_descr->eol = 1;
1306
1307 /* Setup DMA contexts. */
1308 (*int_op)->ctx_out.next = NULL;
1309 (*int_op)->ctx_out.eol = 1;
1310 (*int_op)->ctx_out.intr = 0;
1311 (*int_op)->ctx_out.store_mode = 0;
1312 (*int_op)->ctx_out.en = 0;
1313 (*int_op)->ctx_out.dis = 0;
1314 (*int_op)->ctx_out.md0 = 0;
1315 (*int_op)->ctx_out.md1 = 0;
1316 (*int_op)->ctx_out.md2 = 0;
1317 (*int_op)->ctx_out.md3 = 0;
1318 (*int_op)->ctx_out.md4 = 0;
1319 (*int_op)->ctx_out.saved_data = (dma_descr_data*)virt_to_phys((*int_op)->cdesc_out->dma_descr);
1320 (*int_op)->ctx_out.saved_data_buf = (*int_op)->cdesc_out->dma_descr->buf; /* Already physical address. */
1321
1322 (*int_op)->ctx_in.next = NULL;
1323 (*int_op)->ctx_in.eol = 1;
1324 (*int_op)->ctx_in.intr = 0;
1325 (*int_op)->ctx_in.store_mode = 0;
1326 (*int_op)->ctx_in.en = 0;
1327 (*int_op)->ctx_in.dis = 0;
1328 (*int_op)->ctx_in.md0 = 0;
1329 (*int_op)->ctx_in.md1 = 0;
1330 (*int_op)->ctx_in.md2 = 0;
1331 (*int_op)->ctx_in.md3 = 0;
1332 (*int_op)->ctx_in.md4 = 0;
1333
1334 (*int_op)->ctx_in.saved_data = (dma_descr_data*)virt_to_phys((*int_op)->cdesc_in->dma_descr);
1335 (*int_op)->ctx_in.saved_data_buf = (*int_op)->cdesc_in->dma_descr->buf; /* Already physical address. */
1336
1337 DEBUG(printk("cryptocop_setup_dma_list: done\n"));
1338 return 0;
1339
1340error_cleanup:
1341 {
1342 /* Free all allocated resources. */
1343 struct cryptocop_dma_desc *tmp_cdesc;
1344 while (digest_ctx.pad_descs){
1345 tmp_cdesc = digest_ctx.pad_descs->next;
1346 free_cdesc(digest_ctx.pad_descs);
1347 digest_ctx.pad_descs = tmp_cdesc;
1348 }
1349 while (csum_ctx.pad_descs){
1350 tmp_cdesc = csum_ctx.pad_descs->next;
1351 free_cdesc(csum_ctx.pad_descs);
1352 csum_ctx.pad_descs = tmp_cdesc;
1353 }
1354 assert(cipher_ctx.pad_descs == NULL); /* The ciphers are never padded. */
1355
1356 if (*int_op != NULL) delete_internal_operation(*int_op);
1357 }
1358 DEBUG_API(printk("cryptocop_setup_dma_list: done with error %d\n", failed));
1359 return failed;
1360}
1361
1362
1363static void delete_internal_operation(struct cryptocop_int_operation *iop)
1364{
1365 void *ptr = iop->alloc_ptr;
1366 struct cryptocop_dma_desc *cd = iop->cdesc_out;
1367 struct cryptocop_dma_desc *next;
1368
1369 DEBUG(printk("delete_internal_operation: iop=0x%p, alloc_ptr=0x%p\n", iop, ptr));
1370
1371 while (cd) {
1372 next = cd->next;
1373 free_cdesc(cd);
1374 cd = next;
1375 }
1376 cd = iop->cdesc_in;
1377 while (cd) {
1378 next = cd->next;
1379 free_cdesc(cd);
1380 cd = next;
1381 }
1382 kfree(ptr);
1383}
1384
1385#define MD5_MIN_PAD_LENGTH (9)
1386#define MD5_PAD_LENGTH_FIELD_LENGTH (8)
1387
1388static int create_md5_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length)
1389{
1390 size_t padlen = MD5_BLOCK_LENGTH - (hashed_length % MD5_BLOCK_LENGTH);
1391 unsigned char *p;
1392 int i;
1393 unsigned long long int bit_length = hashed_length << 3;
1394
1395 if (padlen < MD5_MIN_PAD_LENGTH) padlen += MD5_BLOCK_LENGTH;
1396
1397 p = kmalloc(padlen, alloc_flag);
91a120d0 1398 if (!p) return -ENOMEM;
51533b61
MS		 1399
1400 *p = 0x80;
1401 memset(p+1, 0, padlen - 1);
1402
1403 DEBUG(printk("create_md5_pad: hashed_length=%lld bits == %lld bytes\n", bit_length, hashed_length));
1404
1405 i = padlen - MD5_PAD_LENGTH_FIELD_LENGTH;
1406 while (bit_length != 0){
1407 p[i++] = bit_length % 0x100;
1408 bit_length >>= 8;
1409 }
1410
1411 *pad = (char*)p;
1412 *pad_length = padlen;
1413
1414 return 0;
1415}
1416
1417#define SHA1_MIN_PAD_LENGTH (9)
1418#define SHA1_PAD_LENGTH_FIELD_LENGTH (8)
1419
1420static int create_sha1_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length)
1421{
1422 size_t padlen = SHA1_BLOCK_LENGTH - (hashed_length % SHA1_BLOCK_LENGTH);
1423 unsigned char *p;
1424 int i;
1425 unsigned long long int bit_length = hashed_length << 3;
1426
1427 if (padlen < SHA1_MIN_PAD_LENGTH) padlen += SHA1_BLOCK_LENGTH;
1428
1429 p = kmalloc(padlen, alloc_flag);
91a120d0 1430 if (!p) return -ENOMEM;
51533b61
MS		 1431
1432 *p = 0x80;
1433 memset(p+1, 0, padlen - 1);
1434
1435 DEBUG(printk("create_sha1_pad: hashed_length=%lld bits == %lld bytes\n", bit_length, hashed_length));
1436
1437 i = padlen - 1;
1438 while (bit_length != 0){
1439 p[i--] = bit_length % 0x100;
1440 bit_length >>= 8;
1441 }
1442
1443 *pad = (char*)p;
1444 *pad_length = padlen;
1445
1446 return 0;
1447}
1448
1449
1450static int transform_ok(struct cryptocop_transform_init *tinit)
1451{
1452 switch (tinit->alg){
1453 case cryptocop_alg_csum:
1454 switch (tinit->csum_mode){
1455 case cryptocop_csum_le:
1456 case cryptocop_csum_be:
1457 break;
1458 default:
1459 DEBUG_API(printk("transform_ok: Bad mode set for csum transform\n"));
1460 return -EINVAL;
1461 }
1462 case cryptocop_alg_mem2mem:
1463 case cryptocop_alg_md5:
1464 case cryptocop_alg_sha1:
1465 if (tinit->keylen != 0) {
1466 DEBUG_API(printk("transform_ok: non-zero keylength, %d, for a digest/csum algorithm\n", tinit->keylen));
1467 return -EINVAL; /* This check is a bit strict. */
1468 }
1469 break;
1470 case cryptocop_alg_des:
1471 if (tinit->keylen != 64) {
1472 DEBUG_API(printk("transform_ok: keylen %d invalid for DES\n", tinit->keylen));
1473 return -EINVAL;
1474 }
1475 break;
1476 case cryptocop_alg_3des:
1477 if (tinit->keylen != 192) {
1478 DEBUG_API(printk("transform_ok: keylen %d invalid for 3DES\n", tinit->keylen));
1479 return -EINVAL;
1480 }
1481 break;
1482 case cryptocop_alg_aes:
1483 if (tinit->keylen != 128 && tinit->keylen != 192 && tinit->keylen != 256) {
1484 DEBUG_API(printk("transform_ok: keylen %d invalid for AES\n", tinit->keylen));
1485 return -EINVAL;
1486 }
1487 break;
1488 case cryptocop_no_alg:
1489 default:
1490 DEBUG_API(printk("transform_ok: no such algorithm %d\n", tinit->alg));
1491 return -EINVAL;
1492 }
1493
1494 switch (tinit->alg){
1495 case cryptocop_alg_des:
1496 case cryptocop_alg_3des:
1497 case cryptocop_alg_aes:
1498 if (tinit->cipher_mode != cryptocop_cipher_mode_ecb && tinit->cipher_mode != cryptocop_cipher_mode_cbc) return -EINVAL;
1499 default:
1500 break;
1501 }
1502 return 0;
1503}
1504
1505
1506int cryptocop_new_session(cryptocop_session_id *sid, struct cryptocop_transform_init *tinit, int alloc_flag)
1507{
1508 struct cryptocop_session *sess;
1509 struct cryptocop_transform_init *tfrm_in = tinit;
1510 struct cryptocop_transform_init *tmp_in;
1511 int no_tfrms = 0;
1512 int i;
1513 unsigned long int flags;
1514
1515 init_stream_coprocessor(); /* For safety if we are called early */
1516
1517 while (tfrm_in){
1518 int err;
1519 ++no_tfrms;
1520 if ((err = transform_ok(tfrm_in))) {
1521 DEBUG_API(printk("cryptocop_new_session, bad transform\n"));
1522 return err;
1523 }
1524 tfrm_in = tfrm_in->next;
1525 }
1526 if (0 == no_tfrms) {
1527 DEBUG_API(printk("cryptocop_new_session, no transforms specified\n"));
1528 return -EINVAL;
1529 }
1530
1531 sess = kmalloc(sizeof(struct cryptocop_session), alloc_flag);
1532 if (!sess){
1533 DEBUG_API(printk("cryptocop_new_session, kmalloc cryptocop_session\n"));
1534 return -ENOMEM;
1535 }
1536
1537 sess->tfrm_ctx = kmalloc(no_tfrms * sizeof(struct cryptocop_transform_ctx), alloc_flag);
1538 if (!sess->tfrm_ctx) {
1539 DEBUG_API(printk("cryptocop_new_session, kmalloc cryptocop_transform_ctx\n"));
1540 kfree(sess);
1541 return -ENOMEM;
1542 }
1543
1544 tfrm_in = tinit;
1545 for (i = 0; i < no_tfrms; i++){
1546 tmp_in = tfrm_in->next;
1547 while (tmp_in){
1548 if (tmp_in->tid == tfrm_in->tid) {
1549 DEBUG_API(printk("cryptocop_new_session, duplicate transform ids\n"));
1550 kfree(sess->tfrm_ctx);
1551 kfree(sess);
1552 return -EINVAL;
1553 }
1554 tmp_in = tmp_in->next;
1555 }
1556 memcpy(&sess->tfrm_ctx[i].init, tfrm_in, sizeof(struct cryptocop_transform_init));
1557 sess->tfrm_ctx[i].dec_key_set = 0;
1558 sess->tfrm_ctx[i].next = &sess->tfrm_ctx[i] + 1;
1559
1560 tfrm_in = tfrm_in->next;
1561 }
1562 sess->tfrm_ctx[i-1].next = NULL;
1563
1564 spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1565 sess->sid = next_sid;
1566 next_sid++;
1567 /* TODO If we are really paranoid we should do duplicate check to handle sid wraparound.
1568 * OTOH 2^64 is a really large number of session. */
1569 if (next_sid == 0) next_sid = 1;
1570
1571 /* Prepend to session list. */
1572 sess->next = cryptocop_sessions;
1573 cryptocop_sessions = sess;
1574 spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1575 *sid = sess->sid;
1576 return 0;
1577}
1578
1579
1580int cryptocop_free_session(cryptocop_session_id sid)
1581{
1582 struct cryptocop_transform_ctx *tc;
1583 struct cryptocop_session *sess = NULL;
1584 struct cryptocop_session *psess = NULL;
1585 unsigned long int flags;
1586 int i;
1587 LIST_HEAD(remove_list);
1588 struct list_head *node, *tmp;
1589 struct cryptocop_prio_job *pj;
1590
1591 DEBUG(printk("cryptocop_free_session: sid=%lld\n", sid));
1592
1593 spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1594 sess = cryptocop_sessions;
1595 while (sess && sess->sid != sid){
1596 psess = sess;
1597 sess = sess->next;
1598 }
1599 if (sess){
1600 if (psess){
1601 psess->next = sess->next;
1602 } else {
1603 cryptocop_sessions = sess->next;
1604 }
1605 }
1606 spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1607
1608 if (!sess) return -EINVAL;
1609
1610 /* Remove queued jobs. */
1611 spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
1612
1613 for (i = 0; i < cryptocop_prio_no_prios; i++){
1614 if (!list_empty(&(cryptocop_job_queues[i].jobs))){
1615 list_for_each_safe(node, tmp, &(cryptocop_job_queues[i].jobs)) {
1616 pj = list_entry(node, struct cryptocop_prio_job, node);
1617 if (pj->oper->sid == sid) {
1618 list_move_tail(node, &remove_list);
1619 }
1620 }
1621 }
1622 }
1623 spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
1624
1625 list_for_each_safe(node, tmp, &remove_list) {
1626 list_del(node);
1627 pj = list_entry(node, struct cryptocop_prio_job, node);
1628 pj->oper->operation_status = -EAGAIN; /* EAGAIN is not ideal for job/session terminated but it's the best choice I know of. */
1629 DEBUG(printk("cryptocop_free_session: pj=0x%p, pj->oper=0x%p, pj->iop=0x%p\n", pj, pj->oper, pj->iop));
1630 pj->oper->cb(pj->oper, pj->oper->cb_data);
1631 delete_internal_operation(pj->iop);
1632 kfree(pj);
1633 }
1634
1635 tc = sess->tfrm_ctx;
1636 /* Erase keying data. */
1637 while (tc){
1638 DEBUG(printk("cryptocop_free_session: memset keys, tfrm id=%d\n", tc->init.tid));
1639 memset(tc->init.key, 0xff, CRYPTOCOP_MAX_KEY_LENGTH);
1640 memset(tc->dec_key, 0xff, CRYPTOCOP_MAX_KEY_LENGTH);
1641 tc = tc->next;
1642 }
1643 kfree(sess->tfrm_ctx);
1644 kfree(sess);
1645
1646 return 0;
1647}
1648
1649static struct cryptocop_session *get_session(cryptocop_session_id sid)
1650{
1651 struct cryptocop_session *sess;
1652 unsigned long int flags;
1653
1654 spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1655 sess = cryptocop_sessions;
1656 while (sess && (sess->sid != sid)){
1657 sess = sess->next;
1658 }
1659 spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1660
1661 return sess;
1662}
1663
1664static struct cryptocop_transform_ctx *get_transform_ctx(struct cryptocop_session *sess, cryptocop_tfrm_id tid)
1665{
1666 struct cryptocop_transform_ctx *tc = sess->tfrm_ctx;
1667
1668 DEBUG(printk("get_transform_ctx, sess=0x%p, tid=%d\n", sess, tid));
1669 assert(sess != NULL);
1670 while (tc && tc->init.tid != tid){
1671 DEBUG(printk("tc=0x%p, tc->next=0x%p\n", tc, tc->next));
1672 tc = tc->next;
1673 }
1674 DEBUG(printk("get_transform_ctx, returning tc=0x%p\n", tc));
1675 return tc;
1676}
1677
1678
1679
1680/* The AES s-transform matrix (s-box). */
1681static const u8 aes_sbox[256] = {
1682 99, 124, 119, 123, 242, 107, 111, 197, 48, 1, 103, 43, 254, 215, 171, 118,
1683 202, 130, 201, 125, 250, 89, 71, 240, 173, 212, 162, 175, 156, 164, 114, 192,
1684 183, 253, 147, 38, 54, 63, 247, 204, 52, 165, 229, 241, 113, 216, 49, 21,
1685 4, 199, 35, 195, 24, 150, 5, 154, 7, 18, 128, 226, 235, 39, 178, 117,
1686 9, 131, 44, 26, 27, 110, 90, 160, 82, 59, 214, 179, 41, 227, 47, 132,
1687 83, 209, 0, 237, 32, 252, 177, 91, 106, 203, 190, 57, 74, 76, 88, 207,
1688 208, 239, 170, 251, 67, 77, 51, 133, 69, 249, 2, 127, 80, 60, 159, 168,
1689 81, 163, 64, 143, 146, 157, 56, 245, 188, 182, 218, 33, 16, 255, 243, 210,
1690 205, 12, 19, 236, 95, 151, 68, 23, 196, 167, 126, 61, 100, 93, 25, 115,
1691 96, 129, 79, 220, 34, 42, 144, 136, 70, 238, 184, 20, 222, 94, 11, 219,
1692 224, 50, 58, 10, 73, 6, 36, 92, 194, 211, 172, 98, 145, 149, 228, 121,
1693 231, 200, 55, 109, 141, 213, 78, 169, 108, 86, 244, 234, 101, 122, 174, 8,
1694 186, 120, 37, 46, 28, 166, 180, 198, 232, 221, 116, 31, 75, 189, 139, 138,
1695 112, 62, 181, 102, 72, 3, 246, 14, 97, 53, 87, 185, 134, 193, 29, 158,
1696 225, 248, 152, 17, 105, 217, 142, 148, 155, 30, 135, 233, 206, 85, 40, 223,
1697 140, 161, 137, 13, 191, 230, 66, 104, 65, 153, 45, 15, 176, 84, 187, 22
1698};
1699
1700/* AES has a 32 bit word round constants for each round in the
1701 * key schedule. round_constant[i] is really Rcon[i+1] in FIPS187.
1702 */
1703static u32 round_constant[11] = {
1704 0x01000000, 0x02000000, 0x04000000, 0x08000000,
1705 0x10000000, 0x20000000, 0x40000000, 0x80000000,
1706 0x1B000000, 0x36000000, 0x6C000000
1707};
1708
1709/* Apply the s-box to each of the four occtets in w. */
1710static u32 aes_ks_subword(const u32 w)
1711{
1712 u8 bytes[4];
1713
1714 *(u32*)(&bytes[0]) = w;
1715 bytes[0] = aes_sbox[bytes[0]];
1716 bytes[1] = aes_sbox[bytes[1]];
1717 bytes[2] = aes_sbox[bytes[2]];
1718 bytes[3] = aes_sbox[bytes[3]];
1719 return *(u32*)(&bytes[0]);
1720}
1721
1722/* The encrypt (forward) Rijndael key schedule algorithm pseudo code:
1723 * (Note that AES words are 32 bit long)
1724 *
1725 * KeyExpansion(byte key[4*Nk], word w[Nb*(Nr+1)], Nk){
1726 * word temp
1727 * i = 0
1728 * while (i < Nk) {
1729 * w[i] = word(key[4*i, 4*i + 1, 4*i + 2, 4*i + 3])
1730 * i = i + 1
1731 * }
1732 * i = Nk
1733 *
1734 * while (i < (Nb * (Nr + 1))) {
1735 * temp = w[i - 1]
1736 * if ((i mod Nk) == 0) {
1737 * temp = SubWord(RotWord(temp)) xor Rcon[i/Nk]
1738 * }
1739 * else if ((Nk > 6) && ((i mod Nk) == 4)) {
1740 * temp = SubWord(temp)
1741 * }
1742 * w[i] = w[i - Nk] xor temp
1743 * }
1744 * RotWord(t) does a 8 bit cyclic shift left on a 32 bit word.
1745 * SubWord(t) applies the AES s-box individually to each octet
1746 * in a 32 bit word.
1747 *
1748 * For AES Nk can have the values 4, 6, and 8 (corresponding to
1749 * values for Nr of 10, 12, and 14). Nb is always 4.
1750 *
1751 * To construct w[i], w[i - 1] and w[i - Nk] must be
1752 * available. Consequently we must keep a state of the last Nk words
1753 * to be able to create the last round keys.
1754 */
1755static void get_aes_decrypt_key(unsigned char *dec_key, const unsigned char *key, unsigned int keylength)
1756{
1757 u32 temp;
1758 u32 w_ring[8]; /* nk is max 8, use elements 0..(nk - 1) as a ringbuffer */
1759 u8 w_last_ix;
1760 int i;
1761 u8 nr, nk;
1762
1763 switch (keylength){
1764 case 128:
1765 nk = 4;
1766 nr = 10;
1767 break;
1768 case 192:
1769 nk = 6;
1770 nr = 12;
1771 break;
1772 case 256:
1773 nk = 8;
1774 nr = 14;
1775 break;
1776 default:
1777 panic("stream co-processor: bad aes key length in get_aes_decrypt_key\n");
1778 };
1779
1780 /* Need to do host byte order correction here since key is byte oriented and the
1781 * kx algorithm is word (u32) oriented. */
1782 for (i = 0; i < nk; i+=1) {
1783 w_ring[i] = be32_to_cpu(*(u32*)&key[4*i]);
1784 }
1785
1786 i = (int)nk;
1787 w_last_ix = i - 1;
1788 while (i < (4 * (nr + 2))) {
1789 temp = w_ring[w_last_ix];
1790 if (!(i % nk)) {
1791 /* RotWord(temp) */
1792 temp = (temp << 8) | (temp >> 24);
1793 temp = aes_ks_subword(temp);
1794 temp ^= round_constant[i/nk - 1];
1795 } else if ((nk > 6) && ((i % nk) == 4)) {
1796 temp = aes_ks_subword(temp);
1797 }
1798 w_last_ix = (w_last_ix + 1) % nk; /* This is the same as (i-Nk) mod Nk */
1799 temp ^= w_ring[w_last_ix];
1800 w_ring[w_last_ix] = temp;
1801
1802 /* We need the round keys for round Nr+1 and Nr+2 (round key
1803 * Nr+2 is the round key beyond the last one used when
1804 * encrypting). Rounds are numbered starting from 0, Nr=10
1805 * implies 11 rounds are used in encryption/decryption.
1806 */
1807 if (i >= (4 * nr)) {
1808 /* Need to do host byte order correction here, the key
1809 * is byte oriented. */
1810 *(u32*)dec_key = cpu_to_be32(temp);
1811 dec_key += 4;
1812 }
1813 ++i;
1814 }
1815}
1816
1817
1818/**** Job/operation management. ****/
1819
1820int cryptocop_job_queue_insert_csum(struct cryptocop_operation *operation)
1821{
1822 return cryptocop_job_queue_insert(cryptocop_prio_kernel_csum, operation);
1823}
1824
1825int cryptocop_job_queue_insert_crypto(struct cryptocop_operation *operation)
1826{
1827 return cryptocop_job_queue_insert(cryptocop_prio_kernel, operation);
1828}
1829
1830int cryptocop_job_queue_insert_user_job(struct cryptocop_operation *operation)
1831{
1832 return cryptocop_job_queue_insert(cryptocop_prio_user, operation);
1833}
1834
1835static int cryptocop_job_queue_insert(cryptocop_queue_priority prio, struct cryptocop_operation *operation)
1836{
1837 int ret;
1838 struct cryptocop_prio_job *pj = NULL;
1839 unsigned long int flags;
1840
1841 DEBUG(printk("cryptocop_job_queue_insert(%d, 0x%p)\n", prio, operation));
1842
1843 if (!operation || !operation->cb){
1844 DEBUG_API(printk("cryptocop_job_queue_insert oper=0x%p, NULL operation or callback\n", operation));
1845 return -EINVAL;
1846 }
1847
1848 if ((ret = cryptocop_job_setup(&pj, operation)) != 0){
1849 DEBUG_API(printk("cryptocop_job_queue_insert: job setup failed\n"));
1850 return ret;
1851 }
1852 assert(pj != NULL);
1853
1854 spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
1855 list_add_tail(&pj->node, &cryptocop_job_queues[prio].jobs);
1856 spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
1857
1858 /* Make sure a job is running */
1859 cryptocop_start_job();
1860 return 0;
1861}
1862
1863static void cryptocop_do_tasklet(unsigned long unused);
1864DECLARE_TASKLET (cryptocop_tasklet, cryptocop_do_tasklet, 0);
1865
1866static void cryptocop_do_tasklet(unsigned long unused)
1867{
1868 struct list_head *node;
1869 struct cryptocop_prio_job *pj = NULL;
1870 unsigned long flags;
1871
1872 DEBUG(printk("cryptocop_do_tasklet: entering\n"));
1873
1874 do {
1875 spin_lock_irqsave(&cryptocop_completed_jobs_lock, flags);
1876 if (!list_empty(&cryptocop_completed_jobs)){
1877 node = cryptocop_completed_jobs.next;
1878 list_del(node);
1879 pj = list_entry(node, struct cryptocop_prio_job, node);
1880 } else {
1881 pj = NULL;
1882 }
1883 spin_unlock_irqrestore(&cryptocop_completed_jobs_lock, flags);
1884 if (pj) {
1885 assert(pj->oper != NULL);
1886
1887 /* Notify consumer of operation completeness. */
1888 DEBUG(printk("cryptocop_do_tasklet: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
1889
1890 pj->oper->operation_status = 0; /* Job is completed. */
1891 pj->oper->cb(pj->oper, pj->oper->cb_data);
1892 delete_internal_operation(pj->iop);
1893 kfree(pj);
1894 }
1895 } while (pj != NULL);
1896
1897 DEBUG(printk("cryptocop_do_tasklet: exiting\n"));
1898}
1899
1900static irqreturn_t
635c45c1 1901dma_done_interrupt(int irq, void *dev_id)
51533b61
MS		 1902{
1903 struct cryptocop_prio_job *done_job;
1904 reg_dma_rw_ack_intr ack_intr = {
1905 .data = 1,
1906 };
1907
635c45c1 1908 REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
51533b61
MS		 1909
1910 DEBUG(printk("cryptocop DMA done\n"));
1911
1912 spin_lock(&running_job_lock);
1913 if (cryptocop_running_job == NULL){
1914 printk("stream co-processor got interrupt when not busy\n");
1915 spin_unlock(&running_job_lock);
1916 return IRQ_HANDLED;
1917 }
1918 done_job = cryptocop_running_job;
1919 cryptocop_running_job = NULL;
1920 spin_unlock(&running_job_lock);
1921
1922 /* Start processing a job. */
1923 if (!spin_trylock(&cryptocop_process_lock)){
1924 DEBUG(printk("cryptocop irq handler, not starting a job\n"));
1925 } else {
1926 cryptocop_start_job();
1927 spin_unlock(&cryptocop_process_lock);
1928 }
1929
1930 done_job->oper->operation_status = 0; /* Job is completed. */
1931 if (done_job->oper->fast_callback){
1932 /* This operation wants callback from interrupt. */
1933 done_job->oper->cb(done_job->oper, done_job->oper->cb_data);
1934 delete_internal_operation(done_job->iop);
1935 kfree(done_job);
1936 } else {
1937 spin_lock(&cryptocop_completed_jobs_lock);
1938 list_add_tail(&(done_job->node), &cryptocop_completed_jobs);
1939 spin_unlock(&cryptocop_completed_jobs_lock);
1940 tasklet_schedule(&cryptocop_tasklet);
1941 }
1942
1943 DEBUG(printk("cryptocop leave irq handler\n"));
1944 return IRQ_HANDLED;
1945}
1946
1947
1948/* Setup interrupts and DMA channels. */
1949static int init_cryptocop(void)
1950{
1951 unsigned long flags;
51533b61
MS		 1952 reg_dma_rw_cfg dma_cfg = {.en = 1};
1953 reg_dma_rw_intr_mask intr_mask_in = {.data = regk_dma_yes}; /* Only want descriptor interrupts from the DMA in channel. */
1954 reg_dma_rw_ack_intr ack_intr = {.data = 1,.in_eop = 1 };
1955 reg_strcop_rw_cfg strcop_cfg = {
1956 .ipend = regk_strcop_little,
1957 .td1 = regk_strcop_e,
1958 .td2 = regk_strcop_d,
1959 .td3 = regk_strcop_e,
1960 .ignore_sync = 0,
1961 .en = 1
1962 };
1963
635c45c1
JN		 1964 if (request_irq(DMA_IRQ, dma_done_interrupt, 0,
1965 "stream co-processor DMA", NULL))
1966 panic("request_irq stream co-processor irq dma9");
51533b61 1967
635c45c1
JN		 1968 (void)crisv32_request_dma(OUT_DMA, "strcop", DMA_PANIC_ON_ERROR,
1969 0, dma_strp);
1970 (void)crisv32_request_dma(IN_DMA, "strcop", DMA_PANIC_ON_ERROR,
1971 0, dma_strp);
51533b61
MS		 1972
1973 local_irq_save(flags);
1974
1975 /* Reset and enable the cryptocop. */
1976 strcop_cfg.en = 0;
1977 REG_WR(strcop, regi_strcop, rw_cfg, strcop_cfg);
1978 strcop_cfg.en = 1;
1979 REG_WR(strcop, regi_strcop, rw_cfg, strcop_cfg);
1980
51533b61 1981 /* Enable DMAs. */
635c45c1
JN		 1982 REG_WR(dma, IN_DMA_INST, rw_cfg, dma_cfg); /* input DMA */
1983 REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_cfg); /* output DMA */
51533b61
MS		 1984
1985 /* Set up wordsize = 4 for DMAs. */
635c45c1
JN		 1986 DMA_WR_CMD(OUT_DMA_INST, regk_dma_set_w_size4);
1987 DMA_WR_CMD(IN_DMA_INST, regk_dma_set_w_size4);
51533b61
MS		 1988
1989 /* Enable interrupts. */
635c45c1 1990 REG_WR(dma, IN_DMA_INST, rw_intr_mask, intr_mask_in);
51533b61
MS		 1991
1992 /* Clear intr ack. */
635c45c1 1993 REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
51533b61
MS		 1994
1995 local_irq_restore(flags);
1996
1997 return 0;
1998}
1999
2000/* Free used cryptocop hw resources (interrupt and DMA channels). */
2001static void release_cryptocop(void)
2002{
2003 unsigned long flags;
51533b61
MS		 2004 reg_dma_rw_cfg dma_cfg = {.en = 0};
2005 reg_dma_rw_intr_mask intr_mask_in = {0};
2006 reg_dma_rw_ack_intr ack_intr = {.data = 1,.in_eop = 1 };
2007
2008 local_irq_save(flags);
2009
2010 /* Clear intr ack. */
635c45c1 2011 REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
51533b61
MS		 2012
2013 /* Disable DMAs. */
635c45c1
JN		 2014 REG_WR(dma, IN_DMA_INST, rw_cfg, dma_cfg); /* input DMA */
2015 REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_cfg); /* output DMA */
51533b61
MS		 2016
2017 /* Disable interrupts. */
635c45c1 2018 REG_WR(dma, IN_DMA_INST, rw_intr_mask, intr_mask_in);
51533b61
MS		 2019
2020 local_irq_restore(flags);
2021
635c45c1 2022 free_irq(DMA_IRQ, NULL);
51533b61 2023
635c45c1
JN		 2024 (void)crisv32_free_dma(OUT_DMA);
2025 (void)crisv32_free_dma(IN_DMA);
51533b61
MS		 2026}
2027
2028
2029/* Init job queue. */
2030static int cryptocop_job_queue_init(void)
2031{
2032 int i;
2033
2034 INIT_LIST_HEAD(&cryptocop_completed_jobs);
2035
2036 for (i = 0; i < cryptocop_prio_no_prios; i++){
2037 cryptocop_job_queues[i].prio = (cryptocop_queue_priority)i;
2038 INIT_LIST_HEAD(&cryptocop_job_queues[i].jobs);
2039 }
2040 return 0;
2041}
2042
2043
2044static void cryptocop_job_queue_close(void)
2045{
2046 struct list_head *node, *tmp;
2047 struct cryptocop_prio_job *pj = NULL;
2048 unsigned long int process_flags, flags;
2049 int i;
2050
2051 /* FIXME: This is as yet untested code. */
2052
2053 /* Stop strcop from getting an operation to process while we are closing the
2054 module. */
2055 spin_lock_irqsave(&cryptocop_process_lock, process_flags);
2056
2057 /* Empty the job queue. */
51533b61
MS		 2058 for (i = 0; i < cryptocop_prio_no_prios; i++){
2059 if (!list_empty(&(cryptocop_job_queues[i].jobs))){
2060 list_for_each_safe(node, tmp, &(cryptocop_job_queues[i].jobs)) {
2061 pj = list_entry(node, struct cryptocop_prio_job, node);
2062 list_del(node);
2063
2064 /* Call callback to notify consumer of job removal. */
2065 DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2066 pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2067 pj->oper->cb(pj->oper, pj->oper->cb_data);
2068
2069 delete_internal_operation(pj->iop);
2070 kfree(pj);
2071 }
2072 }
2073 }
2074 spin_unlock_irqrestore(&cryptocop_process_lock, process_flags);
2075
2076 /* Remove the running job, if any. */
2077 spin_lock_irqsave(&running_job_lock, flags);
2078 if (cryptocop_running_job){
2079 reg_strcop_rw_cfg rw_cfg;
2080 reg_dma_rw_cfg dma_out_cfg, dma_in_cfg;
2081
2082 /* Stop DMA. */
635c45c1 2083 dma_out_cfg = REG_RD(dma, OUT_DMA_INST, rw_cfg);
51533b61 2084 dma_out_cfg.en = regk_dma_no;
635c45c1 2085 REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_out_cfg);
51533b61 2086
635c45c1 2087 dma_in_cfg = REG_RD(dma, IN_DMA_INST, rw_cfg);
51533b61 2088 dma_in_cfg.en = regk_dma_no;
635c45c1 2089 REG_WR(dma, IN_DMA_INST, rw_cfg, dma_in_cfg);
51533b61
MS		 2090
2091 /* Disble the cryptocop. */
2092 rw_cfg = REG_RD(strcop, regi_strcop, rw_cfg);
2093 rw_cfg.en = 0;
2094 REG_WR(strcop, regi_strcop, rw_cfg, rw_cfg);
2095
2096 pj = cryptocop_running_job;
2097 cryptocop_running_job = NULL;
2098
2099 /* Call callback to notify consumer of job removal. */
2100 DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2101 pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2102 pj->oper->cb(pj->oper, pj->oper->cb_data);
2103
2104 delete_internal_operation(pj->iop);
2105 kfree(pj);
2106 }
2107 spin_unlock_irqrestore(&running_job_lock, flags);
2108
2109 /* Remove completed jobs, if any. */
2110 spin_lock_irqsave(&cryptocop_completed_jobs_lock, flags);
2111
2112 list_for_each_safe(node, tmp, &cryptocop_completed_jobs) {
2113 pj = list_entry(node, struct cryptocop_prio_job, node);
2114 list_del(node);
2115 /* Call callback to notify consumer of job removal. */
2116 DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2117 pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2118 pj->oper->cb(pj->oper, pj->oper->cb_data);
2119
2120 delete_internal_operation(pj->iop);
2121 kfree(pj);
2122 }
2123 spin_unlock_irqrestore(&cryptocop_completed_jobs_lock, flags);
2124}
2125
2126
2127static void cryptocop_start_job(void)
2128{
2129 int i;
2130 struct cryptocop_prio_job *pj;
2131 unsigned long int flags;
2132 unsigned long int running_job_flags;
2133 reg_strcop_rw_cfg rw_cfg = {.en = 1, .ignore_sync = 0};
2134
2135 DEBUG(printk("cryptocop_start_job: entering\n"));
2136
2137 spin_lock_irqsave(&running_job_lock, running_job_flags);
2138 if (cryptocop_running_job != NULL){
2139 /* Already running. */
2140 DEBUG(printk("cryptocop_start_job: already running, exit\n"));
2141 spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2142 return;
2143 }
2144 spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
2145
2146 /* Check the queues in priority order. */
2147 for (i = cryptocop_prio_kernel_csum; (i < cryptocop_prio_no_prios) && list_empty(&cryptocop_job_queues[i].jobs); i++);
2148 if (i == cryptocop_prio_no_prios) {
2149 spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
2150 spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2151 DEBUG(printk("cryptocop_start_job: no jobs to run\n"));
2152 return; /* No jobs to run */
2153 }
2154 DEBUG(printk("starting job for prio %d\n", i));
2155
2156 /* TODO: Do not starve lower priority jobs. Let in a lower
2157 * prio job for every N-th processed higher prio job or some
2158 * other scheduling policy. This could reasonably be
2159 * tweakable since the optimal balance would depend on the
2160 * type of load on the system. */
2161
2162 /* Pull the DMA lists from the job and start the DMA client. */
2163 pj = list_entry(cryptocop_job_queues[i].jobs.next, struct cryptocop_prio_job, node);
2164 list_del(&pj->node);
2165 spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
2166 cryptocop_running_job = pj;
2167
2168 /* Set config register (3DES and CSUM modes). */
2169 switch (pj->iop->tdes_mode){
2170 case cryptocop_3des_eee:
2171 rw_cfg.td1 = regk_strcop_e;
2172 rw_cfg.td2 = regk_strcop_e;
2173 rw_cfg.td3 = regk_strcop_e;
2174 break;
2175 case cryptocop_3des_eed:
2176 rw_cfg.td1 = regk_strcop_e;
2177 rw_cfg.td2 = regk_strcop_e;
2178 rw_cfg.td3 = regk_strcop_d;
2179 break;
2180 case cryptocop_3des_ede:
2181 rw_cfg.td1 = regk_strcop_e;
2182 rw_cfg.td2 = regk_strcop_d;
2183 rw_cfg.td3 = regk_strcop_e;
2184 break;
2185 case cryptocop_3des_edd:
2186 rw_cfg.td1 = regk_strcop_e;
2187 rw_cfg.td2 = regk_strcop_d;
2188 rw_cfg.td3 = regk_strcop_d;
2189 break;
2190 case cryptocop_3des_dee:
2191 rw_cfg.td1 = regk_strcop_d;
2192 rw_cfg.td2 = regk_strcop_e;
2193 rw_cfg.td3 = regk_strcop_e;
2194 break;
2195 case cryptocop_3des_ded:
2196 rw_cfg.td1 = regk_strcop_d;
2197 rw_cfg.td2 = regk_strcop_e;
2198 rw_cfg.td3 = regk_strcop_d;
2199 break;
2200 case cryptocop_3des_dde:
2201 rw_cfg.td1 = regk_strcop_d;
2202 rw_cfg.td2 = regk_strcop_d;
2203 rw_cfg.td3 = regk_strcop_e;
2204 break;
2205 case cryptocop_3des_ddd:
2206 rw_cfg.td1 = regk_strcop_d;
2207 rw_cfg.td2 = regk_strcop_d;
2208 rw_cfg.td3 = regk_strcop_d;
2209 break;
2210 default:
2211 DEBUG(printk("cryptocop_setup_dma_list: bad 3DES mode\n"));
2212 }
2213 switch (pj->iop->csum_mode){
2214 case cryptocop_csum_le:
2215 rw_cfg.ipend = regk_strcop_little;
2216 break;
2217 case cryptocop_csum_be:
2218 rw_cfg.ipend = regk_strcop_big;
2219 break;
2220 default:
2221 DEBUG(printk("cryptocop_setup_dma_list: bad checksum mode\n"));
2222 }
2223 REG_WR(strcop, regi_strcop, rw_cfg, rw_cfg);
2224
2225 DEBUG(printk("cryptocop_start_job: starting DMA, new cryptocop_running_job=0x%p\n"
2226 "ctx_in: 0x%p, phys: 0x%p\n"
2227 "ctx_out: 0x%p, phys: 0x%p\n",
2228 pj,
2229 &pj->iop->ctx_in, (char*)virt_to_phys(&pj->iop->ctx_in),
2230 &pj->iop->ctx_out, (char*)virt_to_phys(&pj->iop->ctx_out)));
2231
2232 /* Start input DMA. */
635c45c1
JN		 2233 flush_dma_context(&pj->iop->ctx_in);
2234 DMA_START_CONTEXT(IN_DMA_INST, virt_to_phys(&pj->iop->ctx_in));
51533b61
MS		 2235
2236 /* Start output DMA. */
635c45c1 2237 DMA_START_CONTEXT(OUT_DMA_INST, virt_to_phys(&pj->iop->ctx_out));
51533b61
MS		 2238
2239 spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2240 DEBUG(printk("cryptocop_start_job: exiting\n"));
2241}
2242
2243
2244static int cryptocop_job_setup(struct cryptocop_prio_job **pj, struct cryptocop_operation *operation)
2245{
2246 int err;
2247 int alloc_flag = operation->in_interrupt ? GFP_ATOMIC : GFP_KERNEL;
2248 void *iop_alloc_ptr = NULL;
2249
2250 *pj = kmalloc(sizeof (struct cryptocop_prio_job), alloc_flag);
2251 if (!*pj) return -ENOMEM;
2252
2253 DEBUG(printk("cryptocop_job_setup: operation=0x%p\n", operation));
2254
2255 (*pj)->oper = operation;
2256 DEBUG(printk("cryptocop_job_setup, cb=0x%p cb_data=0x%p\n", (*pj)->oper->cb, (*pj)->oper->cb_data));
2257
2258 if (operation->use_dmalists) {
2259 DEBUG(print_user_dma_lists(&operation->list_op));
2260 if (!operation->list_op.inlist || !operation->list_op.outlist || !operation->list_op.out_data_buf || !operation->list_op.in_data_buf){
2261 DEBUG_API(printk("cryptocop_job_setup: bad indata (use_dmalists)\n"));
2262 kfree(*pj);
2263 return -EINVAL;
2264 }
2265 iop_alloc_ptr = kmalloc(DESCR_ALLOC_PAD + sizeof(struct cryptocop_int_operation), alloc_flag);
2266 if (!iop_alloc_ptr) {
2267 DEBUG_API(printk("cryptocop_job_setup: kmalloc cryptocop_int_operation\n"));
2268 kfree(*pj);
2269 return -ENOMEM;
2270 }
2271 (*pj)->iop = (struct cryptocop_int_operation*)(((unsigned long int)(iop_alloc_ptr + DESCR_ALLOC_PAD + offsetof(struct cryptocop_int_operation, ctx_out)) & ~0x0000001F) - offsetof(struct cryptocop_int_operation, ctx_out));
2272 DEBUG(memset((*pj)->iop, 0xff, sizeof(struct cryptocop_int_operation)));
2273 (*pj)->iop->alloc_ptr = iop_alloc_ptr;
2274 (*pj)->iop->sid = operation->sid;
2275 (*pj)->iop->cdesc_out = NULL;
2276 (*pj)->iop->cdesc_in = NULL;
2277 (*pj)->iop->tdes_mode = operation->list_op.tdes_mode;
2278 (*pj)->iop->csum_mode = operation->list_op.csum_mode;
2279 (*pj)->iop->ddesc_out = operation->list_op.outlist;
2280 (*pj)->iop->ddesc_in = operation->list_op.inlist;
2281
2282 /* Setup DMA contexts. */
2283 (*pj)->iop->ctx_out.next = NULL;
2284 (*pj)->iop->ctx_out.eol = 1;
2285 (*pj)->iop->ctx_out.saved_data = operation->list_op.outlist;
2286 (*pj)->iop->ctx_out.saved_data_buf = operation->list_op.out_data_buf;
2287
2288 (*pj)->iop->ctx_in.next = NULL;
2289 (*pj)->iop->ctx_in.eol = 1;
2290 (*pj)->iop->ctx_in.saved_data = operation->list_op.inlist;
2291 (*pj)->iop->ctx_in.saved_data_buf = operation->list_op.in_data_buf;
2292 } else {
2293 if ((err = cryptocop_setup_dma_list(operation, &(*pj)->iop, alloc_flag))) {
2294 DEBUG_API(printk("cryptocop_job_setup: cryptocop_setup_dma_list failed %d\n", err));
2295 kfree(*pj);
2296 return err;
2297 }
2298 }
2299 DEBUG(print_dma_descriptors((*pj)->iop));
2300
2301 DEBUG(printk("cryptocop_job_setup, DMA list setup successful\n"));
2302
2303 return 0;
2304}
2305
51533b61
MS		 2306static int cryptocop_open(struct inode *inode, struct file *filp)
2307{
32ea086b 2308 int p = iminor(inode);
51533b61
MS		 2309
2310 if (p != CRYPTOCOP_MINOR) return -EINVAL;
2311
2312 filp->private_data = NULL;
2313 return 0;
2314}
2315
2316
2317static int cryptocop_release(struct inode *inode, struct file *filp)
2318{
2319 struct cryptocop_private *dev = filp->private_data;
2320 struct cryptocop_private *dev_next;
2321
2322 while (dev){
2323 dev_next = dev->next;
2324 if (dev->sid != CRYPTOCOP_SESSION_ID_NONE) {
2325 (void)cryptocop_free_session(dev->sid);
2326 }
2327 kfree(dev);
2328 dev = dev_next;
2329 }
2330
2331 return 0;
2332}
2333
2334
2335static int cryptocop_ioctl_close_session(struct inode *inode, struct file *filp,
2336 unsigned int cmd, unsigned long arg)
2337{
2338 struct cryptocop_private *dev = filp->private_data;
2339 struct cryptocop_private *prev_dev = NULL;
2340 struct strcop_session_op *sess_op = (struct strcop_session_op *)arg;
2341 struct strcop_session_op sop;
2342 int err;
2343
2344 DEBUG(printk("cryptocop_ioctl_close_session\n"));
2345
2346 if (!access_ok(VERIFY_READ, sess_op, sizeof(struct strcop_session_op)))
2347 return -EFAULT;
2348 err = copy_from_user(&sop, sess_op, sizeof(struct strcop_session_op));
2349 if (err) return -EFAULT;
2350
2351 while (dev && (dev->sid != sop.ses_id)) {
2352 prev_dev = dev;
2353 dev = dev->next;
2354 }
2355 if (dev){
2356 if (prev_dev){
2357 prev_dev->next = dev->next;
2358 } else {
2359 filp->private_data = dev->next;
2360 }
2361 err = cryptocop_free_session(dev->sid);
2362 if (err) return -EFAULT;
2363 } else {
2364 DEBUG_API(printk("cryptocop_ioctl_close_session: session %lld not found\n", sop.ses_id));
2365 return -EINVAL;
2366 }
2367 return 0;
2368}
2369
2370
2371static void ioctl_process_job_callback(struct cryptocop_operation *op, void*cb_data)
2372{
2373 struct ioctl_job_cb_ctx *jc = (struct ioctl_job_cb_ctx *)cb_data;
2374
2375 DEBUG(printk("ioctl_process_job_callback: op=0x%p, cb_data=0x%p\n", op, cb_data));
2376
2377 jc->processed = 1;
2378 wake_up(&cryptocop_ioc_process_wq);
2379}
2380
2381
2382#define CRYPTOCOP_IOCTL_CIPHER_TID (1)
2383#define CRYPTOCOP_IOCTL_DIGEST_TID (2)
2384#define CRYPTOCOP_IOCTL_CSUM_TID (3)
2385
2386static size_t first_cfg_change_ix(struct strcop_crypto_op *crp_op)
2387{
2388 size_t ch_ix = 0;
2389
2390 if (crp_op->do_cipher) ch_ix = crp_op->cipher_start;
2391 if (crp_op->do_digest && (crp_op->digest_start < ch_ix)) ch_ix = crp_op->digest_start;
2392 if (crp_op->do_csum && (crp_op->csum_start < ch_ix)) ch_ix = crp_op->csum_start;
2393
2394 DEBUG(printk("first_cfg_change_ix: ix=%d\n", ch_ix));
2395 return ch_ix;
2396}
2397
2398
2399static size_t next_cfg_change_ix(struct strcop_crypto_op *crp_op, size_t ix)
2400{
2401 size_t ch_ix = INT_MAX;
2402 size_t tmp_ix = 0;
2403
2404 if (crp_op->do_cipher && ((crp_op->cipher_start + crp_op->cipher_len) > ix)){
2405 if (crp_op->cipher_start > ix) {
2406 ch_ix = crp_op->cipher_start;
2407 } else {
2408 ch_ix = crp_op->cipher_start + crp_op->cipher_len;
2409 }
2410 }
2411 if (crp_op->do_digest && ((crp_op->digest_start + crp_op->digest_len) > ix)){
2412 if (crp_op->digest_start > ix) {
2413 tmp_ix = crp_op->digest_start;
2414 } else {
2415 tmp_ix = crp_op->digest_start + crp_op->digest_len;
2416 }
2417 if (tmp_ix < ch_ix) ch_ix = tmp_ix;
2418 }
2419 if (crp_op->do_csum && ((crp_op->csum_start + crp_op->csum_len) > ix)){
2420 if (crp_op->csum_start > ix) {
2421 tmp_ix = crp_op->csum_start;
2422 } else {
2423 tmp_ix = crp_op->csum_start + crp_op->csum_len;
2424 }
2425 if (tmp_ix < ch_ix) ch_ix = tmp_ix;
2426 }
2427 if (ch_ix == INT_MAX) ch_ix = ix;
2428 DEBUG(printk("next_cfg_change_ix prev ix=%d, next ix=%d\n", ix, ch_ix));
2429 return ch_ix;
2430}
2431
2432
2433/* Map map_length bytes from the pages starting on *pageix and *pageoffset to iovecs starting on *iovix.
2434 * Return -1 for ok, 0 for fail. */
2435static int map_pages_to_iovec(struct iovec *iov, int iovlen, int *iovix, struct page **pages, int nopages, int *pageix, int *pageoffset, int map_length )
2436{
2437 int tmplen;
2438
2439 assert(iov != NULL);
2440 assert(iovix != NULL);
2441 assert(pages != NULL);
2442 assert(pageix != NULL);
2443 assert(pageoffset != NULL);
2444
2445 DEBUG(printk("map_pages_to_iovec, map_length=%d, iovlen=%d, *iovix=%d, nopages=%d, *pageix=%d, *pageoffset=%d\n", map_length, iovlen, *iovix, nopages, *pageix, *pageoffset));
2446
2447 while (map_length > 0){
2448 DEBUG(printk("map_pages_to_iovec, map_length=%d, iovlen=%d, *iovix=%d, nopages=%d, *pageix=%d, *pageoffset=%d\n", map_length, iovlen, *iovix, nopages, *pageix, *pageoffset));
2449 if (*iovix >= iovlen){
2450 DEBUG_API(printk("map_page_to_iovec: *iovix=%d >= iovlen=%d\n", *iovix, iovlen));
2451 return 0;
2452 }
2453 if (*pageix >= nopages){
2454 DEBUG_API(printk("map_page_to_iovec: *pageix=%d >= nopages=%d\n", *pageix, nopages));
2455 return 0;
2456 }
2457 iov[*iovix].iov_base = (unsigned char*)page_address(pages[*pageix]) + *pageoffset;
2458 tmplen = PAGE_SIZE - *pageoffset;
2459 if (tmplen < map_length){
2460 (*pageoffset) = 0;
2461 (*pageix)++;
2462 } else {
2463 tmplen = map_length;
2464 (*pageoffset) += map_length;
2465 }
2466 DEBUG(printk("mapping %d bytes from page %d (or %d) to iovec %d\n", tmplen, *pageix, *pageix-1, *iovix));
2467 iov[*iovix].iov_len = tmplen;
2468 map_length -= tmplen;
2469 (*iovix)++;
2470 }
2471 DEBUG(printk("map_page_to_iovec, exit, *iovix=%d\n", *iovix));
2472 return -1;
2473}
2474
2475
2476
2477static int cryptocop_ioctl_process(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
2478{
2479 int i;
2480 struct cryptocop_private *dev = filp->private_data;
2481 struct strcop_crypto_op *crp_oper = (struct strcop_crypto_op *)arg;
2482 struct strcop_crypto_op oper = {0};
2483 int err = 0;
2484 struct cryptocop_operation *cop = NULL;
2485
2486 struct ioctl_job_cb_ctx *jc = NULL;
2487
2488 struct page **inpages = NULL;
2489 struct page **outpages = NULL;
2490 int noinpages = 0;
2491 int nooutpages = 0;
2492
2493 struct cryptocop_desc descs[5]; /* Max 5 descriptors are needed, there are three transforms that
2494 * can get connected/disconnected on different places in the indata. */
2495 struct cryptocop_desc_cfg dcfgs[5*3];
2496 int desc_ix = 0;
2497 int dcfg_ix = 0;
2498 struct cryptocop_tfrm_cfg ciph_tcfg = {0};
2499 struct cryptocop_tfrm_cfg digest_tcfg = {0};
2500 struct cryptocop_tfrm_cfg csum_tcfg = {0};
2501
2502 unsigned char *digest_result = NULL;
2503 int digest_length = 0;
2504 int cblocklen = 0;
2505 unsigned char csum_result[CSUM_BLOCK_LENGTH];
2506 struct cryptocop_session *sess;
2507
2508 int iovlen = 0;
2509 int iovix = 0;
2510 int pageix = 0;
2511 int pageoffset = 0;
2512
2513 size_t prev_ix = 0;
2514 size_t next_ix;
2515
2516 int cipher_active, digest_active, csum_active;
2517 int end_digest, end_csum;
2518 int digest_done = 0;
2519 int cipher_done = 0;
2520 int csum_done = 0;
2521
2522 DEBUG(printk("cryptocop_ioctl_process\n"));
2523
2524 if (!access_ok(VERIFY_WRITE, crp_oper, sizeof(struct strcop_crypto_op))){
2525 DEBUG_API(printk("cryptocop_ioctl_process: !access_ok crp_oper!\n"));
2526 return -EFAULT;
2527 }
2528 if (copy_from_user(&oper, crp_oper, sizeof(struct strcop_crypto_op))) {
2529 DEBUG_API(printk("cryptocop_ioctl_process: copy_from_user\n"));
2530 return -EFAULT;
2531 }
2532 DEBUG(print_strcop_crypto_op(&oper));
2533
2534 while (dev && dev->sid != oper.ses_id) dev = dev->next;
2535 if (!dev){
2536 DEBUG_API(printk("cryptocop_ioctl_process: session %lld not found\n", oper.ses_id));
2537 return -EINVAL;
2538 }
2539
2540 /* Check buffers. */
2541 if (((oper.indata + oper.inlen) < oper.indata) || ((oper.cipher_outdata + oper.cipher_outlen) < oper.cipher_outdata)){
2542 DEBUG_API(printk("cryptocop_ioctl_process: user buffers wrapped around, bad user!\n"));
2543 return -EINVAL;
2544 }
2545
2546 if (!access_ok(VERIFY_WRITE, oper.cipher_outdata, oper.cipher_outlen)){
2547 DEBUG_API(printk("cryptocop_ioctl_process: !access_ok out data!\n"));
2548 return -EFAULT;
2549 }
2550 if (!access_ok(VERIFY_READ, oper.indata, oper.inlen)){
2551 DEBUG_API(printk("cryptocop_ioctl_process: !access_ok in data!\n"));
2552 return -EFAULT;
2553 }
2554
2555 cop = kmalloc(sizeof(struct cryptocop_operation), GFP_KERNEL);
2556 if (!cop) {
2557 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc\n"));
2558 return -ENOMEM;
2559 }
2560 jc = kmalloc(sizeof(struct ioctl_job_cb_ctx), GFP_KERNEL);
2561 if (!jc) {
2562 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc\n"));
2563 err = -ENOMEM;
2564 goto error_cleanup;
2565 }
2566 jc->processed = 0;
2567
2568 cop->cb_data = jc;
2569 cop->cb = ioctl_process_job_callback;
2570 cop->operation_status = 0;
2571 cop->use_dmalists = 0;
2572 cop->in_interrupt = 0;
2573 cop->fast_callback = 0;
2574 cop->tfrm_op.tfrm_cfg = NULL;
2575 cop->tfrm_op.desc = NULL;
2576 cop->tfrm_op.indata = NULL;
2577 cop->tfrm_op.incount = 0;
2578 cop->tfrm_op.inlen = 0;
2579 cop->tfrm_op.outdata = NULL;
2580 cop->tfrm_op.outcount = 0;
2581 cop->tfrm_op.outlen = 0;
2582
2583 sess = get_session(oper.ses_id);
2584 if (!sess){
2585 DEBUG_API(printk("cryptocop_ioctl_process: bad session id.\n"));
2586 kfree(cop);
2587 kfree(jc);
2588 return -EINVAL;
2589 }
2590
2591 if (oper.do_cipher) {
2592 unsigned int cipher_outlen = 0;
2593 struct cryptocop_transform_ctx *tc = get_transform_ctx(sess, CRYPTOCOP_IOCTL_CIPHER_TID);
2594 if (!tc) {
2595 DEBUG_API(printk("cryptocop_ioctl_process: no cipher transform in session.\n"));
2596 err = -EINVAL;
2597 goto error_cleanup;
2598 }
2599 ciph_tcfg.tid = CRYPTOCOP_IOCTL_CIPHER_TID;
2600 ciph_tcfg.inject_ix = 0;
2601 ciph_tcfg.flags = 0;
2602 if ((oper.cipher_start < 0) || (oper.cipher_len <= 0) || (oper.cipher_start > oper.inlen) || ((oper.cipher_start + oper.cipher_len) > oper.inlen)){
2603 DEBUG_API(printk("cryptocop_ioctl_process: bad cipher length\n"));
2604 kfree(cop);
2605 kfree(jc);
2606 return -EINVAL;
2607 }
2608 cblocklen = tc->init.alg == cryptocop_alg_aes ? AES_BLOCK_LENGTH : DES_BLOCK_LENGTH;
2609 if (oper.cipher_len % cblocklen) {
2610 kfree(cop);
2611 kfree(jc);
2612 DEBUG_API(printk("cryptocop_ioctl_process: cipher inlength not multiple of block length.\n"));
2613 return -EINVAL;
2614 }
2615 cipher_outlen = oper.cipher_len;
2616 if (tc->init.cipher_mode == cryptocop_cipher_mode_cbc){
2617 if (oper.cipher_explicit) {
2618 ciph_tcfg.flags |= CRYPTOCOP_EXPLICIT_IV;
2619 memcpy(ciph_tcfg.iv, oper.cipher_iv, cblocklen);
2620 } else {
2621 cipher_outlen = oper.cipher_len - cblocklen;
2622 }
2623 } else {
2624 if (oper.cipher_explicit){
2625 kfree(cop);
2626 kfree(jc);
2627 DEBUG_API(printk("cryptocop_ioctl_process: explicit_iv when not CBC mode\n"));
2628 return -EINVAL;
2629 }
2630 }
2631 if (oper.cipher_outlen != cipher_outlen) {
2632 kfree(cop);
2633 kfree(jc);
2634 DEBUG_API(printk("cryptocop_ioctl_process: cipher_outlen incorrect, should be %d not %d.\n", cipher_outlen, oper.cipher_outlen));
2635 return -EINVAL;
2636 }
2637
2638 if (oper.decrypt){
2639 ciph_tcfg.flags |= CRYPTOCOP_DECRYPT;
2640 } else {
2641 ciph_tcfg.flags |= CRYPTOCOP_ENCRYPT;
2642 }
2643 ciph_tcfg.next = cop->tfrm_op.tfrm_cfg;
2644 cop->tfrm_op.tfrm_cfg = &ciph_tcfg;
2645 }
2646 if (oper.do_digest){
2647 struct cryptocop_transform_ctx *tc = get_transform_ctx(sess, CRYPTOCOP_IOCTL_DIGEST_TID);
2648 if (!tc) {
2649 DEBUG_API(printk("cryptocop_ioctl_process: no digest transform in session.\n"));
2650 err = -EINVAL;
2651 goto error_cleanup;
2652 }
2653 digest_length = tc->init.alg == cryptocop_alg_md5 ? 16 : 20;
2654 digest_result = kmalloc(digest_length, GFP_KERNEL);
2655 if (!digest_result) {
2656 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc digest_result\n"));
2657 err = -EINVAL;
2658 goto error_cleanup;
2659 }
2660 DEBUG(memset(digest_result, 0xff, digest_length));
2661
2662 digest_tcfg.tid = CRYPTOCOP_IOCTL_DIGEST_TID;
2663 digest_tcfg.inject_ix = 0;
2664 ciph_tcfg.inject_ix += digest_length;
2665 if ((oper.digest_start < 0) || (oper.digest_len <= 0) || (oper.digest_start > oper.inlen) || ((oper.digest_start + oper.digest_len) > oper.inlen)){
2666 DEBUG_API(printk("cryptocop_ioctl_process: bad digest length\n"));
2667 err = -EINVAL;
2668 goto error_cleanup;
2669 }
2670
2671 digest_tcfg.next = cop->tfrm_op.tfrm_cfg;
2672 cop->tfrm_op.tfrm_cfg = &digest_tcfg;
2673 }
2674 if (oper.do_csum){
2675 csum_tcfg.tid = CRYPTOCOP_IOCTL_CSUM_TID;
2676 csum_tcfg.inject_ix = digest_length;
2677 ciph_tcfg.inject_ix += 2;
2678
2679 if ((oper.csum_start < 0) || (oper.csum_len <= 0) || (oper.csum_start > oper.inlen) || ((oper.csum_start + oper.csum_len) > oper.inlen)){
2680 DEBUG_API(printk("cryptocop_ioctl_process: bad csum length\n"));
2681 kfree(cop);
2682 kfree(jc);
2683 return -EINVAL;
2684 }
2685
2686 csum_tcfg.next = cop->tfrm_op.tfrm_cfg;
2687 cop->tfrm_op.tfrm_cfg = &csum_tcfg;
2688 }
2689
2690 prev_ix = first_cfg_change_ix(&oper);
2691 if (prev_ix > oper.inlen) {
2692 DEBUG_API(printk("cryptocop_ioctl_process: length mismatch\n"));
2693 nooutpages = noinpages = 0;
2694 err = -EINVAL;
2695 goto error_cleanup;
2696 }
2697 DEBUG(printk("cryptocop_ioctl_process: inlen=%d, cipher_outlen=%d\n", oper.inlen, oper.cipher_outlen));
2698
2699 /* Map user pages for in and out data of the operation. */
2700 noinpages = (((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK) + oper.inlen - 1 - prev_ix + ~PAGE_MASK) >> PAGE_SHIFT;
2701 DEBUG(printk("cryptocop_ioctl_process: noinpages=%d\n", noinpages));
2702 inpages = kmalloc(noinpages * sizeof(struct page*), GFP_KERNEL);
2703 if (!inpages){
2704 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc inpages\n"));
2705 nooutpages = noinpages = 0;
2706 err = -ENOMEM;
2707 goto error_cleanup;
2708 }
2709 if (oper.do_cipher){
2710 nooutpages = (((unsigned long int)oper.cipher_outdata & ~PAGE_MASK) + oper.cipher_outlen - 1 + ~PAGE_MASK) >> PAGE_SHIFT;
2711 DEBUG(printk("cryptocop_ioctl_process: nooutpages=%d\n", nooutpages));
2712 outpages = kmalloc(nooutpages * sizeof(struct page*), GFP_KERNEL);
2713 if (!outpages){
2714 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc outpages\n"));
2715 nooutpages = noinpages = 0;
2716 err = -ENOMEM;
2717 goto error_cleanup;
2718 }
2719 }
2720
2721 /* Acquire the mm page semaphore. */
2722 down_read(&current->mm->mmap_sem);
2723
2724 err = get_user_pages(current,
2725 current->mm,
2726 (unsigned long int)(oper.indata + prev_ix),
2727 noinpages,
2728 0, /* read access only for in data */
2729 0, /* no force */
2730 inpages,
2731 NULL);
2732
2733 if (err < 0) {
2734 up_read(&current->mm->mmap_sem);
2735 nooutpages = noinpages = 0;
2736 DEBUG_API(printk("cryptocop_ioctl_process: get_user_pages indata\n"));
2737 goto error_cleanup;
2738 }
2739 noinpages = err;
2740 if (oper.do_cipher){
2741 err = get_user_pages(current,
2742 current->mm,
2743 (unsigned long int)oper.cipher_outdata,
2744 nooutpages,
2745 1, /* write access for out data */
2746 0, /* no force */
2747 outpages,
2748 NULL);
2749 up_read(&current->mm->mmap_sem);
2750 if (err < 0) {
2751 nooutpages = 0;
2752 DEBUG_API(printk("cryptocop_ioctl_process: get_user_pages outdata\n"));
2753 goto error_cleanup;
2754 }
2755 nooutpages = err;
2756 } else {
2757 up_read(&current->mm->mmap_sem);
2758 }
2759
2760 /* Add 6 to nooutpages to make room for possibly inserted buffers for storing digest and
2761 * csum output and splits when units are (dis-)connected. */
2762 cop->tfrm_op.indata = kmalloc((noinpages) * sizeof(struct iovec), GFP_KERNEL);
2763 cop->tfrm_op.outdata = kmalloc((6 + nooutpages) * sizeof(struct iovec), GFP_KERNEL);
2764 if (!cop->tfrm_op.indata || !cop->tfrm_op.outdata) {
2765 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc iovecs\n"));
2766 err = -ENOMEM;
2767 goto error_cleanup;
2768 }
2769
2770 cop->tfrm_op.inlen = oper.inlen - prev_ix;
2771 cop->tfrm_op.outlen = 0;
2772 if (oper.do_cipher) cop->tfrm_op.outlen += oper.cipher_outlen;
2773 if (oper.do_digest) cop->tfrm_op.outlen += digest_length;
2774 if (oper.do_csum) cop->tfrm_op.outlen += 2;
2775
2776 /* Setup the in iovecs. */
2777 cop->tfrm_op.incount = noinpages;
2778 if (noinpages > 1){
2779 size_t tmplen = cop->tfrm_op.inlen;
2780
2781 cop->tfrm_op.indata[0].iov_len = PAGE_SIZE - ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2782 cop->tfrm_op.indata[0].iov_base = (unsigned char*)page_address(inpages[0]) + ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2783 tmplen -= cop->tfrm_op.indata[0].iov_len;
2784 for (i = 1; i<noinpages; i++){
2785 cop->tfrm_op.indata[i].iov_len = tmplen < PAGE_SIZE ? tmplen : PAGE_SIZE;
2786 cop->tfrm_op.indata[i].iov_base = (unsigned char*)page_address(inpages[i]);
2787 tmplen -= PAGE_SIZE;
2788 }
2789 } else {
2790 cop->tfrm_op.indata[0].iov_len = oper.inlen - prev_ix;
2791 cop->tfrm_op.indata[0].iov_base = (unsigned char*)page_address(inpages[0]) + ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2792 }
2793
2794 iovlen = nooutpages + 6;
2795 pageoffset = oper.do_cipher ? ((unsigned long int)oper.cipher_outdata & ~PAGE_MASK) : 0;
2796
2797 next_ix = next_cfg_change_ix(&oper, prev_ix);
2798 if (prev_ix == next_ix){
2799 DEBUG_API(printk("cryptocop_ioctl_process: length configuration broken.\n"));
2800 err = -EINVAL; /* This should be impossible barring bugs. */
2801 goto error_cleanup;
2802 }
2803 while (prev_ix != next_ix){
2804 end_digest = end_csum = cipher_active = digest_active = csum_active = 0;
2805 descs[desc_ix].cfg = NULL;
2806 descs[desc_ix].length = next_ix - prev_ix;
2807
2808 if (oper.do_cipher && (oper.cipher_start < next_ix) && (prev_ix < (oper.cipher_start + oper.cipher_len))) {
2809 dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_CIPHER_TID;
2810 dcfgs[dcfg_ix].src = cryptocop_source_dma;
2811 cipher_active = 1;
2812
2813 if (next_ix == (oper.cipher_start + oper.cipher_len)){
2814 cipher_done = 1;
2815 dcfgs[dcfg_ix].last = 1;
2816 } else {
2817 dcfgs[dcfg_ix].last = 0;
2818 }
2819 dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2820 descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2821 ++dcfg_ix;
2822 }
2823 if (oper.do_digest && (oper.digest_start < next_ix) && (prev_ix < (oper.digest_start + oper.digest_len))) {
2824 digest_active = 1;
2825 dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_DIGEST_TID;
2826 dcfgs[dcfg_ix].src = cryptocop_source_dma;
2827 if (next_ix == (oper.digest_start + oper.digest_len)){
2828 assert(!digest_done);
2829 digest_done = 1;
2830 dcfgs[dcfg_ix].last = 1;
2831 } else {
2832 dcfgs[dcfg_ix].last = 0;
2833 }
2834 dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2835 descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2836 ++dcfg_ix;
2837 }
2838 if (oper.do_csum && (oper.csum_start < next_ix) && (prev_ix < (oper.csum_start + oper.csum_len))){
2839 csum_active = 1;
2840 dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_CSUM_TID;
2841 dcfgs[dcfg_ix].src = cryptocop_source_dma;
2842 if (next_ix == (oper.csum_start + oper.csum_len)){
2843 csum_done = 1;
2844 dcfgs[dcfg_ix].last = 1;
2845 } else {
2846 dcfgs[dcfg_ix].last = 0;
2847 }
2848 dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2849 descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2850 ++dcfg_ix;
2851 }
2852 if (!descs[desc_ix].cfg){
2853 DEBUG_API(printk("cryptocop_ioctl_process: data segment %d (%d to %d) had no active transforms\n", desc_ix, prev_ix, next_ix));
2854 err = -EINVAL;
2855 goto error_cleanup;
2856 }
2857 descs[desc_ix].next = &(descs[desc_ix]) + 1;
2858 ++desc_ix;
2859 prev_ix = next_ix;
2860 next_ix = next_cfg_change_ix(&oper, prev_ix);
2861 }
2862 if (desc_ix > 0){
2863 descs[desc_ix-1].next = NULL;
2864 } else {
2865 descs[0].next = NULL;
2866 }
2867 if (oper.do_digest) {
2868 DEBUG(printk("cryptocop_ioctl_process: mapping %d byte digest output to iovec %d\n", digest_length, iovix));
2869 /* Add outdata iovec, length == <length of type of digest> */
2870 cop->tfrm_op.outdata[iovix].iov_base = digest_result;
2871 cop->tfrm_op.outdata[iovix].iov_len = digest_length;
2872 ++iovix;
2873 }
2874 if (oper.do_csum) {
2875 /* Add outdata iovec, length == 2, the length of csum. */
2876 DEBUG(printk("cryptocop_ioctl_process: mapping 2 byte csum output to iovec %d\n", iovix));
2877 /* Add outdata iovec, length == <length of type of digest> */
2878 cop->tfrm_op.outdata[iovix].iov_base = csum_result;
2879 cop->tfrm_op.outdata[iovix].iov_len = 2;
2880 ++iovix;
2881 }
2882 if (oper.do_cipher) {
2883 if (!map_pages_to_iovec(cop->tfrm_op.outdata, iovlen, &iovix, outpages, nooutpages, &pageix, &pageoffset, oper.cipher_outlen)){
2884 DEBUG_API(printk("cryptocop_ioctl_process: failed to map pages to iovec.\n"));
2885 err = -ENOSYS; /* This should be impossible barring bugs. */
2886 goto error_cleanup;
2887 }
2888 }
2889 DEBUG(printk("cryptocop_ioctl_process: setting cop->tfrm_op.outcount %d\n", iovix));
2890 cop->tfrm_op.outcount = iovix;
2891 assert(iovix <= (nooutpages + 6));
2892
2893 cop->sid = oper.ses_id;
2894 cop->tfrm_op.desc = &descs[0];
2895
2896 DEBUG(printk("cryptocop_ioctl_process: inserting job, cb_data=0x%p\n", cop->cb_data));
2897
2898 if ((err = cryptocop_job_queue_insert_user_job(cop)) != 0) {
2899 DEBUG_API(printk("cryptocop_ioctl_process: insert job %d\n", err));
2900 err = -EINVAL;
2901 goto error_cleanup;
2902 }
2903
2904 DEBUG(printk("cryptocop_ioctl_process: begin wait for result\n"));
2905
2906 wait_event(cryptocop_ioc_process_wq, (jc->processed != 0));
2907 DEBUG(printk("cryptocop_ioctl_process: end wait for result\n"));
2908 if (!jc->processed){
2909 printk(KERN_WARNING "cryptocop_ioctl_process: job not processed at completion\n");
2910 err = -EIO;
2911 goto error_cleanup;
2912 }
2913
2914 /* Job process done. Cipher output should already be correct in job so no post processing of outdata. */
2915 DEBUG(printk("cryptocop_ioctl_process: operation_status = %d\n", cop->operation_status));
2916 if (cop->operation_status == 0){
2917 if (oper.do_digest){
2918 DEBUG(printk("cryptocop_ioctl_process: copy %d bytes digest to user\n", digest_length));
2919 err = copy_to_user((unsigned char*)crp_oper + offsetof(struct strcop_crypto_op, digest), digest_result, digest_length);
2920 if (0 != err){
2921 DEBUG_API(printk("cryptocop_ioctl_process: copy_to_user, digest length %d, err %d\n", digest_length, err));
2922 err = -EFAULT;
2923 goto error_cleanup;
2924 }
2925 }
2926 if (oper.do_csum){
2927 DEBUG(printk("cryptocop_ioctl_process: copy 2 bytes checksum to user\n"));
2928 err = copy_to_user((unsigned char*)crp_oper + offsetof(struct strcop_crypto_op, csum), csum_result, 2);
2929 if (0 != err){
2930 DEBUG_API(printk("cryptocop_ioctl_process: copy_to_user, csum, err %d\n", err));
2931 err = -EFAULT;
2932 goto error_cleanup;
2933 }
2934 }
2935 err = 0;
2936 } else {
2937 DEBUG(printk("cryptocop_ioctl_process: returning err = operation_status = %d\n", cop->operation_status));
2938 err = cop->operation_status;
2939 }
2940
2941 error_cleanup:
2942 /* Release page caches. */
2943 for (i = 0; i < noinpages; i++){
2944 put_page(inpages[i]);
2945 }
2946 for (i = 0; i < nooutpages; i++){
2947 int spdl_err;
2948 /* Mark output pages dirty. */
2949 spdl_err = set_page_dirty_lock(outpages[i]);
4741c9fd 2950 DEBUG(if (spdl_err < 0)printk("cryptocop_ioctl_process: set_page_dirty_lock returned %d\n", spdl_err));
51533b61
MS		 2951 }
2952 for (i = 0; i < nooutpages; i++){
2953 put_page(outpages[i]);
2954 }
2955
b2325fe1
JJ		 2956 kfree(digest_result);
2957 kfree(inpages);
2958 kfree(outpages);
51533b61 2959 if (cop){
b2325fe1
JJ		 2960 kfree(cop->tfrm_op.indata);
2961 kfree(cop->tfrm_op.outdata);
51533b61
MS		 2962 kfree(cop);
2963 }
b2325fe1 2964 kfree(jc);
51533b61
MS		 2965
2966 DEBUG(print_lock_status());
2967
2968 return err;
2969}
2970
2971
2972static int cryptocop_ioctl_create_session(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
2973{
2974 cryptocop_session_id sid;
2975 int err;
2976 struct cryptocop_private *dev;
2977 struct strcop_session_op *sess_op = (struct strcop_session_op *)arg;
2978 struct strcop_session_op sop;
2979 struct cryptocop_transform_init *tis = NULL;
2980 struct cryptocop_transform_init ti_cipher = {0};
2981 struct cryptocop_transform_init ti_digest = {0};
2982 struct cryptocop_transform_init ti_csum = {0};
2983
2984 if (!access_ok(VERIFY_WRITE, sess_op, sizeof(struct strcop_session_op)))
2985 return -EFAULT;
2986 err = copy_from_user(&sop, sess_op, sizeof(struct strcop_session_op));
2987 if (err) return -EFAULT;
2988 if (sop.cipher != cryptocop_cipher_none) {
2989 if (!access_ok(VERIFY_READ, sop.key, sop.keylen)) return -EFAULT;
2990 }
2991 DEBUG(printk("cryptocop_ioctl_create_session, sess_op:\n"));
2992
2993 DEBUG(printk("\tcipher:%d\n"
2994 "\tcipher_mode:%d\n"
2995 "\tdigest:%d\n"
2996 "\tcsum:%d\n",
2997 (int)sop.cipher,
2998 (int)sop.cmode,
2999 (int)sop.digest,
3000 (int)sop.csum));
3001
3002 if (sop.cipher != cryptocop_cipher_none){
3003 /* Init the cipher. */
3004 switch (sop.cipher){
3005 case cryptocop_cipher_des:
3006 ti_cipher.alg = cryptocop_alg_des;
3007 break;
3008 case cryptocop_cipher_3des:
3009 ti_cipher.alg = cryptocop_alg_3des;
3010 break;
3011 case cryptocop_cipher_aes:
3012 ti_cipher.alg = cryptocop_alg_aes;
3013 break;
3014 default:
3015 DEBUG_API(printk("create session, bad cipher algorithm %d\n", sop.cipher));
3016 return -EINVAL;
3017 };
3018 DEBUG(printk("setting cipher transform %d\n", ti_cipher.alg));
3019 copy_from_user(ti_cipher.key, sop.key, sop.keylen/8);
3020 ti_cipher.keylen = sop.keylen;
3021 switch (sop.cmode){
3022 case cryptocop_cipher_mode_cbc:
3023 case cryptocop_cipher_mode_ecb:
3024 ti_cipher.cipher_mode = sop.cmode;
3025 break;
3026 default:
3027 DEBUG_API(printk("create session, bad cipher mode %d\n", sop.cmode));
3028 return -EINVAL;
3029 }
3030 DEBUG(printk("cryptocop_ioctl_create_session: setting CBC mode %d\n", ti_cipher.cipher_mode));
3031 switch (sop.des3_mode){
3032 case cryptocop_3des_eee:
3033 case cryptocop_3des_eed:
3034 case cryptocop_3des_ede:
3035 case cryptocop_3des_edd:
3036 case cryptocop_3des_dee:
3037 case cryptocop_3des_ded:
3038 case cryptocop_3des_dde:
3039 case cryptocop_3des_ddd:
3040 ti_cipher.tdes_mode = sop.des3_mode;
3041 break;
3042 default:
3043 DEBUG_API(printk("create session, bad 3DES mode %d\n", sop.des3_mode));
3044 return -EINVAL;
3045 }
3046 ti_cipher.tid = CRYPTOCOP_IOCTL_CIPHER_TID;
3047 ti_cipher.next = tis;
3048 tis = &ti_cipher;
3049 } /* if (sop.cipher != cryptocop_cipher_none) */
3050 if (sop.digest != cryptocop_digest_none){
3051 DEBUG(printk("setting digest transform\n"));
3052 switch (sop.digest){
3053 case cryptocop_digest_md5:
3054 ti_digest.alg = cryptocop_alg_md5;
3055 break;
3056 case cryptocop_digest_sha1:
3057 ti_digest.alg = cryptocop_alg_sha1;
3058 break;
3059 default:
3060 DEBUG_API(printk("create session, bad digest algorithm %d\n", sop.digest));
3061 return -EINVAL;
3062 }
3063 ti_digest.tid = CRYPTOCOP_IOCTL_DIGEST_TID;
3064 ti_digest.next = tis;
3065 tis = &ti_digest;
3066 } /* if (sop.digest != cryptocop_digest_none) */
3067 if (sop.csum != cryptocop_csum_none){
3068 DEBUG(printk("setting csum transform\n"));
3069 switch (sop.csum){
3070 case cryptocop_csum_le:
3071 case cryptocop_csum_be:
3072 ti_csum.csum_mode = sop.csum;
3073 break;
3074 default:
3075 DEBUG_API(printk("create session, bad checksum algorithm %d\n", sop.csum));
3076 return -EINVAL;
3077 }
3078 ti_csum.alg = cryptocop_alg_csum;
3079 ti_csum.tid = CRYPTOCOP_IOCTL_CSUM_TID;
3080 ti_csum.next = tis;
3081 tis = &ti_csum;
3082 } /* (sop.csum != cryptocop_csum_none) */
3083 dev = kmalloc(sizeof(struct cryptocop_private), GFP_KERNEL);
3084 if (!dev){
3085 DEBUG_API(printk("create session, alloc dev\n"));
3086 return -ENOMEM;
3087 }
3088
3089 err = cryptocop_new_session(&sid, tis, GFP_KERNEL);
3090 DEBUG({ if (err) printk("create session, cryptocop_new_session %d\n", err);});
3091
3092 if (err) {
3093 kfree(dev);
3094 return err;
3095 }
3096 sess_op->ses_id = sid;
3097 dev->sid = sid;
3098 dev->next = filp->private_data;
3099 filp->private_data = dev;
3100
3101 return 0;
3102}
3103
90276a1a
JN		 3104static long cryptocop_ioctl_unlocked(struct inode *inode,
3105 struct file *filp, unsigned int cmd, unsigned long arg)
51533b61
MS		 3106{
3107 int err = 0;
3108 if (_IOC_TYPE(cmd) != ETRAXCRYPTOCOP_IOCTYPE) {
3109 DEBUG_API(printk("cryptocop_ioctl: wrong type\n"));
3110 return -ENOTTY;
3111 }
3112 if (_IOC_NR(cmd) > CRYPTOCOP_IO_MAXNR){
3113 return -ENOTTY;
3114 }
3115 /* Access check of the argument. Some commands, e.g. create session and process op,
3116 needs additional checks. Those are handled in the command handling functions. */
3117 if (_IOC_DIR(cmd) & _IOC_READ)
3118 err = !access_ok(VERIFY_WRITE, (void *)arg, _IOC_SIZE(cmd));
3119 else if (_IOC_DIR(cmd) & _IOC_WRITE)
3120 err = !access_ok(VERIFY_READ, (void *)arg, _IOC_SIZE(cmd));
3121 if (err) return -EFAULT;
3122
3123 switch (cmd) {
3124 case CRYPTOCOP_IO_CREATE_SESSION:
3125 return cryptocop_ioctl_create_session(inode, filp, cmd, arg);
3126 case CRYPTOCOP_IO_CLOSE_SESSION:
3127 return cryptocop_ioctl_close_session(inode, filp, cmd, arg);
3128 case CRYPTOCOP_IO_PROCESS_OP:
3129 return cryptocop_ioctl_process(inode, filp, cmd, arg);
3130 default:
3131 DEBUG_API(printk("cryptocop_ioctl: unknown command\n"));
3132 return -ENOTTY;
3133 }
3134 return 0;
3135}
3136
90276a1a
JN		 3137static long
3138cryptocop_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
3139{
3140 struct inode *inode = file->f_path.dentry->d_inode;
3141 long ret;
3142
3143 lock_kernel();
3144 ret = cryptocop_ioctl_unlocked(inode, filp, cmd, arg);
3145 unlock_kernel();
3146
3147 return ret;
3148}
3149
51533b61
MS		 3150
3151#ifdef LDEBUG
3152static void print_dma_descriptors(struct cryptocop_int_operation *iop)
3153{
3154 struct cryptocop_dma_desc *cdesc_out = iop->cdesc_out;
3155 struct cryptocop_dma_desc *cdesc_in = iop->cdesc_in;
3156 int i;
3157
3158 printk("print_dma_descriptors start\n");
3159
3160 printk("iop:\n");
3161 printk("\tsid: 0x%lld\n", iop->sid);
3162
3163 printk("\tcdesc_out: 0x%p\n", iop->cdesc_out);
3164 printk("\tcdesc_in: 0x%p\n", iop->cdesc_in);
3165 printk("\tddesc_out: 0x%p\n", iop->ddesc_out);
3166 printk("\tddesc_in: 0x%p\n", iop->ddesc_in);
3167
3168 printk("\niop->ctx_out: 0x%p phys: 0x%p\n", &iop->ctx_out, (char*)virt_to_phys(&iop->ctx_out));
3169 printk("\tnext: 0x%p\n"
3170 "\tsaved_data: 0x%p\n"
3171 "\tsaved_data_buf: 0x%p\n",
3172 iop->ctx_out.next,
3173 iop->ctx_out.saved_data,
3174 iop->ctx_out.saved_data_buf);
3175
3176 printk("\niop->ctx_in: 0x%p phys: 0x%p\n", &iop->ctx_in, (char*)virt_to_phys(&iop->ctx_in));
3177 printk("\tnext: 0x%p\n"
3178 "\tsaved_data: 0x%p\n"
3179 "\tsaved_data_buf: 0x%p\n",
3180 iop->ctx_in.next,
3181 iop->ctx_in.saved_data,
3182 iop->ctx_in.saved_data_buf);
3183
3184 i = 0;
3185 while (cdesc_out) {
3186 dma_descr_data *td;
3187 printk("cdesc_out %d, desc=0x%p\n", i, cdesc_out->dma_descr);
3188 printk("\n\tvirt_to_phys(desc): 0x%p\n", (char*)virt_to_phys(cdesc_out->dma_descr));
3189 td = cdesc_out->dma_descr;
3190 printk("\n\tbuf: 0x%p\n"
3191 "\tafter: 0x%p\n"
3192 "\tmd: 0x%04x\n"
3193 "\tnext: 0x%p\n",
3194 td->buf,
3195 td->after,
3196 td->md,
3197 td->next);
3198 printk("flags:\n"
3199 "\twait:\t%d\n"
3200 "\teol:\t%d\n"
3201 "\touteop:\t%d\n"
3202 "\tineop:\t%d\n"
3203 "\tintr:\t%d\n",
3204 td->wait,
3205 td->eol,
3206 td->out_eop,
3207 td->in_eop,
3208 td->intr);
3209 cdesc_out = cdesc_out->next;
3210 i++;
3211 }
3212 i = 0;
3213 while (cdesc_in) {
3214 dma_descr_data *td;
3215 printk("cdesc_in %d, desc=0x%p\n", i, cdesc_in->dma_descr);
3216 printk("\n\tvirt_to_phys(desc): 0x%p\n", (char*)virt_to_phys(cdesc_in->dma_descr));
3217 td = cdesc_in->dma_descr;
3218 printk("\n\tbuf: 0x%p\n"
3219 "\tafter: 0x%p\n"
3220 "\tmd: 0x%04x\n"
3221 "\tnext: 0x%p\n",
3222 td->buf,
3223 td->after,
3224 td->md,
3225 td->next);
3226 printk("flags:\n"
3227 "\twait:\t%d\n"
3228 "\teol:\t%d\n"
3229 "\touteop:\t%d\n"
3230 "\tineop:\t%d\n"
3231 "\tintr:\t%d\n",
3232 td->wait,
3233 td->eol,
3234 td->out_eop,
3235 td->in_eop,
3236 td->intr);
3237 cdesc_in = cdesc_in->next;
3238 i++;
3239 }
3240
3241 printk("print_dma_descriptors end\n");
3242}
3243
3244
3245static void print_strcop_crypto_op(struct strcop_crypto_op *cop)
3246{
3247 printk("print_strcop_crypto_op, 0x%p\n", cop);
3248
3249 /* Indata. */
3250 printk("indata=0x%p\n"
3251 "inlen=%d\n"
3252 "do_cipher=%d\n"
3253 "decrypt=%d\n"
3254 "cipher_explicit=%d\n"
3255 "cipher_start=%d\n"
3256 "cipher_len=%d\n"
3257 "outdata=0x%p\n"
3258 "outlen=%d\n",
3259 cop->indata,
3260 cop->inlen,
3261 cop->do_cipher,
3262 cop->decrypt,
3263 cop->cipher_explicit,
3264 cop->cipher_start,
3265 cop->cipher_len,
3266 cop->cipher_outdata,
3267 cop->cipher_outlen);
3268
3269 printk("do_digest=%d\n"
3270 "digest_start=%d\n"
3271 "digest_len=%d\n",
3272 cop->do_digest,
3273 cop->digest_start,
3274 cop->digest_len);
3275
3276 printk("do_csum=%d\n"
3277 "csum_start=%d\n"
3278 "csum_len=%d\n",
3279 cop->do_csum,
3280 cop->csum_start,
3281 cop->csum_len);
3282}
3283
3284static void print_cryptocop_operation(struct cryptocop_operation *cop)
3285{
3286 struct cryptocop_desc *d;
3287 struct cryptocop_tfrm_cfg *tc;
3288 struct cryptocop_desc_cfg *dc;
3289 int i;
3290
3291 printk("print_cryptocop_operation, cop=0x%p\n\n", cop);
3292 printk("sid: %lld\n", cop->sid);
3293 printk("operation_status=%d\n"
3294 "use_dmalists=%d\n"
3295 "in_interrupt=%d\n"
3296 "fast_callback=%d\n",
3297 cop->operation_status,
3298 cop->use_dmalists,
3299 cop->in_interrupt,
3300 cop->fast_callback);
3301
3302 if (cop->use_dmalists){
3303 print_user_dma_lists(&cop->list_op);
3304 } else {
3305 printk("cop->tfrm_op\n"
3306 "tfrm_cfg=0x%p\n"
3307 "desc=0x%p\n"
3308 "indata=0x%p\n"
3309 "incount=%d\n"
3310 "inlen=%d\n"
3311 "outdata=0x%p\n"
3312 "outcount=%d\n"
3313 "outlen=%d\n\n",
3314 cop->tfrm_op.tfrm_cfg,
3315 cop->tfrm_op.desc,
3316 cop->tfrm_op.indata,
3317 cop->tfrm_op.incount,
3318 cop->tfrm_op.inlen,
3319 cop->tfrm_op.outdata,
3320 cop->tfrm_op.outcount,
3321 cop->tfrm_op.outlen);
3322
3323 tc = cop->tfrm_op.tfrm_cfg;
3324 while (tc){
3325 printk("tfrm_cfg, 0x%p\n"
3326 "tid=%d\n"
3327 "flags=%d\n"
3328 "inject_ix=%d\n"
3329 "next=0x%p\n",
3330 tc,
3331 tc->tid,
3332 tc->flags,
3333 tc->inject_ix,
3334 tc->next);
3335 tc = tc->next;
3336 }
3337 d = cop->tfrm_op.desc;
3338 while (d){
3339 printk("\n======================desc, 0x%p\n"
3340 "length=%d\n"
3341 "cfg=0x%p\n"
3342 "next=0x%p\n",
3343 d,
3344 d->length,
3345 d->cfg,
3346 d->next);
3347 dc = d->cfg;
3348 while (dc){
3349 printk("=========desc_cfg, 0x%p\n"
3350 "tid=%d\n"
3351 "src=%d\n"
3352 "last=%d\n"
3353 "next=0x%p\n",
3354 dc,
3355 dc->tid,
3356 dc->src,
3357 dc->last,
3358 dc->next);
3359 dc = dc->next;
3360 }
3361 d = d->next;
3362 }
3363 printk("\n====iniov\n");
3364 for (i = 0; i < cop->tfrm_op.incount; i++){
3365 printk("indata[%d]\n"
3366 "base=0x%p\n"
3367 "len=%d\n",
3368 i,
3369 cop->tfrm_op.indata[i].iov_base,
3370 cop->tfrm_op.indata[i].iov_len);
3371 }
3372 printk("\n====outiov\n");
3373 for (i = 0; i < cop->tfrm_op.outcount; i++){
3374 printk("outdata[%d]\n"
3375 "base=0x%p\n"
3376 "len=%d\n",
3377 i,
3378 cop->tfrm_op.outdata[i].iov_base,
3379 cop->tfrm_op.outdata[i].iov_len);
3380 }
3381 }
3382 printk("------------end print_cryptocop_operation\n");
3383}
3384
3385
3386static void print_user_dma_lists(struct cryptocop_dma_list_operation *dma_op)
3387{
3388 dma_descr_data *dd;
3389 int i;
3390
3391 printk("print_user_dma_lists, dma_op=0x%p\n", dma_op);
3392
3393 printk("out_data_buf = 0x%p, phys_to_virt(out_data_buf) = 0x%p\n", dma_op->out_data_buf, phys_to_virt((unsigned long int)dma_op->out_data_buf));
3394 printk("in_data_buf = 0x%p, phys_to_virt(in_data_buf) = 0x%p\n", dma_op->in_data_buf, phys_to_virt((unsigned long int)dma_op->in_data_buf));
3395
3396 printk("##############outlist\n");
3397 dd = phys_to_virt((unsigned long int)dma_op->outlist);
3398 i = 0;
3399 while (dd != NULL) {
3400 printk("#%d phys_to_virt(desc) 0x%p\n", i, dd);
3401 printk("\n\tbuf: 0x%p\n"
3402 "\tafter: 0x%p\n"
3403 "\tmd: 0x%04x\n"
3404 "\tnext: 0x%p\n",
3405 dd->buf,
3406 dd->after,
3407 dd->md,
3408 dd->next);
3409 printk("flags:\n"
3410 "\twait:\t%d\n"
3411 "\teol:\t%d\n"
3412 "\touteop:\t%d\n"
3413 "\tineop:\t%d\n"
3414 "\tintr:\t%d\n",
3415 dd->wait,
3416 dd->eol,
3417 dd->out_eop,
3418 dd->in_eop,
3419 dd->intr);
3420 if (dd->eol)
3421 dd = NULL;
3422 else
3423 dd = phys_to_virt((unsigned long int)dd->next);
3424 ++i;
3425 }
3426
3427 printk("##############inlist\n");
3428 dd = phys_to_virt((unsigned long int)dma_op->inlist);
3429 i = 0;
3430 while (dd != NULL) {
3431 printk("#%d phys_to_virt(desc) 0x%p\n", i, dd);
3432 printk("\n\tbuf: 0x%p\n"
3433 "\tafter: 0x%p\n"
3434 "\tmd: 0x%04x\n"
3435 "\tnext: 0x%p\n",
3436 dd->buf,
3437 dd->after,
3438 dd->md,
3439 dd->next);
3440 printk("flags:\n"
3441 "\twait:\t%d\n"
3442 "\teol:\t%d\n"
3443 "\touteop:\t%d\n"
3444 "\tineop:\t%d\n"
3445 "\tintr:\t%d\n",
3446 dd->wait,
3447 dd->eol,
3448 dd->out_eop,
3449 dd->in_eop,
3450 dd->intr);
3451 if (dd->eol)
3452 dd = NULL;
3453 else
3454 dd = phys_to_virt((unsigned long int)dd->next);
3455 ++i;
3456 }
3457}
3458
3459
3460static void print_lock_status(void)
3461{
3462 printk("**********************print_lock_status\n");
3463 printk("cryptocop_completed_jobs_lock %d\n", spin_is_locked(&cryptocop_completed_jobs_lock));
3464 printk("cryptocop_job_queue_lock %d\n", spin_is_locked(&cryptocop_job_queue_lock));
3465 printk("descr_pool_lock %d\n", spin_is_locked(&descr_pool_lock));
3466 printk("cryptocop_sessions_lock %d\n", spin_is_locked(cryptocop_sessions_lock));
3467 printk("running_job_lock %d\n", spin_is_locked(running_job_lock));
3468 printk("cryptocop_process_lock %d\n", spin_is_locked(cryptocop_process_lock));
3469}
3470#endif /* LDEBUG */
3471
3472
3473static const char cryptocop_name[] = "ETRAX FS stream co-processor";
3474
3475static int init_stream_coprocessor(void)
3476{
3477 int err;
3478 int i;
3479 static int initialized = 0;
3480
3481 if (initialized)
3482 return 0;
3483
3484 initialized = 1;
3485
3486 printk("ETRAX FS stream co-processor driver v0.01, (c) 2003 Axis Communications AB\n");
3487
3488 err = register_chrdev(CRYPTOCOP_MAJOR, cryptocop_name, &cryptocop_fops);
3489 if (err < 0) {
3490 printk(KERN_ERR "stream co-processor: could not get major number.\n");
3491 return err;
3492 }
3493
3494 err = init_cryptocop();
3495 if (err) {
3496 (void)unregister_chrdev(CRYPTOCOP_MAJOR, cryptocop_name);
3497 return err;
3498 }
3499 err = cryptocop_job_queue_init();
3500 if (err) {
3501 release_cryptocop();
3502 (void)unregister_chrdev(CRYPTOCOP_MAJOR, cryptocop_name);
3503 return err;
3504 }
3505 /* Init the descriptor pool. */
3506 for (i = 0; i < CRYPTOCOP_DESCRIPTOR_POOL_SIZE - 1; i++) {
3507 descr_pool[i].from_pool = 1;
3508 descr_pool[i].next = &descr_pool[i + 1];
3509 }
3510 descr_pool[i].from_pool = 1;
3511 descr_pool[i].next = NULL;
3512 descr_pool_free_list = &descr_pool[0];
3513 descr_pool_no_free = CRYPTOCOP_DESCRIPTOR_POOL_SIZE;
3514
3515 spin_lock_init(&cryptocop_completed_jobs_lock);
3516 spin_lock_init(&cryptocop_job_queue_lock);
3517 spin_lock_init(&descr_pool_lock);
3518 spin_lock_init(&cryptocop_sessions_lock);
3519 spin_lock_init(&running_job_lock);
3520 spin_lock_init(&cryptocop_process_lock);
3521
3522 cryptocop_sessions = NULL;
3523 next_sid = 1;
3524
3525 cryptocop_running_job = NULL;
3526
3527 printk("stream co-processor: init done.\n");
3528 return 0;
3529}
3530
3531static void __exit exit_stream_coprocessor(void)
3532{
3533 release_cryptocop();
3534 cryptocop_job_queue_close();
3535}
3536
3537module_init(init_stream_coprocessor);
3538module_exit(exit_stream_coprocessor);
3539
Linux 6.x block layer and io_uring tree(s)