projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
dmaengine: bcm-sba-raid: Improve sba_issue_pending() run duration
[linux-2.6-block.git] / drivers / dma / bcm-sba-raid.c
CommitLineData
743e1c8f
AP		 1/*
2 * Copyright (C) 2017 Broadcom
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 */
8
9/*
10 * Broadcom SBA RAID Driver
11 *
12 * The Broadcom stream buffer accelerator (SBA) provides offloading
13 * capabilities for RAID operations. The SBA offload engine is accessible
14 * via Broadcom SoC specific ring manager. Two or more offload engines
15 * can share same Broadcom SoC specific ring manager due to this Broadcom
16 * SoC specific ring manager driver is implemented as a mailbox controller
17 * driver and offload engine drivers are implemented as mallbox clients.
18 *
19 * Typically, Broadcom SoC specific ring manager will implement larger
20 * number of hardware rings over one or more SBA hardware devices. By
21 * design, the internal buffer size of SBA hardware device is limited
22 * but all offload operations supported by SBA can be broken down into
23 * multiple small size requests and executed parallely on multiple SBA
24 * hardware devices for achieving high through-put.
25 *
26 * The Broadcom SBA RAID driver does not require any register programming
27 * except submitting request to SBA hardware device via mailbox channels.
28 * This driver implements a DMA device with one DMA channel using a set
29 * of mailbox channels provided by Broadcom SoC specific ring manager
30 * driver. To exploit parallelism (as described above), all DMA request
31 * coming to SBA RAID DMA channel are broken down to smaller requests
32 * and submitted to multiple mailbox channels in round-robin fashion.
33 * For having more SBA DMA channels, we can create more SBA device nodes
34 * in Broadcom SoC specific DTS based on number of hardware rings supported
35 * by Broadcom SoC ring manager.
36 */
37
38#include <linux/bitops.h>
39#include <linux/dma-mapping.h>
40#include <linux/dmaengine.h>
41#include <linux/list.h>
42#include <linux/mailbox_client.h>
43#include <linux/mailbox/brcm-message.h>
44#include <linux/module.h>
45#include <linux/of_device.h>
46#include <linux/slab.h>
47#include <linux/raid/pq.h>
48
49#include "dmaengine.h"
50
e897091a
AP		 51/* ====== Driver macros and defines ===== */
52
743e1c8f
AP		 53#define SBA_TYPE_SHIFT 48
54#define SBA_TYPE_MASK GENMASK(1, 0)
55#define SBA_TYPE_A 0x0
56#define SBA_TYPE_B 0x2
57#define SBA_TYPE_C 0x3
58#define SBA_USER_DEF_SHIFT 32
59#define SBA_USER_DEF_MASK GENMASK(15, 0)
60#define SBA_R_MDATA_SHIFT 24
61#define SBA_R_MDATA_MASK GENMASK(7, 0)
62#define SBA_C_MDATA_MS_SHIFT 18
63#define SBA_C_MDATA_MS_MASK GENMASK(1, 0)
64#define SBA_INT_SHIFT 17
65#define SBA_INT_MASK BIT(0)
66#define SBA_RESP_SHIFT 16
67#define SBA_RESP_MASK BIT(0)
68#define SBA_C_MDATA_SHIFT 8
69#define SBA_C_MDATA_MASK GENMASK(7, 0)
70#define SBA_C_MDATA_BNUMx_SHIFT(__bnum) (2 * (__bnum))
71#define SBA_C_MDATA_BNUMx_MASK GENMASK(1, 0)
72#define SBA_C_MDATA_DNUM_SHIFT 5
73#define SBA_C_MDATA_DNUM_MASK GENMASK(4, 0)
74#define SBA_C_MDATA_LS(__v) ((__v) & 0xff)
75#define SBA_C_MDATA_MS(__v) (((__v) >> 8) & 0x3)
76#define SBA_CMD_SHIFT 0
77#define SBA_CMD_MASK GENMASK(3, 0)
78#define SBA_CMD_ZERO_BUFFER 0x4
79#define SBA_CMD_ZERO_ALL_BUFFERS 0x8
80#define SBA_CMD_LOAD_BUFFER 0x9
81#define SBA_CMD_XOR 0xa
82#define SBA_CMD_GALOIS_XOR 0xb
83#define SBA_CMD_WRITE_BUFFER 0xc
84#define SBA_CMD_GALOIS 0xe
85
5346aafc
AP		 86#define SBA_MAX_REQ_PER_MBOX_CHANNEL 8192
87
743e1c8f
AP		 88/* Driver helper macros */
89#define to_sba_request(tx) \
90 container_of(tx, struct sba_request, tx)
91#define to_sba_device(dchan) \
92 container_of(dchan, struct sba_device, dma_chan)
93
e897091a
AP		 94/* ===== Driver data structures ===== */
95
57a28508
AP		 96enum sba_request_flags {
97 SBA_REQUEST_STATE_FREE = 0x001,
98 SBA_REQUEST_STATE_ALLOCED = 0x002,
99 SBA_REQUEST_STATE_PENDING = 0x004,
100 SBA_REQUEST_STATE_ACTIVE = 0x008,
101 SBA_REQUEST_STATE_RECEIVED = 0x010,
102 SBA_REQUEST_STATE_COMPLETED = 0x020,
103 SBA_REQUEST_STATE_ABORTED = 0x040,
104 SBA_REQUEST_STATE_MASK = 0x0ff,
105 SBA_REQUEST_FENCE = 0x100,
743e1c8f
AP		 106};
107
108struct sba_request {
109 /* Global state */
110 struct list_head node;
111 struct sba_device *sba;
57a28508 112 u32 flags;
743e1c8f
AP		 113 /* Chained requests management */
114 struct sba_request *first;
115 struct list_head next;
743e1c8f
AP		 116 atomic_t next_pending_count;
117 /* BRCM message data */
743e1c8f
AP		 118 struct brcm_message msg;
119 struct dma_async_tx_descriptor tx;
5655e00f
AP		 120 /* SBA commands */
121 struct brcm_sba_command cmds[0];
743e1c8f
AP		 122};
123
124enum sba_version {
125 SBA_VER_1 = 0,
126 SBA_VER_2
127};
128
129struct sba_device {
130 /* Underlying device */
131 struct device *dev;
132 /* DT configuration parameters */
133 enum sba_version ver;
134 /* Derived configuration parameters */
135 u32 max_req;
136 u32 hw_buf_size;
137 u32 hw_resp_size;
138 u32 max_pq_coefs;
139 u32 max_pq_srcs;
140 u32 max_cmd_per_req;
141 u32 max_xor_srcs;
142 u32 max_resp_pool_size;
143 u32 max_cmds_pool_size;
144 /* Maibox client and Mailbox channels */
145 struct mbox_client client;
146 int mchans_count;
147 atomic_t mchans_current;
148 struct mbox_chan **mchans;
149 struct device *mbox_dev;
150 /* DMA device and DMA channel */
151 struct dma_device dma_dev;
152 struct dma_chan dma_chan;
153 /* DMA channel resources */
154 void *resp_base;
155 dma_addr_t resp_dma_base;
156 void *cmds_base;
157 dma_addr_t cmds_dma_base;
158 spinlock_t reqs_lock;
743e1c8f
AP		 159 bool reqs_fence;
160 struct list_head reqs_alloc_list;
161 struct list_head reqs_pending_list;
162 struct list_head reqs_active_list;
163 struct list_head reqs_received_list;
164 struct list_head reqs_completed_list;
165 struct list_head reqs_aborted_list;
166 struct list_head reqs_free_list;
743e1c8f
AP		 167};
168
e897091a 169/* ====== Command helper routines ===== */
743e1c8f
AP		 170
171static inline u64 __pure sba_cmd_enc(u64 cmd, u32 val, u32 shift, u32 mask)
172{
173 cmd &= ~((u64)mask << shift);
174 cmd |= ((u64)(val & mask) << shift);
175 return cmd;
176}
177
178static inline u32 __pure sba_cmd_load_c_mdata(u32 b0)
179{
180 return b0 & SBA_C_MDATA_BNUMx_MASK;
181}
182
183static inline u32 __pure sba_cmd_write_c_mdata(u32 b0)
184{
185 return b0 & SBA_C_MDATA_BNUMx_MASK;
186}
187
188static inline u32 __pure sba_cmd_xor_c_mdata(u32 b1, u32 b0)
189{
190 return (b0 & SBA_C_MDATA_BNUMx_MASK) |
191 ((b1 & SBA_C_MDATA_BNUMx_MASK) << SBA_C_MDATA_BNUMx_SHIFT(1));
192}
193
194static inline u32 __pure sba_cmd_pq_c_mdata(u32 d, u32 b1, u32 b0)
195{
196 return (b0 & SBA_C_MDATA_BNUMx_MASK) |
197 ((b1 & SBA_C_MDATA_BNUMx_MASK) << SBA_C_MDATA_BNUMx_SHIFT(1)) |
198 ((d & SBA_C_MDATA_DNUM_MASK) << SBA_C_MDATA_DNUM_SHIFT);
199}
200
e897091a 201/* ====== General helper routines ===== */
743e1c8f
AP		 202
203static struct sba_request *sba_alloc_request(struct sba_device *sba)
204{
205 unsigned long flags;
206 struct sba_request *req = NULL;
207
208 spin_lock_irqsave(&sba->reqs_lock, flags);
743e1c8f
AP		 209 req = list_first_entry_or_null(&sba->reqs_free_list,
210 struct sba_request, node);
abfa251a 211 if (req)
743e1c8f 212 list_move_tail(&req->node, &sba->reqs_alloc_list);
743e1c8f 213 spin_unlock_irqrestore(&sba->reqs_lock, flags);
e4274cfa
AP		 214 if (!req)
215 return NULL;
216
57a28508 217 req->flags = SBA_REQUEST_STATE_ALLOCED;
e4274cfa
AP		 218 req->first = req;
219 INIT_LIST_HEAD(&req->next);
e4274cfa
AP		 220 atomic_set(&req->next_pending_count, 1);
221
222 dma_async_tx_descriptor_init(&req->tx, &sba->dma_chan);
743e1c8f
AP		 223
224 return req;
225}
226
227/* Note: Must be called with sba->reqs_lock held */
228static void _sba_pending_request(struct sba_device *sba,
229 struct sba_request *req)
230{
231 lockdep_assert_held(&sba->reqs_lock);
57a28508
AP		 232 req->flags &= ~SBA_REQUEST_STATE_MASK;
233 req->flags |= SBA_REQUEST_STATE_PENDING;
743e1c8f
AP		 234 list_move_tail(&req->node, &sba->reqs_pending_list);
235 if (list_empty(&sba->reqs_active_list))
236 sba->reqs_fence = false;
237}
238
239/* Note: Must be called with sba->reqs_lock held */
240static bool _sba_active_request(struct sba_device *sba,
241 struct sba_request *req)
242{
243 lockdep_assert_held(&sba->reqs_lock);
244 if (list_empty(&sba->reqs_active_list))
245 sba->reqs_fence = false;
246 if (sba->reqs_fence)
247 return false;
57a28508
AP		 248 req->flags &= ~SBA_REQUEST_STATE_MASK;
249 req->flags |= SBA_REQUEST_STATE_ACTIVE;
743e1c8f 250 list_move_tail(&req->node, &sba->reqs_active_list);
57a28508 251 if (req->flags & SBA_REQUEST_FENCE)
743e1c8f
AP		 252 sba->reqs_fence = true;
253 return true;
254}
255
256/* Note: Must be called with sba->reqs_lock held */
257static void _sba_abort_request(struct sba_device *sba,
258 struct sba_request *req)
259{
260 lockdep_assert_held(&sba->reqs_lock);
57a28508
AP		 261 req->flags &= ~SBA_REQUEST_STATE_MASK;
262 req->flags |= SBA_REQUEST_STATE_ABORTED;
743e1c8f
AP		 263 list_move_tail(&req->node, &sba->reqs_aborted_list);
264 if (list_empty(&sba->reqs_active_list))
265 sba->reqs_fence = false;
266}
267
268/* Note: Must be called with sba->reqs_lock held */
269static void _sba_free_request(struct sba_device *sba,
270 struct sba_request *req)
271{
272 lockdep_assert_held(&sba->reqs_lock);
57a28508
AP		 273 req->flags &= ~SBA_REQUEST_STATE_MASK;
274 req->flags |= SBA_REQUEST_STATE_FREE;
743e1c8f
AP		 275 list_move_tail(&req->node, &sba->reqs_free_list);
276 if (list_empty(&sba->reqs_active_list))
277 sba->reqs_fence = false;
743e1c8f
AP		 278}
279
f8338514
AP		 280/* Note: Must be called with sba->reqs_lock held */
281static void _sba_complete_request(struct sba_device *sba,
282 struct sba_request *req)
743e1c8f 283{
f8338514 284 lockdep_assert_held(&sba->reqs_lock);
57a28508 285 req->flags &= ~SBA_REQUEST_STATE_MASK;
f8338514
AP		 286 req->flags |= SBA_REQUEST_STATE_COMPLETED;
287 list_move_tail(&req->node, &sba->reqs_completed_list);
288 if (list_empty(&sba->reqs_active_list))
289 sba->reqs_fence = false;
743e1c8f
AP		 290}
291
f8338514
AP		 292/* Note: Must be called with sba->reqs_lock held */
293static void _sba_received_request(struct sba_device *sba,
294 struct sba_request *req)
743e1c8f 295{
f8338514 296 lockdep_assert_held(&sba->reqs_lock);
57a28508 297 req->flags &= ~SBA_REQUEST_STATE_MASK;
f8338514
AP		 298 req->flags |= SBA_REQUEST_STATE_RECEIVED;
299 list_move_tail(&req->node, &sba->reqs_received_list);
743e1c8f
AP		 300 if (list_empty(&sba->reqs_active_list))
301 sba->reqs_fence = false;
743e1c8f
AP		 302}
303
304static void sba_free_chained_requests(struct sba_request *req)
305{
306 unsigned long flags;
307 struct sba_request *nreq;
308 struct sba_device *sba = req->sba;
309
310 spin_lock_irqsave(&sba->reqs_lock, flags);
311
312 _sba_free_request(sba, req);
313 list_for_each_entry(nreq, &req->next, next)
314 _sba_free_request(sba, nreq);
315
316 spin_unlock_irqrestore(&sba->reqs_lock, flags);
317}
318
319static void sba_chain_request(struct sba_request *first,
320 struct sba_request *req)
321{
322 unsigned long flags;
323 struct sba_device *sba = req->sba;
324
325 spin_lock_irqsave(&sba->reqs_lock, flags);
326
327 list_add_tail(&req->next, &first->next);
328 req->first = first;
10f1a330 329 atomic_inc(&first->next_pending_count);
743e1c8f
AP		 330
331 spin_unlock_irqrestore(&sba->reqs_lock, flags);
332}
333
334static void sba_cleanup_nonpending_requests(struct sba_device *sba)
335{
336 unsigned long flags;
337 struct sba_request *req, *req1;
338
339 spin_lock_irqsave(&sba->reqs_lock, flags);
340
341 /* Freeup all alloced request */
342 list_for_each_entry_safe(req, req1, &sba->reqs_alloc_list, node)
343 _sba_free_request(sba, req);
344
345 /* Freeup all received request */
346 list_for_each_entry_safe(req, req1, &sba->reqs_received_list, node)
347 _sba_free_request(sba, req);
348
349 /* Freeup all completed request */
350 list_for_each_entry_safe(req, req1, &sba->reqs_completed_list, node)
351 _sba_free_request(sba, req);
352
353 /* Set all active requests as aborted */
354 list_for_each_entry_safe(req, req1, &sba->reqs_active_list, node)
355 _sba_abort_request(sba, req);
356
357 /*
358 * Note: We expect that aborted request will be eventually
359 * freed by sba_receive_message()
360 */
361
362 spin_unlock_irqrestore(&sba->reqs_lock, flags);
363}
364
365static void sba_cleanup_pending_requests(struct sba_device *sba)
366{
367 unsigned long flags;
368 struct sba_request *req, *req1;
369
370 spin_lock_irqsave(&sba->reqs_lock, flags);
371
372 /* Freeup all pending request */
373 list_for_each_entry_safe(req, req1, &sba->reqs_pending_list, node)
374 _sba_free_request(sba, req);
375
376 spin_unlock_irqrestore(&sba->reqs_lock, flags);
377}
378
743e1c8f
AP		 379static int sba_send_mbox_request(struct sba_device *sba,
380 struct sba_request *req)
381{
382 int mchans_idx, ret = 0;
383
384 /* Select mailbox channel in round-robin fashion */
385 mchans_idx = atomic_inc_return(&sba->mchans_current);
386 mchans_idx = mchans_idx % sba->mchans_count;
387
388 /* Send message for the request */
389 req->msg.error = 0;
390 ret = mbox_send_message(sba->mchans[mchans_idx], &req->msg);
391 if (ret < 0) {
392 dev_err(sba->dev, "send message failed with error %d", ret);
393 return ret;
394 }
395 ret = req->msg.error;
396 if (ret < 0) {
397 dev_err(sba->dev, "message error %d", ret);
398 return ret;
399 }
400
401 return 0;
402}
403
f8338514 404static void sba_process_deferred_requests(struct sba_device *sba)
743e1c8f
AP		 405{
406 int ret;
f8338514 407 u32 count;
743e1c8f 408 unsigned long flags;
f8338514
AP		 409 struct sba_request *req;
410 struct dma_async_tx_descriptor *tx;
743e1c8f
AP		 411
412 spin_lock_irqsave(&sba->reqs_lock, flags);
413
f8338514
AP		 414 /* Count pending requests */
415 count = 0;
416 list_for_each_entry(req, &sba->reqs_pending_list, node)
417 count++;
418
419 /* Process pending requests */
420 while (!list_empty(&sba->reqs_pending_list) && count) {
421 /* Get the first pending request */
422 req = list_first_entry(&sba->reqs_pending_list,
423 struct sba_request, node);
424
743e1c8f
AP		 425 /* Try to make request active */
426 if (!_sba_active_request(sba, req))
427 break;
428
429 /* Send request to mailbox channel */
430 spin_unlock_irqrestore(&sba->reqs_lock, flags);
431 ret = sba_send_mbox_request(sba, req);
432 spin_lock_irqsave(&sba->reqs_lock, flags);
433
434 /* If something went wrong then keep request pending */
435 if (ret < 0) {
436 _sba_pending_request(sba, req);
437 break;
438 }
f8338514
AP		 439
440 count--;
441 }
442
443 /* Count completed requests */
444 count = 0;
445 list_for_each_entry(req, &sba->reqs_completed_list, node)
446 count++;
447
448 /* Process completed requests */
449 while (!list_empty(&sba->reqs_completed_list) && count) {
450 req = list_first_entry(&sba->reqs_completed_list,
451 struct sba_request, node);
452 list_del_init(&req->node);
453 tx = &req->tx;
454
455 spin_unlock_irqrestore(&sba->reqs_lock, flags);
456
457 WARN_ON(tx->cookie < 0);
458 if (tx->cookie > 0) {
459 dma_cookie_complete(tx);
460 dmaengine_desc_get_callback_invoke(tx, NULL);
461 dma_descriptor_unmap(tx);
462 tx->callback = NULL;
463 tx->callback_result = NULL;
464 }
465
466 dma_run_dependencies(tx);
467
468 spin_lock_irqsave(&sba->reqs_lock, flags);
469
470 /* If waiting for 'ack' then move to completed list */
471 if (!async_tx_test_ack(&req->tx))
472 _sba_complete_request(sba, req);
473 else
474 _sba_free_request(sba, req);
475
476 count--;
743e1c8f
AP		 477 }
478
f8338514
AP		 479 /* Re-check pending and completed work */
480 count = 0;
481 if (!list_empty(&sba->reqs_pending_list) ||
482 !list_empty(&sba->reqs_completed_list))
483 count = 1;
484
743e1c8f
AP		 485 spin_unlock_irqrestore(&sba->reqs_lock, flags);
486}
487
f8338514
AP		 488static void sba_process_received_request(struct sba_device *sba,
489 struct sba_request *req)
490{
491 unsigned long flags;
492
493 spin_lock_irqsave(&sba->reqs_lock, flags);
494
495 /* Mark request as received */
496 _sba_received_request(sba, req);
497
498 /* Update request */
499 if (!atomic_dec_return(&req->first->next_pending_count))
500 _sba_complete_request(sba, req->first);
501 if (req->first != req)
502 _sba_free_request(sba, req);
503
504 spin_unlock_irqrestore(&sba->reqs_lock, flags);
505}
506
507/* ====== DMAENGINE callbacks ===== */
508
509static void sba_free_chan_resources(struct dma_chan *dchan)
510{
511 /*
512 * Channel resources are pre-alloced so we just free-up
513 * whatever we can so that we can re-use pre-alloced
514 * channel resources next time.
515 */
516 sba_cleanup_nonpending_requests(to_sba_device(dchan));
517}
518
519static int sba_device_terminate_all(struct dma_chan *dchan)
520{
521 /* Cleanup all pending requests */
522 sba_cleanup_pending_requests(to_sba_device(dchan));
523
524 return 0;
525}
526
527static void sba_issue_pending(struct dma_chan *dchan)
528{
529 struct sba_device *sba = to_sba_device(dchan);
530
531 /* Process deferred requests */
532 sba_process_deferred_requests(sba);
533}
534
743e1c8f
AP		 535static dma_cookie_t sba_tx_submit(struct dma_async_tx_descriptor *tx)
536{
537 unsigned long flags;
538 dma_cookie_t cookie;
539 struct sba_device *sba;
540 struct sba_request *req, *nreq;
541
542 if (unlikely(!tx))
543 return -EINVAL;
544
545 sba = to_sba_device(tx->chan);
546 req = to_sba_request(tx);
547
548 /* Assign cookie and mark all chained requests pending */
549 spin_lock_irqsave(&sba->reqs_lock, flags);
550 cookie = dma_cookie_assign(tx);
551 _sba_pending_request(sba, req);
552 list_for_each_entry(nreq, &req->next, next)
553 _sba_pending_request(sba, nreq);
554 spin_unlock_irqrestore(&sba->reqs_lock, flags);
555
556 return cookie;
557}
558
559static enum dma_status sba_tx_status(struct dma_chan *dchan,
560 dma_cookie_t cookie,
561 struct dma_tx_state *txstate)
562{
563 int mchan_idx;
564 enum dma_status ret;
565 struct sba_device *sba = to_sba_device(dchan);
566
567 for (mchan_idx = 0; mchan_idx < sba->mchans_count; mchan_idx++)
568 mbox_client_peek_data(sba->mchans[mchan_idx]);
569
570 ret = dma_cookie_status(dchan, cookie, txstate);
571 if (ret == DMA_COMPLETE)
572 return ret;
573
574 return dma_cookie_status(dchan, cookie, txstate);
575}
576
577static void sba_fillup_interrupt_msg(struct sba_request *req,
578 struct brcm_sba_command *cmds,
579 struct brcm_message *msg)
580{
581 u64 cmd;
582 u32 c_mdata;
e7ae72aa 583 dma_addr_t resp_dma = req->tx.phys;
743e1c8f
AP		 584 struct brcm_sba_command *cmdsp = cmds;
585
586 /* Type-B command to load dummy data into buf0 */
587 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
588 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
589 cmd = sba_cmd_enc(cmd, req->sba->hw_resp_size,
590 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
591 c_mdata = sba_cmd_load_c_mdata(0);
592 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
593 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
594 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
595 SBA_CMD_SHIFT, SBA_CMD_MASK);
596 cmdsp->cmd = cmd;
597 *cmdsp->cmd_dma = cpu_to_le64(cmd);
598 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
e7ae72aa 599 cmdsp->data = resp_dma;
743e1c8f
AP		 600 cmdsp->data_len = req->sba->hw_resp_size;
601 cmdsp++;
602
603 /* Type-A command to write buf0 to dummy location */
604 cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
605 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
606 cmd = sba_cmd_enc(cmd, req->sba->hw_resp_size,
607 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
608 cmd = sba_cmd_enc(cmd, 0x1,
609 SBA_RESP_SHIFT, SBA_RESP_MASK);
610 c_mdata = sba_cmd_write_c_mdata(0);
611 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
612 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
613 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
614 SBA_CMD_SHIFT, SBA_CMD_MASK);
615 cmdsp->cmd = cmd;
616 *cmdsp->cmd_dma = cpu_to_le64(cmd);
617 cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
618 if (req->sba->hw_resp_size) {
619 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
e7ae72aa 620 cmdsp->resp = resp_dma;
743e1c8f
AP		 621 cmdsp->resp_len = req->sba->hw_resp_size;
622 }
623 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
e7ae72aa 624 cmdsp->data = resp_dma;
743e1c8f
AP		 625 cmdsp->data_len = req->sba->hw_resp_size;
626 cmdsp++;
627
628 /* Fillup brcm_message */
629 msg->type = BRCM_MESSAGE_SBA;
630 msg->sba.cmds = cmds;
631 msg->sba.cmds_count = cmdsp - cmds;
632 msg->ctx = req;
633 msg->error = 0;
634}
635
636static struct dma_async_tx_descriptor *
637sba_prep_dma_interrupt(struct dma_chan *dchan, unsigned long flags)
638{
639 struct sba_request *req = NULL;
640 struct sba_device *sba = to_sba_device(dchan);
641
642 /* Alloc new request */
643 req = sba_alloc_request(sba);
644 if (!req)
645 return NULL;
646
647 /*
648 * Force fence so that no requests are submitted
649 * until DMA callback for this request is invoked.
650 */
57a28508 651 req->flags |= SBA_REQUEST_FENCE;
743e1c8f
AP		 652
653 /* Fillup request message */
654 sba_fillup_interrupt_msg(req, req->cmds, &req->msg);
655
656 /* Init async_tx descriptor */
657 req->tx.flags = flags;
658 req->tx.cookie = -EBUSY;
659
1fc63cb4 660 return &req->tx;
743e1c8f
AP		 661}
662
663static void sba_fillup_memcpy_msg(struct sba_request *req,
664 struct brcm_sba_command *cmds,
665 struct brcm_message *msg,
666 dma_addr_t msg_offset, size_t msg_len,
667 dma_addr_t dst, dma_addr_t src)
668{
669 u64 cmd;
670 u32 c_mdata;
e7ae72aa 671 dma_addr_t resp_dma = req->tx.phys;
743e1c8f
AP		 672 struct brcm_sba_command *cmdsp = cmds;
673
674 /* Type-B command to load data into buf0 */
675 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
676 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
677 cmd = sba_cmd_enc(cmd, msg_len,
678 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
679 c_mdata = sba_cmd_load_c_mdata(0);
680 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
681 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
682 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
683 SBA_CMD_SHIFT, SBA_CMD_MASK);
684 cmdsp->cmd = cmd;
685 *cmdsp->cmd_dma = cpu_to_le64(cmd);
686 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
687 cmdsp->data = src + msg_offset;
688 cmdsp->data_len = msg_len;
689 cmdsp++;
690
691 /* Type-A command to write buf0 */
692 cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
693 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
694 cmd = sba_cmd_enc(cmd, msg_len,
695 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
696 cmd = sba_cmd_enc(cmd, 0x1,
697 SBA_RESP_SHIFT, SBA_RESP_MASK);
698 c_mdata = sba_cmd_write_c_mdata(0);
699 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
700 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
701 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
702 SBA_CMD_SHIFT, SBA_CMD_MASK);
703 cmdsp->cmd = cmd;
704 *cmdsp->cmd_dma = cpu_to_le64(cmd);
705 cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
706 if (req->sba->hw_resp_size) {
707 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
e7ae72aa 708 cmdsp->resp = resp_dma;
743e1c8f
AP		 709 cmdsp->resp_len = req->sba->hw_resp_size;
710 }
711 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
712 cmdsp->data = dst + msg_offset;
713 cmdsp->data_len = msg_len;
714 cmdsp++;
715
716 /* Fillup brcm_message */
717 msg->type = BRCM_MESSAGE_SBA;
718 msg->sba.cmds = cmds;
719 msg->sba.cmds_count = cmdsp - cmds;
720 msg->ctx = req;
721 msg->error = 0;
722}
723
724static struct sba_request *
725sba_prep_dma_memcpy_req(struct sba_device *sba,
726 dma_addr_t off, dma_addr_t dst, dma_addr_t src,
727 size_t len, unsigned long flags)
728{
729 struct sba_request *req = NULL;
730
731 /* Alloc new request */
732 req = sba_alloc_request(sba);
733 if (!req)
734 return NULL;
57a28508
AP		 735 if (flags & DMA_PREP_FENCE)
736 req->flags |= SBA_REQUEST_FENCE;
743e1c8f
AP		 737
738 /* Fillup request message */
739 sba_fillup_memcpy_msg(req, req->cmds, &req->msg,
740 off, len, dst, src);
741
742 /* Init async_tx descriptor */
743 req->tx.flags = flags;
744 req->tx.cookie = -EBUSY;
745
746 return req;
747}
748
749static struct dma_async_tx_descriptor *
750sba_prep_dma_memcpy(struct dma_chan *dchan, dma_addr_t dst, dma_addr_t src,
751 size_t len, unsigned long flags)
752{
753 size_t req_len;
754 dma_addr_t off = 0;
755 struct sba_device *sba = to_sba_device(dchan);
756 struct sba_request *first = NULL, *req;
757
758 /* Create chained requests where each request is upto hw_buf_size */
759 while (len) {
760 req_len = (len < sba->hw_buf_size) ? len : sba->hw_buf_size;
761
762 req = sba_prep_dma_memcpy_req(sba, off, dst, src,
763 req_len, flags);
764 if (!req) {
765 if (first)
766 sba_free_chained_requests(first);
767 return NULL;
768 }
769
770 if (first)
771 sba_chain_request(first, req);
772 else
773 first = req;
774
775 off += req_len;
776 len -= req_len;
777 }
778
779 return (first) ? &first->tx : NULL;
780}
781
782static void sba_fillup_xor_msg(struct sba_request *req,
783 struct brcm_sba_command *cmds,
784 struct brcm_message *msg,
785 dma_addr_t msg_offset, size_t msg_len,
786 dma_addr_t dst, dma_addr_t *src, u32 src_cnt)
787{
788 u64 cmd;
789 u32 c_mdata;
790 unsigned int i;
e7ae72aa 791 dma_addr_t resp_dma = req->tx.phys;
743e1c8f
AP		 792 struct brcm_sba_command *cmdsp = cmds;
793
794 /* Type-B command to load data into buf0 */
795 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
796 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
797 cmd = sba_cmd_enc(cmd, msg_len,
798 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
799 c_mdata = sba_cmd_load_c_mdata(0);
800 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
801 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
802 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
803 SBA_CMD_SHIFT, SBA_CMD_MASK);
804 cmdsp->cmd = cmd;
805 *cmdsp->cmd_dma = cpu_to_le64(cmd);
806 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
807 cmdsp->data = src[0] + msg_offset;
808 cmdsp->data_len = msg_len;
809 cmdsp++;
810
811 /* Type-B commands to xor data with buf0 and put it back in buf0 */
812 for (i = 1; i < src_cnt; i++) {
813 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
814 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
815 cmd = sba_cmd_enc(cmd, msg_len,
816 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
817 c_mdata = sba_cmd_xor_c_mdata(0, 0);
818 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
819 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
820 cmd = sba_cmd_enc(cmd, SBA_CMD_XOR,
821 SBA_CMD_SHIFT, SBA_CMD_MASK);
822 cmdsp->cmd = cmd;
823 *cmdsp->cmd_dma = cpu_to_le64(cmd);
824 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
825 cmdsp->data = src[i] + msg_offset;
826 cmdsp->data_len = msg_len;
827 cmdsp++;
828 }
829
830 /* Type-A command to write buf0 */
831 cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
832 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
833 cmd = sba_cmd_enc(cmd, msg_len,
834 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
835 cmd = sba_cmd_enc(cmd, 0x1,
836 SBA_RESP_SHIFT, SBA_RESP_MASK);
837 c_mdata = sba_cmd_write_c_mdata(0);
838 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
839 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
840 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
841 SBA_CMD_SHIFT, SBA_CMD_MASK);
842 cmdsp->cmd = cmd;
843 *cmdsp->cmd_dma = cpu_to_le64(cmd);
844 cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
845 if (req->sba->hw_resp_size) {
846 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
e7ae72aa 847 cmdsp->resp = resp_dma;
743e1c8f
AP		 848 cmdsp->resp_len = req->sba->hw_resp_size;
849 }
850 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
851 cmdsp->data = dst + msg_offset;
852 cmdsp->data_len = msg_len;
853 cmdsp++;
854
855 /* Fillup brcm_message */
856 msg->type = BRCM_MESSAGE_SBA;
857 msg->sba.cmds = cmds;
858 msg->sba.cmds_count = cmdsp - cmds;
859 msg->ctx = req;
860 msg->error = 0;
861}
862
dd2bceb0 863static struct sba_request *
743e1c8f
AP		 864sba_prep_dma_xor_req(struct sba_device *sba,
865 dma_addr_t off, dma_addr_t dst, dma_addr_t *src,
866 u32 src_cnt, size_t len, unsigned long flags)
867{
868 struct sba_request *req = NULL;
869
870 /* Alloc new request */
871 req = sba_alloc_request(sba);
872 if (!req)
873 return NULL;
57a28508
AP		 874 if (flags & DMA_PREP_FENCE)
875 req->flags |= SBA_REQUEST_FENCE;
743e1c8f
AP		 876
877 /* Fillup request message */
878 sba_fillup_xor_msg(req, req->cmds, &req->msg,
879 off, len, dst, src, src_cnt);
880
881 /* Init async_tx descriptor */
882 req->tx.flags = flags;
883 req->tx.cookie = -EBUSY;
884
885 return req;
886}
887
888static struct dma_async_tx_descriptor *
889sba_prep_dma_xor(struct dma_chan *dchan, dma_addr_t dst, dma_addr_t *src,
890 u32 src_cnt, size_t len, unsigned long flags)
891{
892 size_t req_len;
893 dma_addr_t off = 0;
894 struct sba_device *sba = to_sba_device(dchan);
895 struct sba_request *first = NULL, *req;
896
897 /* Sanity checks */
898 if (unlikely(src_cnt > sba->max_xor_srcs))
899 return NULL;
900
901 /* Create chained requests where each request is upto hw_buf_size */
902 while (len) {
903 req_len = (len < sba->hw_buf_size) ? len : sba->hw_buf_size;
904
905 req = sba_prep_dma_xor_req(sba, off, dst, src, src_cnt,
906 req_len, flags);
907 if (!req) {
908 if (first)
909 sba_free_chained_requests(first);
910 return NULL;
911 }
912
913 if (first)
914 sba_chain_request(first, req);
915 else
916 first = req;
917
918 off += req_len;
919 len -= req_len;
920 }
921
922 return (first) ? &first->tx : NULL;
923}
924
925static void sba_fillup_pq_msg(struct sba_request *req,
926 bool pq_continue,
927 struct brcm_sba_command *cmds,
928 struct brcm_message *msg,
929 dma_addr_t msg_offset, size_t msg_len,
930 dma_addr_t *dst_p, dma_addr_t *dst_q,
931 const u8 *scf, dma_addr_t *src, u32 src_cnt)
932{
933 u64 cmd;
934 u32 c_mdata;
935 unsigned int i;
e7ae72aa 936 dma_addr_t resp_dma = req->tx.phys;
743e1c8f
AP		 937 struct brcm_sba_command *cmdsp = cmds;
938
939 if (pq_continue) {
940 /* Type-B command to load old P into buf0 */
941 if (dst_p) {
942 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
943 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
944 cmd = sba_cmd_enc(cmd, msg_len,
945 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
946 c_mdata = sba_cmd_load_c_mdata(0);
947 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
948 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
949 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
950 SBA_CMD_SHIFT, SBA_CMD_MASK);
951 cmdsp->cmd = cmd;
952 *cmdsp->cmd_dma = cpu_to_le64(cmd);
953 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
954 cmdsp->data = *dst_p + msg_offset;
955 cmdsp->data_len = msg_len;
956 cmdsp++;
957 }
958
959 /* Type-B command to load old Q into buf1 */
960 if (dst_q) {
961 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
962 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
963 cmd = sba_cmd_enc(cmd, msg_len,
964 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
965 c_mdata = sba_cmd_load_c_mdata(1);
966 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
967 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
968 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
969 SBA_CMD_SHIFT, SBA_CMD_MASK);
970 cmdsp->cmd = cmd;
971 *cmdsp->cmd_dma = cpu_to_le64(cmd);
972 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
973 cmdsp->data = *dst_q + msg_offset;
974 cmdsp->data_len = msg_len;
975 cmdsp++;
976 }
977 } else {
978 /* Type-A command to zero all buffers */
979 cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
980 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
981 cmd = sba_cmd_enc(cmd, msg_len,
982 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
983 cmd = sba_cmd_enc(cmd, SBA_CMD_ZERO_ALL_BUFFERS,
984 SBA_CMD_SHIFT, SBA_CMD_MASK);
985 cmdsp->cmd = cmd;
986 *cmdsp->cmd_dma = cpu_to_le64(cmd);
987 cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
988 cmdsp++;
989 }
990
991 /* Type-B commands for generate P onto buf0 and Q onto buf1 */
992 for (i = 0; i < src_cnt; i++) {
993 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
994 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
995 cmd = sba_cmd_enc(cmd, msg_len,
996 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
997 c_mdata = sba_cmd_pq_c_mdata(raid6_gflog[scf[i]], 1, 0);
998 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
999 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1000 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_MS(c_mdata),
1001 SBA_C_MDATA_MS_SHIFT, SBA_C_MDATA_MS_MASK);
1002 cmd = sba_cmd_enc(cmd, SBA_CMD_GALOIS_XOR,
1003 SBA_CMD_SHIFT, SBA_CMD_MASK);
1004 cmdsp->cmd = cmd;
1005 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1006 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
1007 cmdsp->data = src[i] + msg_offset;
1008 cmdsp->data_len = msg_len;
1009 cmdsp++;
1010 }
1011
1012 /* Type-A command to write buf0 */
1013 if (dst_p) {
1014 cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
1015 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1016 cmd = sba_cmd_enc(cmd, msg_len,
1017 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1018 cmd = sba_cmd_enc(cmd, 0x1,
1019 SBA_RESP_SHIFT, SBA_RESP_MASK);
1020 c_mdata = sba_cmd_write_c_mdata(0);
1021 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1022 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1023 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
1024 SBA_CMD_SHIFT, SBA_CMD_MASK);
1025 cmdsp->cmd = cmd;
1026 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1027 cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
1028 if (req->sba->hw_resp_size) {
1029 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
e7ae72aa 1030 cmdsp->resp = resp_dma;
743e1c8f
AP		 1031 cmdsp->resp_len = req->sba->hw_resp_size;
1032 }
1033 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
1034 cmdsp->data = *dst_p + msg_offset;
1035 cmdsp->data_len = msg_len;
1036 cmdsp++;
1037 }
1038
1039 /* Type-A command to write buf1 */
1040 if (dst_q) {
1041 cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
1042 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1043 cmd = sba_cmd_enc(cmd, msg_len,
1044 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1045 cmd = sba_cmd_enc(cmd, 0x1,
1046 SBA_RESP_SHIFT, SBA_RESP_MASK);
1047 c_mdata = sba_cmd_write_c_mdata(1);
1048 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1049 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1050 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
1051 SBA_CMD_SHIFT, SBA_CMD_MASK);
1052 cmdsp->cmd = cmd;
1053 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1054 cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
1055 if (req->sba->hw_resp_size) {
1056 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
e7ae72aa 1057 cmdsp->resp = resp_dma;
743e1c8f
AP		 1058 cmdsp->resp_len = req->sba->hw_resp_size;
1059 }
1060 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
1061 cmdsp->data = *dst_q + msg_offset;
1062 cmdsp->data_len = msg_len;
1063 cmdsp++;
1064 }
1065
1066 /* Fillup brcm_message */
1067 msg->type = BRCM_MESSAGE_SBA;
1068 msg->sba.cmds = cmds;
1069 msg->sba.cmds_count = cmdsp - cmds;
1070 msg->ctx = req;
1071 msg->error = 0;
1072}
1073
dd2bceb0 1074static struct sba_request *
743e1c8f
AP		 1075sba_prep_dma_pq_req(struct sba_device *sba, dma_addr_t off,
1076 dma_addr_t *dst_p, dma_addr_t *dst_q, dma_addr_t *src,
1077 u32 src_cnt, const u8 *scf, size_t len, unsigned long flags)
1078{
1079 struct sba_request *req = NULL;
1080
1081 /* Alloc new request */
1082 req = sba_alloc_request(sba);
1083 if (!req)
1084 return NULL;
57a28508
AP		 1085 if (flags & DMA_PREP_FENCE)
1086 req->flags |= SBA_REQUEST_FENCE;
743e1c8f
AP		 1087
1088 /* Fillup request messages */
1089 sba_fillup_pq_msg(req, dmaf_continue(flags),
1090 req->cmds, &req->msg,
1091 off, len, dst_p, dst_q, scf, src, src_cnt);
1092
1093 /* Init async_tx descriptor */
1094 req->tx.flags = flags;
1095 req->tx.cookie = -EBUSY;
1096
1097 return req;
1098}
1099
1100static void sba_fillup_pq_single_msg(struct sba_request *req,
1101 bool pq_continue,
1102 struct brcm_sba_command *cmds,
1103 struct brcm_message *msg,
1104 dma_addr_t msg_offset, size_t msg_len,
1105 dma_addr_t *dst_p, dma_addr_t *dst_q,
1106 dma_addr_t src, u8 scf)
1107{
1108 u64 cmd;
1109 u32 c_mdata;
1110 u8 pos, dpos = raid6_gflog[scf];
e7ae72aa 1111 dma_addr_t resp_dma = req->tx.phys;
743e1c8f
AP		 1112 struct brcm_sba_command *cmdsp = cmds;
1113
1114 if (!dst_p)
1115 goto skip_p;
1116
1117 if (pq_continue) {
1118 /* Type-B command to load old P into buf0 */
1119 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
1120 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1121 cmd = sba_cmd_enc(cmd, msg_len,
1122 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1123 c_mdata = sba_cmd_load_c_mdata(0);
1124 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1125 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1126 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
1127 SBA_CMD_SHIFT, SBA_CMD_MASK);
1128 cmdsp->cmd = cmd;
1129 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1130 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
1131 cmdsp->data = *dst_p + msg_offset;
1132 cmdsp->data_len = msg_len;
1133 cmdsp++;
1134
1135 /*
1136 * Type-B commands to xor data with buf0 and put it
1137 * back in buf0
1138 */
1139 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
1140 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1141 cmd = sba_cmd_enc(cmd, msg_len,
1142 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1143 c_mdata = sba_cmd_xor_c_mdata(0, 0);
1144 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1145 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1146 cmd = sba_cmd_enc(cmd, SBA_CMD_XOR,
1147 SBA_CMD_SHIFT, SBA_CMD_MASK);
1148 cmdsp->cmd = cmd;
1149 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1150 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
1151 cmdsp->data = src + msg_offset;
1152 cmdsp->data_len = msg_len;
1153 cmdsp++;
1154 } else {
1155 /* Type-B command to load old P into buf0 */
1156 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
1157 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1158 cmd = sba_cmd_enc(cmd, msg_len,
1159 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1160 c_mdata = sba_cmd_load_c_mdata(0);
1161 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1162 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1163 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
1164 SBA_CMD_SHIFT, SBA_CMD_MASK);
1165 cmdsp->cmd = cmd;
1166 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1167 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
1168 cmdsp->data = src + msg_offset;
1169 cmdsp->data_len = msg_len;
1170 cmdsp++;
1171 }
1172
1173 /* Type-A command to write buf0 */
1174 cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
1175 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1176 cmd = sba_cmd_enc(cmd, msg_len,
1177 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1178 cmd = sba_cmd_enc(cmd, 0x1,
1179 SBA_RESP_SHIFT, SBA_RESP_MASK);
1180 c_mdata = sba_cmd_write_c_mdata(0);
1181 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1182 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1183 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
1184 SBA_CMD_SHIFT, SBA_CMD_MASK);
1185 cmdsp->cmd = cmd;
1186 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1187 cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
1188 if (req->sba->hw_resp_size) {
1189 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
e7ae72aa 1190 cmdsp->resp = resp_dma;
743e1c8f
AP		 1191 cmdsp->resp_len = req->sba->hw_resp_size;
1192 }
1193 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
1194 cmdsp->data = *dst_p + msg_offset;
1195 cmdsp->data_len = msg_len;
1196 cmdsp++;
1197
1198skip_p:
1199 if (!dst_q)
1200 goto skip_q;
1201
1202 /* Type-A command to zero all buffers */
1203 cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
1204 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1205 cmd = sba_cmd_enc(cmd, msg_len,
1206 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1207 cmd = sba_cmd_enc(cmd, SBA_CMD_ZERO_ALL_BUFFERS,
1208 SBA_CMD_SHIFT, SBA_CMD_MASK);
1209 cmdsp->cmd = cmd;
1210 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1211 cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
1212 cmdsp++;
1213
1214 if (dpos == 255)
1215 goto skip_q_computation;
1216 pos = (dpos < req->sba->max_pq_coefs) ?
1217 dpos : (req->sba->max_pq_coefs - 1);
1218
1219 /*
1220 * Type-B command to generate initial Q from data
1221 * and store output into buf0
1222 */
1223 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
1224 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1225 cmd = sba_cmd_enc(cmd, msg_len,
1226 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1227 c_mdata = sba_cmd_pq_c_mdata(pos, 0, 0);
1228 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1229 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1230 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_MS(c_mdata),
1231 SBA_C_MDATA_MS_SHIFT, SBA_C_MDATA_MS_MASK);
1232 cmd = sba_cmd_enc(cmd, SBA_CMD_GALOIS,
1233 SBA_CMD_SHIFT, SBA_CMD_MASK);
1234 cmdsp->cmd = cmd;
1235 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1236 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
1237 cmdsp->data = src + msg_offset;
1238 cmdsp->data_len = msg_len;
1239 cmdsp++;
1240
1241 dpos -= pos;
1242
1243 /* Multiple Type-A command to generate final Q */
1244 while (dpos) {
1245 pos = (dpos < req->sba->max_pq_coefs) ?
1246 dpos : (req->sba->max_pq_coefs - 1);
1247
1248 /*
1249 * Type-A command to generate Q with buf0 and
1250 * buf1 store result in buf0
1251 */
1252 cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
1253 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1254 cmd = sba_cmd_enc(cmd, msg_len,
1255 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1256 c_mdata = sba_cmd_pq_c_mdata(pos, 0, 1);
1257 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1258 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1259 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_MS(c_mdata),
1260 SBA_C_MDATA_MS_SHIFT, SBA_C_MDATA_MS_MASK);
1261 cmd = sba_cmd_enc(cmd, SBA_CMD_GALOIS,
1262 SBA_CMD_SHIFT, SBA_CMD_MASK);
1263 cmdsp->cmd = cmd;
1264 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1265 cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
1266 cmdsp++;
1267
1268 dpos -= pos;
1269 }
1270
1271skip_q_computation:
1272 if (pq_continue) {
1273 /*
1274 * Type-B command to XOR previous output with
1275 * buf0 and write it into buf0
1276 */
1277 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
1278 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1279 cmd = sba_cmd_enc(cmd, msg_len,
1280 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1281 c_mdata = sba_cmd_xor_c_mdata(0, 0);
1282 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1283 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1284 cmd = sba_cmd_enc(cmd, SBA_CMD_XOR,
1285 SBA_CMD_SHIFT, SBA_CMD_MASK);
1286 cmdsp->cmd = cmd;
1287 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1288 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
1289 cmdsp->data = *dst_q + msg_offset;
1290 cmdsp->data_len = msg_len;
1291 cmdsp++;
1292 }
1293
1294 /* Type-A command to write buf0 */
1295 cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
1296 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1297 cmd = sba_cmd_enc(cmd, msg_len,
1298 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1299 cmd = sba_cmd_enc(cmd, 0x1,
1300 SBA_RESP_SHIFT, SBA_RESP_MASK);
1301 c_mdata = sba_cmd_write_c_mdata(0);
1302 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1303 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1304 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
1305 SBA_CMD_SHIFT, SBA_CMD_MASK);
1306 cmdsp->cmd = cmd;
1307 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1308 cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
1309 if (req->sba->hw_resp_size) {
1310 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
e7ae72aa 1311 cmdsp->resp = resp_dma;
743e1c8f
AP		 1312 cmdsp->resp_len = req->sba->hw_resp_size;
1313 }
1314 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
1315 cmdsp->data = *dst_q + msg_offset;
1316 cmdsp->data_len = msg_len;
1317 cmdsp++;
1318
1319skip_q:
1320 /* Fillup brcm_message */
1321 msg->type = BRCM_MESSAGE_SBA;
1322 msg->sba.cmds = cmds;
1323 msg->sba.cmds_count = cmdsp - cmds;
1324 msg->ctx = req;
1325 msg->error = 0;
1326}
1327
dd2bceb0 1328static struct sba_request *
743e1c8f
AP		 1329sba_prep_dma_pq_single_req(struct sba_device *sba, dma_addr_t off,
1330 dma_addr_t *dst_p, dma_addr_t *dst_q,
1331 dma_addr_t src, u8 scf, size_t len,
1332 unsigned long flags)
1333{
1334 struct sba_request *req = NULL;
1335
1336 /* Alloc new request */
1337 req = sba_alloc_request(sba);
1338 if (!req)
1339 return NULL;
57a28508
AP		 1340 if (flags & DMA_PREP_FENCE)
1341 req->flags |= SBA_REQUEST_FENCE;
743e1c8f
AP		 1342
1343 /* Fillup request messages */
1344 sba_fillup_pq_single_msg(req, dmaf_continue(flags),
1345 req->cmds, &req->msg, off, len,
1346 dst_p, dst_q, src, scf);
1347
1348 /* Init async_tx descriptor */
1349 req->tx.flags = flags;
1350 req->tx.cookie = -EBUSY;
1351
1352 return req;
1353}
1354
1355static struct dma_async_tx_descriptor *
1356sba_prep_dma_pq(struct dma_chan *dchan, dma_addr_t *dst, dma_addr_t *src,
1357 u32 src_cnt, const u8 *scf, size_t len, unsigned long flags)
1358{
1359 u32 i, dst_q_index;
1360 size_t req_len;
1361 bool slow = false;
1362 dma_addr_t off = 0;
1363 dma_addr_t *dst_p = NULL, *dst_q = NULL;
1364 struct sba_device *sba = to_sba_device(dchan);
1365 struct sba_request *first = NULL, *req;
1366
1367 /* Sanity checks */
1368 if (unlikely(src_cnt > sba->max_pq_srcs))
1369 return NULL;
1370 for (i = 0; i < src_cnt; i++)
1371 if (sba->max_pq_coefs <= raid6_gflog[scf[i]])
1372 slow = true;
1373
1374 /* Figure-out P and Q destination addresses */
1375 if (!(flags & DMA_PREP_PQ_DISABLE_P))
1376 dst_p = &dst[0];
1377 if (!(flags & DMA_PREP_PQ_DISABLE_Q))
1378 dst_q = &dst[1];
1379
1380 /* Create chained requests where each request is upto hw_buf_size */
1381 while (len) {
1382 req_len = (len < sba->hw_buf_size) ? len : sba->hw_buf_size;
1383
1384 if (slow) {
1385 dst_q_index = src_cnt;
1386
1387 if (dst_q) {
1388 for (i = 0; i < src_cnt; i++) {
1389 if (*dst_q == src[i]) {
1390 dst_q_index = i;
1391 break;
1392 }
1393 }
1394 }
1395
1396 if (dst_q_index < src_cnt) {
1397 i = dst_q_index;
1398 req = sba_prep_dma_pq_single_req(sba,
1399 off, dst_p, dst_q, src[i], scf[i],
1400 req_len, flags | DMA_PREP_FENCE);
1401 if (!req)
1402 goto fail;
1403
1404 if (first)
1405 sba_chain_request(first, req);
1406 else
1407 first = req;
1408
1409 flags |= DMA_PREP_CONTINUE;
1410 }
1411
1412 for (i = 0; i < src_cnt; i++) {
1413 if (dst_q_index == i)
1414 continue;
1415
1416 req = sba_prep_dma_pq_single_req(sba,
1417 off, dst_p, dst_q, src[i], scf[i],
1418 req_len, flags | DMA_PREP_FENCE);
1419 if (!req)
1420 goto fail;
1421
1422 if (first)
1423 sba_chain_request(first, req);
1424 else
1425 first = req;
1426
1427 flags |= DMA_PREP_CONTINUE;
1428 }
1429 } else {
1430 req = sba_prep_dma_pq_req(sba, off,
1431 dst_p, dst_q, src, src_cnt,
1432 scf, req_len, flags);
1433 if (!req)
1434 goto fail;
1435
1436 if (first)
1437 sba_chain_request(first, req);
1438 else
1439 first = req;
1440 }
1441
1442 off += req_len;
1443 len -= req_len;
1444 }
1445
1446 return (first) ? &first->tx : NULL;
1447
1448fail:
1449 if (first)
1450 sba_free_chained_requests(first);
1451 return NULL;
1452}
1453
1454/* ====== Mailbox callbacks ===== */
1455
743e1c8f
AP		 1456static void sba_receive_message(struct mbox_client *cl, void *msg)
1457{
743e1c8f 1458 struct brcm_message *m = msg;
f8338514 1459 struct sba_request *req = m->ctx;
743e1c8f
AP		 1460 struct sba_device *sba = req->sba;
1461
1462 /* Error count if message has error */
1463 if (m->error < 0)
1464 dev_err(sba->dev, "%s got message with error %d",
1465 dma_chan_name(&sba->dma_chan), m->error);
1466
f8338514
AP		 1467 /* Process received request */
1468 sba_process_received_request(sba, req);
743e1c8f 1469
f8338514
AP		 1470 /* Process deferred requests */
1471 sba_process_deferred_requests(sba);
743e1c8f
AP		 1472}
1473
1474/* ====== Platform driver routines ===== */
1475
1476static int sba_prealloc_channel_resources(struct sba_device *sba)
1477{
e7ae72aa 1478 int i, j, ret = 0;
743e1c8f
AP		 1479 struct sba_request *req = NULL;
1480
1481 sba->resp_base = dma_alloc_coherent(sba->dma_dev.dev,
1482 sba->max_resp_pool_size,
1483 &sba->resp_dma_base, GFP_KERNEL);
1484 if (!sba->resp_base)
1485 return -ENOMEM;
1486
1487 sba->cmds_base = dma_alloc_coherent(sba->dma_dev.dev,
1488 sba->max_cmds_pool_size,
1489 &sba->cmds_dma_base, GFP_KERNEL);
1490 if (!sba->cmds_base) {
1491 ret = -ENOMEM;
1492 goto fail_free_resp_pool;
1493 }
1494
1495 spin_lock_init(&sba->reqs_lock);
1496 sba->reqs_fence = false;
1497 INIT_LIST_HEAD(&sba->reqs_alloc_list);
1498 INIT_LIST_HEAD(&sba->reqs_pending_list);
1499 INIT_LIST_HEAD(&sba->reqs_active_list);
1500 INIT_LIST_HEAD(&sba->reqs_received_list);
1501 INIT_LIST_HEAD(&sba->reqs_completed_list);
1502 INIT_LIST_HEAD(&sba->reqs_aborted_list);
1503 INIT_LIST_HEAD(&sba->reqs_free_list);
1504
e7ae72aa 1505 for (i = 0; i < sba->max_req; i++) {
5655e00f
AP		 1506 req = devm_kzalloc(sba->dev,
1507 sizeof(*req) +
1508 sba->max_cmd_per_req * sizeof(req->cmds[0]),
1509 GFP_KERNEL);
1510 if (!req) {
1511 ret = -ENOMEM;
1512 goto fail_free_cmds_pool;
1513 }
743e1c8f
AP		 1514 INIT_LIST_HEAD(&req->node);
1515 req->sba = sba;
57a28508 1516 req->flags = SBA_REQUEST_STATE_FREE;
743e1c8f 1517 INIT_LIST_HEAD(&req->next);
743e1c8f 1518 atomic_set(&req->next_pending_count, 0);
743e1c8f
AP		 1519 for (j = 0; j < sba->max_cmd_per_req; j++) {
1520 req->cmds[j].cmd = 0;
1521 req->cmds[j].cmd_dma = sba->cmds_base +
1522 (i * sba->max_cmd_per_req + j) * sizeof(u64);
1523 req->cmds[j].cmd_dma_addr = sba->cmds_dma_base +
1524 (i * sba->max_cmd_per_req + j) * sizeof(u64);
1525 req->cmds[j].flags = 0;
1526 }
1527 memset(&req->msg, 0, sizeof(req->msg));
1528 dma_async_tx_descriptor_init(&req->tx, &sba->dma_chan);
1529 req->tx.tx_submit = sba_tx_submit;
e7ae72aa 1530 req->tx.phys = sba->resp_dma_base + i * sba->hw_resp_size;
743e1c8f
AP		 1531 list_add_tail(&req->node, &sba->reqs_free_list);
1532 }
1533
743e1c8f
AP		 1534 return 0;
1535
1536fail_free_cmds_pool:
1537 dma_free_coherent(sba->dma_dev.dev,
1538 sba->max_cmds_pool_size,
1539 sba->cmds_base, sba->cmds_dma_base);
1540fail_free_resp_pool:
1541 dma_free_coherent(sba->dma_dev.dev,
1542 sba->max_resp_pool_size,
1543 sba->resp_base, sba->resp_dma_base);
1544 return ret;
1545}
1546
1547static void sba_freeup_channel_resources(struct sba_device *sba)
1548{
1549 dmaengine_terminate_all(&sba->dma_chan);
1550 dma_free_coherent(sba->dma_dev.dev, sba->max_cmds_pool_size,
1551 sba->cmds_base, sba->cmds_dma_base);
1552 dma_free_coherent(sba->dma_dev.dev, sba->max_resp_pool_size,
1553 sba->resp_base, sba->resp_dma_base);
1554 sba->resp_base = NULL;
1555 sba->resp_dma_base = 0;
1556}
1557
1558static int sba_async_register(struct sba_device *sba)
1559{
1560 int ret;
1561 struct dma_device *dma_dev = &sba->dma_dev;
1562
1563 /* Initialize DMA channel cookie */
1564 sba->dma_chan.device = dma_dev;
1565 dma_cookie_init(&sba->dma_chan);
1566
1567 /* Initialize DMA device capability mask */
1568 dma_cap_zero(dma_dev->cap_mask);
1569 dma_cap_set(DMA_INTERRUPT, dma_dev->cap_mask);
1570 dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
1571 dma_cap_set(DMA_XOR, dma_dev->cap_mask);
1572 dma_cap_set(DMA_PQ, dma_dev->cap_mask);
1573
1574 /*
1575 * Set mailbox channel device as the base device of
1576 * our dma_device because the actual memory accesses
1577 * will be done by mailbox controller
1578 */
1579 dma_dev->dev = sba->mbox_dev;
1580
1581 /* Set base prep routines */
1582 dma_dev->device_free_chan_resources = sba_free_chan_resources;
1583 dma_dev->device_terminate_all = sba_device_terminate_all;
1584 dma_dev->device_issue_pending = sba_issue_pending;
1585 dma_dev->device_tx_status = sba_tx_status;
1586
1587 /* Set interrupt routine */
1588 if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask))
1589 dma_dev->device_prep_dma_interrupt = sba_prep_dma_interrupt;
1590
1591 /* Set memcpy routine */
1592 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask))
1593 dma_dev->device_prep_dma_memcpy = sba_prep_dma_memcpy;
1594
1595 /* Set xor routine and capability */
1596 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
1597 dma_dev->device_prep_dma_xor = sba_prep_dma_xor;
1598 dma_dev->max_xor = sba->max_xor_srcs;
1599 }
1600
1601 /* Set pq routine and capability */
1602 if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
1603 dma_dev->device_prep_dma_pq = sba_prep_dma_pq;
1604 dma_set_maxpq(dma_dev, sba->max_pq_srcs, 0);
1605 }
1606
1607 /* Initialize DMA device channel list */
1608 INIT_LIST_HEAD(&dma_dev->channels);
1609 list_add_tail(&sba->dma_chan.device_node, &dma_dev->channels);
1610
1611 /* Register with Linux async DMA framework*/
1612 ret = dma_async_device_register(dma_dev);
1613 if (ret) {
1614 dev_err(sba->dev, "async device register error %d", ret);
1615 return ret;
1616 }
1617
1618 dev_info(sba->dev, "%s capabilities: %s%s%s%s\n",
1619 dma_chan_name(&sba->dma_chan),
1620 dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "interrupt " : "",
1621 dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "memcpy " : "",
1622 dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "",
1623 dma_has_cap(DMA_PQ, dma_dev->cap_mask) ? "pq " : "");
1624
1625 return 0;
1626}
1627
1628static int sba_probe(struct platform_device *pdev)
1629{
1630 int i, ret = 0, mchans_count;
1631 struct sba_device *sba;
1632 struct platform_device *mbox_pdev;
1633 struct of_phandle_args args;
1634
1635 /* Allocate main SBA struct */
1636 sba = devm_kzalloc(&pdev->dev, sizeof(*sba), GFP_KERNEL);
1637 if (!sba)
1638 return -ENOMEM;
1639
1640 sba->dev = &pdev->dev;
1641 platform_set_drvdata(pdev, sba);
1642
5346aafc
AP		 1643 /* Number of channels equals number of mailbox channels */
1644 ret = of_count_phandle_with_args(pdev->dev.of_node,
1645 "mboxes", "#mbox-cells");
1646 if (ret <= 0)
1647 return -ENODEV;
1648 mchans_count = ret;
1649
743e1c8f
AP		 1650 /* Determine SBA version from DT compatible string */
1651 if (of_device_is_compatible(sba->dev->of_node, "brcm,iproc-sba"))
1652 sba->ver = SBA_VER_1;
1653 else if (of_device_is_compatible(sba->dev->of_node,
1654 "brcm,iproc-sba-v2"))
1655 sba->ver = SBA_VER_2;
1656 else
1657 return -ENODEV;
1658
1659 /* Derived Configuration parameters */
1660 switch (sba->ver) {
1661 case SBA_VER_1:
743e1c8f
AP		 1662 sba->hw_buf_size = 4096;
1663 sba->hw_resp_size = 8;
1664 sba->max_pq_coefs = 6;
1665 sba->max_pq_srcs = 6;
1666 break;
1667 case SBA_VER_2:
743e1c8f
AP		 1668 sba->hw_buf_size = 4096;
1669 sba->hw_resp_size = 8;
1670 sba->max_pq_coefs = 30;
1671 /*
1672 * We can support max_pq_srcs == max_pq_coefs because
1673 * we are limited by number of SBA commands that we can
1674 * fit in one message for underlying ring manager HW.
1675 */
1676 sba->max_pq_srcs = 12;
1677 break;
1678 default:
1679 return -EINVAL;
1680 }
5346aafc 1681 sba->max_req = SBA_MAX_REQ_PER_MBOX_CHANNEL * mchans_count;
743e1c8f
AP		 1682 sba->max_cmd_per_req = sba->max_pq_srcs + 3;
1683 sba->max_xor_srcs = sba->max_cmd_per_req - 1;
1684 sba->max_resp_pool_size = sba->max_req * sba->hw_resp_size;
1685 sba->max_cmds_pool_size = sba->max_req *
1686 sba->max_cmd_per_req * sizeof(u64);
1687
1688 /* Setup mailbox client */
1689 sba->client.dev = &pdev->dev;
1690 sba->client.rx_callback = sba_receive_message;
1691 sba->client.tx_block = false;
1692 sba->client.knows_txdone = false;
1693 sba->client.tx_tout = 0;
1694
743e1c8f 1695 /* Allocate mailbox channel array */
5346aafc 1696 sba->mchans = devm_kcalloc(&pdev->dev, mchans_count,
743e1c8f
AP		 1697 sizeof(*sba->mchans), GFP_KERNEL);
1698 if (!sba->mchans)
1699 return -ENOMEM;
1700
1701 /* Request mailbox channels */
5346aafc 1702 sba->mchans_count = 0;
743e1c8f
AP		 1703 for (i = 0; i < mchans_count; i++) {
1704 sba->mchans[i] = mbox_request_channel(&sba->client, i);
1705 if (IS_ERR(sba->mchans[i])) {
1706 ret = PTR_ERR(sba->mchans[i]);
1707 goto fail_free_mchans;
1708 }
1709 sba->mchans_count++;
1710 }
5346aafc 1711 atomic_set(&sba->mchans_current, 0);
743e1c8f
AP		 1712
1713 /* Find-out underlying mailbox device */
1714 ret = of_parse_phandle_with_args(pdev->dev.of_node,
1715 "mboxes", "#mbox-cells", 0, &args);
1716 if (ret)
1717 goto fail_free_mchans;
1718 mbox_pdev = of_find_device_by_node(args.np);
1719 of_node_put(args.np);
1720 if (!mbox_pdev) {
1721 ret = -ENODEV;
1722 goto fail_free_mchans;
1723 }
1724 sba->mbox_dev = &mbox_pdev->dev;
1725
1726 /* All mailbox channels should be of same ring manager device */
1727 for (i = 1; i < mchans_count; i++) {
1728 ret = of_parse_phandle_with_args(pdev->dev.of_node,
1729 "mboxes", "#mbox-cells", i, &args);
1730 if (ret)
1731 goto fail_free_mchans;
1732 mbox_pdev = of_find_device_by_node(args.np);
1733 of_node_put(args.np);
1734 if (sba->mbox_dev != &mbox_pdev->dev) {
1735 ret = -EINVAL;
1736 goto fail_free_mchans;
1737 }
1738 }
1739
1740 /* Register DMA device with linux async framework */
1741 ret = sba_async_register(sba);
1742 if (ret)
1743 goto fail_free_mchans;
1744
1745 /* Prealloc channel resource */
1746 ret = sba_prealloc_channel_resources(sba);
1747 if (ret)
1748 goto fail_async_dev_unreg;
1749
1750 /* Print device info */
1751 dev_info(sba->dev, "%s using SBAv%d and %d mailbox channels",
1752 dma_chan_name(&sba->dma_chan), sba->ver+1,
1753 sba->mchans_count);
1754
1755 return 0;
1756
1757fail_async_dev_unreg:
1758 dma_async_device_unregister(&sba->dma_dev);
1759fail_free_mchans:
1760 for (i = 0; i < sba->mchans_count; i++)
1761 mbox_free_channel(sba->mchans[i]);
1762 return ret;
1763}
1764
1765static int sba_remove(struct platform_device *pdev)
1766{
1767 int i;
1768 struct sba_device *sba = platform_get_drvdata(pdev);
1769
1770 sba_freeup_channel_resources(sba);
1771
1772 dma_async_device_unregister(&sba->dma_dev);
1773
1774 for (i = 0; i < sba->mchans_count; i++)
1775 mbox_free_channel(sba->mchans[i]);
1776
1777 return 0;
1778}
1779
1780static const struct of_device_id sba_of_match[] = {
1781 { .compatible = "brcm,iproc-sba", },
1782 { .compatible = "brcm,iproc-sba-v2", },
1783 {},
1784};
1785MODULE_DEVICE_TABLE(of, sba_of_match);
1786
1787static struct platform_driver sba_driver = {
1788 .probe = sba_probe,
1789 .remove = sba_remove,
1790 .driver = {
1791 .name = "bcm-sba-raid",
1792 .of_match_table = sba_of_match,
1793 },
1794};
1795module_platform_driver(sba_driver);
1796
1797MODULE_DESCRIPTION("Broadcom SBA RAID driver");
1798MODULE_AUTHOR("Anup Patel <anup.patel@broadcom.com>");
1799MODULE_LICENSE("GPL v2");
Linux 6.x block layer and io_uring tree(s)