projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
dmaengine: bcm-sba-raid: Minor improvments in comments
[linux-block.git] / drivers / dma / bcm-sba-raid.c
CommitLineData
743e1c8f
AP		 1/*
2 * Copyright (C) 2017 Broadcom
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 */
8
9/*
10 * Broadcom SBA RAID Driver
11 *
12 * The Broadcom stream buffer accelerator (SBA) provides offloading
13 * capabilities for RAID operations. The SBA offload engine is accessible
14 * via Broadcom SoC specific ring manager. Two or more offload engines
15 * can share same Broadcom SoC specific ring manager due to this Broadcom
16 * SoC specific ring manager driver is implemented as a mailbox controller
17 * driver and offload engine drivers are implemented as mallbox clients.
18 *
19 * Typically, Broadcom SoC specific ring manager will implement larger
20 * number of hardware rings over one or more SBA hardware devices. By
21 * design, the internal buffer size of SBA hardware device is limited
22 * but all offload operations supported by SBA can be broken down into
23 * multiple small size requests and executed parallely on multiple SBA
24 * hardware devices for achieving high through-put.
25 *
26 * The Broadcom SBA RAID driver does not require any register programming
27 * except submitting request to SBA hardware device via mailbox channels.
28 * This driver implements a DMA device with one DMA channel using a set
29 * of mailbox channels provided by Broadcom SoC specific ring manager
30 * driver. To exploit parallelism (as described above), all DMA request
31 * coming to SBA RAID DMA channel are broken down to smaller requests
32 * and submitted to multiple mailbox channels in round-robin fashion.
33 * For having more SBA DMA channels, we can create more SBA device nodes
34 * in Broadcom SoC specific DTS based on number of hardware rings supported
35 * by Broadcom SoC ring manager.
36 */
37
38#include <linux/bitops.h>
39#include <linux/dma-mapping.h>
40#include <linux/dmaengine.h>
41#include <linux/list.h>
42#include <linux/mailbox_client.h>
43#include <linux/mailbox/brcm-message.h>
44#include <linux/module.h>
45#include <linux/of_device.h>
46#include <linux/slab.h>
47#include <linux/raid/pq.h>
48
49#include "dmaengine.h"
50
e897091a
AP		 51/* ====== Driver macros and defines ===== */
52
743e1c8f
AP		 53#define SBA_TYPE_SHIFT 48
54#define SBA_TYPE_MASK GENMASK(1, 0)
55#define SBA_TYPE_A 0x0
56#define SBA_TYPE_B 0x2
57#define SBA_TYPE_C 0x3
58#define SBA_USER_DEF_SHIFT 32
59#define SBA_USER_DEF_MASK GENMASK(15, 0)
60#define SBA_R_MDATA_SHIFT 24
61#define SBA_R_MDATA_MASK GENMASK(7, 0)
62#define SBA_C_MDATA_MS_SHIFT 18
63#define SBA_C_MDATA_MS_MASK GENMASK(1, 0)
64#define SBA_INT_SHIFT 17
65#define SBA_INT_MASK BIT(0)
66#define SBA_RESP_SHIFT 16
67#define SBA_RESP_MASK BIT(0)
68#define SBA_C_MDATA_SHIFT 8
69#define SBA_C_MDATA_MASK GENMASK(7, 0)
70#define SBA_C_MDATA_BNUMx_SHIFT(__bnum) (2 * (__bnum))
71#define SBA_C_MDATA_BNUMx_MASK GENMASK(1, 0)
72#define SBA_C_MDATA_DNUM_SHIFT 5
73#define SBA_C_MDATA_DNUM_MASK GENMASK(4, 0)
74#define SBA_C_MDATA_LS(__v) ((__v) & 0xff)
75#define SBA_C_MDATA_MS(__v) (((__v) >> 8) & 0x3)
76#define SBA_CMD_SHIFT 0
77#define SBA_CMD_MASK GENMASK(3, 0)
78#define SBA_CMD_ZERO_BUFFER 0x4
79#define SBA_CMD_ZERO_ALL_BUFFERS 0x8
80#define SBA_CMD_LOAD_BUFFER 0x9
81#define SBA_CMD_XOR 0xa
82#define SBA_CMD_GALOIS_XOR 0xb
83#define SBA_CMD_WRITE_BUFFER 0xc
84#define SBA_CMD_GALOIS 0xe
85
86/* Driver helper macros */
87#define to_sba_request(tx) \
88 container_of(tx, struct sba_request, tx)
89#define to_sba_device(dchan) \
90 container_of(dchan, struct sba_device, dma_chan)
91
e897091a
AP		 92/* ===== Driver data structures ===== */
93
743e1c8f
AP		 94enum sba_request_state {
95 SBA_REQUEST_STATE_FREE = 1,
96 SBA_REQUEST_STATE_ALLOCED = 2,
97 SBA_REQUEST_STATE_PENDING = 3,
98 SBA_REQUEST_STATE_ACTIVE = 4,
99 SBA_REQUEST_STATE_RECEIVED = 5,
100 SBA_REQUEST_STATE_COMPLETED = 6,
101 SBA_REQUEST_STATE_ABORTED = 7,
102};
103
104struct sba_request {
105 /* Global state */
106 struct list_head node;
107 struct sba_device *sba;
108 enum sba_request_state state;
109 bool fence;
110 /* Chained requests management */
111 struct sba_request *first;
112 struct list_head next;
113 unsigned int next_count;
114 atomic_t next_pending_count;
115 /* BRCM message data */
116 void *resp;
117 dma_addr_t resp_dma;
118 struct brcm_sba_command *cmds;
119 struct brcm_message msg;
120 struct dma_async_tx_descriptor tx;
121};
122
123enum sba_version {
124 SBA_VER_1 = 0,
125 SBA_VER_2
126};
127
128struct sba_device {
129 /* Underlying device */
130 struct device *dev;
131 /* DT configuration parameters */
132 enum sba_version ver;
133 /* Derived configuration parameters */
134 u32 max_req;
135 u32 hw_buf_size;
136 u32 hw_resp_size;
137 u32 max_pq_coefs;
138 u32 max_pq_srcs;
139 u32 max_cmd_per_req;
140 u32 max_xor_srcs;
141 u32 max_resp_pool_size;
142 u32 max_cmds_pool_size;
143 /* Maibox client and Mailbox channels */
144 struct mbox_client client;
145 int mchans_count;
146 atomic_t mchans_current;
147 struct mbox_chan **mchans;
148 struct device *mbox_dev;
149 /* DMA device and DMA channel */
150 struct dma_device dma_dev;
151 struct dma_chan dma_chan;
152 /* DMA channel resources */
153 void *resp_base;
154 dma_addr_t resp_dma_base;
155 void *cmds_base;
156 dma_addr_t cmds_dma_base;
157 spinlock_t reqs_lock;
158 struct sba_request *reqs;
159 bool reqs_fence;
160 struct list_head reqs_alloc_list;
161 struct list_head reqs_pending_list;
162 struct list_head reqs_active_list;
163 struct list_head reqs_received_list;
164 struct list_head reqs_completed_list;
165 struct list_head reqs_aborted_list;
166 struct list_head reqs_free_list;
167 int reqs_free_count;
168};
169
e897091a 170/* ====== Command helper routines ===== */
743e1c8f
AP		 171
172static inline u64 __pure sba_cmd_enc(u64 cmd, u32 val, u32 shift, u32 mask)
173{
174 cmd &= ~((u64)mask << shift);
175 cmd |= ((u64)(val & mask) << shift);
176 return cmd;
177}
178
179static inline u32 __pure sba_cmd_load_c_mdata(u32 b0)
180{
181 return b0 & SBA_C_MDATA_BNUMx_MASK;
182}
183
184static inline u32 __pure sba_cmd_write_c_mdata(u32 b0)
185{
186 return b0 & SBA_C_MDATA_BNUMx_MASK;
187}
188
189static inline u32 __pure sba_cmd_xor_c_mdata(u32 b1, u32 b0)
190{
191 return (b0 & SBA_C_MDATA_BNUMx_MASK) |
192 ((b1 & SBA_C_MDATA_BNUMx_MASK) << SBA_C_MDATA_BNUMx_SHIFT(1));
193}
194
195static inline u32 __pure sba_cmd_pq_c_mdata(u32 d, u32 b1, u32 b0)
196{
197 return (b0 & SBA_C_MDATA_BNUMx_MASK) |
198 ((b1 & SBA_C_MDATA_BNUMx_MASK) << SBA_C_MDATA_BNUMx_SHIFT(1)) |
199 ((d & SBA_C_MDATA_DNUM_MASK) << SBA_C_MDATA_DNUM_SHIFT);
200}
201
e897091a 202/* ====== General helper routines ===== */
743e1c8f
AP		 203
204static struct sba_request *sba_alloc_request(struct sba_device *sba)
205{
206 unsigned long flags;
207 struct sba_request *req = NULL;
208
209 spin_lock_irqsave(&sba->reqs_lock, flags);
210
211 req = list_first_entry_or_null(&sba->reqs_free_list,
212 struct sba_request, node);
213 if (req) {
214 list_move_tail(&req->node, &sba->reqs_alloc_list);
215 req->state = SBA_REQUEST_STATE_ALLOCED;
216 req->fence = false;
217 req->first = req;
218 INIT_LIST_HEAD(&req->next);
219 req->next_count = 1;
220 atomic_set(&req->next_pending_count, 1);
221
222 sba->reqs_free_count--;
223
224 dma_async_tx_descriptor_init(&req->tx, &sba->dma_chan);
225 }
226
227 spin_unlock_irqrestore(&sba->reqs_lock, flags);
228
229 return req;
230}
231
232/* Note: Must be called with sba->reqs_lock held */
233static void _sba_pending_request(struct sba_device *sba,
234 struct sba_request *req)
235{
236 lockdep_assert_held(&sba->reqs_lock);
237 req->state = SBA_REQUEST_STATE_PENDING;
238 list_move_tail(&req->node, &sba->reqs_pending_list);
239 if (list_empty(&sba->reqs_active_list))
240 sba->reqs_fence = false;
241}
242
243/* Note: Must be called with sba->reqs_lock held */
244static bool _sba_active_request(struct sba_device *sba,
245 struct sba_request *req)
246{
247 lockdep_assert_held(&sba->reqs_lock);
248 if (list_empty(&sba->reqs_active_list))
249 sba->reqs_fence = false;
250 if (sba->reqs_fence)
251 return false;
252 req->state = SBA_REQUEST_STATE_ACTIVE;
253 list_move_tail(&req->node, &sba->reqs_active_list);
254 if (req->fence)
255 sba->reqs_fence = true;
256 return true;
257}
258
259/* Note: Must be called with sba->reqs_lock held */
260static void _sba_abort_request(struct sba_device *sba,
261 struct sba_request *req)
262{
263 lockdep_assert_held(&sba->reqs_lock);
264 req->state = SBA_REQUEST_STATE_ABORTED;
265 list_move_tail(&req->node, &sba->reqs_aborted_list);
266 if (list_empty(&sba->reqs_active_list))
267 sba->reqs_fence = false;
268}
269
270/* Note: Must be called with sba->reqs_lock held */
271static void _sba_free_request(struct sba_device *sba,
272 struct sba_request *req)
273{
274 lockdep_assert_held(&sba->reqs_lock);
275 req->state = SBA_REQUEST_STATE_FREE;
276 list_move_tail(&req->node, &sba->reqs_free_list);
277 if (list_empty(&sba->reqs_active_list))
278 sba->reqs_fence = false;
279 sba->reqs_free_count++;
280}
281
282static void sba_received_request(struct sba_request *req)
283{
284 unsigned long flags;
285 struct sba_device *sba = req->sba;
286
287 spin_lock_irqsave(&sba->reqs_lock, flags);
288 req->state = SBA_REQUEST_STATE_RECEIVED;
289 list_move_tail(&req->node, &sba->reqs_received_list);
290 spin_unlock_irqrestore(&sba->reqs_lock, flags);
291}
292
293static void sba_complete_chained_requests(struct sba_request *req)
294{
295 unsigned long flags;
296 struct sba_request *nreq;
297 struct sba_device *sba = req->sba;
298
299 spin_lock_irqsave(&sba->reqs_lock, flags);
300
301 req->state = SBA_REQUEST_STATE_COMPLETED;
302 list_move_tail(&req->node, &sba->reqs_completed_list);
303 list_for_each_entry(nreq, &req->next, next) {
304 nreq->state = SBA_REQUEST_STATE_COMPLETED;
305 list_move_tail(&nreq->node, &sba->reqs_completed_list);
306 }
307 if (list_empty(&sba->reqs_active_list))
308 sba->reqs_fence = false;
309
310 spin_unlock_irqrestore(&sba->reqs_lock, flags);
311}
312
313static void sba_free_chained_requests(struct sba_request *req)
314{
315 unsigned long flags;
316 struct sba_request *nreq;
317 struct sba_device *sba = req->sba;
318
319 spin_lock_irqsave(&sba->reqs_lock, flags);
320
321 _sba_free_request(sba, req);
322 list_for_each_entry(nreq, &req->next, next)
323 _sba_free_request(sba, nreq);
324
325 spin_unlock_irqrestore(&sba->reqs_lock, flags);
326}
327
328static void sba_chain_request(struct sba_request *first,
329 struct sba_request *req)
330{
331 unsigned long flags;
332 struct sba_device *sba = req->sba;
333
334 spin_lock_irqsave(&sba->reqs_lock, flags);
335
336 list_add_tail(&req->next, &first->next);
337 req->first = first;
338 first->next_count++;
339 atomic_set(&first->next_pending_count, first->next_count);
340
341 spin_unlock_irqrestore(&sba->reqs_lock, flags);
342}
343
344static void sba_cleanup_nonpending_requests(struct sba_device *sba)
345{
346 unsigned long flags;
347 struct sba_request *req, *req1;
348
349 spin_lock_irqsave(&sba->reqs_lock, flags);
350
351 /* Freeup all alloced request */
352 list_for_each_entry_safe(req, req1, &sba->reqs_alloc_list, node)
353 _sba_free_request(sba, req);
354
355 /* Freeup all received request */
356 list_for_each_entry_safe(req, req1, &sba->reqs_received_list, node)
357 _sba_free_request(sba, req);
358
359 /* Freeup all completed request */
360 list_for_each_entry_safe(req, req1, &sba->reqs_completed_list, node)
361 _sba_free_request(sba, req);
362
363 /* Set all active requests as aborted */
364 list_for_each_entry_safe(req, req1, &sba->reqs_active_list, node)
365 _sba_abort_request(sba, req);
366
367 /*
368 * Note: We expect that aborted request will be eventually
369 * freed by sba_receive_message()
370 */
371
372 spin_unlock_irqrestore(&sba->reqs_lock, flags);
373}
374
375static void sba_cleanup_pending_requests(struct sba_device *sba)
376{
377 unsigned long flags;
378 struct sba_request *req, *req1;
379
380 spin_lock_irqsave(&sba->reqs_lock, flags);
381
382 /* Freeup all pending request */
383 list_for_each_entry_safe(req, req1, &sba->reqs_pending_list, node)
384 _sba_free_request(sba, req);
385
386 spin_unlock_irqrestore(&sba->reqs_lock, flags);
387}
388
389/* ====== DMAENGINE callbacks ===== */
390
391static void sba_free_chan_resources(struct dma_chan *dchan)
392{
393 /*
394 * Channel resources are pre-alloced so we just free-up
395 * whatever we can so that we can re-use pre-alloced
396 * channel resources next time.
397 */
398 sba_cleanup_nonpending_requests(to_sba_device(dchan));
399}
400
401static int sba_device_terminate_all(struct dma_chan *dchan)
402{
403 /* Cleanup all pending requests */
404 sba_cleanup_pending_requests(to_sba_device(dchan));
405
406 return 0;
407}
408
409static int sba_send_mbox_request(struct sba_device *sba,
410 struct sba_request *req)
411{
412 int mchans_idx, ret = 0;
413
414 /* Select mailbox channel in round-robin fashion */
415 mchans_idx = atomic_inc_return(&sba->mchans_current);
416 mchans_idx = mchans_idx % sba->mchans_count;
417
418 /* Send message for the request */
419 req->msg.error = 0;
420 ret = mbox_send_message(sba->mchans[mchans_idx], &req->msg);
421 if (ret < 0) {
422 dev_err(sba->dev, "send message failed with error %d", ret);
423 return ret;
424 }
425 ret = req->msg.error;
426 if (ret < 0) {
427 dev_err(sba->dev, "message error %d", ret);
428 return ret;
429 }
430
431 return 0;
432}
433
434static void sba_issue_pending(struct dma_chan *dchan)
435{
436 int ret;
437 unsigned long flags;
438 struct sba_request *req, *req1;
439 struct sba_device *sba = to_sba_device(dchan);
440
441 spin_lock_irqsave(&sba->reqs_lock, flags);
442
443 /* Process all pending request */
444 list_for_each_entry_safe(req, req1, &sba->reqs_pending_list, node) {
445 /* Try to make request active */
446 if (!_sba_active_request(sba, req))
447 break;
448
449 /* Send request to mailbox channel */
450 spin_unlock_irqrestore(&sba->reqs_lock, flags);
451 ret = sba_send_mbox_request(sba, req);
452 spin_lock_irqsave(&sba->reqs_lock, flags);
453
454 /* If something went wrong then keep request pending */
455 if (ret < 0) {
456 _sba_pending_request(sba, req);
457 break;
458 }
459 }
460
461 spin_unlock_irqrestore(&sba->reqs_lock, flags);
462}
463
464static dma_cookie_t sba_tx_submit(struct dma_async_tx_descriptor *tx)
465{
466 unsigned long flags;
467 dma_cookie_t cookie;
468 struct sba_device *sba;
469 struct sba_request *req, *nreq;
470
471 if (unlikely(!tx))
472 return -EINVAL;
473
474 sba = to_sba_device(tx->chan);
475 req = to_sba_request(tx);
476
477 /* Assign cookie and mark all chained requests pending */
478 spin_lock_irqsave(&sba->reqs_lock, flags);
479 cookie = dma_cookie_assign(tx);
480 _sba_pending_request(sba, req);
481 list_for_each_entry(nreq, &req->next, next)
482 _sba_pending_request(sba, nreq);
483 spin_unlock_irqrestore(&sba->reqs_lock, flags);
484
485 return cookie;
486}
487
488static enum dma_status sba_tx_status(struct dma_chan *dchan,
489 dma_cookie_t cookie,
490 struct dma_tx_state *txstate)
491{
492 int mchan_idx;
493 enum dma_status ret;
494 struct sba_device *sba = to_sba_device(dchan);
495
496 for (mchan_idx = 0; mchan_idx < sba->mchans_count; mchan_idx++)
497 mbox_client_peek_data(sba->mchans[mchan_idx]);
498
499 ret = dma_cookie_status(dchan, cookie, txstate);
500 if (ret == DMA_COMPLETE)
501 return ret;
502
503 return dma_cookie_status(dchan, cookie, txstate);
504}
505
506static void sba_fillup_interrupt_msg(struct sba_request *req,
507 struct brcm_sba_command *cmds,
508 struct brcm_message *msg)
509{
510 u64 cmd;
511 u32 c_mdata;
512 struct brcm_sba_command *cmdsp = cmds;
513
514 /* Type-B command to load dummy data into buf0 */
515 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
516 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
517 cmd = sba_cmd_enc(cmd, req->sba->hw_resp_size,
518 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
519 c_mdata = sba_cmd_load_c_mdata(0);
520 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
521 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
522 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
523 SBA_CMD_SHIFT, SBA_CMD_MASK);
524 cmdsp->cmd = cmd;
525 *cmdsp->cmd_dma = cpu_to_le64(cmd);
526 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
527 cmdsp->data = req->resp_dma;
528 cmdsp->data_len = req->sba->hw_resp_size;
529 cmdsp++;
530
531 /* Type-A command to write buf0 to dummy location */
532 cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
533 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
534 cmd = sba_cmd_enc(cmd, req->sba->hw_resp_size,
535 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
536 cmd = sba_cmd_enc(cmd, 0x1,
537 SBA_RESP_SHIFT, SBA_RESP_MASK);
538 c_mdata = sba_cmd_write_c_mdata(0);
539 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
540 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
541 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
542 SBA_CMD_SHIFT, SBA_CMD_MASK);
543 cmdsp->cmd = cmd;
544 *cmdsp->cmd_dma = cpu_to_le64(cmd);
545 cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
546 if (req->sba->hw_resp_size) {
547 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
548 cmdsp->resp = req->resp_dma;
549 cmdsp->resp_len = req->sba->hw_resp_size;
550 }
551 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
552 cmdsp->data = req->resp_dma;
553 cmdsp->data_len = req->sba->hw_resp_size;
554 cmdsp++;
555
556 /* Fillup brcm_message */
557 msg->type = BRCM_MESSAGE_SBA;
558 msg->sba.cmds = cmds;
559 msg->sba.cmds_count = cmdsp - cmds;
560 msg->ctx = req;
561 msg->error = 0;
562}
563
564static struct dma_async_tx_descriptor *
565sba_prep_dma_interrupt(struct dma_chan *dchan, unsigned long flags)
566{
567 struct sba_request *req = NULL;
568 struct sba_device *sba = to_sba_device(dchan);
569
570 /* Alloc new request */
571 req = sba_alloc_request(sba);
572 if (!req)
573 return NULL;
574
575 /*
576 * Force fence so that no requests are submitted
577 * until DMA callback for this request is invoked.
578 */
579 req->fence = true;
580
581 /* Fillup request message */
582 sba_fillup_interrupt_msg(req, req->cmds, &req->msg);
583
584 /* Init async_tx descriptor */
585 req->tx.flags = flags;
586 req->tx.cookie = -EBUSY;
587
1fc63cb4 588 return &req->tx;
743e1c8f
AP		 589}
590
591static void sba_fillup_memcpy_msg(struct sba_request *req,
592 struct brcm_sba_command *cmds,
593 struct brcm_message *msg,
594 dma_addr_t msg_offset, size_t msg_len,
595 dma_addr_t dst, dma_addr_t src)
596{
597 u64 cmd;
598 u32 c_mdata;
599 struct brcm_sba_command *cmdsp = cmds;
600
601 /* Type-B command to load data into buf0 */
602 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
603 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
604 cmd = sba_cmd_enc(cmd, msg_len,
605 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
606 c_mdata = sba_cmd_load_c_mdata(0);
607 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
608 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
609 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
610 SBA_CMD_SHIFT, SBA_CMD_MASK);
611 cmdsp->cmd = cmd;
612 *cmdsp->cmd_dma = cpu_to_le64(cmd);
613 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
614 cmdsp->data = src + msg_offset;
615 cmdsp->data_len = msg_len;
616 cmdsp++;
617
618 /* Type-A command to write buf0 */
619 cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
620 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
621 cmd = sba_cmd_enc(cmd, msg_len,
622 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
623 cmd = sba_cmd_enc(cmd, 0x1,
624 SBA_RESP_SHIFT, SBA_RESP_MASK);
625 c_mdata = sba_cmd_write_c_mdata(0);
626 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
627 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
628 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
629 SBA_CMD_SHIFT, SBA_CMD_MASK);
630 cmdsp->cmd = cmd;
631 *cmdsp->cmd_dma = cpu_to_le64(cmd);
632 cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
633 if (req->sba->hw_resp_size) {
634 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
635 cmdsp->resp = req->resp_dma;
636 cmdsp->resp_len = req->sba->hw_resp_size;
637 }
638 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
639 cmdsp->data = dst + msg_offset;
640 cmdsp->data_len = msg_len;
641 cmdsp++;
642
643 /* Fillup brcm_message */
644 msg->type = BRCM_MESSAGE_SBA;
645 msg->sba.cmds = cmds;
646 msg->sba.cmds_count = cmdsp - cmds;
647 msg->ctx = req;
648 msg->error = 0;
649}
650
651static struct sba_request *
652sba_prep_dma_memcpy_req(struct sba_device *sba,
653 dma_addr_t off, dma_addr_t dst, dma_addr_t src,
654 size_t len, unsigned long flags)
655{
656 struct sba_request *req = NULL;
657
658 /* Alloc new request */
659 req = sba_alloc_request(sba);
660 if (!req)
661 return NULL;
662 req->fence = (flags & DMA_PREP_FENCE) ? true : false;
663
664 /* Fillup request message */
665 sba_fillup_memcpy_msg(req, req->cmds, &req->msg,
666 off, len, dst, src);
667
668 /* Init async_tx descriptor */
669 req->tx.flags = flags;
670 req->tx.cookie = -EBUSY;
671
672 return req;
673}
674
675static struct dma_async_tx_descriptor *
676sba_prep_dma_memcpy(struct dma_chan *dchan, dma_addr_t dst, dma_addr_t src,
677 size_t len, unsigned long flags)
678{
679 size_t req_len;
680 dma_addr_t off = 0;
681 struct sba_device *sba = to_sba_device(dchan);
682 struct sba_request *first = NULL, *req;
683
684 /* Create chained requests where each request is upto hw_buf_size */
685 while (len) {
686 req_len = (len < sba->hw_buf_size) ? len : sba->hw_buf_size;
687
688 req = sba_prep_dma_memcpy_req(sba, off, dst, src,
689 req_len, flags);
690 if (!req) {
691 if (first)
692 sba_free_chained_requests(first);
693 return NULL;
694 }
695
696 if (first)
697 sba_chain_request(first, req);
698 else
699 first = req;
700
701 off += req_len;
702 len -= req_len;
703 }
704
705 return (first) ? &first->tx : NULL;
706}
707
708static void sba_fillup_xor_msg(struct sba_request *req,
709 struct brcm_sba_command *cmds,
710 struct brcm_message *msg,
711 dma_addr_t msg_offset, size_t msg_len,
712 dma_addr_t dst, dma_addr_t *src, u32 src_cnt)
713{
714 u64 cmd;
715 u32 c_mdata;
716 unsigned int i;
717 struct brcm_sba_command *cmdsp = cmds;
718
719 /* Type-B command to load data into buf0 */
720 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
721 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
722 cmd = sba_cmd_enc(cmd, msg_len,
723 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
724 c_mdata = sba_cmd_load_c_mdata(0);
725 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
726 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
727 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
728 SBA_CMD_SHIFT, SBA_CMD_MASK);
729 cmdsp->cmd = cmd;
730 *cmdsp->cmd_dma = cpu_to_le64(cmd);
731 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
732 cmdsp->data = src[0] + msg_offset;
733 cmdsp->data_len = msg_len;
734 cmdsp++;
735
736 /* Type-B commands to xor data with buf0 and put it back in buf0 */
737 for (i = 1; i < src_cnt; i++) {
738 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
739 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
740 cmd = sba_cmd_enc(cmd, msg_len,
741 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
742 c_mdata = sba_cmd_xor_c_mdata(0, 0);
743 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
744 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
745 cmd = sba_cmd_enc(cmd, SBA_CMD_XOR,
746 SBA_CMD_SHIFT, SBA_CMD_MASK);
747 cmdsp->cmd = cmd;
748 *cmdsp->cmd_dma = cpu_to_le64(cmd);
749 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
750 cmdsp->data = src[i] + msg_offset;
751 cmdsp->data_len = msg_len;
752 cmdsp++;
753 }
754
755 /* Type-A command to write buf0 */
756 cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
757 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
758 cmd = sba_cmd_enc(cmd, msg_len,
759 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
760 cmd = sba_cmd_enc(cmd, 0x1,
761 SBA_RESP_SHIFT, SBA_RESP_MASK);
762 c_mdata = sba_cmd_write_c_mdata(0);
763 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
764 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
765 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
766 SBA_CMD_SHIFT, SBA_CMD_MASK);
767 cmdsp->cmd = cmd;
768 *cmdsp->cmd_dma = cpu_to_le64(cmd);
769 cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
770 if (req->sba->hw_resp_size) {
771 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
772 cmdsp->resp = req->resp_dma;
773 cmdsp->resp_len = req->sba->hw_resp_size;
774 }
775 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
776 cmdsp->data = dst + msg_offset;
777 cmdsp->data_len = msg_len;
778 cmdsp++;
779
780 /* Fillup brcm_message */
781 msg->type = BRCM_MESSAGE_SBA;
782 msg->sba.cmds = cmds;
783 msg->sba.cmds_count = cmdsp - cmds;
784 msg->ctx = req;
785 msg->error = 0;
786}
787
dd2bceb0 788static struct sba_request *
743e1c8f
AP		 789sba_prep_dma_xor_req(struct sba_device *sba,
790 dma_addr_t off, dma_addr_t dst, dma_addr_t *src,
791 u32 src_cnt, size_t len, unsigned long flags)
792{
793 struct sba_request *req = NULL;
794
795 /* Alloc new request */
796 req = sba_alloc_request(sba);
797 if (!req)
798 return NULL;
799 req->fence = (flags & DMA_PREP_FENCE) ? true : false;
800
801 /* Fillup request message */
802 sba_fillup_xor_msg(req, req->cmds, &req->msg,
803 off, len, dst, src, src_cnt);
804
805 /* Init async_tx descriptor */
806 req->tx.flags = flags;
807 req->tx.cookie = -EBUSY;
808
809 return req;
810}
811
812static struct dma_async_tx_descriptor *
813sba_prep_dma_xor(struct dma_chan *dchan, dma_addr_t dst, dma_addr_t *src,
814 u32 src_cnt, size_t len, unsigned long flags)
815{
816 size_t req_len;
817 dma_addr_t off = 0;
818 struct sba_device *sba = to_sba_device(dchan);
819 struct sba_request *first = NULL, *req;
820
821 /* Sanity checks */
822 if (unlikely(src_cnt > sba->max_xor_srcs))
823 return NULL;
824
825 /* Create chained requests where each request is upto hw_buf_size */
826 while (len) {
827 req_len = (len < sba->hw_buf_size) ? len : sba->hw_buf_size;
828
829 req = sba_prep_dma_xor_req(sba, off, dst, src, src_cnt,
830 req_len, flags);
831 if (!req) {
832 if (first)
833 sba_free_chained_requests(first);
834 return NULL;
835 }
836
837 if (first)
838 sba_chain_request(first, req);
839 else
840 first = req;
841
842 off += req_len;
843 len -= req_len;
844 }
845
846 return (first) ? &first->tx : NULL;
847}
848
849static void sba_fillup_pq_msg(struct sba_request *req,
850 bool pq_continue,
851 struct brcm_sba_command *cmds,
852 struct brcm_message *msg,
853 dma_addr_t msg_offset, size_t msg_len,
854 dma_addr_t *dst_p, dma_addr_t *dst_q,
855 const u8 *scf, dma_addr_t *src, u32 src_cnt)
856{
857 u64 cmd;
858 u32 c_mdata;
859 unsigned int i;
860 struct brcm_sba_command *cmdsp = cmds;
861
862 if (pq_continue) {
863 /* Type-B command to load old P into buf0 */
864 if (dst_p) {
865 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
866 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
867 cmd = sba_cmd_enc(cmd, msg_len,
868 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
869 c_mdata = sba_cmd_load_c_mdata(0);
870 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
871 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
872 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
873 SBA_CMD_SHIFT, SBA_CMD_MASK);
874 cmdsp->cmd = cmd;
875 *cmdsp->cmd_dma = cpu_to_le64(cmd);
876 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
877 cmdsp->data = *dst_p + msg_offset;
878 cmdsp->data_len = msg_len;
879 cmdsp++;
880 }
881
882 /* Type-B command to load old Q into buf1 */
883 if (dst_q) {
884 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
885 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
886 cmd = sba_cmd_enc(cmd, msg_len,
887 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
888 c_mdata = sba_cmd_load_c_mdata(1);
889 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
890 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
891 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
892 SBA_CMD_SHIFT, SBA_CMD_MASK);
893 cmdsp->cmd = cmd;
894 *cmdsp->cmd_dma = cpu_to_le64(cmd);
895 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
896 cmdsp->data = *dst_q + msg_offset;
897 cmdsp->data_len = msg_len;
898 cmdsp++;
899 }
900 } else {
901 /* Type-A command to zero all buffers */
902 cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
903 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
904 cmd = sba_cmd_enc(cmd, msg_len,
905 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
906 cmd = sba_cmd_enc(cmd, SBA_CMD_ZERO_ALL_BUFFERS,
907 SBA_CMD_SHIFT, SBA_CMD_MASK);
908 cmdsp->cmd = cmd;
909 *cmdsp->cmd_dma = cpu_to_le64(cmd);
910 cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
911 cmdsp++;
912 }
913
914 /* Type-B commands for generate P onto buf0 and Q onto buf1 */
915 for (i = 0; i < src_cnt; i++) {
916 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
917 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
918 cmd = sba_cmd_enc(cmd, msg_len,
919 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
920 c_mdata = sba_cmd_pq_c_mdata(raid6_gflog[scf[i]], 1, 0);
921 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
922 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
923 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_MS(c_mdata),
924 SBA_C_MDATA_MS_SHIFT, SBA_C_MDATA_MS_MASK);
925 cmd = sba_cmd_enc(cmd, SBA_CMD_GALOIS_XOR,
926 SBA_CMD_SHIFT, SBA_CMD_MASK);
927 cmdsp->cmd = cmd;
928 *cmdsp->cmd_dma = cpu_to_le64(cmd);
929 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
930 cmdsp->data = src[i] + msg_offset;
931 cmdsp->data_len = msg_len;
932 cmdsp++;
933 }
934
935 /* Type-A command to write buf0 */
936 if (dst_p) {
937 cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
938 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
939 cmd = sba_cmd_enc(cmd, msg_len,
940 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
941 cmd = sba_cmd_enc(cmd, 0x1,
942 SBA_RESP_SHIFT, SBA_RESP_MASK);
943 c_mdata = sba_cmd_write_c_mdata(0);
944 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
945 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
946 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
947 SBA_CMD_SHIFT, SBA_CMD_MASK);
948 cmdsp->cmd = cmd;
949 *cmdsp->cmd_dma = cpu_to_le64(cmd);
950 cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
951 if (req->sba->hw_resp_size) {
952 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
953 cmdsp->resp = req->resp_dma;
954 cmdsp->resp_len = req->sba->hw_resp_size;
955 }
956 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
957 cmdsp->data = *dst_p + msg_offset;
958 cmdsp->data_len = msg_len;
959 cmdsp++;
960 }
961
962 /* Type-A command to write buf1 */
963 if (dst_q) {
964 cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
965 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
966 cmd = sba_cmd_enc(cmd, msg_len,
967 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
968 cmd = sba_cmd_enc(cmd, 0x1,
969 SBA_RESP_SHIFT, SBA_RESP_MASK);
970 c_mdata = sba_cmd_write_c_mdata(1);
971 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
972 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
973 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
974 SBA_CMD_SHIFT, SBA_CMD_MASK);
975 cmdsp->cmd = cmd;
976 *cmdsp->cmd_dma = cpu_to_le64(cmd);
977 cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
978 if (req->sba->hw_resp_size) {
979 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
980 cmdsp->resp = req->resp_dma;
981 cmdsp->resp_len = req->sba->hw_resp_size;
982 }
983 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
984 cmdsp->data = *dst_q + msg_offset;
985 cmdsp->data_len = msg_len;
986 cmdsp++;
987 }
988
989 /* Fillup brcm_message */
990 msg->type = BRCM_MESSAGE_SBA;
991 msg->sba.cmds = cmds;
992 msg->sba.cmds_count = cmdsp - cmds;
993 msg->ctx = req;
994 msg->error = 0;
995}
996
dd2bceb0 997static struct sba_request *
743e1c8f
AP		 998sba_prep_dma_pq_req(struct sba_device *sba, dma_addr_t off,
999 dma_addr_t *dst_p, dma_addr_t *dst_q, dma_addr_t *src,
1000 u32 src_cnt, const u8 *scf, size_t len, unsigned long flags)
1001{
1002 struct sba_request *req = NULL;
1003
1004 /* Alloc new request */
1005 req = sba_alloc_request(sba);
1006 if (!req)
1007 return NULL;
1008 req->fence = (flags & DMA_PREP_FENCE) ? true : false;
1009
1010 /* Fillup request messages */
1011 sba_fillup_pq_msg(req, dmaf_continue(flags),
1012 req->cmds, &req->msg,
1013 off, len, dst_p, dst_q, scf, src, src_cnt);
1014
1015 /* Init async_tx descriptor */
1016 req->tx.flags = flags;
1017 req->tx.cookie = -EBUSY;
1018
1019 return req;
1020}
1021
1022static void sba_fillup_pq_single_msg(struct sba_request *req,
1023 bool pq_continue,
1024 struct brcm_sba_command *cmds,
1025 struct brcm_message *msg,
1026 dma_addr_t msg_offset, size_t msg_len,
1027 dma_addr_t *dst_p, dma_addr_t *dst_q,
1028 dma_addr_t src, u8 scf)
1029{
1030 u64 cmd;
1031 u32 c_mdata;
1032 u8 pos, dpos = raid6_gflog[scf];
1033 struct brcm_sba_command *cmdsp = cmds;
1034
1035 if (!dst_p)
1036 goto skip_p;
1037
1038 if (pq_continue) {
1039 /* Type-B command to load old P into buf0 */
1040 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
1041 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1042 cmd = sba_cmd_enc(cmd, msg_len,
1043 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1044 c_mdata = sba_cmd_load_c_mdata(0);
1045 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1046 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1047 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
1048 SBA_CMD_SHIFT, SBA_CMD_MASK);
1049 cmdsp->cmd = cmd;
1050 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1051 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
1052 cmdsp->data = *dst_p + msg_offset;
1053 cmdsp->data_len = msg_len;
1054 cmdsp++;
1055
1056 /*
1057 * Type-B commands to xor data with buf0 and put it
1058 * back in buf0
1059 */
1060 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
1061 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1062 cmd = sba_cmd_enc(cmd, msg_len,
1063 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1064 c_mdata = sba_cmd_xor_c_mdata(0, 0);
1065 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1066 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1067 cmd = sba_cmd_enc(cmd, SBA_CMD_XOR,
1068 SBA_CMD_SHIFT, SBA_CMD_MASK);
1069 cmdsp->cmd = cmd;
1070 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1071 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
1072 cmdsp->data = src + msg_offset;
1073 cmdsp->data_len = msg_len;
1074 cmdsp++;
1075 } else {
1076 /* Type-B command to load old P into buf0 */
1077 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
1078 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1079 cmd = sba_cmd_enc(cmd, msg_len,
1080 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1081 c_mdata = sba_cmd_load_c_mdata(0);
1082 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1083 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1084 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
1085 SBA_CMD_SHIFT, SBA_CMD_MASK);
1086 cmdsp->cmd = cmd;
1087 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1088 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
1089 cmdsp->data = src + msg_offset;
1090 cmdsp->data_len = msg_len;
1091 cmdsp++;
1092 }
1093
1094 /* Type-A command to write buf0 */
1095 cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
1096 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1097 cmd = sba_cmd_enc(cmd, msg_len,
1098 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1099 cmd = sba_cmd_enc(cmd, 0x1,
1100 SBA_RESP_SHIFT, SBA_RESP_MASK);
1101 c_mdata = sba_cmd_write_c_mdata(0);
1102 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1103 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1104 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
1105 SBA_CMD_SHIFT, SBA_CMD_MASK);
1106 cmdsp->cmd = cmd;
1107 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1108 cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
1109 if (req->sba->hw_resp_size) {
1110 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
1111 cmdsp->resp = req->resp_dma;
1112 cmdsp->resp_len = req->sba->hw_resp_size;
1113 }
1114 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
1115 cmdsp->data = *dst_p + msg_offset;
1116 cmdsp->data_len = msg_len;
1117 cmdsp++;
1118
1119skip_p:
1120 if (!dst_q)
1121 goto skip_q;
1122
1123 /* Type-A command to zero all buffers */
1124 cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
1125 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1126 cmd = sba_cmd_enc(cmd, msg_len,
1127 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1128 cmd = sba_cmd_enc(cmd, SBA_CMD_ZERO_ALL_BUFFERS,
1129 SBA_CMD_SHIFT, SBA_CMD_MASK);
1130 cmdsp->cmd = cmd;
1131 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1132 cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
1133 cmdsp++;
1134
1135 if (dpos == 255)
1136 goto skip_q_computation;
1137 pos = (dpos < req->sba->max_pq_coefs) ?
1138 dpos : (req->sba->max_pq_coefs - 1);
1139
1140 /*
1141 * Type-B command to generate initial Q from data
1142 * and store output into buf0
1143 */
1144 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
1145 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1146 cmd = sba_cmd_enc(cmd, msg_len,
1147 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1148 c_mdata = sba_cmd_pq_c_mdata(pos, 0, 0);
1149 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1150 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1151 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_MS(c_mdata),
1152 SBA_C_MDATA_MS_SHIFT, SBA_C_MDATA_MS_MASK);
1153 cmd = sba_cmd_enc(cmd, SBA_CMD_GALOIS,
1154 SBA_CMD_SHIFT, SBA_CMD_MASK);
1155 cmdsp->cmd = cmd;
1156 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1157 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
1158 cmdsp->data = src + msg_offset;
1159 cmdsp->data_len = msg_len;
1160 cmdsp++;
1161
1162 dpos -= pos;
1163
1164 /* Multiple Type-A command to generate final Q */
1165 while (dpos) {
1166 pos = (dpos < req->sba->max_pq_coefs) ?
1167 dpos : (req->sba->max_pq_coefs - 1);
1168
1169 /*
1170 * Type-A command to generate Q with buf0 and
1171 * buf1 store result in buf0
1172 */
1173 cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
1174 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1175 cmd = sba_cmd_enc(cmd, msg_len,
1176 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1177 c_mdata = sba_cmd_pq_c_mdata(pos, 0, 1);
1178 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1179 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1180 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_MS(c_mdata),
1181 SBA_C_MDATA_MS_SHIFT, SBA_C_MDATA_MS_MASK);
1182 cmd = sba_cmd_enc(cmd, SBA_CMD_GALOIS,
1183 SBA_CMD_SHIFT, SBA_CMD_MASK);
1184 cmdsp->cmd = cmd;
1185 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1186 cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
1187 cmdsp++;
1188
1189 dpos -= pos;
1190 }
1191
1192skip_q_computation:
1193 if (pq_continue) {
1194 /*
1195 * Type-B command to XOR previous output with
1196 * buf0 and write it into buf0
1197 */
1198 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
1199 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1200 cmd = sba_cmd_enc(cmd, msg_len,
1201 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1202 c_mdata = sba_cmd_xor_c_mdata(0, 0);
1203 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1204 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1205 cmd = sba_cmd_enc(cmd, SBA_CMD_XOR,
1206 SBA_CMD_SHIFT, SBA_CMD_MASK);
1207 cmdsp->cmd = cmd;
1208 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1209 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
1210 cmdsp->data = *dst_q + msg_offset;
1211 cmdsp->data_len = msg_len;
1212 cmdsp++;
1213 }
1214
1215 /* Type-A command to write buf0 */
1216 cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
1217 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1218 cmd = sba_cmd_enc(cmd, msg_len,
1219 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1220 cmd = sba_cmd_enc(cmd, 0x1,
1221 SBA_RESP_SHIFT, SBA_RESP_MASK);
1222 c_mdata = sba_cmd_write_c_mdata(0);
1223 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1224 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1225 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
1226 SBA_CMD_SHIFT, SBA_CMD_MASK);
1227 cmdsp->cmd = cmd;
1228 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1229 cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
1230 if (req->sba->hw_resp_size) {
1231 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
1232 cmdsp->resp = req->resp_dma;
1233 cmdsp->resp_len = req->sba->hw_resp_size;
1234 }
1235 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
1236 cmdsp->data = *dst_q + msg_offset;
1237 cmdsp->data_len = msg_len;
1238 cmdsp++;
1239
1240skip_q:
1241 /* Fillup brcm_message */
1242 msg->type = BRCM_MESSAGE_SBA;
1243 msg->sba.cmds = cmds;
1244 msg->sba.cmds_count = cmdsp - cmds;
1245 msg->ctx = req;
1246 msg->error = 0;
1247}
1248
dd2bceb0 1249static struct sba_request *
743e1c8f
AP		 1250sba_prep_dma_pq_single_req(struct sba_device *sba, dma_addr_t off,
1251 dma_addr_t *dst_p, dma_addr_t *dst_q,
1252 dma_addr_t src, u8 scf, size_t len,
1253 unsigned long flags)
1254{
1255 struct sba_request *req = NULL;
1256
1257 /* Alloc new request */
1258 req = sba_alloc_request(sba);
1259 if (!req)
1260 return NULL;
1261 req->fence = (flags & DMA_PREP_FENCE) ? true : false;
1262
1263 /* Fillup request messages */
1264 sba_fillup_pq_single_msg(req, dmaf_continue(flags),
1265 req->cmds, &req->msg, off, len,
1266 dst_p, dst_q, src, scf);
1267
1268 /* Init async_tx descriptor */
1269 req->tx.flags = flags;
1270 req->tx.cookie = -EBUSY;
1271
1272 return req;
1273}
1274
1275static struct dma_async_tx_descriptor *
1276sba_prep_dma_pq(struct dma_chan *dchan, dma_addr_t *dst, dma_addr_t *src,
1277 u32 src_cnt, const u8 *scf, size_t len, unsigned long flags)
1278{
1279 u32 i, dst_q_index;
1280 size_t req_len;
1281 bool slow = false;
1282 dma_addr_t off = 0;
1283 dma_addr_t *dst_p = NULL, *dst_q = NULL;
1284 struct sba_device *sba = to_sba_device(dchan);
1285 struct sba_request *first = NULL, *req;
1286
1287 /* Sanity checks */
1288 if (unlikely(src_cnt > sba->max_pq_srcs))
1289 return NULL;
1290 for (i = 0; i < src_cnt; i++)
1291 if (sba->max_pq_coefs <= raid6_gflog[scf[i]])
1292 slow = true;
1293
1294 /* Figure-out P and Q destination addresses */
1295 if (!(flags & DMA_PREP_PQ_DISABLE_P))
1296 dst_p = &dst[0];
1297 if (!(flags & DMA_PREP_PQ_DISABLE_Q))
1298 dst_q = &dst[1];
1299
1300 /* Create chained requests where each request is upto hw_buf_size */
1301 while (len) {
1302 req_len = (len < sba->hw_buf_size) ? len : sba->hw_buf_size;
1303
1304 if (slow) {
1305 dst_q_index = src_cnt;
1306
1307 if (dst_q) {
1308 for (i = 0; i < src_cnt; i++) {
1309 if (*dst_q == src[i]) {
1310 dst_q_index = i;
1311 break;
1312 }
1313 }
1314 }
1315
1316 if (dst_q_index < src_cnt) {
1317 i = dst_q_index;
1318 req = sba_prep_dma_pq_single_req(sba,
1319 off, dst_p, dst_q, src[i], scf[i],
1320 req_len, flags | DMA_PREP_FENCE);
1321 if (!req)
1322 goto fail;
1323
1324 if (first)
1325 sba_chain_request(first, req);
1326 else
1327 first = req;
1328
1329 flags |= DMA_PREP_CONTINUE;
1330 }
1331
1332 for (i = 0; i < src_cnt; i++) {
1333 if (dst_q_index == i)
1334 continue;
1335
1336 req = sba_prep_dma_pq_single_req(sba,
1337 off, dst_p, dst_q, src[i], scf[i],
1338 req_len, flags | DMA_PREP_FENCE);
1339 if (!req)
1340 goto fail;
1341
1342 if (first)
1343 sba_chain_request(first, req);
1344 else
1345 first = req;
1346
1347 flags |= DMA_PREP_CONTINUE;
1348 }
1349 } else {
1350 req = sba_prep_dma_pq_req(sba, off,
1351 dst_p, dst_q, src, src_cnt,
1352 scf, req_len, flags);
1353 if (!req)
1354 goto fail;
1355
1356 if (first)
1357 sba_chain_request(first, req);
1358 else
1359 first = req;
1360 }
1361
1362 off += req_len;
1363 len -= req_len;
1364 }
1365
1366 return (first) ? &first->tx : NULL;
1367
1368fail:
1369 if (first)
1370 sba_free_chained_requests(first);
1371 return NULL;
1372}
1373
1374/* ====== Mailbox callbacks ===== */
1375
1376static void sba_dma_tx_actions(struct sba_request *req)
1377{
1378 struct dma_async_tx_descriptor *tx = &req->tx;
1379
1380 WARN_ON(tx->cookie < 0);
1381
1382 if (tx->cookie > 0) {
1383 dma_cookie_complete(tx);
1384
1385 /*
1386 * Call the callback (must not sleep or submit new
1387 * operations to this channel)
1388 */
1389 if (tx->callback)
1390 tx->callback(tx->callback_param);
1391
1392 dma_descriptor_unmap(tx);
1393 }
1394
1395 /* Run dependent operations */
1396 dma_run_dependencies(tx);
1397
1398 /* If waiting for 'ack' then move to completed list */
1399 if (!async_tx_test_ack(&req->tx))
1400 sba_complete_chained_requests(req);
1401 else
1402 sba_free_chained_requests(req);
1403}
1404
1405static void sba_receive_message(struct mbox_client *cl, void *msg)
1406{
1407 unsigned long flags;
1408 struct brcm_message *m = msg;
1409 struct sba_request *req = m->ctx, *req1;
1410 struct sba_device *sba = req->sba;
1411
1412 /* Error count if message has error */
1413 if (m->error < 0)
1414 dev_err(sba->dev, "%s got message with error %d",
1415 dma_chan_name(&sba->dma_chan), m->error);
1416
1417 /* Mark request as received */
1418 sba_received_request(req);
1419
1420 /* Wait for all chained requests to be completed */
1421 if (atomic_dec_return(&req->first->next_pending_count))
1422 goto done;
1423
1424 /* Point to first request */
1425 req = req->first;
1426
1427 /* Update request */
1428 if (req->state == SBA_REQUEST_STATE_RECEIVED)
1429 sba_dma_tx_actions(req);
1430 else
1431 sba_free_chained_requests(req);
1432
1433 spin_lock_irqsave(&sba->reqs_lock, flags);
1434
1435 /* Re-check all completed request waiting for 'ack' */
1436 list_for_each_entry_safe(req, req1, &sba->reqs_completed_list, node) {
1437 spin_unlock_irqrestore(&sba->reqs_lock, flags);
1438 sba_dma_tx_actions(req);
1439 spin_lock_irqsave(&sba->reqs_lock, flags);
1440 }
1441
1442 spin_unlock_irqrestore(&sba->reqs_lock, flags);
1443
1444done:
1445 /* Try to submit pending request */
1446 sba_issue_pending(&sba->dma_chan);
1447}
1448
1449/* ====== Platform driver routines ===== */
1450
1451static int sba_prealloc_channel_resources(struct sba_device *sba)
1452{
1453 int i, j, p, ret = 0;
1454 struct sba_request *req = NULL;
1455
1456 sba->resp_base = dma_alloc_coherent(sba->dma_dev.dev,
1457 sba->max_resp_pool_size,
1458 &sba->resp_dma_base, GFP_KERNEL);
1459 if (!sba->resp_base)
1460 return -ENOMEM;
1461
1462 sba->cmds_base = dma_alloc_coherent(sba->dma_dev.dev,
1463 sba->max_cmds_pool_size,
1464 &sba->cmds_dma_base, GFP_KERNEL);
1465 if (!sba->cmds_base) {
1466 ret = -ENOMEM;
1467 goto fail_free_resp_pool;
1468 }
1469
1470 spin_lock_init(&sba->reqs_lock);
1471 sba->reqs_fence = false;
1472 INIT_LIST_HEAD(&sba->reqs_alloc_list);
1473 INIT_LIST_HEAD(&sba->reqs_pending_list);
1474 INIT_LIST_HEAD(&sba->reqs_active_list);
1475 INIT_LIST_HEAD(&sba->reqs_received_list);
1476 INIT_LIST_HEAD(&sba->reqs_completed_list);
1477 INIT_LIST_HEAD(&sba->reqs_aborted_list);
1478 INIT_LIST_HEAD(&sba->reqs_free_list);
1479
1480 sba->reqs = devm_kcalloc(sba->dev, sba->max_req,
1481 sizeof(*req), GFP_KERNEL);
1482 if (!sba->reqs) {
1483 ret = -ENOMEM;
1484 goto fail_free_cmds_pool;
1485 }
1486
1487 for (i = 0, p = 0; i < sba->max_req; i++) {
1488 req = &sba->reqs[i];
1489 INIT_LIST_HEAD(&req->node);
1490 req->sba = sba;
1491 req->state = SBA_REQUEST_STATE_FREE;
1492 INIT_LIST_HEAD(&req->next);
1493 req->next_count = 1;
1494 atomic_set(&req->next_pending_count, 0);
1495 req->fence = false;
1496 req->resp = sba->resp_base + p;
1497 req->resp_dma = sba->resp_dma_base + p;
1498 p += sba->hw_resp_size;
1499 req->cmds = devm_kcalloc(sba->dev, sba->max_cmd_per_req,
1500 sizeof(*req->cmds), GFP_KERNEL);
1501 if (!req->cmds) {
1502 ret = -ENOMEM;
1503 goto fail_free_cmds_pool;
1504 }
1505 for (j = 0; j < sba->max_cmd_per_req; j++) {
1506 req->cmds[j].cmd = 0;
1507 req->cmds[j].cmd_dma = sba->cmds_base +
1508 (i * sba->max_cmd_per_req + j) * sizeof(u64);
1509 req->cmds[j].cmd_dma_addr = sba->cmds_dma_base +
1510 (i * sba->max_cmd_per_req + j) * sizeof(u64);
1511 req->cmds[j].flags = 0;
1512 }
1513 memset(&req->msg, 0, sizeof(req->msg));
1514 dma_async_tx_descriptor_init(&req->tx, &sba->dma_chan);
1515 req->tx.tx_submit = sba_tx_submit;
1516 req->tx.phys = req->resp_dma;
1517 list_add_tail(&req->node, &sba->reqs_free_list);
1518 }
1519
1520 sba->reqs_free_count = sba->max_req;
1521
1522 return 0;
1523
1524fail_free_cmds_pool:
1525 dma_free_coherent(sba->dma_dev.dev,
1526 sba->max_cmds_pool_size,
1527 sba->cmds_base, sba->cmds_dma_base);
1528fail_free_resp_pool:
1529 dma_free_coherent(sba->dma_dev.dev,
1530 sba->max_resp_pool_size,
1531 sba->resp_base, sba->resp_dma_base);
1532 return ret;
1533}
1534
1535static void sba_freeup_channel_resources(struct sba_device *sba)
1536{
1537 dmaengine_terminate_all(&sba->dma_chan);
1538 dma_free_coherent(sba->dma_dev.dev, sba->max_cmds_pool_size,
1539 sba->cmds_base, sba->cmds_dma_base);
1540 dma_free_coherent(sba->dma_dev.dev, sba->max_resp_pool_size,
1541 sba->resp_base, sba->resp_dma_base);
1542 sba->resp_base = NULL;
1543 sba->resp_dma_base = 0;
1544}
1545
1546static int sba_async_register(struct sba_device *sba)
1547{
1548 int ret;
1549 struct dma_device *dma_dev = &sba->dma_dev;
1550
1551 /* Initialize DMA channel cookie */
1552 sba->dma_chan.device = dma_dev;
1553 dma_cookie_init(&sba->dma_chan);
1554
1555 /* Initialize DMA device capability mask */
1556 dma_cap_zero(dma_dev->cap_mask);
1557 dma_cap_set(DMA_INTERRUPT, dma_dev->cap_mask);
1558 dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
1559 dma_cap_set(DMA_XOR, dma_dev->cap_mask);
1560 dma_cap_set(DMA_PQ, dma_dev->cap_mask);
1561
1562 /*
1563 * Set mailbox channel device as the base device of
1564 * our dma_device because the actual memory accesses
1565 * will be done by mailbox controller
1566 */
1567 dma_dev->dev = sba->mbox_dev;
1568
1569 /* Set base prep routines */
1570 dma_dev->device_free_chan_resources = sba_free_chan_resources;
1571 dma_dev->device_terminate_all = sba_device_terminate_all;
1572 dma_dev->device_issue_pending = sba_issue_pending;
1573 dma_dev->device_tx_status = sba_tx_status;
1574
1575 /* Set interrupt routine */
1576 if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask))
1577 dma_dev->device_prep_dma_interrupt = sba_prep_dma_interrupt;
1578
1579 /* Set memcpy routine */
1580 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask))
1581 dma_dev->device_prep_dma_memcpy = sba_prep_dma_memcpy;
1582
1583 /* Set xor routine and capability */
1584 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
1585 dma_dev->device_prep_dma_xor = sba_prep_dma_xor;
1586 dma_dev->max_xor = sba->max_xor_srcs;
1587 }
1588
1589 /* Set pq routine and capability */
1590 if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
1591 dma_dev->device_prep_dma_pq = sba_prep_dma_pq;
1592 dma_set_maxpq(dma_dev, sba->max_pq_srcs, 0);
1593 }
1594
1595 /* Initialize DMA device channel list */
1596 INIT_LIST_HEAD(&dma_dev->channels);
1597 list_add_tail(&sba->dma_chan.device_node, &dma_dev->channels);
1598
1599 /* Register with Linux async DMA framework*/
1600 ret = dma_async_device_register(dma_dev);
1601 if (ret) {
1602 dev_err(sba->dev, "async device register error %d", ret);
1603 return ret;
1604 }
1605
1606 dev_info(sba->dev, "%s capabilities: %s%s%s%s\n",
1607 dma_chan_name(&sba->dma_chan),
1608 dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "interrupt " : "",
1609 dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "memcpy " : "",
1610 dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "",
1611 dma_has_cap(DMA_PQ, dma_dev->cap_mask) ? "pq " : "");
1612
1613 return 0;
1614}
1615
1616static int sba_probe(struct platform_device *pdev)
1617{
1618 int i, ret = 0, mchans_count;
1619 struct sba_device *sba;
1620 struct platform_device *mbox_pdev;
1621 struct of_phandle_args args;
1622
1623 /* Allocate main SBA struct */
1624 sba = devm_kzalloc(&pdev->dev, sizeof(*sba), GFP_KERNEL);
1625 if (!sba)
1626 return -ENOMEM;
1627
1628 sba->dev = &pdev->dev;
1629 platform_set_drvdata(pdev, sba);
1630
1631 /* Determine SBA version from DT compatible string */
1632 if (of_device_is_compatible(sba->dev->of_node, "brcm,iproc-sba"))
1633 sba->ver = SBA_VER_1;
1634 else if (of_device_is_compatible(sba->dev->of_node,
1635 "brcm,iproc-sba-v2"))
1636 sba->ver = SBA_VER_2;
1637 else
1638 return -ENODEV;
1639
1640 /* Derived Configuration parameters */
1641 switch (sba->ver) {
1642 case SBA_VER_1:
1643 sba->max_req = 1024;
1644 sba->hw_buf_size = 4096;
1645 sba->hw_resp_size = 8;
1646 sba->max_pq_coefs = 6;
1647 sba->max_pq_srcs = 6;
1648 break;
1649 case SBA_VER_2:
1650 sba->max_req = 1024;
1651 sba->hw_buf_size = 4096;
1652 sba->hw_resp_size = 8;
1653 sba->max_pq_coefs = 30;
1654 /*
1655 * We can support max_pq_srcs == max_pq_coefs because
1656 * we are limited by number of SBA commands that we can
1657 * fit in one message for underlying ring manager HW.
1658 */
1659 sba->max_pq_srcs = 12;
1660 break;
1661 default:
1662 return -EINVAL;
1663 }
1664 sba->max_cmd_per_req = sba->max_pq_srcs + 3;
1665 sba->max_xor_srcs = sba->max_cmd_per_req - 1;
1666 sba->max_resp_pool_size = sba->max_req * sba->hw_resp_size;
1667 sba->max_cmds_pool_size = sba->max_req *
1668 sba->max_cmd_per_req * sizeof(u64);
1669
1670 /* Setup mailbox client */
1671 sba->client.dev = &pdev->dev;
1672 sba->client.rx_callback = sba_receive_message;
1673 sba->client.tx_block = false;
1674 sba->client.knows_txdone = false;
1675 sba->client.tx_tout = 0;
1676
1677 /* Number of channels equals number of mailbox channels */
1678 ret = of_count_phandle_with_args(pdev->dev.of_node,
1679 "mboxes", "#mbox-cells");
1680 if (ret <= 0)
1681 return -ENODEV;
1682 mchans_count = ret;
1683 sba->mchans_count = 0;
1684 atomic_set(&sba->mchans_current, 0);
1685
1686 /* Allocate mailbox channel array */
1687 sba->mchans = devm_kcalloc(&pdev->dev, sba->mchans_count,
1688 sizeof(*sba->mchans), GFP_KERNEL);
1689 if (!sba->mchans)
1690 return -ENOMEM;
1691
1692 /* Request mailbox channels */
1693 for (i = 0; i < mchans_count; i++) {
1694 sba->mchans[i] = mbox_request_channel(&sba->client, i);
1695 if (IS_ERR(sba->mchans[i])) {
1696 ret = PTR_ERR(sba->mchans[i]);
1697 goto fail_free_mchans;
1698 }
1699 sba->mchans_count++;
1700 }
1701
1702 /* Find-out underlying mailbox device */
1703 ret = of_parse_phandle_with_args(pdev->dev.of_node,
1704 "mboxes", "#mbox-cells", 0, &args);
1705 if (ret)
1706 goto fail_free_mchans;
1707 mbox_pdev = of_find_device_by_node(args.np);
1708 of_node_put(args.np);
1709 if (!mbox_pdev) {
1710 ret = -ENODEV;
1711 goto fail_free_mchans;
1712 }
1713 sba->mbox_dev = &mbox_pdev->dev;
1714
1715 /* All mailbox channels should be of same ring manager device */
1716 for (i = 1; i < mchans_count; i++) {
1717 ret = of_parse_phandle_with_args(pdev->dev.of_node,
1718 "mboxes", "#mbox-cells", i, &args);
1719 if (ret)
1720 goto fail_free_mchans;
1721 mbox_pdev = of_find_device_by_node(args.np);
1722 of_node_put(args.np);
1723 if (sba->mbox_dev != &mbox_pdev->dev) {
1724 ret = -EINVAL;
1725 goto fail_free_mchans;
1726 }
1727 }
1728
1729 /* Register DMA device with linux async framework */
1730 ret = sba_async_register(sba);
1731 if (ret)
1732 goto fail_free_mchans;
1733
1734 /* Prealloc channel resource */
1735 ret = sba_prealloc_channel_resources(sba);
1736 if (ret)
1737 goto fail_async_dev_unreg;
1738
1739 /* Print device info */
1740 dev_info(sba->dev, "%s using SBAv%d and %d mailbox channels",
1741 dma_chan_name(&sba->dma_chan), sba->ver+1,
1742 sba->mchans_count);
1743
1744 return 0;
1745
1746fail_async_dev_unreg:
1747 dma_async_device_unregister(&sba->dma_dev);
1748fail_free_mchans:
1749 for (i = 0; i < sba->mchans_count; i++)
1750 mbox_free_channel(sba->mchans[i]);
1751 return ret;
1752}
1753
1754static int sba_remove(struct platform_device *pdev)
1755{
1756 int i;
1757 struct sba_device *sba = platform_get_drvdata(pdev);
1758
1759 sba_freeup_channel_resources(sba);
1760
1761 dma_async_device_unregister(&sba->dma_dev);
1762
1763 for (i = 0; i < sba->mchans_count; i++)
1764 mbox_free_channel(sba->mchans[i]);
1765
1766 return 0;
1767}
1768
1769static const struct of_device_id sba_of_match[] = {
1770 { .compatible = "brcm,iproc-sba", },
1771 { .compatible = "brcm,iproc-sba-v2", },
1772 {},
1773};
1774MODULE_DEVICE_TABLE(of, sba_of_match);
1775
1776static struct platform_driver sba_driver = {
1777 .probe = sba_probe,
1778 .remove = sba_remove,
1779 .driver = {
1780 .name = "bcm-sba-raid",
1781 .of_match_table = sba_of_match,
1782 },
1783};
1784module_platform_driver(sba_driver);
1785
1786MODULE_DESCRIPTION("Broadcom SBA RAID driver");
1787MODULE_AUTHOR("Anup Patel <anup.patel@broadcom.com>");
1788MODULE_LICENSE("GPL v2");
Linux 6.x block layer and io_uring tree(s)