projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
dmaengine: bcm-sba-raid: Remove reqs_free_count from sba_device
[linux-block.git] / drivers / dma / bcm-sba-raid.c
CommitLineData
743e1c8f
AP		 1/*
2 * Copyright (C) 2017 Broadcom
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 */
8
9/*
10 * Broadcom SBA RAID Driver
11 *
12 * The Broadcom stream buffer accelerator (SBA) provides offloading
13 * capabilities for RAID operations. The SBA offload engine is accessible
14 * via Broadcom SoC specific ring manager. Two or more offload engines
15 * can share same Broadcom SoC specific ring manager due to this Broadcom
16 * SoC specific ring manager driver is implemented as a mailbox controller
17 * driver and offload engine drivers are implemented as mallbox clients.
18 *
19 * Typically, Broadcom SoC specific ring manager will implement larger
20 * number of hardware rings over one or more SBA hardware devices. By
21 * design, the internal buffer size of SBA hardware device is limited
22 * but all offload operations supported by SBA can be broken down into
23 * multiple small size requests and executed parallely on multiple SBA
24 * hardware devices for achieving high through-put.
25 *
26 * The Broadcom SBA RAID driver does not require any register programming
27 * except submitting request to SBA hardware device via mailbox channels.
28 * This driver implements a DMA device with one DMA channel using a set
29 * of mailbox channels provided by Broadcom SoC specific ring manager
30 * driver. To exploit parallelism (as described above), all DMA request
31 * coming to SBA RAID DMA channel are broken down to smaller requests
32 * and submitted to multiple mailbox channels in round-robin fashion.
33 * For having more SBA DMA channels, we can create more SBA device nodes
34 * in Broadcom SoC specific DTS based on number of hardware rings supported
35 * by Broadcom SoC ring manager.
36 */
37
38#include <linux/bitops.h>
39#include <linux/dma-mapping.h>
40#include <linux/dmaengine.h>
41#include <linux/list.h>
42#include <linux/mailbox_client.h>
43#include <linux/mailbox/brcm-message.h>
44#include <linux/module.h>
45#include <linux/of_device.h>
46#include <linux/slab.h>
47#include <linux/raid/pq.h>
48
49#include "dmaengine.h"
50
e897091a
AP		 51/* ====== Driver macros and defines ===== */
52
743e1c8f
AP		 53#define SBA_TYPE_SHIFT 48
54#define SBA_TYPE_MASK GENMASK(1, 0)
55#define SBA_TYPE_A 0x0
56#define SBA_TYPE_B 0x2
57#define SBA_TYPE_C 0x3
58#define SBA_USER_DEF_SHIFT 32
59#define SBA_USER_DEF_MASK GENMASK(15, 0)
60#define SBA_R_MDATA_SHIFT 24
61#define SBA_R_MDATA_MASK GENMASK(7, 0)
62#define SBA_C_MDATA_MS_SHIFT 18
63#define SBA_C_MDATA_MS_MASK GENMASK(1, 0)
64#define SBA_INT_SHIFT 17
65#define SBA_INT_MASK BIT(0)
66#define SBA_RESP_SHIFT 16
67#define SBA_RESP_MASK BIT(0)
68#define SBA_C_MDATA_SHIFT 8
69#define SBA_C_MDATA_MASK GENMASK(7, 0)
70#define SBA_C_MDATA_BNUMx_SHIFT(__bnum) (2 * (__bnum))
71#define SBA_C_MDATA_BNUMx_MASK GENMASK(1, 0)
72#define SBA_C_MDATA_DNUM_SHIFT 5
73#define SBA_C_MDATA_DNUM_MASK GENMASK(4, 0)
74#define SBA_C_MDATA_LS(__v) ((__v) & 0xff)
75#define SBA_C_MDATA_MS(__v) (((__v) >> 8) & 0x3)
76#define SBA_CMD_SHIFT 0
77#define SBA_CMD_MASK GENMASK(3, 0)
78#define SBA_CMD_ZERO_BUFFER 0x4
79#define SBA_CMD_ZERO_ALL_BUFFERS 0x8
80#define SBA_CMD_LOAD_BUFFER 0x9
81#define SBA_CMD_XOR 0xa
82#define SBA_CMD_GALOIS_XOR 0xb
83#define SBA_CMD_WRITE_BUFFER 0xc
84#define SBA_CMD_GALOIS 0xe
85
86/* Driver helper macros */
87#define to_sba_request(tx) \
88 container_of(tx, struct sba_request, tx)
89#define to_sba_device(dchan) \
90 container_of(dchan, struct sba_device, dma_chan)
91
e897091a
AP		 92/* ===== Driver data structures ===== */
93
57a28508
AP		 94enum sba_request_flags {
95 SBA_REQUEST_STATE_FREE = 0x001,
96 SBA_REQUEST_STATE_ALLOCED = 0x002,
97 SBA_REQUEST_STATE_PENDING = 0x004,
98 SBA_REQUEST_STATE_ACTIVE = 0x008,
99 SBA_REQUEST_STATE_RECEIVED = 0x010,
100 SBA_REQUEST_STATE_COMPLETED = 0x020,
101 SBA_REQUEST_STATE_ABORTED = 0x040,
102 SBA_REQUEST_STATE_MASK = 0x0ff,
103 SBA_REQUEST_FENCE = 0x100,
743e1c8f
AP		 104};
105
106struct sba_request {
107 /* Global state */
108 struct list_head node;
109 struct sba_device *sba;
57a28508 110 u32 flags;
743e1c8f
AP		 111 /* Chained requests management */
112 struct sba_request *first;
113 struct list_head next;
743e1c8f
AP		 114 atomic_t next_pending_count;
115 /* BRCM message data */
743e1c8f
AP		 116 struct brcm_sba_command *cmds;
117 struct brcm_message msg;
118 struct dma_async_tx_descriptor tx;
119};
120
121enum sba_version {
122 SBA_VER_1 = 0,
123 SBA_VER_2
124};
125
126struct sba_device {
127 /* Underlying device */
128 struct device *dev;
129 /* DT configuration parameters */
130 enum sba_version ver;
131 /* Derived configuration parameters */
132 u32 max_req;
133 u32 hw_buf_size;
134 u32 hw_resp_size;
135 u32 max_pq_coefs;
136 u32 max_pq_srcs;
137 u32 max_cmd_per_req;
138 u32 max_xor_srcs;
139 u32 max_resp_pool_size;
140 u32 max_cmds_pool_size;
141 /* Maibox client and Mailbox channels */
142 struct mbox_client client;
143 int mchans_count;
144 atomic_t mchans_current;
145 struct mbox_chan **mchans;
146 struct device *mbox_dev;
147 /* DMA device and DMA channel */
148 struct dma_device dma_dev;
149 struct dma_chan dma_chan;
150 /* DMA channel resources */
151 void *resp_base;
152 dma_addr_t resp_dma_base;
153 void *cmds_base;
154 dma_addr_t cmds_dma_base;
155 spinlock_t reqs_lock;
156 struct sba_request *reqs;
157 bool reqs_fence;
158 struct list_head reqs_alloc_list;
159 struct list_head reqs_pending_list;
160 struct list_head reqs_active_list;
161 struct list_head reqs_received_list;
162 struct list_head reqs_completed_list;
163 struct list_head reqs_aborted_list;
164 struct list_head reqs_free_list;
743e1c8f
AP		 165};
166
e897091a 167/* ====== Command helper routines ===== */
743e1c8f
AP		 168
169static inline u64 __pure sba_cmd_enc(u64 cmd, u32 val, u32 shift, u32 mask)
170{
171 cmd &= ~((u64)mask << shift);
172 cmd |= ((u64)(val & mask) << shift);
173 return cmd;
174}
175
176static inline u32 __pure sba_cmd_load_c_mdata(u32 b0)
177{
178 return b0 & SBA_C_MDATA_BNUMx_MASK;
179}
180
181static inline u32 __pure sba_cmd_write_c_mdata(u32 b0)
182{
183 return b0 & SBA_C_MDATA_BNUMx_MASK;
184}
185
186static inline u32 __pure sba_cmd_xor_c_mdata(u32 b1, u32 b0)
187{
188 return (b0 & SBA_C_MDATA_BNUMx_MASK) |
189 ((b1 & SBA_C_MDATA_BNUMx_MASK) << SBA_C_MDATA_BNUMx_SHIFT(1));
190}
191
192static inline u32 __pure sba_cmd_pq_c_mdata(u32 d, u32 b1, u32 b0)
193{
194 return (b0 & SBA_C_MDATA_BNUMx_MASK) |
195 ((b1 & SBA_C_MDATA_BNUMx_MASK) << SBA_C_MDATA_BNUMx_SHIFT(1)) |
196 ((d & SBA_C_MDATA_DNUM_MASK) << SBA_C_MDATA_DNUM_SHIFT);
197}
198
e897091a 199/* ====== General helper routines ===== */
743e1c8f
AP		 200
201static struct sba_request *sba_alloc_request(struct sba_device *sba)
202{
203 unsigned long flags;
204 struct sba_request *req = NULL;
205
206 spin_lock_irqsave(&sba->reqs_lock, flags);
743e1c8f
AP		 207 req = list_first_entry_or_null(&sba->reqs_free_list,
208 struct sba_request, node);
abfa251a 209 if (req)
743e1c8f 210 list_move_tail(&req->node, &sba->reqs_alloc_list);
743e1c8f 211 spin_unlock_irqrestore(&sba->reqs_lock, flags);
e4274cfa
AP		 212 if (!req)
213 return NULL;
214
57a28508 215 req->flags = SBA_REQUEST_STATE_ALLOCED;
e4274cfa
AP		 216 req->first = req;
217 INIT_LIST_HEAD(&req->next);
e4274cfa
AP		 218 atomic_set(&req->next_pending_count, 1);
219
220 dma_async_tx_descriptor_init(&req->tx, &sba->dma_chan);
743e1c8f
AP		 221
222 return req;
223}
224
225/* Note: Must be called with sba->reqs_lock held */
226static void _sba_pending_request(struct sba_device *sba,
227 struct sba_request *req)
228{
229 lockdep_assert_held(&sba->reqs_lock);
57a28508
AP		 230 req->flags &= ~SBA_REQUEST_STATE_MASK;
231 req->flags |= SBA_REQUEST_STATE_PENDING;
743e1c8f
AP		 232 list_move_tail(&req->node, &sba->reqs_pending_list);
233 if (list_empty(&sba->reqs_active_list))
234 sba->reqs_fence = false;
235}
236
237/* Note: Must be called with sba->reqs_lock held */
238static bool _sba_active_request(struct sba_device *sba,
239 struct sba_request *req)
240{
241 lockdep_assert_held(&sba->reqs_lock);
242 if (list_empty(&sba->reqs_active_list))
243 sba->reqs_fence = false;
244 if (sba->reqs_fence)
245 return false;
57a28508
AP		 246 req->flags &= ~SBA_REQUEST_STATE_MASK;
247 req->flags |= SBA_REQUEST_STATE_ACTIVE;
743e1c8f 248 list_move_tail(&req->node, &sba->reqs_active_list);
57a28508 249 if (req->flags & SBA_REQUEST_FENCE)
743e1c8f
AP		 250 sba->reqs_fence = true;
251 return true;
252}
253
254/* Note: Must be called with sba->reqs_lock held */
255static void _sba_abort_request(struct sba_device *sba,
256 struct sba_request *req)
257{
258 lockdep_assert_held(&sba->reqs_lock);
57a28508
AP		 259 req->flags &= ~SBA_REQUEST_STATE_MASK;
260 req->flags |= SBA_REQUEST_STATE_ABORTED;
743e1c8f
AP		 261 list_move_tail(&req->node, &sba->reqs_aborted_list);
262 if (list_empty(&sba->reqs_active_list))
263 sba->reqs_fence = false;
264}
265
266/* Note: Must be called with sba->reqs_lock held */
267static void _sba_free_request(struct sba_device *sba,
268 struct sba_request *req)
269{
270 lockdep_assert_held(&sba->reqs_lock);
57a28508
AP		 271 req->flags &= ~SBA_REQUEST_STATE_MASK;
272 req->flags |= SBA_REQUEST_STATE_FREE;
743e1c8f
AP		 273 list_move_tail(&req->node, &sba->reqs_free_list);
274 if (list_empty(&sba->reqs_active_list))
275 sba->reqs_fence = false;
743e1c8f
AP		 276}
277
278static void sba_received_request(struct sba_request *req)
279{
280 unsigned long flags;
281 struct sba_device *sba = req->sba;
282
283 spin_lock_irqsave(&sba->reqs_lock, flags);
57a28508
AP		 284 req->flags &= ~SBA_REQUEST_STATE_MASK;
285 req->flags |= SBA_REQUEST_STATE_RECEIVED;
743e1c8f
AP		 286 list_move_tail(&req->node, &sba->reqs_received_list);
287 spin_unlock_irqrestore(&sba->reqs_lock, flags);
288}
289
290static void sba_complete_chained_requests(struct sba_request *req)
291{
292 unsigned long flags;
293 struct sba_request *nreq;
294 struct sba_device *sba = req->sba;
295
296 spin_lock_irqsave(&sba->reqs_lock, flags);
297
57a28508
AP		 298 req->flags &= ~SBA_REQUEST_STATE_MASK;
299 req->flags |= SBA_REQUEST_STATE_COMPLETED;
743e1c8f
AP		 300 list_move_tail(&req->node, &sba->reqs_completed_list);
301 list_for_each_entry(nreq, &req->next, next) {
57a28508
AP		 302 nreq->flags &= ~SBA_REQUEST_STATE_MASK;
303 nreq->flags |= SBA_REQUEST_STATE_COMPLETED;
743e1c8f
AP		 304 list_move_tail(&nreq->node, &sba->reqs_completed_list);
305 }
306 if (list_empty(&sba->reqs_active_list))
307 sba->reqs_fence = false;
308
309 spin_unlock_irqrestore(&sba->reqs_lock, flags);
310}
311
312static void sba_free_chained_requests(struct sba_request *req)
313{
314 unsigned long flags;
315 struct sba_request *nreq;
316 struct sba_device *sba = req->sba;
317
318 spin_lock_irqsave(&sba->reqs_lock, flags);
319
320 _sba_free_request(sba, req);
321 list_for_each_entry(nreq, &req->next, next)
322 _sba_free_request(sba, nreq);
323
324 spin_unlock_irqrestore(&sba->reqs_lock, flags);
325}
326
327static void sba_chain_request(struct sba_request *first,
328 struct sba_request *req)
329{
330 unsigned long flags;
331 struct sba_device *sba = req->sba;
332
333 spin_lock_irqsave(&sba->reqs_lock, flags);
334
335 list_add_tail(&req->next, &first->next);
336 req->first = first;
10f1a330 337 atomic_inc(&first->next_pending_count);
743e1c8f
AP		 338
339 spin_unlock_irqrestore(&sba->reqs_lock, flags);
340}
341
342static void sba_cleanup_nonpending_requests(struct sba_device *sba)
343{
344 unsigned long flags;
345 struct sba_request *req, *req1;
346
347 spin_lock_irqsave(&sba->reqs_lock, flags);
348
349 /* Freeup all alloced request */
350 list_for_each_entry_safe(req, req1, &sba->reqs_alloc_list, node)
351 _sba_free_request(sba, req);
352
353 /* Freeup all received request */
354 list_for_each_entry_safe(req, req1, &sba->reqs_received_list, node)
355 _sba_free_request(sba, req);
356
357 /* Freeup all completed request */
358 list_for_each_entry_safe(req, req1, &sba->reqs_completed_list, node)
359 _sba_free_request(sba, req);
360
361 /* Set all active requests as aborted */
362 list_for_each_entry_safe(req, req1, &sba->reqs_active_list, node)
363 _sba_abort_request(sba, req);
364
365 /*
366 * Note: We expect that aborted request will be eventually
367 * freed by sba_receive_message()
368 */
369
370 spin_unlock_irqrestore(&sba->reqs_lock, flags);
371}
372
373static void sba_cleanup_pending_requests(struct sba_device *sba)
374{
375 unsigned long flags;
376 struct sba_request *req, *req1;
377
378 spin_lock_irqsave(&sba->reqs_lock, flags);
379
380 /* Freeup all pending request */
381 list_for_each_entry_safe(req, req1, &sba->reqs_pending_list, node)
382 _sba_free_request(sba, req);
383
384 spin_unlock_irqrestore(&sba->reqs_lock, flags);
385}
386
387/* ====== DMAENGINE callbacks ===== */
388
389static void sba_free_chan_resources(struct dma_chan *dchan)
390{
391 /*
392 * Channel resources are pre-alloced so we just free-up
393 * whatever we can so that we can re-use pre-alloced
394 * channel resources next time.
395 */
396 sba_cleanup_nonpending_requests(to_sba_device(dchan));
397}
398
399static int sba_device_terminate_all(struct dma_chan *dchan)
400{
401 /* Cleanup all pending requests */
402 sba_cleanup_pending_requests(to_sba_device(dchan));
403
404 return 0;
405}
406
407static int sba_send_mbox_request(struct sba_device *sba,
408 struct sba_request *req)
409{
410 int mchans_idx, ret = 0;
411
412 /* Select mailbox channel in round-robin fashion */
413 mchans_idx = atomic_inc_return(&sba->mchans_current);
414 mchans_idx = mchans_idx % sba->mchans_count;
415
416 /* Send message for the request */
417 req->msg.error = 0;
418 ret = mbox_send_message(sba->mchans[mchans_idx], &req->msg);
419 if (ret < 0) {
420 dev_err(sba->dev, "send message failed with error %d", ret);
421 return ret;
422 }
423 ret = req->msg.error;
424 if (ret < 0) {
425 dev_err(sba->dev, "message error %d", ret);
426 return ret;
427 }
428
429 return 0;
430}
431
432static void sba_issue_pending(struct dma_chan *dchan)
433{
434 int ret;
435 unsigned long flags;
436 struct sba_request *req, *req1;
437 struct sba_device *sba = to_sba_device(dchan);
438
439 spin_lock_irqsave(&sba->reqs_lock, flags);
440
441 /* Process all pending request */
442 list_for_each_entry_safe(req, req1, &sba->reqs_pending_list, node) {
443 /* Try to make request active */
444 if (!_sba_active_request(sba, req))
445 break;
446
447 /* Send request to mailbox channel */
448 spin_unlock_irqrestore(&sba->reqs_lock, flags);
449 ret = sba_send_mbox_request(sba, req);
450 spin_lock_irqsave(&sba->reqs_lock, flags);
451
452 /* If something went wrong then keep request pending */
453 if (ret < 0) {
454 _sba_pending_request(sba, req);
455 break;
456 }
457 }
458
459 spin_unlock_irqrestore(&sba->reqs_lock, flags);
460}
461
462static dma_cookie_t sba_tx_submit(struct dma_async_tx_descriptor *tx)
463{
464 unsigned long flags;
465 dma_cookie_t cookie;
466 struct sba_device *sba;
467 struct sba_request *req, *nreq;
468
469 if (unlikely(!tx))
470 return -EINVAL;
471
472 sba = to_sba_device(tx->chan);
473 req = to_sba_request(tx);
474
475 /* Assign cookie and mark all chained requests pending */
476 spin_lock_irqsave(&sba->reqs_lock, flags);
477 cookie = dma_cookie_assign(tx);
478 _sba_pending_request(sba, req);
479 list_for_each_entry(nreq, &req->next, next)
480 _sba_pending_request(sba, nreq);
481 spin_unlock_irqrestore(&sba->reqs_lock, flags);
482
483 return cookie;
484}
485
486static enum dma_status sba_tx_status(struct dma_chan *dchan,
487 dma_cookie_t cookie,
488 struct dma_tx_state *txstate)
489{
490 int mchan_idx;
491 enum dma_status ret;
492 struct sba_device *sba = to_sba_device(dchan);
493
494 for (mchan_idx = 0; mchan_idx < sba->mchans_count; mchan_idx++)
495 mbox_client_peek_data(sba->mchans[mchan_idx]);
496
497 ret = dma_cookie_status(dchan, cookie, txstate);
498 if (ret == DMA_COMPLETE)
499 return ret;
500
501 return dma_cookie_status(dchan, cookie, txstate);
502}
503
504static void sba_fillup_interrupt_msg(struct sba_request *req,
505 struct brcm_sba_command *cmds,
506 struct brcm_message *msg)
507{
508 u64 cmd;
509 u32 c_mdata;
e7ae72aa 510 dma_addr_t resp_dma = req->tx.phys;
743e1c8f
AP		 511 struct brcm_sba_command *cmdsp = cmds;
512
513 /* Type-B command to load dummy data into buf0 */
514 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
515 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
516 cmd = sba_cmd_enc(cmd, req->sba->hw_resp_size,
517 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
518 c_mdata = sba_cmd_load_c_mdata(0);
519 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
520 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
521 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
522 SBA_CMD_SHIFT, SBA_CMD_MASK);
523 cmdsp->cmd = cmd;
524 *cmdsp->cmd_dma = cpu_to_le64(cmd);
525 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
e7ae72aa 526 cmdsp->data = resp_dma;
743e1c8f
AP		 527 cmdsp->data_len = req->sba->hw_resp_size;
528 cmdsp++;
529
530 /* Type-A command to write buf0 to dummy location */
531 cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
532 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
533 cmd = sba_cmd_enc(cmd, req->sba->hw_resp_size,
534 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
535 cmd = sba_cmd_enc(cmd, 0x1,
536 SBA_RESP_SHIFT, SBA_RESP_MASK);
537 c_mdata = sba_cmd_write_c_mdata(0);
538 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
539 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
540 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
541 SBA_CMD_SHIFT, SBA_CMD_MASK);
542 cmdsp->cmd = cmd;
543 *cmdsp->cmd_dma = cpu_to_le64(cmd);
544 cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
545 if (req->sba->hw_resp_size) {
546 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
e7ae72aa 547 cmdsp->resp = resp_dma;
743e1c8f
AP		 548 cmdsp->resp_len = req->sba->hw_resp_size;
549 }
550 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
e7ae72aa 551 cmdsp->data = resp_dma;
743e1c8f
AP		 552 cmdsp->data_len = req->sba->hw_resp_size;
553 cmdsp++;
554
555 /* Fillup brcm_message */
556 msg->type = BRCM_MESSAGE_SBA;
557 msg->sba.cmds = cmds;
558 msg->sba.cmds_count = cmdsp - cmds;
559 msg->ctx = req;
560 msg->error = 0;
561}
562
563static struct dma_async_tx_descriptor *
564sba_prep_dma_interrupt(struct dma_chan *dchan, unsigned long flags)
565{
566 struct sba_request *req = NULL;
567 struct sba_device *sba = to_sba_device(dchan);
568
569 /* Alloc new request */
570 req = sba_alloc_request(sba);
571 if (!req)
572 return NULL;
573
574 /*
575 * Force fence so that no requests are submitted
576 * until DMA callback for this request is invoked.
577 */
57a28508 578 req->flags |= SBA_REQUEST_FENCE;
743e1c8f
AP		 579
580 /* Fillup request message */
581 sba_fillup_interrupt_msg(req, req->cmds, &req->msg);
582
583 /* Init async_tx descriptor */
584 req->tx.flags = flags;
585 req->tx.cookie = -EBUSY;
586
1fc63cb4 587 return &req->tx;
743e1c8f
AP		 588}
589
590static void sba_fillup_memcpy_msg(struct sba_request *req,
591 struct brcm_sba_command *cmds,
592 struct brcm_message *msg,
593 dma_addr_t msg_offset, size_t msg_len,
594 dma_addr_t dst, dma_addr_t src)
595{
596 u64 cmd;
597 u32 c_mdata;
e7ae72aa 598 dma_addr_t resp_dma = req->tx.phys;
743e1c8f
AP		 599 struct brcm_sba_command *cmdsp = cmds;
600
601 /* Type-B command to load data into buf0 */
602 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
603 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
604 cmd = sba_cmd_enc(cmd, msg_len,
605 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
606 c_mdata = sba_cmd_load_c_mdata(0);
607 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
608 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
609 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
610 SBA_CMD_SHIFT, SBA_CMD_MASK);
611 cmdsp->cmd = cmd;
612 *cmdsp->cmd_dma = cpu_to_le64(cmd);
613 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
614 cmdsp->data = src + msg_offset;
615 cmdsp->data_len = msg_len;
616 cmdsp++;
617
618 /* Type-A command to write buf0 */
619 cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
620 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
621 cmd = sba_cmd_enc(cmd, msg_len,
622 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
623 cmd = sba_cmd_enc(cmd, 0x1,
624 SBA_RESP_SHIFT, SBA_RESP_MASK);
625 c_mdata = sba_cmd_write_c_mdata(0);
626 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
627 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
628 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
629 SBA_CMD_SHIFT, SBA_CMD_MASK);
630 cmdsp->cmd = cmd;
631 *cmdsp->cmd_dma = cpu_to_le64(cmd);
632 cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
633 if (req->sba->hw_resp_size) {
634 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
e7ae72aa 635 cmdsp->resp = resp_dma;
743e1c8f
AP		 636 cmdsp->resp_len = req->sba->hw_resp_size;
637 }
638 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
639 cmdsp->data = dst + msg_offset;
640 cmdsp->data_len = msg_len;
641 cmdsp++;
642
643 /* Fillup brcm_message */
644 msg->type = BRCM_MESSAGE_SBA;
645 msg->sba.cmds = cmds;
646 msg->sba.cmds_count = cmdsp - cmds;
647 msg->ctx = req;
648 msg->error = 0;
649}
650
651static struct sba_request *
652sba_prep_dma_memcpy_req(struct sba_device *sba,
653 dma_addr_t off, dma_addr_t dst, dma_addr_t src,
654 size_t len, unsigned long flags)
655{
656 struct sba_request *req = NULL;
657
658 /* Alloc new request */
659 req = sba_alloc_request(sba);
660 if (!req)
661 return NULL;
57a28508
AP		 662 if (flags & DMA_PREP_FENCE)
663 req->flags |= SBA_REQUEST_FENCE;
743e1c8f
AP		 664
665 /* Fillup request message */
666 sba_fillup_memcpy_msg(req, req->cmds, &req->msg,
667 off, len, dst, src);
668
669 /* Init async_tx descriptor */
670 req->tx.flags = flags;
671 req->tx.cookie = -EBUSY;
672
673 return req;
674}
675
676static struct dma_async_tx_descriptor *
677sba_prep_dma_memcpy(struct dma_chan *dchan, dma_addr_t dst, dma_addr_t src,
678 size_t len, unsigned long flags)
679{
680 size_t req_len;
681 dma_addr_t off = 0;
682 struct sba_device *sba = to_sba_device(dchan);
683 struct sba_request *first = NULL, *req;
684
685 /* Create chained requests where each request is upto hw_buf_size */
686 while (len) {
687 req_len = (len < sba->hw_buf_size) ? len : sba->hw_buf_size;
688
689 req = sba_prep_dma_memcpy_req(sba, off, dst, src,
690 req_len, flags);
691 if (!req) {
692 if (first)
693 sba_free_chained_requests(first);
694 return NULL;
695 }
696
697 if (first)
698 sba_chain_request(first, req);
699 else
700 first = req;
701
702 off += req_len;
703 len -= req_len;
704 }
705
706 return (first) ? &first->tx : NULL;
707}
708
709static void sba_fillup_xor_msg(struct sba_request *req,
710 struct brcm_sba_command *cmds,
711 struct brcm_message *msg,
712 dma_addr_t msg_offset, size_t msg_len,
713 dma_addr_t dst, dma_addr_t *src, u32 src_cnt)
714{
715 u64 cmd;
716 u32 c_mdata;
717 unsigned int i;
e7ae72aa 718 dma_addr_t resp_dma = req->tx.phys;
743e1c8f
AP		 719 struct brcm_sba_command *cmdsp = cmds;
720
721 /* Type-B command to load data into buf0 */
722 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
723 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
724 cmd = sba_cmd_enc(cmd, msg_len,
725 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
726 c_mdata = sba_cmd_load_c_mdata(0);
727 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
728 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
729 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
730 SBA_CMD_SHIFT, SBA_CMD_MASK);
731 cmdsp->cmd = cmd;
732 *cmdsp->cmd_dma = cpu_to_le64(cmd);
733 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
734 cmdsp->data = src[0] + msg_offset;
735 cmdsp->data_len = msg_len;
736 cmdsp++;
737
738 /* Type-B commands to xor data with buf0 and put it back in buf0 */
739 for (i = 1; i < src_cnt; i++) {
740 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
741 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
742 cmd = sba_cmd_enc(cmd, msg_len,
743 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
744 c_mdata = sba_cmd_xor_c_mdata(0, 0);
745 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
746 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
747 cmd = sba_cmd_enc(cmd, SBA_CMD_XOR,
748 SBA_CMD_SHIFT, SBA_CMD_MASK);
749 cmdsp->cmd = cmd;
750 *cmdsp->cmd_dma = cpu_to_le64(cmd);
751 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
752 cmdsp->data = src[i] + msg_offset;
753 cmdsp->data_len = msg_len;
754 cmdsp++;
755 }
756
757 /* Type-A command to write buf0 */
758 cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
759 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
760 cmd = sba_cmd_enc(cmd, msg_len,
761 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
762 cmd = sba_cmd_enc(cmd, 0x1,
763 SBA_RESP_SHIFT, SBA_RESP_MASK);
764 c_mdata = sba_cmd_write_c_mdata(0);
765 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
766 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
767 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
768 SBA_CMD_SHIFT, SBA_CMD_MASK);
769 cmdsp->cmd = cmd;
770 *cmdsp->cmd_dma = cpu_to_le64(cmd);
771 cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
772 if (req->sba->hw_resp_size) {
773 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
e7ae72aa 774 cmdsp->resp = resp_dma;
743e1c8f
AP		 775 cmdsp->resp_len = req->sba->hw_resp_size;
776 }
777 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
778 cmdsp->data = dst + msg_offset;
779 cmdsp->data_len = msg_len;
780 cmdsp++;
781
782 /* Fillup brcm_message */
783 msg->type = BRCM_MESSAGE_SBA;
784 msg->sba.cmds = cmds;
785 msg->sba.cmds_count = cmdsp - cmds;
786 msg->ctx = req;
787 msg->error = 0;
788}
789
dd2bceb0 790static struct sba_request *
743e1c8f
AP		 791sba_prep_dma_xor_req(struct sba_device *sba,
792 dma_addr_t off, dma_addr_t dst, dma_addr_t *src,
793 u32 src_cnt, size_t len, unsigned long flags)
794{
795 struct sba_request *req = NULL;
796
797 /* Alloc new request */
798 req = sba_alloc_request(sba);
799 if (!req)
800 return NULL;
57a28508
AP		 801 if (flags & DMA_PREP_FENCE)
802 req->flags |= SBA_REQUEST_FENCE;
743e1c8f
AP		 803
804 /* Fillup request message */
805 sba_fillup_xor_msg(req, req->cmds, &req->msg,
806 off, len, dst, src, src_cnt);
807
808 /* Init async_tx descriptor */
809 req->tx.flags = flags;
810 req->tx.cookie = -EBUSY;
811
812 return req;
813}
814
815static struct dma_async_tx_descriptor *
816sba_prep_dma_xor(struct dma_chan *dchan, dma_addr_t dst, dma_addr_t *src,
817 u32 src_cnt, size_t len, unsigned long flags)
818{
819 size_t req_len;
820 dma_addr_t off = 0;
821 struct sba_device *sba = to_sba_device(dchan);
822 struct sba_request *first = NULL, *req;
823
824 /* Sanity checks */
825 if (unlikely(src_cnt > sba->max_xor_srcs))
826 return NULL;
827
828 /* Create chained requests where each request is upto hw_buf_size */
829 while (len) {
830 req_len = (len < sba->hw_buf_size) ? len : sba->hw_buf_size;
831
832 req = sba_prep_dma_xor_req(sba, off, dst, src, src_cnt,
833 req_len, flags);
834 if (!req) {
835 if (first)
836 sba_free_chained_requests(first);
837 return NULL;
838 }
839
840 if (first)
841 sba_chain_request(first, req);
842 else
843 first = req;
844
845 off += req_len;
846 len -= req_len;
847 }
848
849 return (first) ? &first->tx : NULL;
850}
851
852static void sba_fillup_pq_msg(struct sba_request *req,
853 bool pq_continue,
854 struct brcm_sba_command *cmds,
855 struct brcm_message *msg,
856 dma_addr_t msg_offset, size_t msg_len,
857 dma_addr_t *dst_p, dma_addr_t *dst_q,
858 const u8 *scf, dma_addr_t *src, u32 src_cnt)
859{
860 u64 cmd;
861 u32 c_mdata;
862 unsigned int i;
e7ae72aa 863 dma_addr_t resp_dma = req->tx.phys;
743e1c8f
AP		 864 struct brcm_sba_command *cmdsp = cmds;
865
866 if (pq_continue) {
867 /* Type-B command to load old P into buf0 */
868 if (dst_p) {
869 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
870 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
871 cmd = sba_cmd_enc(cmd, msg_len,
872 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
873 c_mdata = sba_cmd_load_c_mdata(0);
874 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
875 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
876 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
877 SBA_CMD_SHIFT, SBA_CMD_MASK);
878 cmdsp->cmd = cmd;
879 *cmdsp->cmd_dma = cpu_to_le64(cmd);
880 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
881 cmdsp->data = *dst_p + msg_offset;
882 cmdsp->data_len = msg_len;
883 cmdsp++;
884 }
885
886 /* Type-B command to load old Q into buf1 */
887 if (dst_q) {
888 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
889 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
890 cmd = sba_cmd_enc(cmd, msg_len,
891 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
892 c_mdata = sba_cmd_load_c_mdata(1);
893 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
894 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
895 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
896 SBA_CMD_SHIFT, SBA_CMD_MASK);
897 cmdsp->cmd = cmd;
898 *cmdsp->cmd_dma = cpu_to_le64(cmd);
899 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
900 cmdsp->data = *dst_q + msg_offset;
901 cmdsp->data_len = msg_len;
902 cmdsp++;
903 }
904 } else {
905 /* Type-A command to zero all buffers */
906 cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
907 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
908 cmd = sba_cmd_enc(cmd, msg_len,
909 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
910 cmd = sba_cmd_enc(cmd, SBA_CMD_ZERO_ALL_BUFFERS,
911 SBA_CMD_SHIFT, SBA_CMD_MASK);
912 cmdsp->cmd = cmd;
913 *cmdsp->cmd_dma = cpu_to_le64(cmd);
914 cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
915 cmdsp++;
916 }
917
918 /* Type-B commands for generate P onto buf0 and Q onto buf1 */
919 for (i = 0; i < src_cnt; i++) {
920 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
921 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
922 cmd = sba_cmd_enc(cmd, msg_len,
923 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
924 c_mdata = sba_cmd_pq_c_mdata(raid6_gflog[scf[i]], 1, 0);
925 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
926 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
927 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_MS(c_mdata),
928 SBA_C_MDATA_MS_SHIFT, SBA_C_MDATA_MS_MASK);
929 cmd = sba_cmd_enc(cmd, SBA_CMD_GALOIS_XOR,
930 SBA_CMD_SHIFT, SBA_CMD_MASK);
931 cmdsp->cmd = cmd;
932 *cmdsp->cmd_dma = cpu_to_le64(cmd);
933 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
934 cmdsp->data = src[i] + msg_offset;
935 cmdsp->data_len = msg_len;
936 cmdsp++;
937 }
938
939 /* Type-A command to write buf0 */
940 if (dst_p) {
941 cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
942 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
943 cmd = sba_cmd_enc(cmd, msg_len,
944 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
945 cmd = sba_cmd_enc(cmd, 0x1,
946 SBA_RESP_SHIFT, SBA_RESP_MASK);
947 c_mdata = sba_cmd_write_c_mdata(0);
948 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
949 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
950 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
951 SBA_CMD_SHIFT, SBA_CMD_MASK);
952 cmdsp->cmd = cmd;
953 *cmdsp->cmd_dma = cpu_to_le64(cmd);
954 cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
955 if (req->sba->hw_resp_size) {
956 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
e7ae72aa 957 cmdsp->resp = resp_dma;
743e1c8f
AP		 958 cmdsp->resp_len = req->sba->hw_resp_size;
959 }
960 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
961 cmdsp->data = *dst_p + msg_offset;
962 cmdsp->data_len = msg_len;
963 cmdsp++;
964 }
965
966 /* Type-A command to write buf1 */
967 if (dst_q) {
968 cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
969 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
970 cmd = sba_cmd_enc(cmd, msg_len,
971 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
972 cmd = sba_cmd_enc(cmd, 0x1,
973 SBA_RESP_SHIFT, SBA_RESP_MASK);
974 c_mdata = sba_cmd_write_c_mdata(1);
975 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
976 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
977 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
978 SBA_CMD_SHIFT, SBA_CMD_MASK);
979 cmdsp->cmd = cmd;
980 *cmdsp->cmd_dma = cpu_to_le64(cmd);
981 cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
982 if (req->sba->hw_resp_size) {
983 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
e7ae72aa 984 cmdsp->resp = resp_dma;
743e1c8f
AP		 985 cmdsp->resp_len = req->sba->hw_resp_size;
986 }
987 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
988 cmdsp->data = *dst_q + msg_offset;
989 cmdsp->data_len = msg_len;
990 cmdsp++;
991 }
992
993 /* Fillup brcm_message */
994 msg->type = BRCM_MESSAGE_SBA;
995 msg->sba.cmds = cmds;
996 msg->sba.cmds_count = cmdsp - cmds;
997 msg->ctx = req;
998 msg->error = 0;
999}
1000
dd2bceb0 1001static struct sba_request *
743e1c8f
AP		 1002sba_prep_dma_pq_req(struct sba_device *sba, dma_addr_t off,
1003 dma_addr_t *dst_p, dma_addr_t *dst_q, dma_addr_t *src,
1004 u32 src_cnt, const u8 *scf, size_t len, unsigned long flags)
1005{
1006 struct sba_request *req = NULL;
1007
1008 /* Alloc new request */
1009 req = sba_alloc_request(sba);
1010 if (!req)
1011 return NULL;
57a28508
AP		 1012 if (flags & DMA_PREP_FENCE)
1013 req->flags |= SBA_REQUEST_FENCE;
743e1c8f
AP		 1014
1015 /* Fillup request messages */
1016 sba_fillup_pq_msg(req, dmaf_continue(flags),
1017 req->cmds, &req->msg,
1018 off, len, dst_p, dst_q, scf, src, src_cnt);
1019
1020 /* Init async_tx descriptor */
1021 req->tx.flags = flags;
1022 req->tx.cookie = -EBUSY;
1023
1024 return req;
1025}
1026
1027static void sba_fillup_pq_single_msg(struct sba_request *req,
1028 bool pq_continue,
1029 struct brcm_sba_command *cmds,
1030 struct brcm_message *msg,
1031 dma_addr_t msg_offset, size_t msg_len,
1032 dma_addr_t *dst_p, dma_addr_t *dst_q,
1033 dma_addr_t src, u8 scf)
1034{
1035 u64 cmd;
1036 u32 c_mdata;
1037 u8 pos, dpos = raid6_gflog[scf];
e7ae72aa 1038 dma_addr_t resp_dma = req->tx.phys;
743e1c8f
AP		 1039 struct brcm_sba_command *cmdsp = cmds;
1040
1041 if (!dst_p)
1042 goto skip_p;
1043
1044 if (pq_continue) {
1045 /* Type-B command to load old P into buf0 */
1046 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
1047 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1048 cmd = sba_cmd_enc(cmd, msg_len,
1049 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1050 c_mdata = sba_cmd_load_c_mdata(0);
1051 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1052 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1053 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
1054 SBA_CMD_SHIFT, SBA_CMD_MASK);
1055 cmdsp->cmd = cmd;
1056 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1057 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
1058 cmdsp->data = *dst_p + msg_offset;
1059 cmdsp->data_len = msg_len;
1060 cmdsp++;
1061
1062 /*
1063 * Type-B commands to xor data with buf0 and put it
1064 * back in buf0
1065 */
1066 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
1067 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1068 cmd = sba_cmd_enc(cmd, msg_len,
1069 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1070 c_mdata = sba_cmd_xor_c_mdata(0, 0);
1071 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1072 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1073 cmd = sba_cmd_enc(cmd, SBA_CMD_XOR,
1074 SBA_CMD_SHIFT, SBA_CMD_MASK);
1075 cmdsp->cmd = cmd;
1076 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1077 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
1078 cmdsp->data = src + msg_offset;
1079 cmdsp->data_len = msg_len;
1080 cmdsp++;
1081 } else {
1082 /* Type-B command to load old P into buf0 */
1083 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
1084 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1085 cmd = sba_cmd_enc(cmd, msg_len,
1086 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1087 c_mdata = sba_cmd_load_c_mdata(0);
1088 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1089 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1090 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
1091 SBA_CMD_SHIFT, SBA_CMD_MASK);
1092 cmdsp->cmd = cmd;
1093 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1094 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
1095 cmdsp->data = src + msg_offset;
1096 cmdsp->data_len = msg_len;
1097 cmdsp++;
1098 }
1099
1100 /* Type-A command to write buf0 */
1101 cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
1102 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1103 cmd = sba_cmd_enc(cmd, msg_len,
1104 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1105 cmd = sba_cmd_enc(cmd, 0x1,
1106 SBA_RESP_SHIFT, SBA_RESP_MASK);
1107 c_mdata = sba_cmd_write_c_mdata(0);
1108 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1109 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1110 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
1111 SBA_CMD_SHIFT, SBA_CMD_MASK);
1112 cmdsp->cmd = cmd;
1113 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1114 cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
1115 if (req->sba->hw_resp_size) {
1116 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
e7ae72aa 1117 cmdsp->resp = resp_dma;
743e1c8f
AP		 1118 cmdsp->resp_len = req->sba->hw_resp_size;
1119 }
1120 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
1121 cmdsp->data = *dst_p + msg_offset;
1122 cmdsp->data_len = msg_len;
1123 cmdsp++;
1124
1125skip_p:
1126 if (!dst_q)
1127 goto skip_q;
1128
1129 /* Type-A command to zero all buffers */
1130 cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
1131 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1132 cmd = sba_cmd_enc(cmd, msg_len,
1133 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1134 cmd = sba_cmd_enc(cmd, SBA_CMD_ZERO_ALL_BUFFERS,
1135 SBA_CMD_SHIFT, SBA_CMD_MASK);
1136 cmdsp->cmd = cmd;
1137 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1138 cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
1139 cmdsp++;
1140
1141 if (dpos == 255)
1142 goto skip_q_computation;
1143 pos = (dpos < req->sba->max_pq_coefs) ?
1144 dpos : (req->sba->max_pq_coefs - 1);
1145
1146 /*
1147 * Type-B command to generate initial Q from data
1148 * and store output into buf0
1149 */
1150 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
1151 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1152 cmd = sba_cmd_enc(cmd, msg_len,
1153 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1154 c_mdata = sba_cmd_pq_c_mdata(pos, 0, 0);
1155 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1156 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1157 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_MS(c_mdata),
1158 SBA_C_MDATA_MS_SHIFT, SBA_C_MDATA_MS_MASK);
1159 cmd = sba_cmd_enc(cmd, SBA_CMD_GALOIS,
1160 SBA_CMD_SHIFT, SBA_CMD_MASK);
1161 cmdsp->cmd = cmd;
1162 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1163 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
1164 cmdsp->data = src + msg_offset;
1165 cmdsp->data_len = msg_len;
1166 cmdsp++;
1167
1168 dpos -= pos;
1169
1170 /* Multiple Type-A command to generate final Q */
1171 while (dpos) {
1172 pos = (dpos < req->sba->max_pq_coefs) ?
1173 dpos : (req->sba->max_pq_coefs - 1);
1174
1175 /*
1176 * Type-A command to generate Q with buf0 and
1177 * buf1 store result in buf0
1178 */
1179 cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
1180 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1181 cmd = sba_cmd_enc(cmd, msg_len,
1182 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1183 c_mdata = sba_cmd_pq_c_mdata(pos, 0, 1);
1184 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1185 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1186 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_MS(c_mdata),
1187 SBA_C_MDATA_MS_SHIFT, SBA_C_MDATA_MS_MASK);
1188 cmd = sba_cmd_enc(cmd, SBA_CMD_GALOIS,
1189 SBA_CMD_SHIFT, SBA_CMD_MASK);
1190 cmdsp->cmd = cmd;
1191 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1192 cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
1193 cmdsp++;
1194
1195 dpos -= pos;
1196 }
1197
1198skip_q_computation:
1199 if (pq_continue) {
1200 /*
1201 * Type-B command to XOR previous output with
1202 * buf0 and write it into buf0
1203 */
1204 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
1205 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1206 cmd = sba_cmd_enc(cmd, msg_len,
1207 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1208 c_mdata = sba_cmd_xor_c_mdata(0, 0);
1209 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1210 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1211 cmd = sba_cmd_enc(cmd, SBA_CMD_XOR,
1212 SBA_CMD_SHIFT, SBA_CMD_MASK);
1213 cmdsp->cmd = cmd;
1214 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1215 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
1216 cmdsp->data = *dst_q + msg_offset;
1217 cmdsp->data_len = msg_len;
1218 cmdsp++;
1219 }
1220
1221 /* Type-A command to write buf0 */
1222 cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
1223 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1224 cmd = sba_cmd_enc(cmd, msg_len,
1225 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1226 cmd = sba_cmd_enc(cmd, 0x1,
1227 SBA_RESP_SHIFT, SBA_RESP_MASK);
1228 c_mdata = sba_cmd_write_c_mdata(0);
1229 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1230 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1231 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
1232 SBA_CMD_SHIFT, SBA_CMD_MASK);
1233 cmdsp->cmd = cmd;
1234 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1235 cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
1236 if (req->sba->hw_resp_size) {
1237 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
e7ae72aa 1238 cmdsp->resp = resp_dma;
743e1c8f
AP		 1239 cmdsp->resp_len = req->sba->hw_resp_size;
1240 }
1241 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
1242 cmdsp->data = *dst_q + msg_offset;
1243 cmdsp->data_len = msg_len;
1244 cmdsp++;
1245
1246skip_q:
1247 /* Fillup brcm_message */
1248 msg->type = BRCM_MESSAGE_SBA;
1249 msg->sba.cmds = cmds;
1250 msg->sba.cmds_count = cmdsp - cmds;
1251 msg->ctx = req;
1252 msg->error = 0;
1253}
1254
dd2bceb0 1255static struct sba_request *
743e1c8f
AP		 1256sba_prep_dma_pq_single_req(struct sba_device *sba, dma_addr_t off,
1257 dma_addr_t *dst_p, dma_addr_t *dst_q,
1258 dma_addr_t src, u8 scf, size_t len,
1259 unsigned long flags)
1260{
1261 struct sba_request *req = NULL;
1262
1263 /* Alloc new request */
1264 req = sba_alloc_request(sba);
1265 if (!req)
1266 return NULL;
57a28508
AP		 1267 if (flags & DMA_PREP_FENCE)
1268 req->flags |= SBA_REQUEST_FENCE;
743e1c8f
AP		 1269
1270 /* Fillup request messages */
1271 sba_fillup_pq_single_msg(req, dmaf_continue(flags),
1272 req->cmds, &req->msg, off, len,
1273 dst_p, dst_q, src, scf);
1274
1275 /* Init async_tx descriptor */
1276 req->tx.flags = flags;
1277 req->tx.cookie = -EBUSY;
1278
1279 return req;
1280}
1281
1282static struct dma_async_tx_descriptor *
1283sba_prep_dma_pq(struct dma_chan *dchan, dma_addr_t *dst, dma_addr_t *src,
1284 u32 src_cnt, const u8 *scf, size_t len, unsigned long flags)
1285{
1286 u32 i, dst_q_index;
1287 size_t req_len;
1288 bool slow = false;
1289 dma_addr_t off = 0;
1290 dma_addr_t *dst_p = NULL, *dst_q = NULL;
1291 struct sba_device *sba = to_sba_device(dchan);
1292 struct sba_request *first = NULL, *req;
1293
1294 /* Sanity checks */
1295 if (unlikely(src_cnt > sba->max_pq_srcs))
1296 return NULL;
1297 for (i = 0; i < src_cnt; i++)
1298 if (sba->max_pq_coefs <= raid6_gflog[scf[i]])
1299 slow = true;
1300
1301 /* Figure-out P and Q destination addresses */
1302 if (!(flags & DMA_PREP_PQ_DISABLE_P))
1303 dst_p = &dst[0];
1304 if (!(flags & DMA_PREP_PQ_DISABLE_Q))
1305 dst_q = &dst[1];
1306
1307 /* Create chained requests where each request is upto hw_buf_size */
1308 while (len) {
1309 req_len = (len < sba->hw_buf_size) ? len : sba->hw_buf_size;
1310
1311 if (slow) {
1312 dst_q_index = src_cnt;
1313
1314 if (dst_q) {
1315 for (i = 0; i < src_cnt; i++) {
1316 if (*dst_q == src[i]) {
1317 dst_q_index = i;
1318 break;
1319 }
1320 }
1321 }
1322
1323 if (dst_q_index < src_cnt) {
1324 i = dst_q_index;
1325 req = sba_prep_dma_pq_single_req(sba,
1326 off, dst_p, dst_q, src[i], scf[i],
1327 req_len, flags | DMA_PREP_FENCE);
1328 if (!req)
1329 goto fail;
1330
1331 if (first)
1332 sba_chain_request(first, req);
1333 else
1334 first = req;
1335
1336 flags |= DMA_PREP_CONTINUE;
1337 }
1338
1339 for (i = 0; i < src_cnt; i++) {
1340 if (dst_q_index == i)
1341 continue;
1342
1343 req = sba_prep_dma_pq_single_req(sba,
1344 off, dst_p, dst_q, src[i], scf[i],
1345 req_len, flags | DMA_PREP_FENCE);
1346 if (!req)
1347 goto fail;
1348
1349 if (first)
1350 sba_chain_request(first, req);
1351 else
1352 first = req;
1353
1354 flags |= DMA_PREP_CONTINUE;
1355 }
1356 } else {
1357 req = sba_prep_dma_pq_req(sba, off,
1358 dst_p, dst_q, src, src_cnt,
1359 scf, req_len, flags);
1360 if (!req)
1361 goto fail;
1362
1363 if (first)
1364 sba_chain_request(first, req);
1365 else
1366 first = req;
1367 }
1368
1369 off += req_len;
1370 len -= req_len;
1371 }
1372
1373 return (first) ? &first->tx : NULL;
1374
1375fail:
1376 if (first)
1377 sba_free_chained_requests(first);
1378 return NULL;
1379}
1380
1381/* ====== Mailbox callbacks ===== */
1382
1383static void sba_dma_tx_actions(struct sba_request *req)
1384{
1385 struct dma_async_tx_descriptor *tx = &req->tx;
1386
1387 WARN_ON(tx->cookie < 0);
1388
1389 if (tx->cookie > 0) {
1390 dma_cookie_complete(tx);
1391
1392 /*
1393 * Call the callback (must not sleep or submit new
1394 * operations to this channel)
1395 */
1396 if (tx->callback)
1397 tx->callback(tx->callback_param);
1398
1399 dma_descriptor_unmap(tx);
1400 }
1401
1402 /* Run dependent operations */
1403 dma_run_dependencies(tx);
1404
1405 /* If waiting for 'ack' then move to completed list */
1406 if (!async_tx_test_ack(&req->tx))
1407 sba_complete_chained_requests(req);
1408 else
1409 sba_free_chained_requests(req);
1410}
1411
1412static void sba_receive_message(struct mbox_client *cl, void *msg)
1413{
1414 unsigned long flags;
1415 struct brcm_message *m = msg;
1416 struct sba_request *req = m->ctx, *req1;
1417 struct sba_device *sba = req->sba;
1418
1419 /* Error count if message has error */
1420 if (m->error < 0)
1421 dev_err(sba->dev, "%s got message with error %d",
1422 dma_chan_name(&sba->dma_chan), m->error);
1423
1424 /* Mark request as received */
1425 sba_received_request(req);
1426
1427 /* Wait for all chained requests to be completed */
1428 if (atomic_dec_return(&req->first->next_pending_count))
1429 goto done;
1430
1431 /* Point to first request */
1432 req = req->first;
1433
1434 /* Update request */
57a28508 1435 if (req->flags & SBA_REQUEST_STATE_RECEIVED)
743e1c8f
AP		 1436 sba_dma_tx_actions(req);
1437 else
1438 sba_free_chained_requests(req);
1439
1440 spin_lock_irqsave(&sba->reqs_lock, flags);
1441
1442 /* Re-check all completed request waiting for 'ack' */
1443 list_for_each_entry_safe(req, req1, &sba->reqs_completed_list, node) {
1444 spin_unlock_irqrestore(&sba->reqs_lock, flags);
1445 sba_dma_tx_actions(req);
1446 spin_lock_irqsave(&sba->reqs_lock, flags);
1447 }
1448
1449 spin_unlock_irqrestore(&sba->reqs_lock, flags);
1450
1451done:
1452 /* Try to submit pending request */
1453 sba_issue_pending(&sba->dma_chan);
1454}
1455
1456/* ====== Platform driver routines ===== */
1457
1458static int sba_prealloc_channel_resources(struct sba_device *sba)
1459{
e7ae72aa 1460 int i, j, ret = 0;
743e1c8f
AP		 1461 struct sba_request *req = NULL;
1462
1463 sba->resp_base = dma_alloc_coherent(sba->dma_dev.dev,
1464 sba->max_resp_pool_size,
1465 &sba->resp_dma_base, GFP_KERNEL);
1466 if (!sba->resp_base)
1467 return -ENOMEM;
1468
1469 sba->cmds_base = dma_alloc_coherent(sba->dma_dev.dev,
1470 sba->max_cmds_pool_size,
1471 &sba->cmds_dma_base, GFP_KERNEL);
1472 if (!sba->cmds_base) {
1473 ret = -ENOMEM;
1474 goto fail_free_resp_pool;
1475 }
1476
1477 spin_lock_init(&sba->reqs_lock);
1478 sba->reqs_fence = false;
1479 INIT_LIST_HEAD(&sba->reqs_alloc_list);
1480 INIT_LIST_HEAD(&sba->reqs_pending_list);
1481 INIT_LIST_HEAD(&sba->reqs_active_list);
1482 INIT_LIST_HEAD(&sba->reqs_received_list);
1483 INIT_LIST_HEAD(&sba->reqs_completed_list);
1484 INIT_LIST_HEAD(&sba->reqs_aborted_list);
1485 INIT_LIST_HEAD(&sba->reqs_free_list);
1486
1487 sba->reqs = devm_kcalloc(sba->dev, sba->max_req,
1488 sizeof(*req), GFP_KERNEL);
1489 if (!sba->reqs) {
1490 ret = -ENOMEM;
1491 goto fail_free_cmds_pool;
1492 }
1493
e7ae72aa 1494 for (i = 0; i < sba->max_req; i++) {
743e1c8f
AP		 1495 req = &sba->reqs[i];
1496 INIT_LIST_HEAD(&req->node);
1497 req->sba = sba;
57a28508 1498 req->flags = SBA_REQUEST_STATE_FREE;
743e1c8f 1499 INIT_LIST_HEAD(&req->next);
743e1c8f 1500 atomic_set(&req->next_pending_count, 0);
743e1c8f
AP		 1501 req->cmds = devm_kcalloc(sba->dev, sba->max_cmd_per_req,
1502 sizeof(*req->cmds), GFP_KERNEL);
1503 if (!req->cmds) {
1504 ret = -ENOMEM;
1505 goto fail_free_cmds_pool;
1506 }
1507 for (j = 0; j < sba->max_cmd_per_req; j++) {
1508 req->cmds[j].cmd = 0;
1509 req->cmds[j].cmd_dma = sba->cmds_base +
1510 (i * sba->max_cmd_per_req + j) * sizeof(u64);
1511 req->cmds[j].cmd_dma_addr = sba->cmds_dma_base +
1512 (i * sba->max_cmd_per_req + j) * sizeof(u64);
1513 req->cmds[j].flags = 0;
1514 }
1515 memset(&req->msg, 0, sizeof(req->msg));
1516 dma_async_tx_descriptor_init(&req->tx, &sba->dma_chan);
1517 req->tx.tx_submit = sba_tx_submit;
e7ae72aa 1518 req->tx.phys = sba->resp_dma_base + i * sba->hw_resp_size;
743e1c8f
AP		 1519 list_add_tail(&req->node, &sba->reqs_free_list);
1520 }
1521
743e1c8f
AP		 1522 return 0;
1523
1524fail_free_cmds_pool:
1525 dma_free_coherent(sba->dma_dev.dev,
1526 sba->max_cmds_pool_size,
1527 sba->cmds_base, sba->cmds_dma_base);
1528fail_free_resp_pool:
1529 dma_free_coherent(sba->dma_dev.dev,
1530 sba->max_resp_pool_size,
1531 sba->resp_base, sba->resp_dma_base);
1532 return ret;
1533}
1534
1535static void sba_freeup_channel_resources(struct sba_device *sba)
1536{
1537 dmaengine_terminate_all(&sba->dma_chan);
1538 dma_free_coherent(sba->dma_dev.dev, sba->max_cmds_pool_size,
1539 sba->cmds_base, sba->cmds_dma_base);
1540 dma_free_coherent(sba->dma_dev.dev, sba->max_resp_pool_size,
1541 sba->resp_base, sba->resp_dma_base);
1542 sba->resp_base = NULL;
1543 sba->resp_dma_base = 0;
1544}
1545
1546static int sba_async_register(struct sba_device *sba)
1547{
1548 int ret;
1549 struct dma_device *dma_dev = &sba->dma_dev;
1550
1551 /* Initialize DMA channel cookie */
1552 sba->dma_chan.device = dma_dev;
1553 dma_cookie_init(&sba->dma_chan);
1554
1555 /* Initialize DMA device capability mask */
1556 dma_cap_zero(dma_dev->cap_mask);
1557 dma_cap_set(DMA_INTERRUPT, dma_dev->cap_mask);
1558 dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
1559 dma_cap_set(DMA_XOR, dma_dev->cap_mask);
1560 dma_cap_set(DMA_PQ, dma_dev->cap_mask);
1561
1562 /*
1563 * Set mailbox channel device as the base device of
1564 * our dma_device because the actual memory accesses
1565 * will be done by mailbox controller
1566 */
1567 dma_dev->dev = sba->mbox_dev;
1568
1569 /* Set base prep routines */
1570 dma_dev->device_free_chan_resources = sba_free_chan_resources;
1571 dma_dev->device_terminate_all = sba_device_terminate_all;
1572 dma_dev->device_issue_pending = sba_issue_pending;
1573 dma_dev->device_tx_status = sba_tx_status;
1574
1575 /* Set interrupt routine */
1576 if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask))
1577 dma_dev->device_prep_dma_interrupt = sba_prep_dma_interrupt;
1578
1579 /* Set memcpy routine */
1580 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask))
1581 dma_dev->device_prep_dma_memcpy = sba_prep_dma_memcpy;
1582
1583 /* Set xor routine and capability */
1584 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
1585 dma_dev->device_prep_dma_xor = sba_prep_dma_xor;
1586 dma_dev->max_xor = sba->max_xor_srcs;
1587 }
1588
1589 /* Set pq routine and capability */
1590 if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
1591 dma_dev->device_prep_dma_pq = sba_prep_dma_pq;
1592 dma_set_maxpq(dma_dev, sba->max_pq_srcs, 0);
1593 }
1594
1595 /* Initialize DMA device channel list */
1596 INIT_LIST_HEAD(&dma_dev->channels);
1597 list_add_tail(&sba->dma_chan.device_node, &dma_dev->channels);
1598
1599 /* Register with Linux async DMA framework*/
1600 ret = dma_async_device_register(dma_dev);
1601 if (ret) {
1602 dev_err(sba->dev, "async device register error %d", ret);
1603 return ret;
1604 }
1605
1606 dev_info(sba->dev, "%s capabilities: %s%s%s%s\n",
1607 dma_chan_name(&sba->dma_chan),
1608 dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "interrupt " : "",
1609 dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "memcpy " : "",
1610 dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "",
1611 dma_has_cap(DMA_PQ, dma_dev->cap_mask) ? "pq " : "");
1612
1613 return 0;
1614}
1615
1616static int sba_probe(struct platform_device *pdev)
1617{
1618 int i, ret = 0, mchans_count;
1619 struct sba_device *sba;
1620 struct platform_device *mbox_pdev;
1621 struct of_phandle_args args;
1622
1623 /* Allocate main SBA struct */
1624 sba = devm_kzalloc(&pdev->dev, sizeof(*sba), GFP_KERNEL);
1625 if (!sba)
1626 return -ENOMEM;
1627
1628 sba->dev = &pdev->dev;
1629 platform_set_drvdata(pdev, sba);
1630
1631 /* Determine SBA version from DT compatible string */
1632 if (of_device_is_compatible(sba->dev->of_node, "brcm,iproc-sba"))
1633 sba->ver = SBA_VER_1;
1634 else if (of_device_is_compatible(sba->dev->of_node,
1635 "brcm,iproc-sba-v2"))
1636 sba->ver = SBA_VER_2;
1637 else
1638 return -ENODEV;
1639
1640 /* Derived Configuration parameters */
1641 switch (sba->ver) {
1642 case SBA_VER_1:
1643 sba->max_req = 1024;
1644 sba->hw_buf_size = 4096;
1645 sba->hw_resp_size = 8;
1646 sba->max_pq_coefs = 6;
1647 sba->max_pq_srcs = 6;
1648 break;
1649 case SBA_VER_2:
1650 sba->max_req = 1024;
1651 sba->hw_buf_size = 4096;
1652 sba->hw_resp_size = 8;
1653 sba->max_pq_coefs = 30;
1654 /*
1655 * We can support max_pq_srcs == max_pq_coefs because
1656 * we are limited by number of SBA commands that we can
1657 * fit in one message for underlying ring manager HW.
1658 */
1659 sba->max_pq_srcs = 12;
1660 break;
1661 default:
1662 return -EINVAL;
1663 }
1664 sba->max_cmd_per_req = sba->max_pq_srcs + 3;
1665 sba->max_xor_srcs = sba->max_cmd_per_req - 1;
1666 sba->max_resp_pool_size = sba->max_req * sba->hw_resp_size;
1667 sba->max_cmds_pool_size = sba->max_req *
1668 sba->max_cmd_per_req * sizeof(u64);
1669
1670 /* Setup mailbox client */
1671 sba->client.dev = &pdev->dev;
1672 sba->client.rx_callback = sba_receive_message;
1673 sba->client.tx_block = false;
1674 sba->client.knows_txdone = false;
1675 sba->client.tx_tout = 0;
1676
1677 /* Number of channels equals number of mailbox channels */
1678 ret = of_count_phandle_with_args(pdev->dev.of_node,
1679 "mboxes", "#mbox-cells");
1680 if (ret <= 0)
1681 return -ENODEV;
1682 mchans_count = ret;
1683 sba->mchans_count = 0;
1684 atomic_set(&sba->mchans_current, 0);
1685
1686 /* Allocate mailbox channel array */
1687 sba->mchans = devm_kcalloc(&pdev->dev, sba->mchans_count,
1688 sizeof(*sba->mchans), GFP_KERNEL);
1689 if (!sba->mchans)
1690 return -ENOMEM;
1691
1692 /* Request mailbox channels */
1693 for (i = 0; i < mchans_count; i++) {
1694 sba->mchans[i] = mbox_request_channel(&sba->client, i);
1695 if (IS_ERR(sba->mchans[i])) {
1696 ret = PTR_ERR(sba->mchans[i]);
1697 goto fail_free_mchans;
1698 }
1699 sba->mchans_count++;
1700 }
1701
1702 /* Find-out underlying mailbox device */
1703 ret = of_parse_phandle_with_args(pdev->dev.of_node,
1704 "mboxes", "#mbox-cells", 0, &args);
1705 if (ret)
1706 goto fail_free_mchans;
1707 mbox_pdev = of_find_device_by_node(args.np);
1708 of_node_put(args.np);
1709 if (!mbox_pdev) {
1710 ret = -ENODEV;
1711 goto fail_free_mchans;
1712 }
1713 sba->mbox_dev = &mbox_pdev->dev;
1714
1715 /* All mailbox channels should be of same ring manager device */
1716 for (i = 1; i < mchans_count; i++) {
1717 ret = of_parse_phandle_with_args(pdev->dev.of_node,
1718 "mboxes", "#mbox-cells", i, &args);
1719 if (ret)
1720 goto fail_free_mchans;
1721 mbox_pdev = of_find_device_by_node(args.np);
1722 of_node_put(args.np);
1723 if (sba->mbox_dev != &mbox_pdev->dev) {
1724 ret = -EINVAL;
1725 goto fail_free_mchans;
1726 }
1727 }
1728
1729 /* Register DMA device with linux async framework */
1730 ret = sba_async_register(sba);
1731 if (ret)
1732 goto fail_free_mchans;
1733
1734 /* Prealloc channel resource */
1735 ret = sba_prealloc_channel_resources(sba);
1736 if (ret)
1737 goto fail_async_dev_unreg;
1738
1739 /* Print device info */
1740 dev_info(sba->dev, "%s using SBAv%d and %d mailbox channels",
1741 dma_chan_name(&sba->dma_chan), sba->ver+1,
1742 sba->mchans_count);
1743
1744 return 0;
1745
1746fail_async_dev_unreg:
1747 dma_async_device_unregister(&sba->dma_dev);
1748fail_free_mchans:
1749 for (i = 0; i < sba->mchans_count; i++)
1750 mbox_free_channel(sba->mchans[i]);
1751 return ret;
1752}
1753
1754static int sba_remove(struct platform_device *pdev)
1755{
1756 int i;
1757 struct sba_device *sba = platform_get_drvdata(pdev);
1758
1759 sba_freeup_channel_resources(sba);
1760
1761 dma_async_device_unregister(&sba->dma_dev);
1762
1763 for (i = 0; i < sba->mchans_count; i++)
1764 mbox_free_channel(sba->mchans[i]);
1765
1766 return 0;
1767}
1768
1769static const struct of_device_id sba_of_match[] = {
1770 { .compatible = "brcm,iproc-sba", },
1771 { .compatible = "brcm,iproc-sba-v2", },
1772 {},
1773};
1774MODULE_DEVICE_TABLE(of, sba_of_match);
1775
1776static struct platform_driver sba_driver = {
1777 .probe = sba_probe,
1778 .remove = sba_remove,
1779 .driver = {
1780 .name = "bcm-sba-raid",
1781 .of_match_table = sba_of_match,
1782 },
1783};
1784module_platform_driver(sba_driver);
1785
1786MODULE_DESCRIPTION("Broadcom SBA RAID driver");
1787MODULE_AUTHOR("Anup Patel <anup.patel@broadcom.com>");
1788MODULE_LICENSE("GPL v2");
Linux 6.x block layer and io_uring tree(s)