Commit | Line | Data |
---|---|---|
ef8c2dab CC |
1 | /* |
2 | * Texas Instruments CPDMA Driver | |
3 | * | |
4 | * Copyright (C) 2010 Texas Instruments | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or | |
7 | * modify it under the terms of the GNU General Public License as | |
8 | * published by the Free Software Foundation version 2. | |
9 | * | |
10 | * This program is distributed "as is" WITHOUT ANY WARRANTY of any | |
11 | * kind, whether express or implied; without even the implied warranty | |
12 | * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
13 | * GNU General Public License for more details. | |
14 | */ | |
15 | #include <linux/kernel.h> | |
16 | #include <linux/spinlock.h> | |
17 | #include <linux/device.h> | |
76fbc247 | 18 | #include <linux/module.h> |
ef8c2dab CC |
19 | #include <linux/slab.h> |
20 | #include <linux/err.h> | |
21 | #include <linux/dma-mapping.h> | |
22 | #include <linux/io.h> | |
817f6d1a | 23 | #include <linux/delay.h> |
742fb20f | 24 | #include <linux/genalloc.h> |
ef8c2dab CC |
25 | #include "davinci_cpdma.h" |
26 | ||
27 | /* DMA Registers */ | |
28 | #define CPDMA_TXIDVER 0x00 | |
29 | #define CPDMA_TXCONTROL 0x04 | |
30 | #define CPDMA_TXTEARDOWN 0x08 | |
31 | #define CPDMA_RXIDVER 0x10 | |
32 | #define CPDMA_RXCONTROL 0x14 | |
33 | #define CPDMA_SOFTRESET 0x1c | |
34 | #define CPDMA_RXTEARDOWN 0x18 | |
8f32b909 | 35 | #define CPDMA_TX_PRI0_RATE 0x30 |
ef8c2dab CC |
36 | #define CPDMA_TXINTSTATRAW 0x80 |
37 | #define CPDMA_TXINTSTATMASKED 0x84 | |
38 | #define CPDMA_TXINTMASKSET 0x88 | |
39 | #define CPDMA_TXINTMASKCLEAR 0x8c | |
40 | #define CPDMA_MACINVECTOR 0x90 | |
41 | #define CPDMA_MACEOIVECTOR 0x94 | |
42 | #define CPDMA_RXINTSTATRAW 0xa0 | |
43 | #define CPDMA_RXINTSTATMASKED 0xa4 | |
44 | #define CPDMA_RXINTMASKSET 0xa8 | |
45 | #define CPDMA_RXINTMASKCLEAR 0xac | |
46 | #define CPDMA_DMAINTSTATRAW 0xb0 | |
47 | #define CPDMA_DMAINTSTATMASKED 0xb4 | |
48 | #define CPDMA_DMAINTMASKSET 0xb8 | |
49 | #define CPDMA_DMAINTMASKCLEAR 0xbc | |
50 | #define CPDMA_DMAINT_HOSTERR BIT(1) | |
51 | ||
52 | /* the following exist only if has_ext_regs is set */ | |
53 | #define CPDMA_DMACONTROL 0x20 | |
54 | #define CPDMA_DMASTATUS 0x24 | |
55 | #define CPDMA_RXBUFFOFS 0x28 | |
56 | #define CPDMA_EM_CONTROL 0x2c | |
57 | ||
58 | /* Descriptor mode bits */ | |
59 | #define CPDMA_DESC_SOP BIT(31) | |
60 | #define CPDMA_DESC_EOP BIT(30) | |
61 | #define CPDMA_DESC_OWNER BIT(29) | |
62 | #define CPDMA_DESC_EOQ BIT(28) | |
63 | #define CPDMA_DESC_TD_COMPLETE BIT(27) | |
64 | #define CPDMA_DESC_PASS_CRC BIT(26) | |
f6e135c8 M |
65 | #define CPDMA_DESC_TO_PORT_EN BIT(20) |
66 | #define CPDMA_TO_PORT_SHIFT 16 | |
67 | #define CPDMA_DESC_PORT_MASK (BIT(18) | BIT(17) | BIT(16)) | |
28a19fe6 | 68 | #define CPDMA_DESC_CRC_LEN 4 |
ef8c2dab CC |
69 | |
70 | #define CPDMA_TEARDOWN_VALUE 0xfffffffc | |
71 | ||
8f32b909 IK |
72 | #define CPDMA_MAX_RLIM_CNT 16384 |
73 | ||
ef8c2dab CC |
74 | struct cpdma_desc { |
75 | /* hardware fields */ | |
76 | u32 hw_next; | |
77 | u32 hw_buffer; | |
78 | u32 hw_len; | |
79 | u32 hw_mode; | |
80 | /* software fields */ | |
81 | void *sw_token; | |
82 | u32 sw_buffer; | |
83 | u32 sw_len; | |
84 | }; | |
85 | ||
86 | struct cpdma_desc_pool { | |
c767db51 | 87 | phys_addr_t phys; |
84092996 | 88 | dma_addr_t hw_addr; |
ef8c2dab CC |
89 | void __iomem *iomap; /* ioremap map */ |
90 | void *cpumap; /* dma_alloc map */ | |
91 | int desc_size, mem_size; | |
aeec3021 | 92 | int num_desc; |
ef8c2dab | 93 | struct device *dev; |
742fb20f | 94 | struct gen_pool *gen_pool; |
ef8c2dab CC |
95 | }; |
96 | ||
97 | enum cpdma_state { | |
98 | CPDMA_STATE_IDLE, | |
99 | CPDMA_STATE_ACTIVE, | |
100 | CPDMA_STATE_TEARDOWN, | |
101 | }; | |
102 | ||
ef8c2dab CC |
103 | struct cpdma_ctlr { |
104 | enum cpdma_state state; | |
105 | struct cpdma_params params; | |
106 | struct device *dev; | |
107 | struct cpdma_desc_pool *pool; | |
108 | spinlock_t lock; | |
109 | struct cpdma_chan *channels[2 * CPDMA_MAX_CHANNELS]; | |
3802dce1 | 110 | int chan_num; |
be034fc1 GS |
111 | int num_rx_desc; /* RX descriptors number */ |
112 | int num_tx_desc; /* TX descriptors number */ | |
ef8c2dab CC |
113 | }; |
114 | ||
115 | struct cpdma_chan { | |
fae50823 M |
116 | struct cpdma_desc __iomem *head, *tail; |
117 | void __iomem *hdp, *cp, *rxfree; | |
ef8c2dab CC |
118 | enum cpdma_state state; |
119 | struct cpdma_ctlr *ctlr; | |
120 | int chan_num; | |
121 | spinlock_t lock; | |
ef8c2dab | 122 | int count; |
742fb20f | 123 | u32 desc_num; |
ef8c2dab CC |
124 | u32 mask; |
125 | cpdma_handler_fn handler; | |
126 | enum dma_data_direction dir; | |
127 | struct cpdma_chan_stats stats; | |
128 | /* offsets into dmaregs */ | |
129 | int int_set, int_clear, td; | |
0fc6432c | 130 | int weight; |
8f32b909 IK |
131 | u32 rate_factor; |
132 | u32 rate; | |
ef8c2dab CC |
133 | }; |
134 | ||
991ddb1f IK |
135 | struct cpdma_control_info { |
136 | u32 reg; | |
137 | u32 shift, mask; | |
138 | int access; | |
139 | #define ACCESS_RO BIT(0) | |
140 | #define ACCESS_WO BIT(1) | |
141 | #define ACCESS_RW (ACCESS_RO | ACCESS_WO) | |
142 | }; | |
143 | ||
144 | static struct cpdma_control_info controls[] = { | |
8f32b909 | 145 | [CPDMA_TX_RLIM] = {CPDMA_DMACONTROL, 8, 0xffff, ACCESS_RW}, |
991ddb1f IK |
146 | [CPDMA_CMD_IDLE] = {CPDMA_DMACONTROL, 3, 1, ACCESS_WO}, |
147 | [CPDMA_COPY_ERROR_FRAMES] = {CPDMA_DMACONTROL, 4, 1, ACCESS_RW}, | |
148 | [CPDMA_RX_OFF_LEN_UPDATE] = {CPDMA_DMACONTROL, 2, 1, ACCESS_RW}, | |
149 | [CPDMA_RX_OWNERSHIP_FLIP] = {CPDMA_DMACONTROL, 1, 1, ACCESS_RW}, | |
150 | [CPDMA_TX_PRIO_FIXED] = {CPDMA_DMACONTROL, 0, 1, ACCESS_RW}, | |
151 | [CPDMA_STAT_IDLE] = {CPDMA_DMASTATUS, 31, 1, ACCESS_RO}, | |
152 | [CPDMA_STAT_TX_ERR_CODE] = {CPDMA_DMASTATUS, 20, 0xf, ACCESS_RW}, | |
153 | [CPDMA_STAT_TX_ERR_CHAN] = {CPDMA_DMASTATUS, 16, 0x7, ACCESS_RW}, | |
154 | [CPDMA_STAT_RX_ERR_CODE] = {CPDMA_DMASTATUS, 12, 0xf, ACCESS_RW}, | |
155 | [CPDMA_STAT_RX_ERR_CHAN] = {CPDMA_DMASTATUS, 8, 0x7, ACCESS_RW}, | |
156 | [CPDMA_RX_BUFFER_OFFSET] = {CPDMA_RXBUFFOFS, 0, 0xffff, ACCESS_RW}, | |
157 | }; | |
158 | ||
925d65e6 IK |
159 | #define tx_chan_num(chan) (chan) |
160 | #define rx_chan_num(chan) ((chan) + CPDMA_MAX_CHANNELS) | |
161 | #define is_rx_chan(chan) ((chan)->chan_num >= CPDMA_MAX_CHANNELS) | |
162 | #define is_tx_chan(chan) (!is_rx_chan(chan)) | |
163 | #define __chan_linear(chan_num) ((chan_num) & (CPDMA_MAX_CHANNELS - 1)) | |
164 | #define chan_linear(chan) __chan_linear((chan)->chan_num) | |
165 | ||
ef8c2dab CC |
166 | /* The following make access to common cpdma_ctlr params more readable */ |
167 | #define dmaregs params.dmaregs | |
168 | #define num_chan params.num_chan | |
169 | ||
170 | /* various accessors */ | |
a6c83ccf GS |
171 | #define dma_reg_read(ctlr, ofs) readl((ctlr)->dmaregs + (ofs)) |
172 | #define chan_read(chan, fld) readl((chan)->fld) | |
173 | #define desc_read(desc, fld) readl(&(desc)->fld) | |
174 | #define dma_reg_write(ctlr, ofs, v) writel(v, (ctlr)->dmaregs + (ofs)) | |
175 | #define chan_write(chan, fld, v) writel(v, (chan)->fld) | |
176 | #define desc_write(desc, fld, v) writel((u32)(v), &(desc)->fld) | |
ef8c2dab | 177 | |
f6e135c8 M |
178 | #define cpdma_desc_to_port(chan, mode, directed) \ |
179 | do { \ | |
180 | if (!is_rx_chan(chan) && ((directed == 1) || \ | |
181 | (directed == 2))) \ | |
182 | mode |= (CPDMA_DESC_TO_PORT_EN | \ | |
183 | (directed << CPDMA_TO_PORT_SHIFT)); \ | |
184 | } while (0) | |
185 | ||
5fcc40a9 | 186 | static void cpdma_desc_pool_destroy(struct cpdma_ctlr *ctlr) |
742fb20f | 187 | { |
5fcc40a9 GS |
188 | struct cpdma_desc_pool *pool = ctlr->pool; |
189 | ||
742fb20f GS |
190 | if (!pool) |
191 | return; | |
192 | ||
aeec3021 GS |
193 | WARN(gen_pool_size(pool->gen_pool) != gen_pool_avail(pool->gen_pool), |
194 | "cpdma_desc_pool size %d != avail %d", | |
195 | gen_pool_size(pool->gen_pool), | |
196 | gen_pool_avail(pool->gen_pool)); | |
742fb20f | 197 | if (pool->cpumap) |
5fcc40a9 | 198 | dma_free_coherent(ctlr->dev, pool->mem_size, pool->cpumap, |
742fb20f | 199 | pool->phys); |
742fb20f GS |
200 | } |
201 | ||
ef8c2dab CC |
202 | /* |
203 | * Utility constructs for a cpdma descriptor pool. Some devices (e.g. davinci | |
204 | * emac) have dedicated on-chip memory for these descriptors. Some other | |
205 | * devices (e.g. cpsw switches) use plain old memory. Descriptor pools | |
206 | * abstract out these details | |
207 | */ | |
5fcc40a9 | 208 | int cpdma_desc_pool_create(struct cpdma_ctlr *ctlr) |
ef8c2dab | 209 | { |
5fcc40a9 | 210 | struct cpdma_params *cpdma_params = &ctlr->params; |
ef8c2dab | 211 | struct cpdma_desc_pool *pool; |
5fcc40a9 | 212 | int ret = -ENOMEM; |
ef8c2dab | 213 | |
5fcc40a9 | 214 | pool = devm_kzalloc(ctlr->dev, sizeof(*pool), GFP_KERNEL); |
ef8c2dab | 215 | if (!pool) |
742fb20f | 216 | goto gen_pool_create_fail; |
5fcc40a9 | 217 | ctlr->pool = pool; |
ef8c2dab | 218 | |
5fcc40a9 GS |
219 | pool->mem_size = cpdma_params->desc_mem_size; |
220 | pool->desc_size = ALIGN(sizeof(struct cpdma_desc), | |
221 | cpdma_params->desc_align); | |
222 | pool->num_desc = pool->mem_size / pool->desc_size; | |
ef8c2dab | 223 | |
90225bf0 GS |
224 | if (cpdma_params->descs_pool_size) { |
225 | /* recalculate memory size required cpdma descriptor pool | |
226 | * basing on number of descriptors specified by user and | |
227 | * if memory size > CPPI internal RAM size (desc_mem_size) | |
228 | * then switch to use DDR | |
229 | */ | |
230 | pool->num_desc = cpdma_params->descs_pool_size; | |
231 | pool->mem_size = pool->desc_size * pool->num_desc; | |
232 | if (pool->mem_size > cpdma_params->desc_mem_size) | |
233 | cpdma_params->desc_mem_phys = 0; | |
234 | } | |
235 | ||
5fcc40a9 GS |
236 | pool->gen_pool = devm_gen_pool_create(ctlr->dev, ilog2(pool->desc_size), |
237 | -1, "cpdma"); | |
742fb20f | 238 | if (IS_ERR(pool->gen_pool)) { |
5fcc40a9 GS |
239 | ret = PTR_ERR(pool->gen_pool); |
240 | dev_err(ctlr->dev, "pool create failed %d\n", ret); | |
742fb20f GS |
241 | goto gen_pool_create_fail; |
242 | } | |
ef8c2dab | 243 | |
5fcc40a9 GS |
244 | if (cpdma_params->desc_mem_phys) { |
245 | pool->phys = cpdma_params->desc_mem_phys; | |
7f3b490a GS |
246 | pool->iomap = devm_ioremap(ctlr->dev, pool->phys, |
247 | pool->mem_size); | |
5fcc40a9 | 248 | pool->hw_addr = cpdma_params->desc_hw_addr; |
ef8c2dab | 249 | } else { |
5fcc40a9 GS |
250 | pool->cpumap = dma_alloc_coherent(ctlr->dev, pool->mem_size, |
251 | &pool->hw_addr, GFP_KERNEL); | |
84092996 AB |
252 | pool->iomap = (void __iomem __force *)pool->cpumap; |
253 | pool->phys = pool->hw_addr; /* assumes no IOMMU, don't use this value */ | |
ef8c2dab CC |
254 | } |
255 | ||
742fb20f GS |
256 | if (!pool->iomap) |
257 | goto gen_pool_create_fail; | |
ef8c2dab | 258 | |
742fb20f GS |
259 | ret = gen_pool_add_virt(pool->gen_pool, (unsigned long)pool->iomap, |
260 | pool->phys, pool->mem_size, -1); | |
261 | if (ret < 0) { | |
5fcc40a9 | 262 | dev_err(ctlr->dev, "pool add failed %d\n", ret); |
742fb20f | 263 | goto gen_pool_add_virt_fail; |
ef8c2dab | 264 | } |
742fb20f | 265 | |
5fcc40a9 | 266 | return 0; |
742fb20f GS |
267 | |
268 | gen_pool_add_virt_fail: | |
5fcc40a9 | 269 | cpdma_desc_pool_destroy(ctlr); |
742fb20f | 270 | gen_pool_create_fail: |
5fcc40a9 GS |
271 | ctlr->pool = NULL; |
272 | return ret; | |
ef8c2dab CC |
273 | } |
274 | ||
275 | static inline dma_addr_t desc_phys(struct cpdma_desc_pool *pool, | |
276 | struct cpdma_desc __iomem *desc) | |
277 | { | |
278 | if (!desc) | |
279 | return 0; | |
c767db51 | 280 | return pool->hw_addr + (__force long)desc - (__force long)pool->iomap; |
ef8c2dab CC |
281 | } |
282 | ||
283 | static inline struct cpdma_desc __iomem * | |
284 | desc_from_phys(struct cpdma_desc_pool *pool, dma_addr_t dma) | |
285 | { | |
6a1fef6d | 286 | return dma ? pool->iomap + dma - pool->hw_addr : NULL; |
ef8c2dab CC |
287 | } |
288 | ||
289 | static struct cpdma_desc __iomem * | |
742fb20f | 290 | cpdma_desc_alloc(struct cpdma_desc_pool *pool) |
ef8c2dab | 291 | { |
aeec3021 GS |
292 | return (struct cpdma_desc __iomem *) |
293 | gen_pool_alloc(pool->gen_pool, pool->desc_size); | |
ef8c2dab CC |
294 | } |
295 | ||
296 | static void cpdma_desc_free(struct cpdma_desc_pool *pool, | |
297 | struct cpdma_desc __iomem *desc, int num_desc) | |
298 | { | |
742fb20f | 299 | gen_pool_free(pool->gen_pool, (unsigned long)desc, pool->desc_size); |
ef8c2dab CC |
300 | } |
301 | ||
991ddb1f IK |
302 | static int _cpdma_control_set(struct cpdma_ctlr *ctlr, int control, int value) |
303 | { | |
304 | struct cpdma_control_info *info = &controls[control]; | |
305 | u32 val; | |
306 | ||
307 | if (!ctlr->params.has_ext_regs) | |
308 | return -ENOTSUPP; | |
309 | ||
310 | if (ctlr->state != CPDMA_STATE_ACTIVE) | |
311 | return -EINVAL; | |
312 | ||
313 | if (control < 0 || control >= ARRAY_SIZE(controls)) | |
314 | return -ENOENT; | |
315 | ||
316 | if ((info->access & ACCESS_WO) != ACCESS_WO) | |
317 | return -EPERM; | |
318 | ||
319 | val = dma_reg_read(ctlr, info->reg); | |
320 | val &= ~(info->mask << info->shift); | |
321 | val |= (value & info->mask) << info->shift; | |
322 | dma_reg_write(ctlr, info->reg, val); | |
323 | ||
324 | return 0; | |
325 | } | |
326 | ||
8f32b909 IK |
327 | static int _cpdma_control_get(struct cpdma_ctlr *ctlr, int control) |
328 | { | |
329 | struct cpdma_control_info *info = &controls[control]; | |
330 | int ret; | |
331 | ||
332 | if (!ctlr->params.has_ext_regs) | |
333 | return -ENOTSUPP; | |
334 | ||
335 | if (ctlr->state != CPDMA_STATE_ACTIVE) | |
336 | return -EINVAL; | |
337 | ||
338 | if (control < 0 || control >= ARRAY_SIZE(controls)) | |
339 | return -ENOENT; | |
340 | ||
341 | if ((info->access & ACCESS_RO) != ACCESS_RO) | |
342 | return -EPERM; | |
343 | ||
344 | ret = (dma_reg_read(ctlr, info->reg) >> info->shift) & info->mask; | |
345 | return ret; | |
346 | } | |
347 | ||
348 | /* cpdma_chan_set_chan_shaper - set shaper for a channel | |
349 | * Has to be called under ctlr lock | |
350 | */ | |
351 | static int cpdma_chan_set_chan_shaper(struct cpdma_chan *chan) | |
352 | { | |
353 | struct cpdma_ctlr *ctlr = chan->ctlr; | |
354 | u32 rate_reg; | |
355 | u32 rmask; | |
356 | int ret; | |
357 | ||
358 | if (!chan->rate) | |
359 | return 0; | |
360 | ||
361 | rate_reg = CPDMA_TX_PRI0_RATE + 4 * chan->chan_num; | |
362 | dma_reg_write(ctlr, rate_reg, chan->rate_factor); | |
363 | ||
364 | rmask = _cpdma_control_get(ctlr, CPDMA_TX_RLIM); | |
365 | rmask |= chan->mask; | |
366 | ||
367 | ret = _cpdma_control_set(ctlr, CPDMA_TX_RLIM, rmask); | |
368 | return ret; | |
369 | } | |
370 | ||
371 | static int cpdma_chan_on(struct cpdma_chan *chan) | |
372 | { | |
373 | struct cpdma_ctlr *ctlr = chan->ctlr; | |
374 | struct cpdma_desc_pool *pool = ctlr->pool; | |
375 | unsigned long flags; | |
376 | ||
377 | spin_lock_irqsave(&chan->lock, flags); | |
378 | if (chan->state != CPDMA_STATE_IDLE) { | |
379 | spin_unlock_irqrestore(&chan->lock, flags); | |
380 | return -EBUSY; | |
381 | } | |
382 | if (ctlr->state != CPDMA_STATE_ACTIVE) { | |
383 | spin_unlock_irqrestore(&chan->lock, flags); | |
384 | return -EINVAL; | |
385 | } | |
386 | dma_reg_write(ctlr, chan->int_set, chan->mask); | |
387 | chan->state = CPDMA_STATE_ACTIVE; | |
388 | if (chan->head) { | |
389 | chan_write(chan, hdp, desc_phys(pool, chan->head)); | |
390 | if (chan->rxfree) | |
391 | chan_write(chan, rxfree, chan->count); | |
392 | } | |
393 | ||
394 | spin_unlock_irqrestore(&chan->lock, flags); | |
395 | return 0; | |
396 | } | |
397 | ||
398 | /* cpdma_chan_fit_rate - set rate for a channel and check if it's possible. | |
399 | * rmask - mask of rate limited channels | |
400 | * Returns min rate in Kb/s | |
401 | */ | |
402 | static int cpdma_chan_fit_rate(struct cpdma_chan *ch, u32 rate, | |
403 | u32 *rmask, int *prio_mode) | |
404 | { | |
405 | struct cpdma_ctlr *ctlr = ch->ctlr; | |
406 | struct cpdma_chan *chan; | |
407 | u32 old_rate = ch->rate; | |
408 | u32 new_rmask = 0; | |
409 | int rlim = 1; | |
410 | int i; | |
411 | ||
412 | *prio_mode = 0; | |
413 | for (i = tx_chan_num(0); i < tx_chan_num(CPDMA_MAX_CHANNELS); i++) { | |
414 | chan = ctlr->channels[i]; | |
415 | if (!chan) { | |
416 | rlim = 0; | |
417 | continue; | |
418 | } | |
419 | ||
420 | if (chan == ch) | |
421 | chan->rate = rate; | |
422 | ||
423 | if (chan->rate) { | |
424 | if (rlim) { | |
425 | new_rmask |= chan->mask; | |
426 | } else { | |
427 | ch->rate = old_rate; | |
428 | dev_err(ctlr->dev, "Prev channel of %dch is not rate limited\n", | |
429 | chan->chan_num); | |
430 | return -EINVAL; | |
431 | } | |
432 | } else { | |
433 | *prio_mode = 1; | |
434 | rlim = 0; | |
435 | } | |
436 | } | |
437 | ||
438 | *rmask = new_rmask; | |
439 | return 0; | |
440 | } | |
441 | ||
442 | static u32 cpdma_chan_set_factors(struct cpdma_ctlr *ctlr, | |
443 | struct cpdma_chan *ch) | |
444 | { | |
445 | u32 delta = UINT_MAX, prev_delta = UINT_MAX, best_delta = UINT_MAX; | |
446 | u32 best_send_cnt = 0, best_idle_cnt = 0; | |
447 | u32 new_rate, best_rate = 0, rate_reg; | |
448 | u64 send_cnt, idle_cnt; | |
449 | u32 min_send_cnt, freq; | |
450 | u64 divident, divisor; | |
451 | ||
452 | if (!ch->rate) { | |
453 | ch->rate_factor = 0; | |
454 | goto set_factor; | |
455 | } | |
456 | ||
457 | freq = ctlr->params.bus_freq_mhz * 1000 * 32; | |
458 | if (!freq) { | |
459 | dev_err(ctlr->dev, "The bus frequency is not set\n"); | |
460 | return -EINVAL; | |
461 | } | |
462 | ||
463 | min_send_cnt = freq - ch->rate; | |
464 | send_cnt = DIV_ROUND_UP(min_send_cnt, ch->rate); | |
465 | while (send_cnt <= CPDMA_MAX_RLIM_CNT) { | |
466 | divident = ch->rate * send_cnt; | |
467 | divisor = min_send_cnt; | |
468 | idle_cnt = DIV_ROUND_CLOSEST_ULL(divident, divisor); | |
469 | ||
470 | divident = freq * idle_cnt; | |
471 | divisor = idle_cnt + send_cnt; | |
472 | new_rate = DIV_ROUND_CLOSEST_ULL(divident, divisor); | |
473 | ||
474 | delta = new_rate >= ch->rate ? new_rate - ch->rate : delta; | |
475 | if (delta < best_delta) { | |
476 | best_delta = delta; | |
477 | best_send_cnt = send_cnt; | |
478 | best_idle_cnt = idle_cnt; | |
479 | best_rate = new_rate; | |
480 | ||
481 | if (!delta) | |
482 | break; | |
483 | } | |
484 | ||
485 | if (prev_delta >= delta) { | |
486 | prev_delta = delta; | |
487 | send_cnt++; | |
488 | continue; | |
489 | } | |
490 | ||
491 | idle_cnt++; | |
492 | divident = freq * idle_cnt; | |
493 | send_cnt = DIV_ROUND_CLOSEST_ULL(divident, ch->rate); | |
494 | send_cnt -= idle_cnt; | |
495 | prev_delta = UINT_MAX; | |
496 | } | |
497 | ||
498 | ch->rate = best_rate; | |
499 | ch->rate_factor = best_send_cnt | (best_idle_cnt << 16); | |
500 | ||
501 | set_factor: | |
502 | rate_reg = CPDMA_TX_PRI0_RATE + 4 * ch->chan_num; | |
503 | dma_reg_write(ctlr, rate_reg, ch->rate_factor); | |
504 | return 0; | |
505 | } | |
506 | ||
ef8c2dab CC |
507 | struct cpdma_ctlr *cpdma_ctlr_create(struct cpdma_params *params) |
508 | { | |
509 | struct cpdma_ctlr *ctlr; | |
510 | ||
e1943128 | 511 | ctlr = devm_kzalloc(params->dev, sizeof(*ctlr), GFP_KERNEL); |
ef8c2dab CC |
512 | if (!ctlr) |
513 | return NULL; | |
514 | ||
515 | ctlr->state = CPDMA_STATE_IDLE; | |
516 | ctlr->params = *params; | |
517 | ctlr->dev = params->dev; | |
3802dce1 | 518 | ctlr->chan_num = 0; |
ef8c2dab CC |
519 | spin_lock_init(&ctlr->lock); |
520 | ||
5fcc40a9 | 521 | if (cpdma_desc_pool_create(ctlr)) |
ef8c2dab | 522 | return NULL; |
be034fc1 GS |
523 | /* split pool equally between RX/TX by default */ |
524 | ctlr->num_tx_desc = ctlr->pool->num_desc / 2; | |
525 | ctlr->num_rx_desc = ctlr->pool->num_desc - ctlr->num_tx_desc; | |
ef8c2dab CC |
526 | |
527 | if (WARN_ON(ctlr->num_chan > CPDMA_MAX_CHANNELS)) | |
528 | ctlr->num_chan = CPDMA_MAX_CHANNELS; | |
529 | return ctlr; | |
530 | } | |
32a6d90b | 531 | EXPORT_SYMBOL_GPL(cpdma_ctlr_create); |
ef8c2dab CC |
532 | |
533 | int cpdma_ctlr_start(struct cpdma_ctlr *ctlr) | |
534 | { | |
8f32b909 | 535 | struct cpdma_chan *chan; |
ef8c2dab | 536 | unsigned long flags; |
8f32b909 | 537 | int i, prio_mode; |
ef8c2dab CC |
538 | |
539 | spin_lock_irqsave(&ctlr->lock, flags); | |
540 | if (ctlr->state != CPDMA_STATE_IDLE) { | |
541 | spin_unlock_irqrestore(&ctlr->lock, flags); | |
542 | return -EBUSY; | |
543 | } | |
544 | ||
545 | if (ctlr->params.has_soft_reset) { | |
817f6d1a | 546 | unsigned timeout = 10 * 100; |
ef8c2dab CC |
547 | |
548 | dma_reg_write(ctlr, CPDMA_SOFTRESET, 1); | |
817f6d1a | 549 | while (timeout) { |
ef8c2dab CC |
550 | if (dma_reg_read(ctlr, CPDMA_SOFTRESET) == 0) |
551 | break; | |
817f6d1a SS |
552 | udelay(10); |
553 | timeout--; | |
ef8c2dab | 554 | } |
817f6d1a | 555 | WARN_ON(!timeout); |
ef8c2dab CC |
556 | } |
557 | ||
558 | for (i = 0; i < ctlr->num_chan; i++) { | |
a6c83ccf GS |
559 | writel(0, ctlr->params.txhdp + 4 * i); |
560 | writel(0, ctlr->params.rxhdp + 4 * i); | |
561 | writel(0, ctlr->params.txcp + 4 * i); | |
562 | writel(0, ctlr->params.rxcp + 4 * i); | |
ef8c2dab CC |
563 | } |
564 | ||
565 | dma_reg_write(ctlr, CPDMA_RXINTMASKCLEAR, 0xffffffff); | |
566 | dma_reg_write(ctlr, CPDMA_TXINTMASKCLEAR, 0xffffffff); | |
567 | ||
568 | dma_reg_write(ctlr, CPDMA_TXCONTROL, 1); | |
569 | dma_reg_write(ctlr, CPDMA_RXCONTROL, 1); | |
570 | ||
571 | ctlr->state = CPDMA_STATE_ACTIVE; | |
572 | ||
8f32b909 | 573 | prio_mode = 0; |
ef8c2dab | 574 | for (i = 0; i < ARRAY_SIZE(ctlr->channels); i++) { |
8f32b909 IK |
575 | chan = ctlr->channels[i]; |
576 | if (chan) { | |
577 | cpdma_chan_set_chan_shaper(chan); | |
578 | cpdma_chan_on(chan); | |
579 | ||
580 | /* off prio mode if all tx channels are rate limited */ | |
581 | if (is_tx_chan(chan) && !chan->rate) | |
582 | prio_mode = 1; | |
583 | } | |
ef8c2dab | 584 | } |
991ddb1f | 585 | |
8f32b909 | 586 | _cpdma_control_set(ctlr, CPDMA_TX_PRIO_FIXED, prio_mode); |
991ddb1f IK |
587 | _cpdma_control_set(ctlr, CPDMA_RX_BUFFER_OFFSET, 0); |
588 | ||
ef8c2dab CC |
589 | spin_unlock_irqrestore(&ctlr->lock, flags); |
590 | return 0; | |
591 | } | |
32a6d90b | 592 | EXPORT_SYMBOL_GPL(cpdma_ctlr_start); |
ef8c2dab CC |
593 | |
594 | int cpdma_ctlr_stop(struct cpdma_ctlr *ctlr) | |
595 | { | |
596 | unsigned long flags; | |
597 | int i; | |
598 | ||
599 | spin_lock_irqsave(&ctlr->lock, flags); | |
b993eec0 | 600 | if (ctlr->state != CPDMA_STATE_ACTIVE) { |
ef8c2dab CC |
601 | spin_unlock_irqrestore(&ctlr->lock, flags); |
602 | return -EINVAL; | |
603 | } | |
604 | ||
605 | ctlr->state = CPDMA_STATE_TEARDOWN; | |
080d5c5a | 606 | spin_unlock_irqrestore(&ctlr->lock, flags); |
ef8c2dab CC |
607 | |
608 | for (i = 0; i < ARRAY_SIZE(ctlr->channels); i++) { | |
609 | if (ctlr->channels[i]) | |
610 | cpdma_chan_stop(ctlr->channels[i]); | |
611 | } | |
612 | ||
080d5c5a | 613 | spin_lock_irqsave(&ctlr->lock, flags); |
ef8c2dab CC |
614 | dma_reg_write(ctlr, CPDMA_RXINTMASKCLEAR, 0xffffffff); |
615 | dma_reg_write(ctlr, CPDMA_TXINTMASKCLEAR, 0xffffffff); | |
616 | ||
617 | dma_reg_write(ctlr, CPDMA_TXCONTROL, 0); | |
618 | dma_reg_write(ctlr, CPDMA_RXCONTROL, 0); | |
619 | ||
620 | ctlr->state = CPDMA_STATE_IDLE; | |
621 | ||
622 | spin_unlock_irqrestore(&ctlr->lock, flags); | |
623 | return 0; | |
624 | } | |
32a6d90b | 625 | EXPORT_SYMBOL_GPL(cpdma_ctlr_stop); |
ef8c2dab | 626 | |
ef8c2dab CC |
627 | int cpdma_ctlr_destroy(struct cpdma_ctlr *ctlr) |
628 | { | |
ef8c2dab CC |
629 | int ret = 0, i; |
630 | ||
631 | if (!ctlr) | |
632 | return -EINVAL; | |
633 | ||
ef8c2dab CC |
634 | if (ctlr->state != CPDMA_STATE_IDLE) |
635 | cpdma_ctlr_stop(ctlr); | |
636 | ||
79876e03 CR |
637 | for (i = 0; i < ARRAY_SIZE(ctlr->channels); i++) |
638 | cpdma_chan_destroy(ctlr->channels[i]); | |
ef8c2dab | 639 | |
5fcc40a9 | 640 | cpdma_desc_pool_destroy(ctlr); |
ef8c2dab CC |
641 | return ret; |
642 | } | |
32a6d90b | 643 | EXPORT_SYMBOL_GPL(cpdma_ctlr_destroy); |
ef8c2dab CC |
644 | |
645 | int cpdma_ctlr_int_ctrl(struct cpdma_ctlr *ctlr, bool enable) | |
646 | { | |
647 | unsigned long flags; | |
648 | int i, reg; | |
649 | ||
650 | spin_lock_irqsave(&ctlr->lock, flags); | |
651 | if (ctlr->state != CPDMA_STATE_ACTIVE) { | |
652 | spin_unlock_irqrestore(&ctlr->lock, flags); | |
653 | return -EINVAL; | |
654 | } | |
655 | ||
656 | reg = enable ? CPDMA_DMAINTMASKSET : CPDMA_DMAINTMASKCLEAR; | |
657 | dma_reg_write(ctlr, reg, CPDMA_DMAINT_HOSTERR); | |
658 | ||
659 | for (i = 0; i < ARRAY_SIZE(ctlr->channels); i++) { | |
660 | if (ctlr->channels[i]) | |
661 | cpdma_chan_int_ctrl(ctlr->channels[i], enable); | |
662 | } | |
663 | ||
664 | spin_unlock_irqrestore(&ctlr->lock, flags); | |
665 | return 0; | |
666 | } | |
6929e24e | 667 | EXPORT_SYMBOL_GPL(cpdma_ctlr_int_ctrl); |
ef8c2dab | 668 | |
510a1e72 | 669 | void cpdma_ctlr_eoi(struct cpdma_ctlr *ctlr, u32 value) |
ef8c2dab | 670 | { |
510a1e72 | 671 | dma_reg_write(ctlr, CPDMA_MACEOIVECTOR, value); |
ef8c2dab | 672 | } |
6929e24e | 673 | EXPORT_SYMBOL_GPL(cpdma_ctlr_eoi); |
ef8c2dab | 674 | |
e05107e6 IK |
675 | u32 cpdma_ctrl_rxchs_state(struct cpdma_ctlr *ctlr) |
676 | { | |
677 | return dma_reg_read(ctlr, CPDMA_RXINTSTATMASKED); | |
678 | } | |
679 | EXPORT_SYMBOL_GPL(cpdma_ctrl_rxchs_state); | |
680 | ||
681 | u32 cpdma_ctrl_txchs_state(struct cpdma_ctlr *ctlr) | |
682 | { | |
683 | return dma_reg_read(ctlr, CPDMA_TXINTSTATMASKED); | |
684 | } | |
685 | EXPORT_SYMBOL_GPL(cpdma_ctrl_txchs_state); | |
686 | ||
0fc6432c IK |
687 | static void cpdma_chan_set_descs(struct cpdma_ctlr *ctlr, |
688 | int rx, int desc_num, | |
689 | int per_ch_desc) | |
690 | { | |
691 | struct cpdma_chan *chan, *most_chan = NULL; | |
692 | int desc_cnt = desc_num; | |
693 | int most_dnum = 0; | |
694 | int min, max, i; | |
695 | ||
696 | if (!desc_num) | |
697 | return; | |
698 | ||
699 | if (rx) { | |
700 | min = rx_chan_num(0); | |
701 | max = rx_chan_num(CPDMA_MAX_CHANNELS); | |
702 | } else { | |
703 | min = tx_chan_num(0); | |
704 | max = tx_chan_num(CPDMA_MAX_CHANNELS); | |
705 | } | |
706 | ||
707 | for (i = min; i < max; i++) { | |
708 | chan = ctlr->channels[i]; | |
709 | if (!chan) | |
710 | continue; | |
711 | ||
712 | if (chan->weight) | |
713 | chan->desc_num = (chan->weight * desc_num) / 100; | |
714 | else | |
715 | chan->desc_num = per_ch_desc; | |
716 | ||
717 | desc_cnt -= chan->desc_num; | |
718 | ||
719 | if (most_dnum < chan->desc_num) { | |
720 | most_dnum = chan->desc_num; | |
721 | most_chan = chan; | |
722 | } | |
723 | } | |
724 | /* use remains */ | |
be034fc1 GS |
725 | if (most_chan) |
726 | most_chan->desc_num += desc_cnt; | |
0fc6432c IK |
727 | } |
728 | ||
3802dce1 IK |
729 | /** |
730 | * cpdma_chan_split_pool - Splits ctrl pool between all channels. | |
731 | * Has to be called under ctlr lock | |
732 | */ | |
be034fc1 | 733 | int cpdma_chan_split_pool(struct cpdma_ctlr *ctlr) |
3802dce1 | 734 | { |
0fc6432c | 735 | int tx_per_ch_desc = 0, rx_per_ch_desc = 0; |
0fc6432c IK |
736 | int free_rx_num = 0, free_tx_num = 0; |
737 | int rx_weight = 0, tx_weight = 0; | |
738 | int tx_desc_num, rx_desc_num; | |
3802dce1 | 739 | struct cpdma_chan *chan; |
be034fc1 | 740 | int i; |
3802dce1 IK |
741 | |
742 | if (!ctlr->chan_num) | |
0fc6432c | 743 | return 0; |
3802dce1 | 744 | |
3802dce1 IK |
745 | for (i = 0; i < ARRAY_SIZE(ctlr->channels); i++) { |
746 | chan = ctlr->channels[i]; | |
0fc6432c IK |
747 | if (!chan) |
748 | continue; | |
749 | ||
750 | if (is_rx_chan(chan)) { | |
751 | if (!chan->weight) | |
752 | free_rx_num++; | |
753 | rx_weight += chan->weight; | |
754 | } else { | |
755 | if (!chan->weight) | |
756 | free_tx_num++; | |
757 | tx_weight += chan->weight; | |
0fc6432c IK |
758 | } |
759 | } | |
760 | ||
761 | if (rx_weight > 100 || tx_weight > 100) | |
762 | return -EINVAL; | |
763 | ||
be034fc1 GS |
764 | tx_desc_num = ctlr->num_tx_desc; |
765 | rx_desc_num = ctlr->num_rx_desc; | |
0fc6432c IK |
766 | |
767 | if (free_tx_num) { | |
768 | tx_per_ch_desc = tx_desc_num - (tx_weight * tx_desc_num) / 100; | |
769 | tx_per_ch_desc /= free_tx_num; | |
770 | } | |
771 | if (free_rx_num) { | |
772 | rx_per_ch_desc = rx_desc_num - (rx_weight * rx_desc_num) / 100; | |
773 | rx_per_ch_desc /= free_rx_num; | |
3802dce1 | 774 | } |
0fc6432c IK |
775 | |
776 | cpdma_chan_set_descs(ctlr, 0, tx_desc_num, tx_per_ch_desc); | |
777 | cpdma_chan_set_descs(ctlr, 1, rx_desc_num, rx_per_ch_desc); | |
778 | ||
779 | return 0; | |
780 | } | |
be034fc1 GS |
781 | EXPORT_SYMBOL_GPL(cpdma_chan_split_pool); |
782 | ||
0fc6432c IK |
783 | |
784 | /* cpdma_chan_set_weight - set weight of a channel in percentage. | |
785 | * Tx and Rx channels have separate weights. That is 100% for RX | |
786 | * and 100% for Tx. The weight is used to split cpdma resources | |
787 | * in correct proportion required by the channels, including number | |
788 | * of descriptors. The channel rate is not enough to know the | |
789 | * weight of a channel as the maximum rate of an interface is needed. | |
790 | * If weight = 0, then channel uses rest of descriptors leaved by | |
791 | * weighted channels. | |
792 | */ | |
793 | int cpdma_chan_set_weight(struct cpdma_chan *ch, int weight) | |
794 | { | |
795 | struct cpdma_ctlr *ctlr = ch->ctlr; | |
796 | unsigned long flags, ch_flags; | |
797 | int ret; | |
798 | ||
799 | spin_lock_irqsave(&ctlr->lock, flags); | |
800 | spin_lock_irqsave(&ch->lock, ch_flags); | |
801 | if (ch->weight == weight) { | |
802 | spin_unlock_irqrestore(&ch->lock, ch_flags); | |
803 | spin_unlock_irqrestore(&ctlr->lock, flags); | |
804 | return 0; | |
805 | } | |
806 | ch->weight = weight; | |
807 | spin_unlock_irqrestore(&ch->lock, ch_flags); | |
808 | ||
809 | /* re-split pool using new channel weight */ | |
810 | ret = cpdma_chan_split_pool(ctlr); | |
811 | spin_unlock_irqrestore(&ctlr->lock, flags); | |
812 | return ret; | |
3802dce1 | 813 | } |
397c5ad1 | 814 | EXPORT_SYMBOL_GPL(cpdma_chan_set_weight); |
3802dce1 | 815 | |
8f32b909 IK |
816 | /* cpdma_chan_get_min_rate - get minimum allowed rate for channel |
817 | * Should be called before cpdma_chan_set_rate. | |
818 | * Returns min rate in Kb/s | |
819 | */ | |
820 | u32 cpdma_chan_get_min_rate(struct cpdma_ctlr *ctlr) | |
821 | { | |
822 | unsigned int divident, divisor; | |
823 | ||
824 | divident = ctlr->params.bus_freq_mhz * 32 * 1000; | |
825 | divisor = 1 + CPDMA_MAX_RLIM_CNT; | |
826 | ||
827 | return DIV_ROUND_UP(divident, divisor); | |
828 | } | |
397c5ad1 | 829 | EXPORT_SYMBOL_GPL(cpdma_chan_get_min_rate); |
8f32b909 IK |
830 | |
831 | /* cpdma_chan_set_rate - limits bandwidth for transmit channel. | |
832 | * The bandwidth * limited channels have to be in order beginning from lowest. | |
833 | * ch - transmit channel the bandwidth is configured for | |
834 | * rate - bandwidth in Kb/s, if 0 - then off shaper | |
835 | */ | |
836 | int cpdma_chan_set_rate(struct cpdma_chan *ch, u32 rate) | |
837 | { | |
8f32b909 | 838 | unsigned long flags, ch_flags; |
e33c2ef1 | 839 | struct cpdma_ctlr *ctlr; |
8f32b909 IK |
840 | int ret, prio_mode; |
841 | u32 rmask; | |
842 | ||
843 | if (!ch || !is_tx_chan(ch)) | |
844 | return -EINVAL; | |
845 | ||
846 | if (ch->rate == rate) | |
847 | return rate; | |
848 | ||
e33c2ef1 | 849 | ctlr = ch->ctlr; |
8f32b909 IK |
850 | spin_lock_irqsave(&ctlr->lock, flags); |
851 | spin_lock_irqsave(&ch->lock, ch_flags); | |
852 | ||
853 | ret = cpdma_chan_fit_rate(ch, rate, &rmask, &prio_mode); | |
854 | if (ret) | |
855 | goto err; | |
856 | ||
857 | ret = cpdma_chan_set_factors(ctlr, ch); | |
858 | if (ret) | |
859 | goto err; | |
860 | ||
861 | spin_unlock_irqrestore(&ch->lock, ch_flags); | |
862 | ||
863 | /* on shapers */ | |
864 | _cpdma_control_set(ctlr, CPDMA_TX_RLIM, rmask); | |
865 | _cpdma_control_set(ctlr, CPDMA_TX_PRIO_FIXED, prio_mode); | |
866 | spin_unlock_irqrestore(&ctlr->lock, flags); | |
867 | return ret; | |
868 | ||
869 | err: | |
870 | spin_unlock_irqrestore(&ch->lock, ch_flags); | |
871 | spin_unlock_irqrestore(&ctlr->lock, flags); | |
872 | return ret; | |
873 | } | |
397c5ad1 | 874 | EXPORT_SYMBOL_GPL(cpdma_chan_set_rate); |
8f32b909 IK |
875 | |
876 | u32 cpdma_chan_get_rate(struct cpdma_chan *ch) | |
877 | { | |
878 | unsigned long flags; | |
879 | u32 rate; | |
880 | ||
881 | spin_lock_irqsave(&ch->lock, flags); | |
882 | rate = ch->rate; | |
883 | spin_unlock_irqrestore(&ch->lock, flags); | |
884 | ||
885 | return rate; | |
886 | } | |
397c5ad1 | 887 | EXPORT_SYMBOL_GPL(cpdma_chan_get_rate); |
8f32b909 | 888 | |
ef8c2dab | 889 | struct cpdma_chan *cpdma_chan_create(struct cpdma_ctlr *ctlr, int chan_num, |
925d65e6 | 890 | cpdma_handler_fn handler, int rx_type) |
ef8c2dab | 891 | { |
925d65e6 | 892 | int offset = chan_num * 4; |
ef8c2dab | 893 | struct cpdma_chan *chan; |
ef8c2dab CC |
894 | unsigned long flags; |
895 | ||
925d65e6 IK |
896 | chan_num = rx_type ? rx_chan_num(chan_num) : tx_chan_num(chan_num); |
897 | ||
ef8c2dab CC |
898 | if (__chan_linear(chan_num) >= ctlr->num_chan) |
899 | return NULL; | |
900 | ||
e1943128 | 901 | chan = devm_kzalloc(ctlr->dev, sizeof(*chan), GFP_KERNEL); |
ef8c2dab | 902 | if (!chan) |
e1943128 | 903 | return ERR_PTR(-ENOMEM); |
ef8c2dab CC |
904 | |
905 | spin_lock_irqsave(&ctlr->lock, flags); | |
e1943128 GC |
906 | if (ctlr->channels[chan_num]) { |
907 | spin_unlock_irqrestore(&ctlr->lock, flags); | |
908 | devm_kfree(ctlr->dev, chan); | |
909 | return ERR_PTR(-EBUSY); | |
910 | } | |
ef8c2dab CC |
911 | |
912 | chan->ctlr = ctlr; | |
913 | chan->state = CPDMA_STATE_IDLE; | |
914 | chan->chan_num = chan_num; | |
915 | chan->handler = handler; | |
8f32b909 | 916 | chan->rate = 0; |
0fc6432c | 917 | chan->weight = 0; |
ef8c2dab CC |
918 | |
919 | if (is_rx_chan(chan)) { | |
920 | chan->hdp = ctlr->params.rxhdp + offset; | |
921 | chan->cp = ctlr->params.rxcp + offset; | |
922 | chan->rxfree = ctlr->params.rxfree + offset; | |
923 | chan->int_set = CPDMA_RXINTMASKSET; | |
924 | chan->int_clear = CPDMA_RXINTMASKCLEAR; | |
925 | chan->td = CPDMA_RXTEARDOWN; | |
926 | chan->dir = DMA_FROM_DEVICE; | |
927 | } else { | |
928 | chan->hdp = ctlr->params.txhdp + offset; | |
929 | chan->cp = ctlr->params.txcp + offset; | |
930 | chan->int_set = CPDMA_TXINTMASKSET; | |
931 | chan->int_clear = CPDMA_TXINTMASKCLEAR; | |
932 | chan->td = CPDMA_TXTEARDOWN; | |
933 | chan->dir = DMA_TO_DEVICE; | |
934 | } | |
935 | chan->mask = BIT(chan_linear(chan)); | |
936 | ||
937 | spin_lock_init(&chan->lock); | |
938 | ||
939 | ctlr->channels[chan_num] = chan; | |
3802dce1 IK |
940 | ctlr->chan_num++; |
941 | ||
942 | cpdma_chan_split_pool(ctlr); | |
943 | ||
ef8c2dab CC |
944 | spin_unlock_irqrestore(&ctlr->lock, flags); |
945 | return chan; | |
ef8c2dab | 946 | } |
32a6d90b | 947 | EXPORT_SYMBOL_GPL(cpdma_chan_create); |
ef8c2dab | 948 | |
3802dce1 | 949 | int cpdma_chan_get_rx_buf_num(struct cpdma_chan *chan) |
1793331e | 950 | { |
3802dce1 IK |
951 | unsigned long flags; |
952 | int desc_num; | |
953 | ||
954 | spin_lock_irqsave(&chan->lock, flags); | |
955 | desc_num = chan->desc_num; | |
956 | spin_unlock_irqrestore(&chan->lock, flags); | |
957 | ||
958 | return desc_num; | |
1793331e IK |
959 | } |
960 | EXPORT_SYMBOL_GPL(cpdma_chan_get_rx_buf_num); | |
961 | ||
ef8c2dab CC |
962 | int cpdma_chan_destroy(struct cpdma_chan *chan) |
963 | { | |
f37c54b6 | 964 | struct cpdma_ctlr *ctlr; |
ef8c2dab CC |
965 | unsigned long flags; |
966 | ||
967 | if (!chan) | |
968 | return -EINVAL; | |
f37c54b6 | 969 | ctlr = chan->ctlr; |
ef8c2dab CC |
970 | |
971 | spin_lock_irqsave(&ctlr->lock, flags); | |
972 | if (chan->state != CPDMA_STATE_IDLE) | |
973 | cpdma_chan_stop(chan); | |
974 | ctlr->channels[chan->chan_num] = NULL; | |
3802dce1 | 975 | ctlr->chan_num--; |
b602e491 | 976 | devm_kfree(ctlr->dev, chan); |
3802dce1 IK |
977 | cpdma_chan_split_pool(ctlr); |
978 | ||
ef8c2dab | 979 | spin_unlock_irqrestore(&ctlr->lock, flags); |
ef8c2dab CC |
980 | return 0; |
981 | } | |
32a6d90b | 982 | EXPORT_SYMBOL_GPL(cpdma_chan_destroy); |
ef8c2dab CC |
983 | |
984 | int cpdma_chan_get_stats(struct cpdma_chan *chan, | |
985 | struct cpdma_chan_stats *stats) | |
986 | { | |
987 | unsigned long flags; | |
988 | if (!chan) | |
989 | return -EINVAL; | |
990 | spin_lock_irqsave(&chan->lock, flags); | |
991 | memcpy(stats, &chan->stats, sizeof(*stats)); | |
992 | spin_unlock_irqrestore(&chan->lock, flags); | |
993 | return 0; | |
994 | } | |
0ca04b63 | 995 | EXPORT_SYMBOL_GPL(cpdma_chan_get_stats); |
ef8c2dab | 996 | |
ef8c2dab CC |
997 | static void __cpdma_chan_submit(struct cpdma_chan *chan, |
998 | struct cpdma_desc __iomem *desc) | |
999 | { | |
1000 | struct cpdma_ctlr *ctlr = chan->ctlr; | |
1001 | struct cpdma_desc __iomem *prev = chan->tail; | |
1002 | struct cpdma_desc_pool *pool = ctlr->pool; | |
1003 | dma_addr_t desc_dma; | |
1004 | u32 mode; | |
1005 | ||
1006 | desc_dma = desc_phys(pool, desc); | |
1007 | ||
1008 | /* simple case - idle channel */ | |
1009 | if (!chan->head) { | |
1010 | chan->stats.head_enqueue++; | |
1011 | chan->head = desc; | |
1012 | chan->tail = desc; | |
1013 | if (chan->state == CPDMA_STATE_ACTIVE) | |
1014 | chan_write(chan, hdp, desc_dma); | |
1015 | return; | |
1016 | } | |
1017 | ||
1018 | /* first chain the descriptor at the tail of the list */ | |
1019 | desc_write(prev, hw_next, desc_dma); | |
1020 | chan->tail = desc; | |
1021 | chan->stats.tail_enqueue++; | |
1022 | ||
1023 | /* next check if EOQ has been triggered already */ | |
1024 | mode = desc_read(prev, hw_mode); | |
1025 | if (((mode & (CPDMA_DESC_EOQ | CPDMA_DESC_OWNER)) == CPDMA_DESC_EOQ) && | |
1026 | (chan->state == CPDMA_STATE_ACTIVE)) { | |
1027 | desc_write(prev, hw_mode, mode & ~CPDMA_DESC_EOQ); | |
1028 | chan_write(chan, hdp, desc_dma); | |
1029 | chan->stats.misqueued++; | |
1030 | } | |
1031 | } | |
1032 | ||
1033 | int cpdma_chan_submit(struct cpdma_chan *chan, void *token, void *data, | |
aef614e1 | 1034 | int len, int directed) |
ef8c2dab CC |
1035 | { |
1036 | struct cpdma_ctlr *ctlr = chan->ctlr; | |
1037 | struct cpdma_desc __iomem *desc; | |
1038 | dma_addr_t buffer; | |
1039 | unsigned long flags; | |
1040 | u32 mode; | |
1041 | int ret = 0; | |
1042 | ||
1043 | spin_lock_irqsave(&chan->lock, flags); | |
1044 | ||
1045 | if (chan->state == CPDMA_STATE_TEARDOWN) { | |
1046 | ret = -EINVAL; | |
1047 | goto unlock_ret; | |
1048 | } | |
1049 | ||
742fb20f GS |
1050 | if (chan->count >= chan->desc_num) { |
1051 | chan->stats.desc_alloc_fail++; | |
1052 | ret = -ENOMEM; | |
1053 | goto unlock_ret; | |
1054 | } | |
1055 | ||
1056 | desc = cpdma_desc_alloc(ctlr->pool); | |
ef8c2dab CC |
1057 | if (!desc) { |
1058 | chan->stats.desc_alloc_fail++; | |
1059 | ret = -ENOMEM; | |
1060 | goto unlock_ret; | |
1061 | } | |
1062 | ||
1063 | if (len < ctlr->params.min_packet_size) { | |
1064 | len = ctlr->params.min_packet_size; | |
1065 | chan->stats.runt_transmit_buff++; | |
1066 | } | |
1067 | ||
1068 | buffer = dma_map_single(ctlr->dev, data, len, chan->dir); | |
14bd0769 SS |
1069 | ret = dma_mapping_error(ctlr->dev, buffer); |
1070 | if (ret) { | |
1071 | cpdma_desc_free(ctlr->pool, desc, 1); | |
1072 | ret = -EINVAL; | |
1073 | goto unlock_ret; | |
1074 | } | |
1075 | ||
ef8c2dab | 1076 | mode = CPDMA_DESC_OWNER | CPDMA_DESC_SOP | CPDMA_DESC_EOP; |
f6e135c8 | 1077 | cpdma_desc_to_port(chan, mode, directed); |
ef8c2dab | 1078 | |
a6c83ccf GS |
1079 | /* Relaxed IO accessors can be used here as there is read barrier |
1080 | * at the end of write sequence. | |
1081 | */ | |
1082 | writel_relaxed(0, &desc->hw_next); | |
1083 | writel_relaxed(buffer, &desc->hw_buffer); | |
1084 | writel_relaxed(len, &desc->hw_len); | |
1085 | writel_relaxed(mode | len, &desc->hw_mode); | |
1086 | writel_relaxed(token, &desc->sw_token); | |
1087 | writel_relaxed(buffer, &desc->sw_buffer); | |
1088 | writel_relaxed(len, &desc->sw_len); | |
1089 | desc_read(desc, sw_len); | |
ef8c2dab CC |
1090 | |
1091 | __cpdma_chan_submit(chan, desc); | |
1092 | ||
1093 | if (chan->state == CPDMA_STATE_ACTIVE && chan->rxfree) | |
1094 | chan_write(chan, rxfree, 1); | |
1095 | ||
1096 | chan->count++; | |
1097 | ||
1098 | unlock_ret: | |
1099 | spin_unlock_irqrestore(&chan->lock, flags); | |
1100 | return ret; | |
1101 | } | |
32a6d90b | 1102 | EXPORT_SYMBOL_GPL(cpdma_chan_submit); |
ef8c2dab | 1103 | |
fae50823 M |
1104 | bool cpdma_check_free_tx_desc(struct cpdma_chan *chan) |
1105 | { | |
fae50823 M |
1106 | struct cpdma_ctlr *ctlr = chan->ctlr; |
1107 | struct cpdma_desc_pool *pool = ctlr->pool; | |
742fb20f GS |
1108 | bool free_tx_desc; |
1109 | unsigned long flags; | |
fae50823 | 1110 | |
742fb20f GS |
1111 | spin_lock_irqsave(&chan->lock, flags); |
1112 | free_tx_desc = (chan->count < chan->desc_num) && | |
1113 | gen_pool_avail(pool->gen_pool); | |
1114 | spin_unlock_irqrestore(&chan->lock, flags); | |
1115 | return free_tx_desc; | |
fae50823 M |
1116 | } |
1117 | EXPORT_SYMBOL_GPL(cpdma_check_free_tx_desc); | |
1118 | ||
ef8c2dab CC |
1119 | static void __cpdma_chan_free(struct cpdma_chan *chan, |
1120 | struct cpdma_desc __iomem *desc, | |
1121 | int outlen, int status) | |
1122 | { | |
1123 | struct cpdma_ctlr *ctlr = chan->ctlr; | |
1124 | struct cpdma_desc_pool *pool = ctlr->pool; | |
1125 | dma_addr_t buff_dma; | |
1126 | int origlen; | |
1127 | void *token; | |
1128 | ||
1129 | token = (void *)desc_read(desc, sw_token); | |
1130 | buff_dma = desc_read(desc, sw_buffer); | |
1131 | origlen = desc_read(desc, sw_len); | |
1132 | ||
1133 | dma_unmap_single(ctlr->dev, buff_dma, origlen, chan->dir); | |
1134 | cpdma_desc_free(pool, desc, 1); | |
1135 | (*chan->handler)(token, outlen, status); | |
1136 | } | |
1137 | ||
1138 | static int __cpdma_chan_process(struct cpdma_chan *chan) | |
1139 | { | |
1140 | struct cpdma_ctlr *ctlr = chan->ctlr; | |
1141 | struct cpdma_desc __iomem *desc; | |
1142 | int status, outlen; | |
b4727e69 | 1143 | int cb_status = 0; |
ef8c2dab CC |
1144 | struct cpdma_desc_pool *pool = ctlr->pool; |
1145 | dma_addr_t desc_dma; | |
1146 | unsigned long flags; | |
1147 | ||
1148 | spin_lock_irqsave(&chan->lock, flags); | |
1149 | ||
1150 | desc = chan->head; | |
1151 | if (!desc) { | |
1152 | chan->stats.empty_dequeue++; | |
1153 | status = -ENOENT; | |
1154 | goto unlock_ret; | |
1155 | } | |
1156 | desc_dma = desc_phys(pool, desc); | |
1157 | ||
a6c83ccf | 1158 | status = desc_read(desc, hw_mode); |
ef8c2dab CC |
1159 | outlen = status & 0x7ff; |
1160 | if (status & CPDMA_DESC_OWNER) { | |
1161 | chan->stats.busy_dequeue++; | |
1162 | status = -EBUSY; | |
1163 | goto unlock_ret; | |
1164 | } | |
28a19fe6 M |
1165 | |
1166 | if (status & CPDMA_DESC_PASS_CRC) | |
1167 | outlen -= CPDMA_DESC_CRC_LEN; | |
1168 | ||
f6e135c8 M |
1169 | status = status & (CPDMA_DESC_EOQ | CPDMA_DESC_TD_COMPLETE | |
1170 | CPDMA_DESC_PORT_MASK); | |
ef8c2dab CC |
1171 | |
1172 | chan->head = desc_from_phys(pool, desc_read(desc, hw_next)); | |
1173 | chan_write(chan, cp, desc_dma); | |
1174 | chan->count--; | |
1175 | chan->stats.good_dequeue++; | |
1176 | ||
12a303e3 | 1177 | if ((status & CPDMA_DESC_EOQ) && chan->head) { |
ef8c2dab CC |
1178 | chan->stats.requeue++; |
1179 | chan_write(chan, hdp, desc_phys(pool, chan->head)); | |
1180 | } | |
1181 | ||
1182 | spin_unlock_irqrestore(&chan->lock, flags); | |
b4727e69 SS |
1183 | if (unlikely(status & CPDMA_DESC_TD_COMPLETE)) |
1184 | cb_status = -ENOSYS; | |
1185 | else | |
1186 | cb_status = status; | |
ef8c2dab | 1187 | |
b4727e69 | 1188 | __cpdma_chan_free(chan, desc, outlen, cb_status); |
ef8c2dab CC |
1189 | return status; |
1190 | ||
1191 | unlock_ret: | |
1192 | spin_unlock_irqrestore(&chan->lock, flags); | |
1193 | return status; | |
1194 | } | |
1195 | ||
1196 | int cpdma_chan_process(struct cpdma_chan *chan, int quota) | |
1197 | { | |
1198 | int used = 0, ret = 0; | |
1199 | ||
1200 | if (chan->state != CPDMA_STATE_ACTIVE) | |
1201 | return -EINVAL; | |
1202 | ||
1203 | while (used < quota) { | |
1204 | ret = __cpdma_chan_process(chan); | |
1205 | if (ret < 0) | |
1206 | break; | |
1207 | used++; | |
1208 | } | |
1209 | return used; | |
1210 | } | |
32a6d90b | 1211 | EXPORT_SYMBOL_GPL(cpdma_chan_process); |
ef8c2dab CC |
1212 | |
1213 | int cpdma_chan_start(struct cpdma_chan *chan) | |
1214 | { | |
8f32b909 IK |
1215 | struct cpdma_ctlr *ctlr = chan->ctlr; |
1216 | unsigned long flags; | |
1217 | int ret; | |
ef8c2dab | 1218 | |
8f32b909 IK |
1219 | spin_lock_irqsave(&ctlr->lock, flags); |
1220 | ret = cpdma_chan_set_chan_shaper(chan); | |
1221 | spin_unlock_irqrestore(&ctlr->lock, flags); | |
1222 | if (ret) | |
1223 | return ret; | |
1224 | ||
1225 | ret = cpdma_chan_on(chan); | |
1226 | if (ret) | |
1227 | return ret; | |
ef8c2dab | 1228 | |
ef8c2dab CC |
1229 | return 0; |
1230 | } | |
32a6d90b | 1231 | EXPORT_SYMBOL_GPL(cpdma_chan_start); |
ef8c2dab CC |
1232 | |
1233 | int cpdma_chan_stop(struct cpdma_chan *chan) | |
1234 | { | |
1235 | struct cpdma_ctlr *ctlr = chan->ctlr; | |
1236 | struct cpdma_desc_pool *pool = ctlr->pool; | |
1237 | unsigned long flags; | |
1238 | int ret; | |
817f6d1a | 1239 | unsigned timeout; |
ef8c2dab CC |
1240 | |
1241 | spin_lock_irqsave(&chan->lock, flags); | |
cd11cf50 | 1242 | if (chan->state == CPDMA_STATE_TEARDOWN) { |
ef8c2dab CC |
1243 | spin_unlock_irqrestore(&chan->lock, flags); |
1244 | return -EINVAL; | |
1245 | } | |
1246 | ||
1247 | chan->state = CPDMA_STATE_TEARDOWN; | |
1248 | dma_reg_write(ctlr, chan->int_clear, chan->mask); | |
1249 | ||
1250 | /* trigger teardown */ | |
b4ad0428 | 1251 | dma_reg_write(ctlr, chan->td, chan_linear(chan)); |
ef8c2dab CC |
1252 | |
1253 | /* wait for teardown complete */ | |
817f6d1a SS |
1254 | timeout = 100 * 100; /* 100 ms */ |
1255 | while (timeout) { | |
ef8c2dab CC |
1256 | u32 cp = chan_read(chan, cp); |
1257 | if ((cp & CPDMA_TEARDOWN_VALUE) == CPDMA_TEARDOWN_VALUE) | |
1258 | break; | |
817f6d1a SS |
1259 | udelay(10); |
1260 | timeout--; | |
ef8c2dab | 1261 | } |
817f6d1a | 1262 | WARN_ON(!timeout); |
ef8c2dab CC |
1263 | chan_write(chan, cp, CPDMA_TEARDOWN_VALUE); |
1264 | ||
1265 | /* handle completed packets */ | |
7746ab0a | 1266 | spin_unlock_irqrestore(&chan->lock, flags); |
ef8c2dab CC |
1267 | do { |
1268 | ret = __cpdma_chan_process(chan); | |
1269 | if (ret < 0) | |
1270 | break; | |
1271 | } while ((ret & CPDMA_DESC_TD_COMPLETE) == 0); | |
7746ab0a | 1272 | spin_lock_irqsave(&chan->lock, flags); |
ef8c2dab CC |
1273 | |
1274 | /* remaining packets haven't been tx/rx'ed, clean them up */ | |
1275 | while (chan->head) { | |
1276 | struct cpdma_desc __iomem *desc = chan->head; | |
1277 | dma_addr_t next_dma; | |
1278 | ||
1279 | next_dma = desc_read(desc, hw_next); | |
1280 | chan->head = desc_from_phys(pool, next_dma); | |
ffb5ba90 | 1281 | chan->count--; |
ef8c2dab CC |
1282 | chan->stats.teardown_dequeue++; |
1283 | ||
1284 | /* issue callback without locks held */ | |
1285 | spin_unlock_irqrestore(&chan->lock, flags); | |
1286 | __cpdma_chan_free(chan, desc, 0, -ENOSYS); | |
1287 | spin_lock_irqsave(&chan->lock, flags); | |
1288 | } | |
1289 | ||
1290 | chan->state = CPDMA_STATE_IDLE; | |
1291 | spin_unlock_irqrestore(&chan->lock, flags); | |
1292 | return 0; | |
1293 | } | |
32a6d90b | 1294 | EXPORT_SYMBOL_GPL(cpdma_chan_stop); |
ef8c2dab CC |
1295 | |
1296 | int cpdma_chan_int_ctrl(struct cpdma_chan *chan, bool enable) | |
1297 | { | |
1298 | unsigned long flags; | |
1299 | ||
1300 | spin_lock_irqsave(&chan->lock, flags); | |
1301 | if (chan->state != CPDMA_STATE_ACTIVE) { | |
1302 | spin_unlock_irqrestore(&chan->lock, flags); | |
1303 | return -EINVAL; | |
1304 | } | |
1305 | ||
1306 | dma_reg_write(chan->ctlr, enable ? chan->int_set : chan->int_clear, | |
1307 | chan->mask); | |
1308 | spin_unlock_irqrestore(&chan->lock, flags); | |
1309 | ||
1310 | return 0; | |
1311 | } | |
1312 | ||
ef8c2dab CC |
1313 | int cpdma_control_get(struct cpdma_ctlr *ctlr, int control) |
1314 | { | |
1315 | unsigned long flags; | |
ef8c2dab CC |
1316 | int ret; |
1317 | ||
1318 | spin_lock_irqsave(&ctlr->lock, flags); | |
8f32b909 | 1319 | ret = _cpdma_control_get(ctlr, control); |
ef8c2dab | 1320 | spin_unlock_irqrestore(&ctlr->lock, flags); |
8f32b909 | 1321 | |
ef8c2dab CC |
1322 | return ret; |
1323 | } | |
1324 | ||
1325 | int cpdma_control_set(struct cpdma_ctlr *ctlr, int control, int value) | |
1326 | { | |
1327 | unsigned long flags; | |
ef8c2dab | 1328 | int ret; |
ef8c2dab CC |
1329 | |
1330 | spin_lock_irqsave(&ctlr->lock, flags); | |
991ddb1f | 1331 | ret = _cpdma_control_set(ctlr, control, value); |
ef8c2dab | 1332 | spin_unlock_irqrestore(&ctlr->lock, flags); |
8f32b909 | 1333 | |
ef8c2dab CC |
1334 | return ret; |
1335 | } | |
6929e24e | 1336 | EXPORT_SYMBOL_GPL(cpdma_control_set); |
4bc21d41 | 1337 | |
be034fc1 GS |
1338 | int cpdma_get_num_rx_descs(struct cpdma_ctlr *ctlr) |
1339 | { | |
1340 | return ctlr->num_rx_desc; | |
1341 | } | |
1342 | EXPORT_SYMBOL_GPL(cpdma_get_num_rx_descs); | |
1343 | ||
1344 | int cpdma_get_num_tx_descs(struct cpdma_ctlr *ctlr) | |
1345 | { | |
1346 | return ctlr->num_tx_desc; | |
1347 | } | |
1348 | EXPORT_SYMBOL_GPL(cpdma_get_num_tx_descs); | |
1349 | ||
1350 | void cpdma_set_num_rx_descs(struct cpdma_ctlr *ctlr, int num_rx_desc) | |
1351 | { | |
1352 | ctlr->num_rx_desc = num_rx_desc; | |
1353 | ctlr->num_tx_desc = ctlr->pool->num_desc - ctlr->num_rx_desc; | |
1354 | } | |
1355 | EXPORT_SYMBOL_GPL(cpdma_set_num_rx_descs); | |
1356 | ||
4bc21d41 | 1357 | MODULE_LICENSE("GPL"); |