Commit | Line | Data |
---|---|---|
f21fb3ed RV |
1 | /********************************************************************** |
2 | * Author: Cavium, Inc. | |
3 | * | |
4 | * Contact: support@cavium.com | |
5 | * Please include "LiquidIO" in the subject. | |
6 | * | |
50579d3d | 7 | * Copyright (c) 2003-2016 Cavium, Inc. |
f21fb3ed RV |
8 | * |
9 | * This file is free software; you can redistribute it and/or modify | |
10 | * it under the terms of the GNU General Public License, Version 2, as | |
11 | * published by the Free Software Foundation. | |
12 | * | |
13 | * This file is distributed in the hope that it will be useful, but | |
14 | * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty | |
15 | * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or | |
16 | * NONINFRINGEMENT. See the GNU General Public License for more | |
17 | * details. | |
f21fb3ed | 18 | **********************************************************************/ |
f21fb3ed | 19 | #include <linux/pci.h> |
f21fb3ed | 20 | #include <linux/netdevice.h> |
5b173cf9 | 21 | #include <linux/vmalloc.h> |
f21fb3ed RV |
22 | #include "liquidio_common.h" |
23 | #include "octeon_droq.h" | |
24 | #include "octeon_iq.h" | |
25 | #include "response_manager.h" | |
26 | #include "octeon_device.h" | |
f21fb3ed RV |
27 | #include "octeon_main.h" |
28 | #include "octeon_network.h" | |
f21fb3ed | 29 | #include "cn66xx_device.h" |
5b823514 | 30 | #include "cn23xx_pf_device.h" |
9003baf0 | 31 | #include "cn23xx_vf_device.h" |
f21fb3ed | 32 | |
f21fb3ed RV |
33 | struct iq_post_status { |
34 | int status; | |
35 | int index; | |
36 | }; | |
37 | ||
38 | static void check_db_timeout(struct work_struct *work); | |
9a96bde4 | 39 | static void __check_db_timeout(struct octeon_device *oct, u64 iq_no); |
f21fb3ed RV |
40 | |
41 | static void (*reqtype_free_fn[MAX_OCTEON_DEVICES][REQTYPE_LAST + 1]) (void *); | |
42 | ||
43 | static inline int IQ_INSTR_MODE_64B(struct octeon_device *oct, int iq_no) | |
44 | { | |
45 | struct octeon_instr_queue *iq = | |
46 | (struct octeon_instr_queue *)oct->instr_queue[iq_no]; | |
47 | return iq->iqcmd_64B; | |
48 | } | |
49 | ||
50 | #define IQ_INSTR_MODE_32B(oct, iq_no) (!IQ_INSTR_MODE_64B(oct, iq_no)) | |
51 | ||
52 | /* Define this to return the request status comaptible to old code */ | |
53 | /*#define OCTEON_USE_OLD_REQ_STATUS*/ | |
54 | ||
55 | /* Return 0 on success, 1 on failure */ | |
56 | int octeon_init_instr_queue(struct octeon_device *oct, | |
26236fa9 RV |
57 | union oct_txpciq txpciq, |
58 | u32 num_descs) | |
f21fb3ed RV |
59 | { |
60 | struct octeon_instr_queue *iq; | |
61 | struct octeon_iq_config *conf = NULL; | |
26236fa9 | 62 | u32 iq_no = (u32)txpciq.s.q_no; |
f21fb3ed RV |
63 | u32 q_size; |
64 | struct cavium_wq *db_wq; | |
b3ca9af0 | 65 | int numa_node = dev_to_node(&oct->pci_dev->dev); |
f21fb3ed RV |
66 | |
67 | if (OCTEON_CN6XXX(oct)) | |
97a25326 | 68 | conf = &(CFG_GET_IQ_CFG(CHIP_CONF(oct, cn6xxx))); |
5b823514 | 69 | else if (OCTEON_CN23XX_PF(oct)) |
97a25326 | 70 | conf = &(CFG_GET_IQ_CFG(CHIP_CONF(oct, cn23xx_pf))); |
9003baf0 RV |
71 | else if (OCTEON_CN23XX_VF(oct)) |
72 | conf = &(CFG_GET_IQ_CFG(CHIP_CONF(oct, cn23xx_vf))); | |
73 | ||
f21fb3ed RV |
74 | if (!conf) { |
75 | dev_err(&oct->pci_dev->dev, "Unsupported Chip %x\n", | |
76 | oct->chip_id); | |
77 | return 1; | |
78 | } | |
79 | ||
f21fb3ed RV |
80 | q_size = (u32)conf->instr_type * num_descs; |
81 | ||
82 | iq = oct->instr_queue[iq_no]; | |
5b823514 | 83 | |
6a885b60 | 84 | iq->oct_dev = oct; |
f21fb3ed | 85 | |
b3ca9af0 | 86 | iq->base_addr = lio_dma_alloc(oct, q_size, &iq->base_addr_dma); |
f21fb3ed RV |
87 | if (!iq->base_addr) { |
88 | dev_err(&oct->pci_dev->dev, "Cannot allocate memory for instr queue %d\n", | |
89 | iq_no); | |
90 | return 1; | |
91 | } | |
92 | ||
93 | iq->max_count = num_descs; | |
94 | ||
95 | /* Initialize a list to holds requests that have been posted to Octeon | |
96 | * but has yet to be fetched by octeon | |
97 | */ | |
26236fa9 RV |
98 | iq->request_list = vmalloc_node((sizeof(*iq->request_list) * num_descs), |
99 | numa_node); | |
100 | if (!iq->request_list) | |
42bc47b3 KC |
101 | iq->request_list = |
102 | vmalloc(array_size(num_descs, | |
103 | sizeof(*iq->request_list))); | |
f21fb3ed RV |
104 | if (!iq->request_list) { |
105 | lio_dma_free(oct, q_size, iq->base_addr, iq->base_addr_dma); | |
106 | dev_err(&oct->pci_dev->dev, "Alloc failed for IQ[%d] nr free list\n", | |
107 | iq_no); | |
108 | return 1; | |
109 | } | |
110 | ||
111 | memset(iq->request_list, 0, sizeof(*iq->request_list) * num_descs); | |
112 | ||
113 | dev_dbg(&oct->pci_dev->dev, "IQ[%d]: base: %p basedma: %llx count: %d\n", | |
114 | iq_no, iq->base_addr, iq->base_addr_dma, iq->max_count); | |
115 | ||
26236fa9 | 116 | iq->txpciq.u64 = txpciq.u64; |
f21fb3ed RV |
117 | iq->fill_threshold = (u32)conf->db_min; |
118 | iq->fill_cnt = 0; | |
119 | iq->host_write_index = 0; | |
120 | iq->octeon_read_index = 0; | |
121 | iq->flush_index = 0; | |
122 | iq->last_db_time = 0; | |
123 | iq->do_auto_flush = 1; | |
124 | iq->db_timeout = (u32)conf->db_timeout; | |
125 | atomic_set(&iq->instr_pending, 0); | |
126 | ||
127 | /* Initialize the spinlock for this instruction queue */ | |
128 | spin_lock_init(&iq->lock); | |
9a96bde4 RV |
129 | spin_lock_init(&iq->post_lock); |
130 | ||
131 | spin_lock_init(&iq->iq_flush_running_lock); | |
f21fb3ed | 132 | |
763185a3 | 133 | oct->io_qmask.iq |= BIT_ULL(iq_no); |
f21fb3ed RV |
134 | |
135 | /* Set the 32B/64B mode for each input queue */ | |
136 | oct->io_qmask.iq64B |= ((conf->instr_type == 64) << iq_no); | |
137 | iq->iqcmd_64B = (conf->instr_type == 64); | |
138 | ||
139 | oct->fn_list.setup_iq_regs(oct, iq_no); | |
140 | ||
aaa76724 BS |
141 | oct->check_db_wq[iq_no].wq = alloc_workqueue("check_iq_db", |
142 | WQ_MEM_RECLAIM, | |
143 | 0); | |
f21fb3ed | 144 | if (!oct->check_db_wq[iq_no].wq) { |
515e752d RV |
145 | vfree(iq->request_list); |
146 | iq->request_list = NULL; | |
f21fb3ed RV |
147 | lio_dma_free(oct, q_size, iq->base_addr, iq->base_addr_dma); |
148 | dev_err(&oct->pci_dev->dev, "check db wq create failed for iq %d\n", | |
149 | iq_no); | |
150 | return 1; | |
151 | } | |
152 | ||
153 | db_wq = &oct->check_db_wq[iq_no]; | |
154 | ||
155 | INIT_DELAYED_WORK(&db_wq->wk.work, check_db_timeout); | |
156 | db_wq->wk.ctxptr = oct; | |
157 | db_wq->wk.ctxul = iq_no; | |
158 | queue_delayed_work(db_wq->wq, &db_wq->wk.work, msecs_to_jiffies(1)); | |
159 | ||
160 | return 0; | |
161 | } | |
162 | ||
163 | int octeon_delete_instr_queue(struct octeon_device *oct, u32 iq_no) | |
164 | { | |
165 | u64 desc_size = 0, q_size; | |
166 | struct octeon_instr_queue *iq = oct->instr_queue[iq_no]; | |
167 | ||
168 | cancel_delayed_work_sync(&oct->check_db_wq[iq_no].wk.work); | |
f21fb3ed RV |
169 | destroy_workqueue(oct->check_db_wq[iq_no].wq); |
170 | ||
171 | if (OCTEON_CN6XXX(oct)) | |
172 | desc_size = | |
97a25326 | 173 | CFG_GET_IQ_INSTR_TYPE(CHIP_CONF(oct, cn6xxx)); |
5b823514 RV |
174 | else if (OCTEON_CN23XX_PF(oct)) |
175 | desc_size = | |
97a25326 | 176 | CFG_GET_IQ_INSTR_TYPE(CHIP_CONF(oct, cn23xx_pf)); |
9003baf0 RV |
177 | else if (OCTEON_CN23XX_VF(oct)) |
178 | desc_size = | |
179 | CFG_GET_IQ_INSTR_TYPE(CHIP_CONF(oct, cn23xx_vf)); | |
f21fb3ed | 180 | |
9686f310 | 181 | vfree(iq->request_list); |
f21fb3ed RV |
182 | |
183 | if (iq->base_addr) { | |
184 | q_size = iq->max_count * desc_size; | |
185 | lio_dma_free(oct, (u32)q_size, iq->base_addr, | |
186 | iq->base_addr_dma); | |
c1550fde IB |
187 | oct->io_qmask.iq &= ~(1ULL << iq_no); |
188 | vfree(oct->instr_queue[iq_no]); | |
189 | oct->instr_queue[iq_no] = NULL; | |
190 | oct->num_iqs--; | |
f21fb3ed RV |
191 | return 0; |
192 | } | |
193 | return 1; | |
194 | } | |
195 | ||
196 | /* Return 0 on success, 1 on failure */ | |
197 | int octeon_setup_iq(struct octeon_device *oct, | |
0cece6c5 RV |
198 | int ifidx, |
199 | int q_index, | |
26236fa9 | 200 | union oct_txpciq txpciq, |
f21fb3ed RV |
201 | u32 num_descs, |
202 | void *app_ctx) | |
203 | { | |
26236fa9 | 204 | u32 iq_no = (u32)txpciq.s.q_no; |
b3ca9af0 | 205 | int numa_node = dev_to_node(&oct->pci_dev->dev); |
26236fa9 | 206 | |
f21fb3ed RV |
207 | if (oct->instr_queue[iq_no]) { |
208 | dev_dbg(&oct->pci_dev->dev, "IQ is in use. Cannot create the IQ: %d again\n", | |
209 | iq_no); | |
26236fa9 | 210 | oct->instr_queue[iq_no]->txpciq.u64 = txpciq.u64; |
f21fb3ed RV |
211 | oct->instr_queue[iq_no]->app_ctx = app_ctx; |
212 | return 0; | |
213 | } | |
214 | oct->instr_queue[iq_no] = | |
26236fa9 RV |
215 | vmalloc_node(sizeof(struct octeon_instr_queue), numa_node); |
216 | if (!oct->instr_queue[iq_no]) | |
217 | oct->instr_queue[iq_no] = | |
218 | vmalloc(sizeof(struct octeon_instr_queue)); | |
f21fb3ed RV |
219 | if (!oct->instr_queue[iq_no]) |
220 | return 1; | |
221 | ||
222 | memset(oct->instr_queue[iq_no], 0, | |
223 | sizeof(struct octeon_instr_queue)); | |
224 | ||
0cece6c5 | 225 | oct->instr_queue[iq_no]->q_index = q_index; |
f21fb3ed | 226 | oct->instr_queue[iq_no]->app_ctx = app_ctx; |
0cece6c5 RV |
227 | oct->instr_queue[iq_no]->ifidx = ifidx; |
228 | ||
26236fa9 | 229 | if (octeon_init_instr_queue(oct, txpciq, num_descs)) { |
f21fb3ed RV |
230 | vfree(oct->instr_queue[iq_no]); |
231 | oct->instr_queue[iq_no] = NULL; | |
232 | return 1; | |
233 | } | |
234 | ||
235 | oct->num_iqs++; | |
c865cdf1 RV |
236 | if (oct->fn_list.enable_io_queues(oct)) |
237 | return 1; | |
238 | ||
f21fb3ed RV |
239 | return 0; |
240 | } | |
241 | ||
242 | int lio_wait_for_instr_fetch(struct octeon_device *oct) | |
243 | { | |
244 | int i, retry = 1000, pending, instr_cnt = 0; | |
245 | ||
246 | do { | |
247 | instr_cnt = 0; | |
248 | ||
63da8404 | 249 | for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) { |
763185a3 | 250 | if (!(oct->io_qmask.iq & BIT_ULL(i))) |
f21fb3ed RV |
251 | continue; |
252 | pending = | |
9ae122c6 | 253 | atomic_read(&oct->instr_queue[i]->instr_pending); |
f21fb3ed RV |
254 | if (pending) |
255 | __check_db_timeout(oct, i); | |
256 | instr_cnt += pending; | |
257 | } | |
258 | ||
259 | if (instr_cnt == 0) | |
260 | break; | |
261 | ||
262 | schedule_timeout_uninterruptible(1); | |
263 | ||
264 | } while (retry-- && instr_cnt); | |
265 | ||
266 | return instr_cnt; | |
267 | } | |
268 | ||
269 | static inline void | |
270 | ring_doorbell(struct octeon_device *oct, struct octeon_instr_queue *iq) | |
271 | { | |
272 | if (atomic_read(&oct->status) == OCT_DEV_RUNNING) { | |
273 | writel(iq->fill_cnt, iq->doorbell_reg); | |
274 | /* make sure doorbell write goes through */ | |
275 | mmiowb(); | |
276 | iq->fill_cnt = 0; | |
277 | iq->last_db_time = jiffies; | |
278 | return; | |
279 | } | |
280 | } | |
281 | ||
c859e21a IB |
282 | void |
283 | octeon_ring_doorbell_locked(struct octeon_device *oct, u32 iq_no) | |
284 | { | |
285 | struct octeon_instr_queue *iq; | |
286 | ||
287 | iq = oct->instr_queue[iq_no]; | |
288 | spin_lock(&iq->post_lock); | |
289 | if (iq->fill_cnt) | |
290 | ring_doorbell(oct, iq); | |
291 | spin_unlock(&iq->post_lock); | |
292 | } | |
293 | ||
f21fb3ed RV |
294 | static inline void __copy_cmd_into_iq(struct octeon_instr_queue *iq, |
295 | u8 *cmd) | |
296 | { | |
297 | u8 *iqptr, cmdsize; | |
298 | ||
299 | cmdsize = ((iq->iqcmd_64B) ? 64 : 32); | |
300 | iqptr = iq->base_addr + (cmdsize * iq->host_write_index); | |
301 | ||
302 | memcpy(iqptr, cmd, cmdsize); | |
303 | } | |
304 | ||
f21fb3ed | 305 | static inline struct iq_post_status |
a7d5a3dc | 306 | __post_command2(struct octeon_instr_queue *iq, u8 *cmd) |
f21fb3ed RV |
307 | { |
308 | struct iq_post_status st; | |
309 | ||
310 | st.status = IQ_SEND_OK; | |
311 | ||
312 | /* This ensures that the read index does not wrap around to the same | |
313 | * position if queue gets full before Octeon could fetch any instr. | |
314 | */ | |
315 | if (atomic_read(&iq->instr_pending) >= (s32)(iq->max_count - 1)) { | |
316 | st.status = IQ_SEND_FAILED; | |
317 | st.index = -1; | |
318 | return st; | |
319 | } | |
320 | ||
321 | if (atomic_read(&iq->instr_pending) >= (s32)(iq->max_count - 2)) | |
322 | st.status = IQ_SEND_STOP; | |
323 | ||
324 | __copy_cmd_into_iq(iq, cmd); | |
325 | ||
326 | /* "index" is returned, host_write_index is modified. */ | |
327 | st.index = iq->host_write_index; | |
97a25326 RV |
328 | iq->host_write_index = incr_index(iq->host_write_index, 1, |
329 | iq->max_count); | |
f21fb3ed RV |
330 | iq->fill_cnt++; |
331 | ||
332 | /* Flush the command into memory. We need to be sure the data is in | |
333 | * memory before indicating that the instruction is pending. | |
334 | */ | |
335 | wmb(); | |
336 | ||
337 | atomic_inc(&iq->instr_pending); | |
338 | ||
339 | return st; | |
340 | } | |
341 | ||
342 | int | |
343 | octeon_register_reqtype_free_fn(struct octeon_device *oct, int reqtype, | |
344 | void (*fn)(void *)) | |
345 | { | |
346 | if (reqtype > REQTYPE_LAST) { | |
347 | dev_err(&oct->pci_dev->dev, "%s: Invalid reqtype: %d\n", | |
348 | __func__, reqtype); | |
349 | return -EINVAL; | |
350 | } | |
351 | ||
352 | reqtype_free_fn[oct->octeon_id][reqtype] = fn; | |
353 | ||
354 | return 0; | |
355 | } | |
356 | ||
357 | static inline void | |
358 | __add_to_request_list(struct octeon_instr_queue *iq, | |
359 | int idx, void *buf, int reqtype) | |
360 | { | |
361 | iq->request_list[idx].buf = buf; | |
362 | iq->request_list[idx].reqtype = reqtype; | |
363 | } | |
364 | ||
a2c64b67 | 365 | /* Can only run in process context */ |
f21fb3ed RV |
366 | int |
367 | lio_process_iq_request_list(struct octeon_device *oct, | |
9a96bde4 | 368 | struct octeon_instr_queue *iq, u32 napi_budget) |
f21fb3ed | 369 | { |
cecd8d81 | 370 | struct cavium_wq *cwq = &oct->dma_comp_wq; |
f21fb3ed RV |
371 | int reqtype; |
372 | void *buf; | |
373 | u32 old = iq->flush_index; | |
374 | u32 inst_count = 0; | |
9a96bde4 | 375 | unsigned int pkts_compl = 0, bytes_compl = 0; |
f21fb3ed RV |
376 | struct octeon_soft_command *sc; |
377 | struct octeon_instr_irh *irh; | |
14866ccd | 378 | unsigned long flags; |
f21fb3ed RV |
379 | |
380 | while (old != iq->octeon_read_index) { | |
381 | reqtype = iq->request_list[old].reqtype; | |
382 | buf = iq->request_list[old].buf; | |
383 | ||
384 | if (reqtype == REQTYPE_NONE) | |
385 | goto skip_this; | |
386 | ||
387 | octeon_update_tx_completion_counters(buf, reqtype, &pkts_compl, | |
388 | &bytes_compl); | |
389 | ||
390 | switch (reqtype) { | |
391 | case REQTYPE_NORESP_NET: | |
392 | case REQTYPE_NORESP_NET_SG: | |
393 | case REQTYPE_RESP_NET_SG: | |
394 | reqtype_free_fn[oct->octeon_id][reqtype](buf); | |
395 | break; | |
396 | case REQTYPE_RESP_NET: | |
397 | case REQTYPE_SOFT_COMMAND: | |
398 | sc = buf; | |
399 | ||
9981328a | 400 | if (OCTEON_CN23XX_PF(oct) || OCTEON_CN23XX_VF(oct)) |
5b823514 RV |
401 | irh = (struct octeon_instr_irh *) |
402 | &sc->cmd.cmd3.irh; | |
403 | else | |
404 | irh = (struct octeon_instr_irh *) | |
405 | &sc->cmd.cmd2.irh; | |
f21fb3ed RV |
406 | if (irh->rflag) { |
407 | /* We're expecting a response from Octeon. | |
408 | * It's up to lio_process_ordered_list() to | |
409 | * process sc. Add sc to the ordered soft | |
410 | * command response list because we expect | |
411 | * a response from Octeon. | |
412 | */ | |
14866ccd RV |
413 | spin_lock_irqsave |
414 | (&oct->response_list | |
415 | [OCTEON_ORDERED_SC_LIST].lock, | |
416 | flags); | |
f21fb3ed RV |
417 | atomic_inc(&oct->response_list |
418 | [OCTEON_ORDERED_SC_LIST]. | |
419 | pending_req_count); | |
420 | list_add_tail(&sc->node, &oct->response_list | |
421 | [OCTEON_ORDERED_SC_LIST].head); | |
14866ccd RV |
422 | spin_unlock_irqrestore |
423 | (&oct->response_list | |
424 | [OCTEON_ORDERED_SC_LIST].lock, | |
425 | flags); | |
f21fb3ed RV |
426 | } else { |
427 | if (sc->callback) { | |
a2c64b67 | 428 | /* This callback must not sleep */ |
f21fb3ed RV |
429 | sc->callback(oct, OCTEON_REQUEST_DONE, |
430 | sc->callback_arg); | |
431 | } | |
432 | } | |
433 | break; | |
434 | default: | |
435 | dev_err(&oct->pci_dev->dev, | |
436 | "%s Unknown reqtype: %d buf: %p at idx %d\n", | |
437 | __func__, reqtype, buf, old); | |
438 | } | |
439 | ||
440 | iq->request_list[old].buf = NULL; | |
441 | iq->request_list[old].reqtype = 0; | |
442 | ||
443 | skip_this: | |
444 | inst_count++; | |
97a25326 | 445 | old = incr_index(old, 1, iq->max_count); |
9a96bde4 RV |
446 | |
447 | if ((napi_budget) && (inst_count >= napi_budget)) | |
448 | break; | |
f21fb3ed RV |
449 | } |
450 | if (bytes_compl) | |
451 | octeon_report_tx_completion_to_bql(iq->app_ctx, pkts_compl, | |
452 | bytes_compl); | |
453 | iq->flush_index = old; | |
454 | ||
cecd8d81 PK |
455 | if (atomic_read(&oct->response_list |
456 | [OCTEON_ORDERED_SC_LIST].pending_req_count)) | |
457 | queue_delayed_work(cwq->wq, &cwq->wk.work, msecs_to_jiffies(1)); | |
458 | ||
f21fb3ed RV |
459 | return inst_count; |
460 | } | |
461 | ||
9a96bde4 RV |
462 | /* Can only be called from process context */ |
463 | int | |
464 | octeon_flush_iq(struct octeon_device *oct, struct octeon_instr_queue *iq, | |
60889869 | 465 | u32 napi_budget) |
f21fb3ed RV |
466 | { |
467 | u32 inst_processed = 0; | |
9a96bde4 RV |
468 | u32 tot_inst_processed = 0; |
469 | int tx_done = 1; | |
f21fb3ed | 470 | |
9a96bde4 RV |
471 | if (!spin_trylock(&iq->iq_flush_running_lock)) |
472 | return tx_done; | |
f21fb3ed | 473 | |
9a96bde4 | 474 | spin_lock_bh(&iq->lock); |
f21fb3ed | 475 | |
9a96bde4 | 476 | iq->octeon_read_index = oct->fn_list.update_iq_read_idx(iq); |
f21fb3ed | 477 | |
60889869 DC |
478 | do { |
479 | /* Process any outstanding IQ packets. */ | |
480 | if (iq->flush_index == iq->octeon_read_index) | |
481 | break; | |
9a96bde4 | 482 | |
60889869 DC |
483 | if (napi_budget) |
484 | inst_processed = | |
485 | lio_process_iq_request_list(oct, iq, | |
486 | napi_budget - | |
487 | tot_inst_processed); | |
488 | else | |
489 | inst_processed = | |
490 | lio_process_iq_request_list(oct, iq, 0); | |
491 | ||
492 | if (inst_processed) { | |
493 | atomic_sub(inst_processed, &iq->instr_pending); | |
494 | iq->stats.instr_processed += inst_processed; | |
495 | } | |
9a96bde4 | 496 | |
60889869 | 497 | tot_inst_processed += inst_processed; |
60889869 | 498 | } while (tot_inst_processed < napi_budget); |
9a96bde4 | 499 | |
60889869 DC |
500 | if (napi_budget && (tot_inst_processed >= napi_budget)) |
501 | tx_done = 0; | |
9a96bde4 RV |
502 | |
503 | iq->last_db_time = jiffies; | |
504 | ||
505 | spin_unlock_bh(&iq->lock); | |
506 | ||
507 | spin_unlock(&iq->iq_flush_running_lock); | |
508 | ||
509 | return tx_done; | |
f21fb3ed RV |
510 | } |
511 | ||
9a96bde4 RV |
512 | /* Process instruction queue after timeout. |
513 | * This routine gets called from a workqueue or when removing the module. | |
514 | */ | |
515 | static void __check_db_timeout(struct octeon_device *oct, u64 iq_no) | |
f21fb3ed RV |
516 | { |
517 | struct octeon_instr_queue *iq; | |
518 | u64 next_time; | |
519 | ||
520 | if (!oct) | |
521 | return; | |
cd8b1eb4 | 522 | |
f21fb3ed RV |
523 | iq = oct->instr_queue[iq_no]; |
524 | if (!iq) | |
525 | return; | |
526 | ||
9a96bde4 RV |
527 | /* return immediately, if no work pending */ |
528 | if (!atomic_read(&iq->instr_pending)) | |
529 | return; | |
f21fb3ed RV |
530 | /* If jiffies - last_db_time < db_timeout do nothing */ |
531 | next_time = iq->last_db_time + iq->db_timeout; | |
532 | if (!time_after(jiffies, (unsigned long)next_time)) | |
533 | return; | |
534 | iq->last_db_time = jiffies; | |
535 | ||
f21fb3ed | 536 | /* Flush the instruction queue */ |
60889869 | 537 | octeon_flush_iq(oct, iq, 0); |
cd8b1eb4 RV |
538 | |
539 | lio_enable_irq(NULL, iq); | |
f21fb3ed RV |
540 | } |
541 | ||
542 | /* Called by the Poll thread at regular intervals to check the instruction | |
543 | * queue for commands to be posted and for commands that were fetched by Octeon. | |
544 | */ | |
545 | static void check_db_timeout(struct work_struct *work) | |
546 | { | |
547 | struct cavium_wk *wk = (struct cavium_wk *)work; | |
548 | struct octeon_device *oct = (struct octeon_device *)wk->ctxptr; | |
a2c64b67 | 549 | u64 iq_no = wk->ctxul; |
f21fb3ed | 550 | struct cavium_wq *db_wq = &oct->check_db_wq[iq_no]; |
55893a63 | 551 | u32 delay = 10; |
f21fb3ed RV |
552 | |
553 | __check_db_timeout(oct, iq_no); | |
55893a63 | 554 | queue_delayed_work(db_wq->wq, &db_wq->wk.work, msecs_to_jiffies(delay)); |
f21fb3ed RV |
555 | } |
556 | ||
557 | int | |
558 | octeon_send_command(struct octeon_device *oct, u32 iq_no, | |
559 | u32 force_db, void *cmd, void *buf, | |
560 | u32 datasize, u32 reqtype) | |
561 | { | |
c859e21a | 562 | int xmit_stopped; |
f21fb3ed RV |
563 | struct iq_post_status st; |
564 | struct octeon_instr_queue *iq = oct->instr_queue[iq_no]; | |
565 | ||
9a96bde4 RV |
566 | /* Get the lock and prevent other tasks and tx interrupt handler from |
567 | * running. | |
568 | */ | |
569 | spin_lock_bh(&iq->post_lock); | |
f21fb3ed | 570 | |
a7d5a3dc | 571 | st = __post_command2(iq, cmd); |
f21fb3ed RV |
572 | |
573 | if (st.status != IQ_SEND_FAILED) { | |
c859e21a | 574 | xmit_stopped = octeon_report_sent_bytes_to_bql(buf, reqtype); |
f21fb3ed RV |
575 | __add_to_request_list(iq, st.index, buf, reqtype); |
576 | INCR_INSTRQUEUE_PKT_COUNT(oct, iq_no, bytes_sent, datasize); | |
577 | INCR_INSTRQUEUE_PKT_COUNT(oct, iq_no, instr_posted, 1); | |
578 | ||
c859e21a IB |
579 | if (iq->fill_cnt >= MAX_OCTEON_FILL_COUNT || force_db || |
580 | xmit_stopped || st.status == IQ_SEND_STOP) | |
f21fb3ed RV |
581 | ring_doorbell(oct, iq); |
582 | } else { | |
583 | INCR_INSTRQUEUE_PKT_COUNT(oct, iq_no, instr_dropped, 1); | |
584 | } | |
585 | ||
9a96bde4 | 586 | spin_unlock_bh(&iq->post_lock); |
f21fb3ed | 587 | |
9a96bde4 RV |
588 | /* This is only done here to expedite packets being flushed |
589 | * for cases where there are no IQ completion interrupts. | |
590 | */ | |
f21fb3ed RV |
591 | |
592 | return st.status; | |
593 | } | |
594 | ||
595 | void | |
596 | octeon_prepare_soft_command(struct octeon_device *oct, | |
597 | struct octeon_soft_command *sc, | |
598 | u8 opcode, | |
599 | u8 subcode, | |
600 | u32 irh_ossp, | |
601 | u64 ossp0, | |
602 | u64 ossp1) | |
603 | { | |
604 | struct octeon_config *oct_cfg; | |
6a885b60 | 605 | struct octeon_instr_ih2 *ih2; |
5b823514 RV |
606 | struct octeon_instr_ih3 *ih3; |
607 | struct octeon_instr_pki_ih3 *pki_ih3; | |
f21fb3ed RV |
608 | struct octeon_instr_irh *irh; |
609 | struct octeon_instr_rdp *rdp; | |
610 | ||
a7d5a3dc RV |
611 | WARN_ON(opcode > 15); |
612 | WARN_ON(subcode > 127); | |
f21fb3ed RV |
613 | |
614 | oct_cfg = octeon_get_conf(oct); | |
615 | ||
9981328a | 616 | if (OCTEON_CN23XX_PF(oct) || OCTEON_CN23XX_VF(oct)) { |
5b823514 RV |
617 | ih3 = (struct octeon_instr_ih3 *)&sc->cmd.cmd3.ih3; |
618 | ||
619 | ih3->pkind = oct->instr_queue[sc->iq_no]->txpciq.s.pkind; | |
620 | ||
621 | pki_ih3 = (struct octeon_instr_pki_ih3 *)&sc->cmd.cmd3.pki_ih3; | |
622 | ||
623 | pki_ih3->w = 1; | |
624 | pki_ih3->raw = 1; | |
625 | pki_ih3->utag = 1; | |
626 | pki_ih3->uqpg = | |
627 | oct->instr_queue[sc->iq_no]->txpciq.s.use_qpg; | |
628 | pki_ih3->utt = 1; | |
629 | pki_ih3->tag = LIO_CONTROL; | |
630 | pki_ih3->tagtype = ATOMIC_TAG; | |
631 | pki_ih3->qpg = | |
697fefc7 IB |
632 | oct->instr_queue[sc->iq_no]->txpciq.s.ctrl_qpg; |
633 | ||
5b823514 RV |
634 | pki_ih3->pm = 0x7; |
635 | pki_ih3->sl = 8; | |
636 | ||
637 | if (sc->datasize) | |
638 | ih3->dlengsz = sc->datasize; | |
639 | ||
640 | irh = (struct octeon_instr_irh *)&sc->cmd.cmd3.irh; | |
641 | irh->opcode = opcode; | |
642 | irh->subcode = subcode; | |
643 | ||
644 | /* opcode/subcode specific parameters (ossp) */ | |
645 | irh->ossp = irh_ossp; | |
646 | sc->cmd.cmd3.ossp[0] = ossp0; | |
647 | sc->cmd.cmd3.ossp[1] = ossp1; | |
648 | ||
649 | if (sc->rdatasize) { | |
650 | rdp = (struct octeon_instr_rdp *)&sc->cmd.cmd3.rdp; | |
651 | rdp->pcie_port = oct->pcie_port; | |
652 | rdp->rlen = sc->rdatasize; | |
653 | ||
654 | irh->rflag = 1; | |
655 | /*PKI IH3*/ | |
656 | /* pki_ih3 irh+ossp[0]+ossp[1]+rdp+rptr = 48 bytes */ | |
657 | ih3->fsz = LIO_SOFTCMDRESP_IH3; | |
658 | } else { | |
659 | irh->rflag = 0; | |
660 | /*PKI IH3*/ | |
661 | /* pki_h3 + irh + ossp[0] + ossp[1] = 32 bytes */ | |
662 | ih3->fsz = LIO_PCICMD_O3; | |
663 | } | |
f21fb3ed | 664 | |
f21fb3ed | 665 | } else { |
5b823514 RV |
666 | ih2 = (struct octeon_instr_ih2 *)&sc->cmd.cmd2.ih2; |
667 | ih2->tagtype = ATOMIC_TAG; | |
668 | ih2->tag = LIO_CONTROL; | |
669 | ih2->raw = 1; | |
670 | ih2->grp = CFG_GET_CTRL_Q_GRP(oct_cfg); | |
671 | ||
672 | if (sc->datasize) { | |
673 | ih2->dlengsz = sc->datasize; | |
674 | ih2->rs = 1; | |
675 | } | |
676 | ||
677 | irh = (struct octeon_instr_irh *)&sc->cmd.cmd2.irh; | |
678 | irh->opcode = opcode; | |
679 | irh->subcode = subcode; | |
680 | ||
681 | /* opcode/subcode specific parameters (ossp) */ | |
682 | irh->ossp = irh_ossp; | |
683 | sc->cmd.cmd2.ossp[0] = ossp0; | |
684 | sc->cmd.cmd2.ossp[1] = ossp1; | |
685 | ||
686 | if (sc->rdatasize) { | |
687 | rdp = (struct octeon_instr_rdp *)&sc->cmd.cmd2.rdp; | |
688 | rdp->pcie_port = oct->pcie_port; | |
689 | rdp->rlen = sc->rdatasize; | |
690 | ||
691 | irh->rflag = 1; | |
692 | /* irh+ossp[0]+ossp[1]+rdp+rptr = 40 bytes */ | |
693 | ih2->fsz = LIO_SOFTCMDRESP_IH2; | |
694 | } else { | |
695 | irh->rflag = 0; | |
696 | /* irh + ossp[0] + ossp[1] = 24 bytes */ | |
697 | ih2->fsz = LIO_PCICMD_O2; | |
698 | } | |
f21fb3ed | 699 | } |
f21fb3ed RV |
700 | } |
701 | ||
702 | int octeon_send_soft_command(struct octeon_device *oct, | |
703 | struct octeon_soft_command *sc) | |
704 | { | |
6a885b60 | 705 | struct octeon_instr_ih2 *ih2; |
5b823514 | 706 | struct octeon_instr_ih3 *ih3; |
f21fb3ed | 707 | struct octeon_instr_irh *irh; |
6a885b60 | 708 | u32 len; |
f21fb3ed | 709 | |
9981328a | 710 | if (OCTEON_CN23XX_PF(oct) || OCTEON_CN23XX_VF(oct)) { |
5b823514 RV |
711 | ih3 = (struct octeon_instr_ih3 *)&sc->cmd.cmd3.ih3; |
712 | if (ih3->dlengsz) { | |
713 | WARN_ON(!sc->dmadptr); | |
714 | sc->cmd.cmd3.dptr = sc->dmadptr; | |
715 | } | |
716 | irh = (struct octeon_instr_irh *)&sc->cmd.cmd3.irh; | |
717 | if (irh->rflag) { | |
718 | WARN_ON(!sc->dmarptr); | |
719 | WARN_ON(!sc->status_word); | |
720 | *sc->status_word = COMPLETION_WORD_INIT; | |
721 | sc->cmd.cmd3.rptr = sc->dmarptr; | |
722 | } | |
723 | len = (u32)ih3->dlengsz; | |
724 | } else { | |
725 | ih2 = (struct octeon_instr_ih2 *)&sc->cmd.cmd2.ih2; | |
726 | if (ih2->dlengsz) { | |
727 | WARN_ON(!sc->dmadptr); | |
728 | sc->cmd.cmd2.dptr = sc->dmadptr; | |
729 | } | |
730 | irh = (struct octeon_instr_irh *)&sc->cmd.cmd2.irh; | |
731 | if (irh->rflag) { | |
732 | WARN_ON(!sc->dmarptr); | |
733 | WARN_ON(!sc->status_word); | |
734 | *sc->status_word = COMPLETION_WORD_INIT; | |
735 | sc->cmd.cmd2.rptr = sc->dmarptr; | |
736 | } | |
737 | len = (u32)ih2->dlengsz; | |
f21fb3ed RV |
738 | } |
739 | ||
740 | if (sc->wait_time) | |
741 | sc->timeout = jiffies + sc->wait_time; | |
742 | ||
6a885b60 RV |
743 | return (octeon_send_command(oct, sc->iq_no, 1, &sc->cmd, sc, |
744 | len, REQTYPE_SOFT_COMMAND)); | |
f21fb3ed RV |
745 | } |
746 | ||
747 | int octeon_setup_sc_buffer_pool(struct octeon_device *oct) | |
748 | { | |
749 | int i; | |
750 | u64 dma_addr; | |
751 | struct octeon_soft_command *sc; | |
752 | ||
753 | INIT_LIST_HEAD(&oct->sc_buf_pool.head); | |
754 | spin_lock_init(&oct->sc_buf_pool.lock); | |
755 | atomic_set(&oct->sc_buf_pool.alloc_buf_count, 0); | |
756 | ||
757 | for (i = 0; i < MAX_SOFT_COMMAND_BUFFERS; i++) { | |
758 | sc = (struct octeon_soft_command *) | |
759 | lio_dma_alloc(oct, | |
760 | SOFT_COMMAND_BUFFER_SIZE, | |
761 | (dma_addr_t *)&dma_addr); | |
515e752d RV |
762 | if (!sc) { |
763 | octeon_free_sc_buffer_pool(oct); | |
f21fb3ed | 764 | return 1; |
515e752d | 765 | } |
f21fb3ed RV |
766 | |
767 | sc->dma_addr = dma_addr; | |
768 | sc->size = SOFT_COMMAND_BUFFER_SIZE; | |
769 | ||
770 | list_add_tail(&sc->node, &oct->sc_buf_pool.head); | |
771 | } | |
772 | ||
773 | return 0; | |
774 | } | |
775 | ||
776 | int octeon_free_sc_buffer_pool(struct octeon_device *oct) | |
777 | { | |
778 | struct list_head *tmp, *tmp2; | |
779 | struct octeon_soft_command *sc; | |
780 | ||
14866ccd | 781 | spin_lock_bh(&oct->sc_buf_pool.lock); |
f21fb3ed RV |
782 | |
783 | list_for_each_safe(tmp, tmp2, &oct->sc_buf_pool.head) { | |
784 | list_del(tmp); | |
785 | ||
786 | sc = (struct octeon_soft_command *)tmp; | |
787 | ||
788 | lio_dma_free(oct, sc->size, sc, sc->dma_addr); | |
789 | } | |
790 | ||
791 | INIT_LIST_HEAD(&oct->sc_buf_pool.head); | |
792 | ||
14866ccd | 793 | spin_unlock_bh(&oct->sc_buf_pool.lock); |
f21fb3ed RV |
794 | |
795 | return 0; | |
796 | } | |
797 | ||
798 | struct octeon_soft_command *octeon_alloc_soft_command(struct octeon_device *oct, | |
799 | u32 datasize, | |
800 | u32 rdatasize, | |
801 | u32 ctxsize) | |
802 | { | |
803 | u64 dma_addr; | |
804 | u32 size; | |
805 | u32 offset = sizeof(struct octeon_soft_command); | |
806 | struct octeon_soft_command *sc = NULL; | |
807 | struct list_head *tmp; | |
808 | ||
a7d5a3dc | 809 | WARN_ON((offset + datasize + rdatasize + ctxsize) > |
f21fb3ed RV |
810 | SOFT_COMMAND_BUFFER_SIZE); |
811 | ||
14866ccd | 812 | spin_lock_bh(&oct->sc_buf_pool.lock); |
f21fb3ed RV |
813 | |
814 | if (list_empty(&oct->sc_buf_pool.head)) { | |
14866ccd | 815 | spin_unlock_bh(&oct->sc_buf_pool.lock); |
f21fb3ed RV |
816 | return NULL; |
817 | } | |
818 | ||
819 | list_for_each(tmp, &oct->sc_buf_pool.head) | |
820 | break; | |
821 | ||
822 | list_del(tmp); | |
823 | ||
824 | atomic_inc(&oct->sc_buf_pool.alloc_buf_count); | |
825 | ||
14866ccd | 826 | spin_unlock_bh(&oct->sc_buf_pool.lock); |
f21fb3ed RV |
827 | |
828 | sc = (struct octeon_soft_command *)tmp; | |
829 | ||
830 | dma_addr = sc->dma_addr; | |
831 | size = sc->size; | |
832 | ||
833 | memset(sc, 0, sc->size); | |
834 | ||
835 | sc->dma_addr = dma_addr; | |
836 | sc->size = size; | |
837 | ||
838 | if (ctxsize) { | |
839 | sc->ctxptr = (u8 *)sc + offset; | |
840 | sc->ctxsize = ctxsize; | |
841 | } | |
842 | ||
843 | /* Start data at 128 byte boundary */ | |
844 | offset = (offset + ctxsize + 127) & 0xffffff80; | |
845 | ||
846 | if (datasize) { | |
847 | sc->virtdptr = (u8 *)sc + offset; | |
848 | sc->dmadptr = dma_addr + offset; | |
849 | sc->datasize = datasize; | |
850 | } | |
851 | ||
852 | /* Start rdata at 128 byte boundary */ | |
853 | offset = (offset + datasize + 127) & 0xffffff80; | |
854 | ||
855 | if (rdatasize) { | |
a7d5a3dc | 856 | WARN_ON(rdatasize < 16); |
f21fb3ed RV |
857 | sc->virtrptr = (u8 *)sc + offset; |
858 | sc->dmarptr = dma_addr + offset; | |
859 | sc->rdatasize = rdatasize; | |
860 | sc->status_word = (u64 *)((u8 *)(sc->virtrptr) + rdatasize - 8); | |
861 | } | |
862 | ||
863 | return sc; | |
864 | } | |
865 | ||
866 | void octeon_free_soft_command(struct octeon_device *oct, | |
867 | struct octeon_soft_command *sc) | |
868 | { | |
14866ccd | 869 | spin_lock_bh(&oct->sc_buf_pool.lock); |
f21fb3ed RV |
870 | |
871 | list_add_tail(&sc->node, &oct->sc_buf_pool.head); | |
872 | ||
873 | atomic_dec(&oct->sc_buf_pool.alloc_buf_count); | |
874 | ||
14866ccd | 875 | spin_unlock_bh(&oct->sc_buf_pool.lock); |
f21fb3ed | 876 | } |