Commit | Line | Data |
---|---|---|
f21fb3ed | 1 | /********************************************************************** |
50579d3d RV |
2 | * Author: Cavium, Inc. |
3 | * | |
4 | * Contact: support@cavium.com | |
5 | * Please include "LiquidIO" in the subject. | |
6 | * | |
7 | * Copyright (c) 2003-2016 Cavium, Inc. | |
8 | * | |
9 | * This file is free software; you can redistribute it and/or modify | |
10 | * it under the terms of the GNU General Public License, Version 2, as | |
11 | * published by the Free Software Foundation. | |
12 | * | |
13 | * This file is distributed in the hope that it will be useful, but | |
14 | * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty | |
15 | * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or | |
16 | * NONINFRINGEMENT. See the GNU General Public License for more details. | |
17 | ***********************************************************************/ | |
f21fb3ed | 18 | #include <linux/pci.h> |
f21fb3ed | 19 | #include <linux/netdevice.h> |
5b173cf9 | 20 | #include <linux/vmalloc.h> |
f21fb3ed RV |
21 | #include "liquidio_common.h" |
22 | #include "octeon_droq.h" | |
23 | #include "octeon_iq.h" | |
24 | #include "response_manager.h" | |
25 | #include "octeon_device.h" | |
f21fb3ed RV |
26 | #include "octeon_main.h" |
27 | #include "octeon_network.h" | |
28 | #include "cn66xx_regs.h" | |
29 | #include "cn66xx_device.h" | |
5b823514 | 30 | #include "cn23xx_pf_device.h" |
9217c3cf | 31 | #include "cn23xx_vf_device.h" |
f21fb3ed | 32 | |
f21fb3ed RV |
33 | struct niclist { |
34 | struct list_head list; | |
35 | void *ptr; | |
36 | }; | |
37 | ||
38 | struct __dispatch { | |
39 | struct list_head list; | |
40 | struct octeon_recv_info *rinfo; | |
41 | octeon_dispatch_fn_t disp_fn; | |
42 | }; | |
43 | ||
44 | /** Get the argument that the user set when registering dispatch | |
45 | * function for a given opcode/subcode. | |
46 | * @param octeon_dev - the octeon device pointer. | |
47 | * @param opcode - the opcode for which the dispatch argument | |
48 | * is to be checked. | |
49 | * @param subcode - the subcode for which the dispatch argument | |
50 | * is to be checked. | |
51 | * @return Success: void * (argument to the dispatch function) | |
52 | * @return Failure: NULL | |
53 | * | |
54 | */ | |
bf534588 VMG |
55 | void *octeon_get_dispatch_arg(struct octeon_device *octeon_dev, |
56 | u16 opcode, u16 subcode) | |
f21fb3ed RV |
57 | { |
58 | int idx; | |
59 | struct list_head *dispatch; | |
60 | void *fn_arg = NULL; | |
61 | u16 combined_opcode = OPCODE_SUBCODE(opcode, subcode); | |
62 | ||
63 | idx = combined_opcode & OCTEON_OPCODE_MASK; | |
64 | ||
65 | spin_lock_bh(&octeon_dev->dispatch.lock); | |
66 | ||
67 | if (octeon_dev->dispatch.count == 0) { | |
68 | spin_unlock_bh(&octeon_dev->dispatch.lock); | |
69 | return NULL; | |
70 | } | |
71 | ||
72 | if (octeon_dev->dispatch.dlist[idx].opcode == combined_opcode) { | |
73 | fn_arg = octeon_dev->dispatch.dlist[idx].arg; | |
74 | } else { | |
75 | list_for_each(dispatch, | |
76 | &octeon_dev->dispatch.dlist[idx].list) { | |
77 | if (((struct octeon_dispatch *)dispatch)->opcode == | |
78 | combined_opcode) { | |
79 | fn_arg = ((struct octeon_dispatch *) | |
80 | dispatch)->arg; | |
81 | break; | |
82 | } | |
83 | } | |
84 | } | |
85 | ||
86 | spin_unlock_bh(&octeon_dev->dispatch.lock); | |
87 | return fn_arg; | |
88 | } | |
89 | ||
cd8b1eb4 | 90 | /** Check for packets on Droq. This function should be called with lock held. |
a2c64b67 RV |
91 | * @param droq - Droq on which count is checked. |
92 | * @return Returns packet count. | |
93 | */ | |
a7d5a3dc | 94 | u32 octeon_droq_check_hw_for_pkts(struct octeon_droq *droq) |
f21fb3ed RV |
95 | { |
96 | u32 pkt_count = 0; | |
cd8b1eb4 | 97 | u32 last_count; |
f21fb3ed RV |
98 | |
99 | pkt_count = readl(droq->pkts_sent_reg); | |
f21fb3ed | 100 | |
cd8b1eb4 RV |
101 | last_count = pkt_count - droq->pkt_count; |
102 | droq->pkt_count = pkt_count; | |
103 | ||
104 | /* we shall write to cnts at napi irq enable or end of droq tasklet */ | |
105 | if (last_count) | |
106 | atomic_add(last_count, &droq->pkts_pending); | |
107 | ||
108 | return last_count; | |
f21fb3ed RV |
109 | } |
110 | ||
111 | static void octeon_droq_compute_max_packet_bufs(struct octeon_droq *droq) | |
112 | { | |
113 | u32 count = 0; | |
114 | ||
115 | /* max_empty_descs is the max. no. of descs that can have no buffers. | |
116 | * If the empty desc count goes beyond this value, we cannot safely | |
117 | * read in a 64K packet sent by Octeon | |
118 | * (64K is max pkt size from Octeon) | |
119 | */ | |
120 | droq->max_empty_descs = 0; | |
121 | ||
122 | do { | |
123 | droq->max_empty_descs++; | |
124 | count += droq->buffer_size; | |
125 | } while (count < (64 * 1024)); | |
126 | ||
127 | droq->max_empty_descs = droq->max_count - droq->max_empty_descs; | |
128 | } | |
129 | ||
130 | static void octeon_droq_reset_indices(struct octeon_droq *droq) | |
131 | { | |
132 | droq->read_idx = 0; | |
133 | droq->write_idx = 0; | |
134 | droq->refill_idx = 0; | |
135 | droq->refill_count = 0; | |
136 | atomic_set(&droq->pkts_pending, 0); | |
137 | } | |
138 | ||
139 | static void | |
140 | octeon_droq_destroy_ring_buffers(struct octeon_device *oct, | |
141 | struct octeon_droq *droq) | |
142 | { | |
143 | u32 i; | |
cabeb13b | 144 | struct octeon_skb_page_info *pg_info; |
f21fb3ed RV |
145 | |
146 | for (i = 0; i < droq->max_count; i++) { | |
cabeb13b | 147 | pg_info = &droq->recv_buf_list[i].pg_info; |
689062a1 RF |
148 | if (!pg_info) |
149 | continue; | |
cabeb13b RV |
150 | |
151 | if (pg_info->dma) | |
152 | lio_unmap_ring(oct->pci_dev, | |
153 | (u64)pg_info->dma); | |
154 | pg_info->dma = 0; | |
155 | ||
156 | if (pg_info->page) | |
157 | recv_buffer_destroy(droq->recv_buf_list[i].buffer, | |
158 | pg_info); | |
159 | ||
cabeb13b | 160 | droq->recv_buf_list[i].buffer = NULL; |
f21fb3ed RV |
161 | } |
162 | ||
163 | octeon_droq_reset_indices(droq); | |
164 | } | |
165 | ||
166 | static int | |
167 | octeon_droq_setup_ring_buffers(struct octeon_device *oct, | |
168 | struct octeon_droq *droq) | |
169 | { | |
170 | u32 i; | |
171 | void *buf; | |
172 | struct octeon_droq_desc *desc_ring = droq->desc_ring; | |
173 | ||
174 | for (i = 0; i < droq->max_count; i++) { | |
cabeb13b | 175 | buf = recv_buffer_alloc(oct, &droq->recv_buf_list[i].pg_info); |
f21fb3ed RV |
176 | |
177 | if (!buf) { | |
178 | dev_err(&oct->pci_dev->dev, "%s buffer alloc failed\n", | |
179 | __func__); | |
cabeb13b | 180 | droq->stats.rx_alloc_failure++; |
f21fb3ed RV |
181 | return -ENOMEM; |
182 | } | |
183 | ||
184 | droq->recv_buf_list[i].buffer = buf; | |
185 | droq->recv_buf_list[i].data = get_rbd(buf); | |
c4ee5d81 | 186 | desc_ring[i].info_ptr = 0; |
f21fb3ed | 187 | desc_ring[i].buffer_ptr = |
cabeb13b | 188 | lio_map_ring(droq->recv_buf_list[i].buffer); |
f21fb3ed RV |
189 | } |
190 | ||
191 | octeon_droq_reset_indices(droq); | |
192 | ||
193 | octeon_droq_compute_max_packet_bufs(droq); | |
194 | ||
195 | return 0; | |
196 | } | |
197 | ||
198 | int octeon_delete_droq(struct octeon_device *oct, u32 q_no) | |
199 | { | |
200 | struct octeon_droq *droq = oct->droq[q_no]; | |
201 | ||
202 | dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no); | |
203 | ||
204 | octeon_droq_destroy_ring_buffers(oct, droq); | |
9686f310 | 205 | vfree(droq->recv_buf_list); |
f21fb3ed | 206 | |
f21fb3ed RV |
207 | if (droq->desc_ring) |
208 | lio_dma_free(oct, (droq->max_count * OCT_DROQ_DESC_SIZE), | |
209 | droq->desc_ring, droq->desc_ring_dma); | |
210 | ||
211 | memset(droq, 0, OCT_DROQ_SIZE); | |
c1550fde IB |
212 | oct->io_qmask.oq &= ~(1ULL << q_no); |
213 | vfree(oct->droq[q_no]); | |
214 | oct->droq[q_no] = NULL; | |
215 | oct->num_oqs--; | |
f21fb3ed RV |
216 | |
217 | return 0; | |
218 | } | |
219 | ||
220 | int octeon_init_droq(struct octeon_device *oct, | |
221 | u32 q_no, | |
222 | u32 num_descs, | |
223 | u32 desc_size, | |
224 | void *app_ctx) | |
225 | { | |
226 | struct octeon_droq *droq; | |
227 | u32 desc_ring_size = 0, c_num_descs = 0, c_buf_size = 0; | |
228 | u32 c_pkts_per_intr = 0, c_refill_threshold = 0; | |
b3ca9af0 | 229 | int numa_node = dev_to_node(&oct->pci_dev->dev); |
f21fb3ed RV |
230 | |
231 | dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no); | |
232 | ||
233 | droq = oct->droq[q_no]; | |
234 | memset(droq, 0, OCT_DROQ_SIZE); | |
235 | ||
236 | droq->oct_dev = oct; | |
237 | droq->q_no = q_no; | |
238 | if (app_ctx) | |
239 | droq->app_ctx = app_ctx; | |
240 | else | |
241 | droq->app_ctx = (void *)(size_t)q_no; | |
242 | ||
243 | c_num_descs = num_descs; | |
244 | c_buf_size = desc_size; | |
245 | if (OCTEON_CN6XXX(oct)) { | |
97a25326 | 246 | struct octeon_config *conf6x = CHIP_CONF(oct, cn6xxx); |
f21fb3ed RV |
247 | |
248 | c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf6x); | |
96ae48b7 RV |
249 | c_refill_threshold = |
250 | (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf6x); | |
5b823514 | 251 | } else if (OCTEON_CN23XX_PF(oct)) { |
97a25326 | 252 | struct octeon_config *conf23 = CHIP_CONF(oct, cn23xx_pf); |
5b823514 | 253 | |
9217c3cf RV |
254 | c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf23); |
255 | c_refill_threshold = (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf23); | |
256 | } else if (OCTEON_CN23XX_VF(oct)) { | |
257 | struct octeon_config *conf23 = CHIP_CONF(oct, cn23xx_vf); | |
258 | ||
5b823514 RV |
259 | c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf23); |
260 | c_refill_threshold = (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf23); | |
96ae48b7 RV |
261 | } else { |
262 | return 1; | |
f21fb3ed RV |
263 | } |
264 | ||
265 | droq->max_count = c_num_descs; | |
266 | droq->buffer_size = c_buf_size; | |
267 | ||
268 | desc_ring_size = droq->max_count * OCT_DROQ_DESC_SIZE; | |
269 | droq->desc_ring = lio_dma_alloc(oct, desc_ring_size, | |
270 | (dma_addr_t *)&droq->desc_ring_dma); | |
271 | ||
272 | if (!droq->desc_ring) { | |
273 | dev_err(&oct->pci_dev->dev, | |
274 | "Output queue %d ring alloc failed\n", q_no); | |
275 | return 1; | |
276 | } | |
277 | ||
278 | dev_dbg(&oct->pci_dev->dev, "droq[%d]: desc_ring: virt: 0x%p, dma: %lx\n", | |
279 | q_no, droq->desc_ring, droq->desc_ring_dma); | |
280 | dev_dbg(&oct->pci_dev->dev, "droq[%d]: num_desc: %d\n", q_no, | |
281 | droq->max_count); | |
282 | ||
f21fb3ed | 283 | droq->recv_buf_list = (struct octeon_recv_buffer *) |
00587f2f | 284 | vzalloc_node(droq->max_count * |
96ae48b7 RV |
285 | OCT_DROQ_RECVBUF_SIZE, |
286 | numa_node); | |
287 | if (!droq->recv_buf_list) | |
288 | droq->recv_buf_list = (struct octeon_recv_buffer *) | |
fad953ce KC |
289 | vzalloc(array_size(droq->max_count, |
290 | OCT_DROQ_RECVBUF_SIZE)); | |
f21fb3ed RV |
291 | if (!droq->recv_buf_list) { |
292 | dev_err(&oct->pci_dev->dev, "Output queue recv buf list alloc failed\n"); | |
293 | goto init_droq_fail; | |
294 | } | |
295 | ||
296 | if (octeon_droq_setup_ring_buffers(oct, droq)) | |
297 | goto init_droq_fail; | |
298 | ||
299 | droq->pkts_per_intr = c_pkts_per_intr; | |
300 | droq->refill_threshold = c_refill_threshold; | |
301 | ||
302 | dev_dbg(&oct->pci_dev->dev, "DROQ INIT: max_empty_descs: %d\n", | |
303 | droq->max_empty_descs); | |
304 | ||
305 | spin_lock_init(&droq->lock); | |
306 | ||
307 | INIT_LIST_HEAD(&droq->dispatch_list); | |
308 | ||
309 | /* For 56xx Pass1, this function won't be called, so no checks. */ | |
310 | oct->fn_list.setup_oq_regs(oct, q_no); | |
311 | ||
763185a3 | 312 | oct->io_qmask.oq |= BIT_ULL(q_no); |
f21fb3ed RV |
313 | |
314 | return 0; | |
315 | ||
316 | init_droq_fail: | |
317 | octeon_delete_droq(oct, q_no); | |
318 | return 1; | |
319 | } | |
320 | ||
321 | /* octeon_create_recv_info | |
322 | * Parameters: | |
323 | * octeon_dev - pointer to the octeon device structure | |
324 | * droq - droq in which the packet arrived. | |
325 | * buf_cnt - no. of buffers used by the packet. | |
326 | * idx - index in the descriptor for the first buffer in the packet. | |
327 | * Description: | |
328 | * Allocates a recv_info_t and copies the buffer addresses for packet data | |
329 | * into the recv_pkt space which starts at an 8B offset from recv_info_t. | |
330 | * Flags the descriptors for refill later. If available descriptors go | |
331 | * below the threshold to receive a 64K pkt, new buffers are first allocated | |
332 | * before the recv_pkt_t is created. | |
333 | * This routine will be called in interrupt context. | |
334 | * Returns: | |
335 | * Success: Pointer to recv_info_t | |
336 | * Failure: NULL. | |
337 | * Locks: | |
338 | * The droq->lock is held when this routine is called. | |
339 | */ | |
340 | static inline struct octeon_recv_info *octeon_create_recv_info( | |
341 | struct octeon_device *octeon_dev, | |
342 | struct octeon_droq *droq, | |
343 | u32 buf_cnt, | |
344 | u32 idx) | |
345 | { | |
346 | struct octeon_droq_info *info; | |
347 | struct octeon_recv_pkt *recv_pkt; | |
348 | struct octeon_recv_info *recv_info; | |
349 | u32 i, bytes_left; | |
cabeb13b | 350 | struct octeon_skb_page_info *pg_info; |
f21fb3ed | 351 | |
c4ee5d81 | 352 | info = (struct octeon_droq_info *)droq->recv_buf_list[idx].data; |
f21fb3ed RV |
353 | |
354 | recv_info = octeon_alloc_recv_info(sizeof(struct __dispatch)); | |
355 | if (!recv_info) | |
356 | return NULL; | |
357 | ||
358 | recv_pkt = recv_info->recv_pkt; | |
359 | recv_pkt->rh = info->rh; | |
360 | recv_pkt->length = (u32)info->length; | |
361 | recv_pkt->buffer_count = (u16)buf_cnt; | |
362 | recv_pkt->octeon_id = (u16)octeon_dev->octeon_id; | |
363 | ||
364 | i = 0; | |
365 | bytes_left = (u32)info->length; | |
366 | ||
367 | while (buf_cnt) { | |
cabeb13b RV |
368 | { |
369 | pg_info = &droq->recv_buf_list[idx].pg_info; | |
370 | ||
371 | lio_unmap_ring(octeon_dev->pci_dev, | |
372 | (u64)pg_info->dma); | |
373 | pg_info->page = NULL; | |
374 | pg_info->dma = 0; | |
375 | } | |
f21fb3ed RV |
376 | |
377 | recv_pkt->buffer_size[i] = | |
378 | (bytes_left >= | |
379 | droq->buffer_size) ? droq->buffer_size : bytes_left; | |
380 | ||
381 | recv_pkt->buffer_ptr[i] = droq->recv_buf_list[idx].buffer; | |
382 | droq->recv_buf_list[idx].buffer = NULL; | |
383 | ||
97a25326 | 384 | idx = incr_index(idx, 1, droq->max_count); |
f21fb3ed RV |
385 | bytes_left -= droq->buffer_size; |
386 | i++; | |
387 | buf_cnt--; | |
388 | } | |
389 | ||
390 | return recv_info; | |
391 | } | |
392 | ||
393 | /* If we were not able to refill all buffers, try to move around | |
394 | * the buffers that were not dispatched. | |
395 | */ | |
396 | static inline u32 | |
397 | octeon_droq_refill_pullup_descs(struct octeon_droq *droq, | |
398 | struct octeon_droq_desc *desc_ring) | |
399 | { | |
400 | u32 desc_refilled = 0; | |
401 | ||
402 | u32 refill_index = droq->refill_idx; | |
403 | ||
404 | while (refill_index != droq->read_idx) { | |
405 | if (droq->recv_buf_list[refill_index].buffer) { | |
406 | droq->recv_buf_list[droq->refill_idx].buffer = | |
407 | droq->recv_buf_list[refill_index].buffer; | |
408 | droq->recv_buf_list[droq->refill_idx].data = | |
409 | droq->recv_buf_list[refill_index].data; | |
410 | desc_ring[droq->refill_idx].buffer_ptr = | |
411 | desc_ring[refill_index].buffer_ptr; | |
412 | droq->recv_buf_list[refill_index].buffer = NULL; | |
413 | desc_ring[refill_index].buffer_ptr = 0; | |
414 | do { | |
97a25326 RV |
415 | droq->refill_idx = incr_index(droq->refill_idx, |
416 | 1, | |
417 | droq->max_count); | |
f21fb3ed RV |
418 | desc_refilled++; |
419 | droq->refill_count--; | |
9ae122c6 | 420 | } while (droq->recv_buf_list[droq->refill_idx].buffer); |
f21fb3ed | 421 | } |
97a25326 | 422 | refill_index = incr_index(refill_index, 1, droq->max_count); |
f21fb3ed RV |
423 | } /* while */ |
424 | return desc_refilled; | |
425 | } | |
426 | ||
427 | /* octeon_droq_refill | |
428 | * Parameters: | |
429 | * droq - droq in which descriptors require new buffers. | |
430 | * Description: | |
431 | * Called during normal DROQ processing in interrupt mode or by the poll | |
432 | * thread to refill the descriptors from which buffers were dispatched | |
433 | * to upper layers. Attempts to allocate new buffers. If that fails, moves | |
434 | * up buffers (that were not dispatched) to form a contiguous ring. | |
435 | * Returns: | |
436 | * No of descriptors refilled. | |
437 | * Locks: | |
438 | * This routine is called with droq->lock held. | |
439 | */ | |
440 | static u32 | |
441 | octeon_droq_refill(struct octeon_device *octeon_dev, struct octeon_droq *droq) | |
442 | { | |
443 | struct octeon_droq_desc *desc_ring; | |
444 | void *buf = NULL; | |
445 | u8 *data; | |
446 | u32 desc_refilled = 0; | |
cabeb13b | 447 | struct octeon_skb_page_info *pg_info; |
f21fb3ed RV |
448 | |
449 | desc_ring = droq->desc_ring; | |
450 | ||
451 | while (droq->refill_count && (desc_refilled < droq->max_count)) { | |
452 | /* If a valid buffer exists (happens if there is no dispatch), | |
453 | * reuse | |
454 | * the buffer, else allocate. | |
455 | */ | |
456 | if (!droq->recv_buf_list[droq->refill_idx].buffer) { | |
cabeb13b RV |
457 | pg_info = |
458 | &droq->recv_buf_list[droq->refill_idx].pg_info; | |
459 | /* Either recycle the existing pages or go for | |
460 | * new page alloc | |
461 | */ | |
462 | if (pg_info->page) | |
463 | buf = recv_buffer_reuse(octeon_dev, pg_info); | |
464 | else | |
465 | buf = recv_buffer_alloc(octeon_dev, pg_info); | |
f21fb3ed RV |
466 | /* If a buffer could not be allocated, no point in |
467 | * continuing | |
468 | */ | |
cabeb13b RV |
469 | if (!buf) { |
470 | droq->stats.rx_alloc_failure++; | |
f21fb3ed | 471 | break; |
cabeb13b | 472 | } |
f21fb3ed RV |
473 | droq->recv_buf_list[droq->refill_idx].buffer = |
474 | buf; | |
475 | data = get_rbd(buf); | |
476 | } else { | |
477 | data = get_rbd(droq->recv_buf_list | |
478 | [droq->refill_idx].buffer); | |
479 | } | |
480 | ||
481 | droq->recv_buf_list[droq->refill_idx].data = data; | |
482 | ||
483 | desc_ring[droq->refill_idx].buffer_ptr = | |
9ae122c6 SB |
484 | lio_map_ring(droq->recv_buf_list[ |
485 | droq->refill_idx].buffer); | |
f21fb3ed | 486 | |
97a25326 RV |
487 | droq->refill_idx = incr_index(droq->refill_idx, 1, |
488 | droq->max_count); | |
f21fb3ed RV |
489 | desc_refilled++; |
490 | droq->refill_count--; | |
491 | } | |
492 | ||
493 | if (droq->refill_count) | |
494 | desc_refilled += | |
495 | octeon_droq_refill_pullup_descs(droq, desc_ring); | |
496 | ||
497 | /* if droq->refill_count | |
498 | * The refill count would not change in pass two. We only moved buffers | |
499 | * to close the gap in the ring, but we would still have the same no. of | |
500 | * buffers to refill. | |
501 | */ | |
502 | return desc_refilled; | |
503 | } | |
504 | ||
031d4f12 SB |
505 | /** check if we can allocate packets to get out of oom. |
506 | * @param droq - Droq being checked. | |
507 | * @return does not return anything | |
508 | */ | |
509 | void octeon_droq_check_oom(struct octeon_droq *droq) | |
510 | { | |
511 | int desc_refilled; | |
512 | struct octeon_device *oct = droq->oct_dev; | |
513 | ||
514 | if (readl(droq->pkts_credit_reg) <= CN23XX_SLI_DEF_BP) { | |
515 | spin_lock_bh(&droq->lock); | |
516 | desc_refilled = octeon_droq_refill(oct, droq); | |
517 | if (desc_refilled) { | |
518 | /* Flush the droq descriptor data to memory to be sure | |
519 | * that when we update the credits the data in memory | |
520 | * is accurate. | |
521 | */ | |
522 | wmb(); | |
523 | writel(desc_refilled, droq->pkts_credit_reg); | |
524 | /* make sure mmio write completes */ | |
525 | mmiowb(); | |
526 | } | |
527 | spin_unlock_bh(&droq->lock); | |
528 | } | |
529 | } | |
530 | ||
f21fb3ed RV |
531 | static inline u32 |
532 | octeon_droq_get_bufcount(u32 buf_size, u32 total_len) | |
533 | { | |
c4ee5d81 | 534 | return ((total_len + buf_size - 1) / buf_size); |
f21fb3ed RV |
535 | } |
536 | ||
537 | static int | |
538 | octeon_droq_dispatch_pkt(struct octeon_device *oct, | |
539 | struct octeon_droq *droq, | |
540 | union octeon_rh *rh, | |
541 | struct octeon_droq_info *info) | |
542 | { | |
543 | u32 cnt; | |
544 | octeon_dispatch_fn_t disp_fn; | |
545 | struct octeon_recv_info *rinfo; | |
546 | ||
547 | cnt = octeon_droq_get_bufcount(droq->buffer_size, (u32)info->length); | |
548 | ||
549 | disp_fn = octeon_get_dispatch(oct, (u16)rh->r.opcode, | |
550 | (u16)rh->r.subcode); | |
551 | if (disp_fn) { | |
552 | rinfo = octeon_create_recv_info(oct, droq, cnt, droq->read_idx); | |
553 | if (rinfo) { | |
554 | struct __dispatch *rdisp = rinfo->rsvd; | |
555 | ||
556 | rdisp->rinfo = rinfo; | |
557 | rdisp->disp_fn = disp_fn; | |
558 | rinfo->recv_pkt->rh = *rh; | |
559 | list_add_tail(&rdisp->list, | |
560 | &droq->dispatch_list); | |
561 | } else { | |
562 | droq->stats.dropped_nomem++; | |
563 | } | |
564 | } else { | |
a2c64b67 RV |
565 | dev_err(&oct->pci_dev->dev, "DROQ: No dispatch function (opcode %u/%u)\n", |
566 | (unsigned int)rh->r.opcode, | |
567 | (unsigned int)rh->r.subcode); | |
f21fb3ed | 568 | droq->stats.dropped_nodispatch++; |
9ded1a51 | 569 | } |
f21fb3ed RV |
570 | |
571 | return cnt; | |
572 | } | |
573 | ||
574 | static inline void octeon_droq_drop_packets(struct octeon_device *oct, | |
575 | struct octeon_droq *droq, | |
576 | u32 cnt) | |
577 | { | |
578 | u32 i = 0, buf_cnt; | |
579 | struct octeon_droq_info *info; | |
580 | ||
581 | for (i = 0; i < cnt; i++) { | |
c4ee5d81 PK |
582 | info = (struct octeon_droq_info *) |
583 | droq->recv_buf_list[droq->read_idx].data; | |
f21fb3ed RV |
584 | octeon_swap_8B_data((u64 *)info, 2); |
585 | ||
586 | if (info->length) { | |
c4ee5d81 | 587 | info->length += OCTNET_FRM_LENGTH_SIZE; |
f21fb3ed RV |
588 | droq->stats.bytes_received += info->length; |
589 | buf_cnt = octeon_droq_get_bufcount(droq->buffer_size, | |
590 | (u32)info->length); | |
591 | } else { | |
592 | dev_err(&oct->pci_dev->dev, "DROQ: In drop: pkt with len 0\n"); | |
593 | buf_cnt = 1; | |
594 | } | |
595 | ||
97a25326 RV |
596 | droq->read_idx = incr_index(droq->read_idx, buf_cnt, |
597 | droq->max_count); | |
f21fb3ed RV |
598 | droq->refill_count += buf_cnt; |
599 | } | |
600 | } | |
601 | ||
602 | static u32 | |
603 | octeon_droq_fast_process_packets(struct octeon_device *oct, | |
604 | struct octeon_droq *droq, | |
605 | u32 pkts_to_process) | |
606 | { | |
607 | struct octeon_droq_info *info; | |
608 | union octeon_rh *rh; | |
609 | u32 pkt, total_len = 0, pkt_count; | |
610 | ||
611 | pkt_count = pkts_to_process; | |
612 | ||
613 | for (pkt = 0; pkt < pkt_count; pkt++) { | |
614 | u32 pkt_len = 0; | |
615 | struct sk_buff *nicbuf = NULL; | |
cabeb13b RV |
616 | struct octeon_skb_page_info *pg_info; |
617 | void *buf; | |
f21fb3ed | 618 | |
c4ee5d81 PK |
619 | info = (struct octeon_droq_info *) |
620 | droq->recv_buf_list[droq->read_idx].data; | |
f21fb3ed RV |
621 | octeon_swap_8B_data((u64 *)info, 2); |
622 | ||
623 | if (!info->length) { | |
624 | dev_err(&oct->pci_dev->dev, | |
625 | "DROQ[%d] idx: %d len:0, pkt_cnt: %d\n", | |
626 | droq->q_no, droq->read_idx, pkt_count); | |
627 | print_hex_dump_bytes("", DUMP_PREFIX_ADDRESS, | |
628 | (u8 *)info, | |
629 | OCT_DROQ_INFO_SIZE); | |
630 | break; | |
631 | } | |
632 | ||
633 | /* Len of resp hdr in included in the received data len. */ | |
f21fb3ed RV |
634 | rh = &info->rh; |
635 | ||
c4ee5d81 PK |
636 | info->length += OCTNET_FRM_LENGTH_SIZE; |
637 | rh->r_dh.len += (ROUNDUP8(OCT_DROQ_INFO_SIZE) / sizeof(u64)); | |
f21fb3ed | 638 | total_len += (u32)info->length; |
97a25326 | 639 | if (opcode_slow_path(rh)) { |
f21fb3ed RV |
640 | u32 buf_cnt; |
641 | ||
642 | buf_cnt = octeon_droq_dispatch_pkt(oct, droq, rh, info); | |
97a25326 RV |
643 | droq->read_idx = incr_index(droq->read_idx, |
644 | buf_cnt, droq->max_count); | |
f21fb3ed RV |
645 | droq->refill_count += buf_cnt; |
646 | } else { | |
647 | if (info->length <= droq->buffer_size) { | |
f21fb3ed RV |
648 | pkt_len = (u32)info->length; |
649 | nicbuf = droq->recv_buf_list[ | |
650 | droq->read_idx].buffer; | |
cabeb13b RV |
651 | pg_info = &droq->recv_buf_list[ |
652 | droq->read_idx].pg_info; | |
653 | if (recv_buffer_recycle(oct, pg_info)) | |
654 | pg_info->page = NULL; | |
f21fb3ed RV |
655 | droq->recv_buf_list[droq->read_idx].buffer = |
656 | NULL; | |
a2c64b67 | 657 | |
97a25326 RV |
658 | droq->read_idx = incr_index(droq->read_idx, 1, |
659 | droq->max_count); | |
f21fb3ed RV |
660 | droq->refill_count++; |
661 | } else { | |
cabeb13b | 662 | nicbuf = octeon_fast_packet_alloc((u32) |
f21fb3ed RV |
663 | info->length); |
664 | pkt_len = 0; | |
665 | /* nicbuf allocation can fail. We'll handle it | |
666 | * inside the loop. | |
667 | */ | |
668 | while (pkt_len < info->length) { | |
cabeb13b | 669 | int cpy_len, idx = droq->read_idx; |
f21fb3ed | 670 | |
cabeb13b RV |
671 | cpy_len = ((pkt_len + droq->buffer_size) |
672 | > info->length) ? | |
f21fb3ed RV |
673 | ((u32)info->length - pkt_len) : |
674 | droq->buffer_size; | |
675 | ||
676 | if (nicbuf) { | |
f21fb3ed RV |
677 | octeon_fast_packet_next(droq, |
678 | nicbuf, | |
679 | cpy_len, | |
cabeb13b | 680 | idx); |
9ae122c6 SB |
681 | buf = droq->recv_buf_list[ |
682 | idx].buffer; | |
cabeb13b RV |
683 | recv_buffer_fast_free(buf); |
684 | droq->recv_buf_list[idx].buffer | |
685 | = NULL; | |
686 | } else { | |
687 | droq->stats.rx_alloc_failure++; | |
f21fb3ed RV |
688 | } |
689 | ||
690 | pkt_len += cpy_len; | |
97a25326 RV |
691 | droq->read_idx = |
692 | incr_index(droq->read_idx, 1, | |
693 | droq->max_count); | |
f21fb3ed RV |
694 | droq->refill_count++; |
695 | } | |
696 | } | |
697 | ||
698 | if (nicbuf) { | |
cabeb13b | 699 | if (droq->ops.fptr) { |
f21fb3ed | 700 | droq->ops.fptr(oct->octeon_id, |
cabeb13b | 701 | nicbuf, pkt_len, |
0cece6c5 RV |
702 | rh, &droq->napi, |
703 | droq->ops.farg); | |
cabeb13b | 704 | } else { |
f21fb3ed | 705 | recv_buffer_free(nicbuf); |
cabeb13b | 706 | } |
f21fb3ed RV |
707 | } |
708 | } | |
709 | ||
710 | if (droq->refill_count >= droq->refill_threshold) { | |
711 | int desc_refilled = octeon_droq_refill(oct, droq); | |
712 | ||
713 | /* Flush the droq descriptor data to memory to be sure | |
cabeb13b RV |
714 | * that when we update the credits the data in memory |
715 | * is accurate. | |
716 | */ | |
f21fb3ed RV |
717 | wmb(); |
718 | writel((desc_refilled), droq->pkts_credit_reg); | |
719 | /* make sure mmio write completes */ | |
720 | mmiowb(); | |
721 | } | |
722 | ||
cabeb13b | 723 | } /* for (each packet)... */ |
f21fb3ed RV |
724 | |
725 | /* Increment refill_count by the number of buffers processed. */ | |
726 | droq->stats.pkts_received += pkt; | |
727 | droq->stats.bytes_received += total_len; | |
728 | ||
729 | if ((droq->ops.drop_on_max) && (pkts_to_process - pkt)) { | |
730 | octeon_droq_drop_packets(oct, droq, (pkts_to_process - pkt)); | |
731 | ||
732 | droq->stats.dropped_toomany += (pkts_to_process - pkt); | |
733 | return pkts_to_process; | |
734 | } | |
735 | ||
736 | return pkt; | |
737 | } | |
738 | ||
739 | int | |
740 | octeon_droq_process_packets(struct octeon_device *oct, | |
741 | struct octeon_droq *droq, | |
742 | u32 budget) | |
743 | { | |
744 | u32 pkt_count = 0, pkts_processed = 0; | |
745 | struct list_head *tmp, *tmp2; | |
746 | ||
cd8b1eb4 RV |
747 | /* Grab the droq lock */ |
748 | spin_lock(&droq->lock); | |
749 | ||
750 | octeon_droq_check_hw_for_pkts(droq); | |
f21fb3ed | 751 | pkt_count = atomic_read(&droq->pkts_pending); |
cd8b1eb4 RV |
752 | |
753 | if (!pkt_count) { | |
754 | spin_unlock(&droq->lock); | |
f21fb3ed | 755 | return 0; |
cd8b1eb4 | 756 | } |
f21fb3ed RV |
757 | |
758 | if (pkt_count > budget) | |
759 | pkt_count = budget; | |
760 | ||
f21fb3ed RV |
761 | pkts_processed = octeon_droq_fast_process_packets(oct, droq, pkt_count); |
762 | ||
763 | atomic_sub(pkts_processed, &droq->pkts_pending); | |
764 | ||
765 | /* Release the spin lock */ | |
766 | spin_unlock(&droq->lock); | |
767 | ||
768 | list_for_each_safe(tmp, tmp2, &droq->dispatch_list) { | |
769 | struct __dispatch *rdisp = (struct __dispatch *)tmp; | |
770 | ||
771 | list_del(tmp); | |
772 | rdisp->disp_fn(rdisp->rinfo, | |
773 | octeon_get_dispatch_arg | |
774 | (oct, | |
775 | (u16)rdisp->rinfo->recv_pkt->rh.r.opcode, | |
776 | (u16)rdisp->rinfo->recv_pkt->rh.r.subcode)); | |
777 | } | |
778 | ||
779 | /* If there are packets pending. schedule tasklet again */ | |
780 | if (atomic_read(&droq->pkts_pending)) | |
781 | return 1; | |
782 | ||
783 | return 0; | |
784 | } | |
785 | ||
786 | /** | |
787 | * Utility function to poll for packets. check_hw_for_packets must be | |
788 | * called before calling this routine. | |
789 | */ | |
790 | ||
5eb297a9 | 791 | int |
f21fb3ed RV |
792 | octeon_droq_process_poll_pkts(struct octeon_device *oct, |
793 | struct octeon_droq *droq, u32 budget) | |
794 | { | |
795 | struct list_head *tmp, *tmp2; | |
796 | u32 pkts_available = 0, pkts_processed = 0; | |
797 | u32 total_pkts_processed = 0; | |
798 | ||
799 | if (budget > droq->max_count) | |
800 | budget = droq->max_count; | |
801 | ||
802 | spin_lock(&droq->lock); | |
803 | ||
804 | while (total_pkts_processed < budget) { | |
cd8b1eb4 RV |
805 | octeon_droq_check_hw_for_pkts(droq); |
806 | ||
97a25326 RV |
807 | pkts_available = min((budget - total_pkts_processed), |
808 | (u32)(atomic_read(&droq->pkts_pending))); | |
f21fb3ed RV |
809 | |
810 | if (pkts_available == 0) | |
811 | break; | |
812 | ||
813 | pkts_processed = | |
814 | octeon_droq_fast_process_packets(oct, droq, | |
815 | pkts_available); | |
816 | ||
817 | atomic_sub(pkts_processed, &droq->pkts_pending); | |
818 | ||
819 | total_pkts_processed += pkts_processed; | |
f21fb3ed RV |
820 | } |
821 | ||
822 | spin_unlock(&droq->lock); | |
823 | ||
824 | list_for_each_safe(tmp, tmp2, &droq->dispatch_list) { | |
825 | struct __dispatch *rdisp = (struct __dispatch *)tmp; | |
826 | ||
827 | list_del(tmp); | |
828 | rdisp->disp_fn(rdisp->rinfo, | |
829 | octeon_get_dispatch_arg | |
830 | (oct, | |
831 | (u16)rdisp->rinfo->recv_pkt->rh.r.opcode, | |
832 | (u16)rdisp->rinfo->recv_pkt->rh.r.subcode)); | |
833 | } | |
834 | ||
835 | return total_pkts_processed; | |
836 | } | |
837 | ||
5eb297a9 | 838 | /* Enable Pkt Interrupt */ |
f21fb3ed | 839 | int |
5eb297a9 | 840 | octeon_enable_irq(struct octeon_device *oct, u32 q_no) |
f21fb3ed | 841 | { |
5eb297a9 IB |
842 | switch (oct->chip_id) { |
843 | case OCTEON_CN66XX: | |
844 | case OCTEON_CN68XX: { | |
845 | struct octeon_cn6xxx *cn6xxx = | |
846 | (struct octeon_cn6xxx *)oct->chip; | |
847 | unsigned long flags; | |
848 | u32 value; | |
f21fb3ed | 849 | |
5eb297a9 IB |
850 | spin_lock_irqsave |
851 | (&cn6xxx->lock_for_droq_int_enb_reg, flags); | |
852 | value = octeon_read_csr(oct, CN6XXX_SLI_PKT_TIME_INT_ENB); | |
853 | value |= (1 << q_no); | |
854 | octeon_write_csr(oct, CN6XXX_SLI_PKT_TIME_INT_ENB, value); | |
855 | value = octeon_read_csr(oct, CN6XXX_SLI_PKT_CNT_INT_ENB); | |
856 | value |= (1 << q_no); | |
857 | octeon_write_csr(oct, CN6XXX_SLI_PKT_CNT_INT_ENB, value); | |
f21fb3ed | 858 | |
5eb297a9 | 859 | /* don't bother flushing the enables */ |
f21fb3ed | 860 | |
5eb297a9 IB |
861 | spin_unlock_irqrestore |
862 | (&cn6xxx->lock_for_droq_int_enb_reg, flags); | |
f21fb3ed | 863 | } |
f21fb3ed | 864 | break; |
5eb297a9 IB |
865 | case OCTEON_CN23XX_PF_VID: |
866 | lio_enable_irq(oct->droq[q_no], oct->instr_queue[q_no]); | |
9ded1a51 | 867 | break; |
9217c3cf | 868 | |
5eb297a9 IB |
869 | case OCTEON_CN23XX_VF_VID: |
870 | lio_enable_irq(oct->droq[q_no], oct->instr_queue[q_no]); | |
9217c3cf | 871 | break; |
5eb297a9 IB |
872 | default: |
873 | dev_err(&oct->pci_dev->dev, "%s Unknown Chip\n", __func__); | |
874 | return 1; | |
f21fb3ed RV |
875 | } |
876 | ||
5eb297a9 | 877 | return 0; |
f21fb3ed RV |
878 | } |
879 | ||
880 | int octeon_register_droq_ops(struct octeon_device *oct, u32 q_no, | |
881 | struct octeon_droq_ops *ops) | |
882 | { | |
883 | struct octeon_droq *droq; | |
884 | unsigned long flags; | |
885 | struct octeon_config *oct_cfg = NULL; | |
886 | ||
887 | oct_cfg = octeon_get_conf(oct); | |
888 | ||
889 | if (!oct_cfg) | |
890 | return -EINVAL; | |
891 | ||
892 | if (!(ops)) { | |
893 | dev_err(&oct->pci_dev->dev, "%s: droq_ops pointer is NULL\n", | |
894 | __func__); | |
895 | return -EINVAL; | |
896 | } | |
897 | ||
898 | if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) { | |
899 | dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n", | |
900 | __func__, q_no, (oct->num_oqs - 1)); | |
901 | return -EINVAL; | |
902 | } | |
903 | ||
904 | droq = oct->droq[q_no]; | |
905 | ||
906 | spin_lock_irqsave(&droq->lock, flags); | |
907 | ||
908 | memcpy(&droq->ops, ops, sizeof(struct octeon_droq_ops)); | |
909 | ||
910 | spin_unlock_irqrestore(&droq->lock, flags); | |
911 | ||
912 | return 0; | |
913 | } | |
914 | ||
915 | int octeon_unregister_droq_ops(struct octeon_device *oct, u32 q_no) | |
916 | { | |
917 | unsigned long flags; | |
918 | struct octeon_droq *droq; | |
919 | struct octeon_config *oct_cfg = NULL; | |
920 | ||
921 | oct_cfg = octeon_get_conf(oct); | |
922 | ||
923 | if (!oct_cfg) | |
924 | return -EINVAL; | |
925 | ||
926 | if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) { | |
927 | dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n", | |
928 | __func__, q_no, oct->num_oqs - 1); | |
929 | return -EINVAL; | |
930 | } | |
931 | ||
932 | droq = oct->droq[q_no]; | |
933 | ||
934 | if (!droq) { | |
935 | dev_info(&oct->pci_dev->dev, | |
936 | "Droq id (%d) not available.\n", q_no); | |
937 | return 0; | |
938 | } | |
939 | ||
940 | spin_lock_irqsave(&droq->lock, flags); | |
941 | ||
942 | droq->ops.fptr = NULL; | |
0cece6c5 | 943 | droq->ops.farg = NULL; |
f21fb3ed RV |
944 | droq->ops.drop_on_max = 0; |
945 | ||
946 | spin_unlock_irqrestore(&droq->lock, flags); | |
947 | ||
948 | return 0; | |
949 | } | |
950 | ||
951 | int octeon_create_droq(struct octeon_device *oct, | |
952 | u32 q_no, u32 num_descs, | |
953 | u32 desc_size, void *app_ctx) | |
954 | { | |
955 | struct octeon_droq *droq; | |
b3ca9af0 | 956 | int numa_node = dev_to_node(&oct->pci_dev->dev); |
f21fb3ed RV |
957 | |
958 | if (oct->droq[q_no]) { | |
959 | dev_dbg(&oct->pci_dev->dev, "Droq already in use. Cannot create droq %d again\n", | |
960 | q_no); | |
961 | return 1; | |
962 | } | |
963 | ||
964 | /* Allocate the DS for the new droq. */ | |
96ae48b7 RV |
965 | droq = vmalloc_node(sizeof(*droq), numa_node); |
966 | if (!droq) | |
967 | droq = vmalloc(sizeof(*droq)); | |
f21fb3ed | 968 | if (!droq) |
515e752d RV |
969 | return -1; |
970 | ||
f21fb3ed RV |
971 | memset(droq, 0, sizeof(struct octeon_droq)); |
972 | ||
973 | /*Disable the pkt o/p for this Q */ | |
974 | octeon_set_droq_pkt_op(oct, q_no, 0); | |
975 | oct->droq[q_no] = droq; | |
976 | ||
977 | /* Initialize the Droq */ | |
515e752d RV |
978 | if (octeon_init_droq(oct, q_no, num_descs, desc_size, app_ctx)) { |
979 | vfree(oct->droq[q_no]); | |
980 | oct->droq[q_no] = NULL; | |
981 | return -1; | |
982 | } | |
f21fb3ed RV |
983 | |
984 | oct->num_oqs++; | |
985 | ||
986 | dev_dbg(&oct->pci_dev->dev, "%s: Total number of OQ: %d\n", __func__, | |
987 | oct->num_oqs); | |
988 | ||
989 | /* Global Droq register settings */ | |
990 | ||
991 | /* As of now not required, as setting are done for all 32 Droqs at | |
992 | * the same time. | |
993 | */ | |
994 | return 0; | |
f21fb3ed | 995 | } |