Commit | Line | Data |
---|---|---|
17d3b21c AM |
1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /**************************************************************************** | |
3 | * Driver for Solarflare network controllers and boards | |
4 | * Copyright 2018 Solarflare Communications Inc. | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or modify it | |
7 | * under the terms of the GNU General Public License version 2 as published | |
8 | * by the Free Software Foundation, incorporated herein by reference. | |
9 | */ | |
10 | ||
11 | #include "net_driver.h" | |
12 | #include "efx.h" | |
13 | #include "nic.h" | |
14 | #include "tx_common.h" | |
15 | ||
16 | static unsigned int efx_tx_cb_page_count(struct efx_tx_queue *tx_queue) | |
17 | { | |
18 | return DIV_ROUND_UP(tx_queue->ptr_mask + 1, | |
19 | PAGE_SIZE >> EFX_TX_CB_ORDER); | |
20 | } | |
21 | ||
22 | int efx_probe_tx_queue(struct efx_tx_queue *tx_queue) | |
23 | { | |
24 | struct efx_nic *efx = tx_queue->efx; | |
25 | unsigned int entries; | |
26 | int rc; | |
27 | ||
28 | /* Create the smallest power-of-two aligned ring */ | |
29 | entries = max(roundup_pow_of_two(efx->txq_entries), EFX_MIN_DMAQ_SIZE); | |
30 | EFX_WARN_ON_PARANOID(entries > EFX_MAX_DMAQ_SIZE); | |
31 | tx_queue->ptr_mask = entries - 1; | |
32 | ||
33 | netif_dbg(efx, probe, efx->net_dev, | |
34 | "creating TX queue %d size %#x mask %#x\n", | |
35 | tx_queue->queue, efx->txq_entries, tx_queue->ptr_mask); | |
36 | ||
37 | /* Allocate software ring */ | |
38 | tx_queue->buffer = kcalloc(entries, sizeof(*tx_queue->buffer), | |
39 | GFP_KERNEL); | |
40 | if (!tx_queue->buffer) | |
41 | return -ENOMEM; | |
42 | ||
43 | tx_queue->cb_page = kcalloc(efx_tx_cb_page_count(tx_queue), | |
44 | sizeof(tx_queue->cb_page[0]), GFP_KERNEL); | |
45 | if (!tx_queue->cb_page) { | |
46 | rc = -ENOMEM; | |
47 | goto fail1; | |
48 | } | |
49 | ||
50 | /* Allocate hardware ring */ | |
51 | rc = efx_nic_probe_tx(tx_queue); | |
52 | if (rc) | |
53 | goto fail2; | |
54 | ||
55 | return 0; | |
56 | ||
57 | fail2: | |
58 | kfree(tx_queue->cb_page); | |
59 | tx_queue->cb_page = NULL; | |
60 | fail1: | |
61 | kfree(tx_queue->buffer); | |
62 | tx_queue->buffer = NULL; | |
63 | return rc; | |
64 | } | |
65 | ||
66 | void efx_init_tx_queue(struct efx_tx_queue *tx_queue) | |
67 | { | |
68 | struct efx_nic *efx = tx_queue->efx; | |
69 | ||
70 | netif_dbg(efx, drv, efx->net_dev, | |
71 | "initialising TX queue %d\n", tx_queue->queue); | |
72 | ||
73 | tx_queue->insert_count = 0; | |
74 | tx_queue->write_count = 0; | |
75 | tx_queue->packet_write_count = 0; | |
76 | tx_queue->old_write_count = 0; | |
77 | tx_queue->read_count = 0; | |
78 | tx_queue->old_read_count = 0; | |
79 | tx_queue->empty_read_count = 0 | EFX_EMPTY_COUNT_VALID; | |
80 | tx_queue->xmit_more_available = false; | |
81 | tx_queue->timestamping = (efx_ptp_use_mac_tx_timestamps(efx) && | |
82 | tx_queue->channel == efx_ptp_channel(efx)); | |
17d3b21c AM |
83 | tx_queue->completed_timestamp_major = 0; |
84 | tx_queue->completed_timestamp_minor = 0; | |
85 | ||
86 | tx_queue->xdp_tx = efx_channel_is_xdp_tx(tx_queue->channel); | |
87 | ||
88 | /* Set up default function pointers. These may get replaced by | |
89 | * efx_nic_init_tx() based off NIC/queue capabilities. | |
90 | */ | |
91 | tx_queue->handle_tso = efx_enqueue_skb_tso; | |
92 | ||
93 | /* Set up TX descriptor ring */ | |
94 | efx_nic_init_tx(tx_queue); | |
95 | ||
96 | tx_queue->initialised = true; | |
97 | } | |
98 | ||
99 | void efx_fini_tx_queue(struct efx_tx_queue *tx_queue) | |
100 | { | |
101 | struct efx_tx_buffer *buffer; | |
102 | ||
103 | netif_dbg(tx_queue->efx, drv, tx_queue->efx->net_dev, | |
104 | "shutting down TX queue %d\n", tx_queue->queue); | |
105 | ||
106 | if (!tx_queue->buffer) | |
107 | return; | |
108 | ||
109 | /* Free any buffers left in the ring */ | |
110 | while (tx_queue->read_count != tx_queue->write_count) { | |
111 | unsigned int pkts_compl = 0, bytes_compl = 0; | |
112 | ||
113 | buffer = &tx_queue->buffer[tx_queue->read_count & tx_queue->ptr_mask]; | |
114 | efx_dequeue_buffer(tx_queue, buffer, &pkts_compl, &bytes_compl); | |
115 | ||
116 | ++tx_queue->read_count; | |
117 | } | |
118 | tx_queue->xmit_more_available = false; | |
119 | netdev_tx_reset_queue(tx_queue->core_txq); | |
120 | } | |
121 | ||
122 | void efx_remove_tx_queue(struct efx_tx_queue *tx_queue) | |
123 | { | |
124 | int i; | |
125 | ||
126 | if (!tx_queue->buffer) | |
127 | return; | |
128 | ||
129 | netif_dbg(tx_queue->efx, drv, tx_queue->efx->net_dev, | |
130 | "destroying TX queue %d\n", tx_queue->queue); | |
131 | efx_nic_remove_tx(tx_queue); | |
132 | ||
133 | if (tx_queue->cb_page) { | |
134 | for (i = 0; i < efx_tx_cb_page_count(tx_queue); i++) | |
135 | efx_nic_free_buffer(tx_queue->efx, | |
136 | &tx_queue->cb_page[i]); | |
137 | kfree(tx_queue->cb_page); | |
138 | tx_queue->cb_page = NULL; | |
139 | } | |
140 | ||
141 | kfree(tx_queue->buffer); | |
142 | tx_queue->buffer = NULL; | |
143 | } | |
144 | ||
145 | void efx_dequeue_buffer(struct efx_tx_queue *tx_queue, | |
146 | struct efx_tx_buffer *buffer, | |
147 | unsigned int *pkts_compl, | |
148 | unsigned int *bytes_compl) | |
149 | { | |
150 | if (buffer->unmap_len) { | |
151 | struct device *dma_dev = &tx_queue->efx->pci_dev->dev; | |
152 | dma_addr_t unmap_addr = buffer->dma_addr - buffer->dma_offset; | |
153 | ||
154 | if (buffer->flags & EFX_TX_BUF_MAP_SINGLE) | |
155 | dma_unmap_single(dma_dev, unmap_addr, buffer->unmap_len, | |
156 | DMA_TO_DEVICE); | |
157 | else | |
158 | dma_unmap_page(dma_dev, unmap_addr, buffer->unmap_len, | |
159 | DMA_TO_DEVICE); | |
160 | buffer->unmap_len = 0; | |
161 | } | |
162 | ||
163 | if (buffer->flags & EFX_TX_BUF_SKB) { | |
164 | struct sk_buff *skb = (struct sk_buff *)buffer->skb; | |
165 | ||
166 | EFX_WARN_ON_PARANOID(!pkts_compl || !bytes_compl); | |
167 | (*pkts_compl)++; | |
168 | (*bytes_compl) += skb->len; | |
169 | if (tx_queue->timestamping && | |
170 | (tx_queue->completed_timestamp_major || | |
171 | tx_queue->completed_timestamp_minor)) { | |
172 | struct skb_shared_hwtstamps hwtstamp; | |
173 | ||
174 | hwtstamp.hwtstamp = | |
175 | efx_ptp_nic_to_kernel_time(tx_queue); | |
176 | skb_tstamp_tx(skb, &hwtstamp); | |
177 | ||
178 | tx_queue->completed_timestamp_major = 0; | |
179 | tx_queue->completed_timestamp_minor = 0; | |
180 | } | |
181 | dev_consume_skb_any((struct sk_buff *)buffer->skb); | |
182 | netif_vdbg(tx_queue->efx, tx_done, tx_queue->efx->net_dev, | |
183 | "TX queue %d transmission id %x complete\n", | |
184 | tx_queue->queue, tx_queue->read_count); | |
185 | } else if (buffer->flags & EFX_TX_BUF_XDP) { | |
186 | xdp_return_frame_rx_napi(buffer->xdpf); | |
187 | } | |
188 | ||
189 | buffer->len = 0; | |
190 | buffer->flags = 0; | |
191 | } | |
192 | ||
b8cd9499 AM |
193 | /* Remove packets from the TX queue |
194 | * | |
195 | * This removes packets from the TX queue, up to and including the | |
196 | * specified index. | |
197 | */ | |
198 | static void efx_dequeue_buffers(struct efx_tx_queue *tx_queue, | |
199 | unsigned int index, | |
200 | unsigned int *pkts_compl, | |
201 | unsigned int *bytes_compl) | |
202 | { | |
203 | struct efx_nic *efx = tx_queue->efx; | |
204 | unsigned int stop_index, read_ptr; | |
205 | ||
206 | stop_index = (index + 1) & tx_queue->ptr_mask; | |
207 | read_ptr = tx_queue->read_count & tx_queue->ptr_mask; | |
208 | ||
209 | while (read_ptr != stop_index) { | |
210 | struct efx_tx_buffer *buffer = &tx_queue->buffer[read_ptr]; | |
211 | ||
3b4f06c7 | 212 | if (!efx_tx_buffer_in_use(buffer)) { |
b8cd9499 | 213 | netif_err(efx, tx_err, efx->net_dev, |
3b4f06c7 | 214 | "TX queue %d spurious TX completion id %d\n", |
b8cd9499 AM |
215 | tx_queue->queue, read_ptr); |
216 | efx_schedule_reset(efx, RESET_TYPE_TX_SKIP); | |
217 | return; | |
218 | } | |
219 | ||
220 | efx_dequeue_buffer(tx_queue, buffer, pkts_compl, bytes_compl); | |
221 | ||
222 | ++tx_queue->read_count; | |
223 | read_ptr = tx_queue->read_count & tx_queue->ptr_mask; | |
224 | } | |
225 | } | |
226 | ||
3b4f06c7 TZ |
227 | void efx_xmit_done_check_empty(struct efx_tx_queue *tx_queue) |
228 | { | |
229 | if ((int)(tx_queue->read_count - tx_queue->old_write_count) >= 0) { | |
230 | tx_queue->old_write_count = READ_ONCE(tx_queue->write_count); | |
231 | if (tx_queue->read_count == tx_queue->old_write_count) { | |
232 | /* Ensure that read_count is flushed. */ | |
233 | smp_mb(); | |
234 | tx_queue->empty_read_count = | |
235 | tx_queue->read_count | EFX_EMPTY_COUNT_VALID; | |
236 | } | |
237 | } | |
238 | } | |
239 | ||
b8cd9499 AM |
240 | void efx_xmit_done(struct efx_tx_queue *tx_queue, unsigned int index) |
241 | { | |
242 | unsigned int fill_level, pkts_compl = 0, bytes_compl = 0; | |
243 | struct efx_nic *efx = tx_queue->efx; | |
244 | struct efx_tx_queue *txq2; | |
245 | ||
246 | EFX_WARN_ON_ONCE_PARANOID(index > tx_queue->ptr_mask); | |
247 | ||
248 | efx_dequeue_buffers(tx_queue, index, &pkts_compl, &bytes_compl); | |
249 | tx_queue->pkts_compl += pkts_compl; | |
250 | tx_queue->bytes_compl += bytes_compl; | |
251 | ||
252 | if (pkts_compl > 1) | |
253 | ++tx_queue->merge_events; | |
254 | ||
255 | /* See if we need to restart the netif queue. This memory | |
256 | * barrier ensures that we write read_count (inside | |
257 | * efx_dequeue_buffers()) before reading the queue status. | |
258 | */ | |
259 | smp_mb(); | |
260 | if (unlikely(netif_tx_queue_stopped(tx_queue->core_txq)) && | |
261 | likely(efx->port_enabled) && | |
262 | likely(netif_device_present(efx->net_dev))) { | |
263 | txq2 = efx_tx_queue_partner(tx_queue); | |
264 | fill_level = max(tx_queue->insert_count - tx_queue->read_count, | |
265 | txq2->insert_count - txq2->read_count); | |
266 | if (fill_level <= efx->txq_wake_thresh) | |
267 | netif_tx_wake_queue(tx_queue->core_txq); | |
268 | } | |
269 | ||
3b4f06c7 | 270 | efx_xmit_done_check_empty(tx_queue); |
b8cd9499 AM |
271 | } |
272 | ||
88f7df35 AM |
273 | /* Remove buffers put into a tx_queue for the current packet. |
274 | * None of the buffers must have an skb attached. | |
275 | */ | |
276 | void efx_enqueue_unwind(struct efx_tx_queue *tx_queue, | |
277 | unsigned int insert_count) | |
278 | { | |
279 | struct efx_tx_buffer *buffer; | |
280 | unsigned int bytes_compl = 0; | |
281 | unsigned int pkts_compl = 0; | |
282 | ||
283 | /* Work backwards until we hit the original insert pointer value */ | |
284 | while (tx_queue->insert_count != insert_count) { | |
285 | --tx_queue->insert_count; | |
286 | buffer = __efx_tx_queue_get_insert_buffer(tx_queue); | |
287 | efx_dequeue_buffer(tx_queue, buffer, &pkts_compl, &bytes_compl); | |
288 | } | |
289 | } | |
290 | ||
17d3b21c AM |
291 | struct efx_tx_buffer *efx_tx_map_chunk(struct efx_tx_queue *tx_queue, |
292 | dma_addr_t dma_addr, size_t len) | |
293 | { | |
294 | const struct efx_nic_type *nic_type = tx_queue->efx->type; | |
295 | struct efx_tx_buffer *buffer; | |
296 | unsigned int dma_len; | |
297 | ||
298 | /* Map the fragment taking account of NIC-dependent DMA limits. */ | |
299 | do { | |
300 | buffer = efx_tx_queue_get_insert_buffer(tx_queue); | |
301 | dma_len = nic_type->tx_limit_len(tx_queue, dma_addr, len); | |
302 | ||
303 | buffer->len = dma_len; | |
304 | buffer->dma_addr = dma_addr; | |
305 | buffer->flags = EFX_TX_BUF_CONT; | |
306 | len -= dma_len; | |
307 | dma_addr += dma_len; | |
308 | ++tx_queue->insert_count; | |
309 | } while (len); | |
310 | ||
311 | return buffer; | |
312 | } | |
313 | ||
314 | /* Map all data from an SKB for DMA and create descriptors on the queue. */ | |
315 | int efx_tx_map_data(struct efx_tx_queue *tx_queue, struct sk_buff *skb, | |
316 | unsigned int segment_count) | |
317 | { | |
318 | struct efx_nic *efx = tx_queue->efx; | |
319 | struct device *dma_dev = &efx->pci_dev->dev; | |
320 | unsigned int frag_index, nr_frags; | |
321 | dma_addr_t dma_addr, unmap_addr; | |
322 | unsigned short dma_flags; | |
323 | size_t len, unmap_len; | |
324 | ||
325 | nr_frags = skb_shinfo(skb)->nr_frags; | |
326 | frag_index = 0; | |
327 | ||
328 | /* Map header data. */ | |
329 | len = skb_headlen(skb); | |
330 | dma_addr = dma_map_single(dma_dev, skb->data, len, DMA_TO_DEVICE); | |
331 | dma_flags = EFX_TX_BUF_MAP_SINGLE; | |
332 | unmap_len = len; | |
333 | unmap_addr = dma_addr; | |
334 | ||
335 | if (unlikely(dma_mapping_error(dma_dev, dma_addr))) | |
336 | return -EIO; | |
337 | ||
338 | if (segment_count) { | |
339 | /* For TSO we need to put the header in to a separate | |
340 | * descriptor. Map this separately if necessary. | |
341 | */ | |
342 | size_t header_len = skb_transport_header(skb) - skb->data + | |
343 | (tcp_hdr(skb)->doff << 2u); | |
344 | ||
345 | if (header_len != len) { | |
346 | tx_queue->tso_long_headers++; | |
347 | efx_tx_map_chunk(tx_queue, dma_addr, header_len); | |
348 | len -= header_len; | |
349 | dma_addr += header_len; | |
350 | } | |
351 | } | |
352 | ||
353 | /* Add descriptors for each fragment. */ | |
354 | do { | |
355 | struct efx_tx_buffer *buffer; | |
356 | skb_frag_t *fragment; | |
357 | ||
358 | buffer = efx_tx_map_chunk(tx_queue, dma_addr, len); | |
359 | ||
360 | /* The final descriptor for a fragment is responsible for | |
361 | * unmapping the whole fragment. | |
362 | */ | |
363 | buffer->flags = EFX_TX_BUF_CONT | dma_flags; | |
364 | buffer->unmap_len = unmap_len; | |
365 | buffer->dma_offset = buffer->dma_addr - unmap_addr; | |
366 | ||
367 | if (frag_index >= nr_frags) { | |
368 | /* Store SKB details with the final buffer for | |
369 | * the completion. | |
370 | */ | |
371 | buffer->skb = skb; | |
372 | buffer->flags = EFX_TX_BUF_SKB | dma_flags; | |
373 | return 0; | |
374 | } | |
375 | ||
376 | /* Move on to the next fragment. */ | |
377 | fragment = &skb_shinfo(skb)->frags[frag_index++]; | |
378 | len = skb_frag_size(fragment); | |
379 | dma_addr = skb_frag_dma_map(dma_dev, fragment, 0, len, | |
380 | DMA_TO_DEVICE); | |
381 | dma_flags = 0; | |
382 | unmap_len = len; | |
383 | unmap_addr = dma_addr; | |
384 | ||
385 | if (unlikely(dma_mapping_error(dma_dev, dma_addr))) | |
386 | return -EIO; | |
387 | } while (1); | |
388 | } | |
389 | ||
390 | unsigned int efx_tx_max_skb_descs(struct efx_nic *efx) | |
391 | { | |
392 | /* Header and payload descriptor for each output segment, plus | |
393 | * one for every input fragment boundary within a segment | |
394 | */ | |
395 | unsigned int max_descs = EFX_TSO_MAX_SEGS * 2 + MAX_SKB_FRAGS; | |
396 | ||
397 | /* Possibly one more per segment for option descriptors */ | |
398 | if (efx_nic_rev(efx) >= EFX_REV_HUNT_A0) | |
399 | max_descs += EFX_TSO_MAX_SEGS; | |
400 | ||
401 | /* Possibly more for PCIe page boundaries within input fragments */ | |
402 | if (PAGE_SIZE > EFX_PAGE_SIZE) | |
403 | max_descs += max_t(unsigned int, MAX_SKB_FRAGS, | |
404 | DIV_ROUND_UP(GSO_MAX_SIZE, EFX_PAGE_SIZE)); | |
405 | ||
406 | return max_descs; | |
407 | } |