projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
net: hns3: add support to set/get rx buf len via ethtool for hns3 driver
[linux-block.git] / drivers / net / ethernet / hisilicon / hns3 / hns3_enet.c
CommitLineData
d71d8381
JS		 1// SPDX-License-Identifier: GPL-2.0+
2// Copyright (c) 2016-2017 Hisilicon Limited.
76ad4f0e
S		 3
4#include <linux/dma-mapping.h>
5#include <linux/etherdevice.h>
6#include <linux/interrupt.h>
ffab9691
JS		 7#ifdef CONFIG_RFS_ACCEL
8#include <linux/cpu_rmap.h>
9#endif
76ad4f0e 10#include <linux/if_vlan.h>
e99a308d 11#include <linux/irq.h>
76ad4f0e
S		 12#include <linux/ip.h>
13#include <linux/ipv6.h>
14#include <linux/module.h>
15#include <linux/pci.h>
6ae4e733 16#include <linux/aer.h>
76ad4f0e
S		 17#include <linux/skbuff.h>
18#include <linux/sctp.h>
76ad4f0e 19#include <net/gre.h>
4721031c 20#include <net/gro.h>
e2ee1c5a 21#include <net/ip6_checksum.h>
30d240df 22#include <net/pkt_cls.h>
a6d53b97 23#include <net/tcp.h>
76ad4f0e 24#include <net/vxlan.h>
a156998f 25#include <net/geneve.h>
76ad4f0e
S		 26
27#include "hnae3.h"
28#include "hns3_enet.h"
698a8954
YL		 29/* All hns3 tracepoints are defined by the include below, which
30 * must be included exactly once across the whole kernel with
31 * CREATE_TRACE_POINTS defined
32 */
33#define CREATE_TRACE_POINTS
34#include "hns3_trace.h"
76ad4f0e 35
9393eb50 36#define hns3_set_field(origin, shift, val) ((origin) |= (val) << (shift))
5f543a54 37#define hns3_tx_bd_count(S) DIV_ROUND_UP(S, HNS3_MAX_BD_SIZE)
cde4ffad 38
b20d7fe5
YL		 39#define hns3_rl_err(fmt, ...) \
40 do { \
41 if (net_ratelimit()) \
42 netdev_err(fmt, ##__VA_ARGS__); \
43 } while (0)
44
f96315f2 45static void hns3_clear_all_ring(struct hnae3_handle *h, bool force);
7b763f3f 46
1db9b1bf 47static const char hns3_driver_name[] = "hns3";
76ad4f0e
S		 48static const char hns3_driver_string[] =
49 "Hisilicon Ethernet Network Driver for Hip08 Family";
50static const char hns3_copyright[] = "Copyright (c) 2017 Huawei Corporation.";
51static struct hnae3_client client;
52
bb87be87
YL		 53static int debug = -1;
54module_param(debug, int, 0);
55MODULE_PARM_DESC(debug, " Network interface message level setting");
56
907676b1
YL		 57static unsigned int tx_spare_buf_size;
58module_param(tx_spare_buf_size, uint, 0400);
59MODULE_PARM_DESC(tx_spare_buf_size, "Size used to allocate tx spare buffer");
60
7459775e
YL		 61static unsigned int tx_sgl = 1;
62module_param(tx_sgl, uint, 0600);
63MODULE_PARM_DESC(tx_sgl, "Minimum number of frags when using dma_map_sg() to optimize the IOMMU mapping");
64
f7ec554b
YL		 65static bool page_pool_enabled = true;
66module_param(page_pool_enabled, bool, 0400);
67
7459775e
YL		 68#define HNS3_SGL_SIZE(nfrag) (sizeof(struct scatterlist) * (nfrag) + \
69 sizeof(struct sg_table))
c74e5035 70#define HNS3_MAX_SGL_SIZE ALIGN(HNS3_SGL_SIZE(HNS3_MAX_TSO_BD_NUM), \
7459775e
YL		 71 dma_get_cache_alignment())
72
bb87be87
YL		 73#define DEFAULT_MSG_LEVEL (NETIF_MSG_PROBE | NETIF_MSG_LINK | \
74 NETIF_MSG_IFDOWN | NETIF_MSG_IFUP)
75
eb977d99
YL		 76#define HNS3_INNER_VLAN_TAG 1
77#define HNS3_OUTER_VLAN_TAG 2
78
36c67349 79#define HNS3_MIN_TX_LEN 33U
d18e8118 80#define HNS3_MIN_TUN_PKT_LEN 65U
36c67349 81
76ad4f0e
S		 82/* hns3_pci_tbl - PCI Device ID Table
83 *
84 * Last entry must be all 0s
85 *
86 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
87 * Class, Class Mask, private data (not used) }
88 */
89static const struct pci_device_id hns3_pci_tbl[] = {
90 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_GE), 0},
91 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE), 0},
e92a0843 92 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA),
2daf4a65 93 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
e92a0843 94 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC),
2daf4a65 95 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
e92a0843 96 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA),
2daf4a65 97 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
e92a0843 98 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC),
2daf4a65 99 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
e92a0843 100 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC),
2daf4a65 101 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
ae6f010c
GH		 102 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_200G_RDMA),
103 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
c155e22b
GH		 104 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_VF), 0},
105 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_RDMA_DCB_PFC_VF),
07acf909 106 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
76ad4f0e 107 /* required last entry */
7f2d4b7f 108 {0,}
76ad4f0e
S		 109};
110MODULE_DEVICE_TABLE(pci, hns3_pci_tbl);
111
79664077
HT		 112#define HNS3_RX_PTYPE_ENTRY(ptype, l, s, t) \
113 { ptype, \
114 l, \
115 CHECKSUM_##s, \
116 HNS3_L3_TYPE_##t, \
117 1 }
118
119#define HNS3_RX_PTYPE_UNUSED_ENTRY(ptype) \
120 { ptype, 0, CHECKSUM_NONE, HNS3_L3_TYPE_PARSE_FAIL, 0 }
121
122static const struct hns3_rx_ptype hns3_rx_ptype_tbl[] = {
123 HNS3_RX_PTYPE_UNUSED_ENTRY(0),
124 HNS3_RX_PTYPE_ENTRY(1, 0, COMPLETE, ARP),
125 HNS3_RX_PTYPE_ENTRY(2, 0, COMPLETE, RARP),
126 HNS3_RX_PTYPE_ENTRY(3, 0, COMPLETE, LLDP),
127 HNS3_RX_PTYPE_ENTRY(4, 0, COMPLETE, PARSE_FAIL),
128 HNS3_RX_PTYPE_ENTRY(5, 0, COMPLETE, PARSE_FAIL),
129 HNS3_RX_PTYPE_ENTRY(6, 0, COMPLETE, PARSE_FAIL),
130 HNS3_RX_PTYPE_ENTRY(7, 0, COMPLETE, CNM),
131 HNS3_RX_PTYPE_ENTRY(8, 0, NONE, PARSE_FAIL),
132 HNS3_RX_PTYPE_UNUSED_ENTRY(9),
133 HNS3_RX_PTYPE_UNUSED_ENTRY(10),
134 HNS3_RX_PTYPE_UNUSED_ENTRY(11),
135 HNS3_RX_PTYPE_UNUSED_ENTRY(12),
136 HNS3_RX_PTYPE_UNUSED_ENTRY(13),
137 HNS3_RX_PTYPE_UNUSED_ENTRY(14),
138 HNS3_RX_PTYPE_UNUSED_ENTRY(15),
139 HNS3_RX_PTYPE_ENTRY(16, 0, COMPLETE, PARSE_FAIL),
140 HNS3_RX_PTYPE_ENTRY(17, 0, COMPLETE, IPV4),
141 HNS3_RX_PTYPE_ENTRY(18, 0, COMPLETE, IPV4),
142 HNS3_RX_PTYPE_ENTRY(19, 0, UNNECESSARY, IPV4),
143 HNS3_RX_PTYPE_ENTRY(20, 0, UNNECESSARY, IPV4),
144 HNS3_RX_PTYPE_ENTRY(21, 0, NONE, IPV4),
145 HNS3_RX_PTYPE_ENTRY(22, 0, UNNECESSARY, IPV4),
146 HNS3_RX_PTYPE_ENTRY(23, 0, NONE, IPV4),
147 HNS3_RX_PTYPE_ENTRY(24, 0, NONE, IPV4),
148 HNS3_RX_PTYPE_ENTRY(25, 0, UNNECESSARY, IPV4),
149 HNS3_RX_PTYPE_UNUSED_ENTRY(26),
150 HNS3_RX_PTYPE_UNUSED_ENTRY(27),
151 HNS3_RX_PTYPE_UNUSED_ENTRY(28),
152 HNS3_RX_PTYPE_ENTRY(29, 0, COMPLETE, PARSE_FAIL),
153 HNS3_RX_PTYPE_ENTRY(30, 0, COMPLETE, PARSE_FAIL),
154 HNS3_RX_PTYPE_ENTRY(31, 0, COMPLETE, IPV4),
155 HNS3_RX_PTYPE_ENTRY(32, 0, COMPLETE, IPV4),
156 HNS3_RX_PTYPE_ENTRY(33, 1, UNNECESSARY, IPV4),
157 HNS3_RX_PTYPE_ENTRY(34, 1, UNNECESSARY, IPV4),
158 HNS3_RX_PTYPE_ENTRY(35, 1, UNNECESSARY, IPV4),
159 HNS3_RX_PTYPE_ENTRY(36, 0, COMPLETE, IPV4),
160 HNS3_RX_PTYPE_ENTRY(37, 0, COMPLETE, IPV4),
161 HNS3_RX_PTYPE_UNUSED_ENTRY(38),
162 HNS3_RX_PTYPE_ENTRY(39, 0, COMPLETE, IPV6),
163 HNS3_RX_PTYPE_ENTRY(40, 0, COMPLETE, IPV6),
164 HNS3_RX_PTYPE_ENTRY(41, 1, UNNECESSARY, IPV6),
165 HNS3_RX_PTYPE_ENTRY(42, 1, UNNECESSARY, IPV6),
166 HNS3_RX_PTYPE_ENTRY(43, 1, UNNECESSARY, IPV6),
167 HNS3_RX_PTYPE_ENTRY(44, 0, COMPLETE, IPV6),
168 HNS3_RX_PTYPE_ENTRY(45, 0, COMPLETE, IPV6),
169 HNS3_RX_PTYPE_UNUSED_ENTRY(46),
170 HNS3_RX_PTYPE_UNUSED_ENTRY(47),
171 HNS3_RX_PTYPE_UNUSED_ENTRY(48),
172 HNS3_RX_PTYPE_UNUSED_ENTRY(49),
173 HNS3_RX_PTYPE_UNUSED_ENTRY(50),
174 HNS3_RX_PTYPE_UNUSED_ENTRY(51),
175 HNS3_RX_PTYPE_UNUSED_ENTRY(52),
176 HNS3_RX_PTYPE_UNUSED_ENTRY(53),
177 HNS3_RX_PTYPE_UNUSED_ENTRY(54),
178 HNS3_RX_PTYPE_UNUSED_ENTRY(55),
179 HNS3_RX_PTYPE_UNUSED_ENTRY(56),
180 HNS3_RX_PTYPE_UNUSED_ENTRY(57),
181 HNS3_RX_PTYPE_UNUSED_ENTRY(58),
182 HNS3_RX_PTYPE_UNUSED_ENTRY(59),
183 HNS3_RX_PTYPE_UNUSED_ENTRY(60),
184 HNS3_RX_PTYPE_UNUSED_ENTRY(61),
185 HNS3_RX_PTYPE_UNUSED_ENTRY(62),
186 HNS3_RX_PTYPE_UNUSED_ENTRY(63),
187 HNS3_RX_PTYPE_UNUSED_ENTRY(64),
188 HNS3_RX_PTYPE_UNUSED_ENTRY(65),
189 HNS3_RX_PTYPE_UNUSED_ENTRY(66),
190 HNS3_RX_PTYPE_UNUSED_ENTRY(67),
191 HNS3_RX_PTYPE_UNUSED_ENTRY(68),
192 HNS3_RX_PTYPE_UNUSED_ENTRY(69),
193 HNS3_RX_PTYPE_UNUSED_ENTRY(70),
194 HNS3_RX_PTYPE_UNUSED_ENTRY(71),
195 HNS3_RX_PTYPE_UNUSED_ENTRY(72),
196 HNS3_RX_PTYPE_UNUSED_ENTRY(73),
197 HNS3_RX_PTYPE_UNUSED_ENTRY(74),
198 HNS3_RX_PTYPE_UNUSED_ENTRY(75),
199 HNS3_RX_PTYPE_UNUSED_ENTRY(76),
200 HNS3_RX_PTYPE_UNUSED_ENTRY(77),
201 HNS3_RX_PTYPE_UNUSED_ENTRY(78),
202 HNS3_RX_PTYPE_UNUSED_ENTRY(79),
203 HNS3_RX_PTYPE_UNUSED_ENTRY(80),
204 HNS3_RX_PTYPE_UNUSED_ENTRY(81),
205 HNS3_RX_PTYPE_UNUSED_ENTRY(82),
206 HNS3_RX_PTYPE_UNUSED_ENTRY(83),
207 HNS3_RX_PTYPE_UNUSED_ENTRY(84),
208 HNS3_RX_PTYPE_UNUSED_ENTRY(85),
209 HNS3_RX_PTYPE_UNUSED_ENTRY(86),
210 HNS3_RX_PTYPE_UNUSED_ENTRY(87),
211 HNS3_RX_PTYPE_UNUSED_ENTRY(88),
212 HNS3_RX_PTYPE_UNUSED_ENTRY(89),
213 HNS3_RX_PTYPE_UNUSED_ENTRY(90),
214 HNS3_RX_PTYPE_UNUSED_ENTRY(91),
215 HNS3_RX_PTYPE_UNUSED_ENTRY(92),
216 HNS3_RX_PTYPE_UNUSED_ENTRY(93),
217 HNS3_RX_PTYPE_UNUSED_ENTRY(94),
218 HNS3_RX_PTYPE_UNUSED_ENTRY(95),
219 HNS3_RX_PTYPE_UNUSED_ENTRY(96),
220 HNS3_RX_PTYPE_UNUSED_ENTRY(97),
221 HNS3_RX_PTYPE_UNUSED_ENTRY(98),
222 HNS3_RX_PTYPE_UNUSED_ENTRY(99),
223 HNS3_RX_PTYPE_UNUSED_ENTRY(100),
224 HNS3_RX_PTYPE_UNUSED_ENTRY(101),
225 HNS3_RX_PTYPE_UNUSED_ENTRY(102),
226 HNS3_RX_PTYPE_UNUSED_ENTRY(103),
227 HNS3_RX_PTYPE_UNUSED_ENTRY(104),
228 HNS3_RX_PTYPE_UNUSED_ENTRY(105),
229 HNS3_RX_PTYPE_UNUSED_ENTRY(106),
230 HNS3_RX_PTYPE_UNUSED_ENTRY(107),
231 HNS3_RX_PTYPE_UNUSED_ENTRY(108),
232 HNS3_RX_PTYPE_UNUSED_ENTRY(109),
233 HNS3_RX_PTYPE_UNUSED_ENTRY(110),
234 HNS3_RX_PTYPE_ENTRY(111, 0, COMPLETE, IPV6),
235 HNS3_RX_PTYPE_ENTRY(112, 0, COMPLETE, IPV6),
236 HNS3_RX_PTYPE_ENTRY(113, 0, UNNECESSARY, IPV6),
237 HNS3_RX_PTYPE_ENTRY(114, 0, UNNECESSARY, IPV6),
238 HNS3_RX_PTYPE_ENTRY(115, 0, NONE, IPV6),
239 HNS3_RX_PTYPE_ENTRY(116, 0, UNNECESSARY, IPV6),
240 HNS3_RX_PTYPE_ENTRY(117, 0, NONE, IPV6),
241 HNS3_RX_PTYPE_ENTRY(118, 0, NONE, IPV6),
242 HNS3_RX_PTYPE_ENTRY(119, 0, UNNECESSARY, IPV6),
243 HNS3_RX_PTYPE_UNUSED_ENTRY(120),
244 HNS3_RX_PTYPE_UNUSED_ENTRY(121),
245 HNS3_RX_PTYPE_UNUSED_ENTRY(122),
246 HNS3_RX_PTYPE_ENTRY(123, 0, COMPLETE, PARSE_FAIL),
247 HNS3_RX_PTYPE_ENTRY(124, 0, COMPLETE, PARSE_FAIL),
248 HNS3_RX_PTYPE_ENTRY(125, 0, COMPLETE, IPV4),
249 HNS3_RX_PTYPE_ENTRY(126, 0, COMPLETE, IPV4),
250 HNS3_RX_PTYPE_ENTRY(127, 1, UNNECESSARY, IPV4),
251 HNS3_RX_PTYPE_ENTRY(128, 1, UNNECESSARY, IPV4),
252 HNS3_RX_PTYPE_ENTRY(129, 1, UNNECESSARY, IPV4),
253 HNS3_RX_PTYPE_ENTRY(130, 0, COMPLETE, IPV4),
254 HNS3_RX_PTYPE_ENTRY(131, 0, COMPLETE, IPV4),
255 HNS3_RX_PTYPE_UNUSED_ENTRY(132),
256 HNS3_RX_PTYPE_ENTRY(133, 0, COMPLETE, IPV6),
257 HNS3_RX_PTYPE_ENTRY(134, 0, COMPLETE, IPV6),
258 HNS3_RX_PTYPE_ENTRY(135, 1, UNNECESSARY, IPV6),
259 HNS3_RX_PTYPE_ENTRY(136, 1, UNNECESSARY, IPV6),
260 HNS3_RX_PTYPE_ENTRY(137, 1, UNNECESSARY, IPV6),
261 HNS3_RX_PTYPE_ENTRY(138, 0, COMPLETE, IPV6),
262 HNS3_RX_PTYPE_ENTRY(139, 0, COMPLETE, IPV6),
263 HNS3_RX_PTYPE_UNUSED_ENTRY(140),
264 HNS3_RX_PTYPE_UNUSED_ENTRY(141),
265 HNS3_RX_PTYPE_UNUSED_ENTRY(142),
266 HNS3_RX_PTYPE_UNUSED_ENTRY(143),
267 HNS3_RX_PTYPE_UNUSED_ENTRY(144),
268 HNS3_RX_PTYPE_UNUSED_ENTRY(145),
269 HNS3_RX_PTYPE_UNUSED_ENTRY(146),
270 HNS3_RX_PTYPE_UNUSED_ENTRY(147),
271 HNS3_RX_PTYPE_UNUSED_ENTRY(148),
272 HNS3_RX_PTYPE_UNUSED_ENTRY(149),
273 HNS3_RX_PTYPE_UNUSED_ENTRY(150),
274 HNS3_RX_PTYPE_UNUSED_ENTRY(151),
275 HNS3_RX_PTYPE_UNUSED_ENTRY(152),
276 HNS3_RX_PTYPE_UNUSED_ENTRY(153),
277 HNS3_RX_PTYPE_UNUSED_ENTRY(154),
278 HNS3_RX_PTYPE_UNUSED_ENTRY(155),
279 HNS3_RX_PTYPE_UNUSED_ENTRY(156),
280 HNS3_RX_PTYPE_UNUSED_ENTRY(157),
281 HNS3_RX_PTYPE_UNUSED_ENTRY(158),
282 HNS3_RX_PTYPE_UNUSED_ENTRY(159),
283 HNS3_RX_PTYPE_UNUSED_ENTRY(160),
284 HNS3_RX_PTYPE_UNUSED_ENTRY(161),
285 HNS3_RX_PTYPE_UNUSED_ENTRY(162),
286 HNS3_RX_PTYPE_UNUSED_ENTRY(163),
287 HNS3_RX_PTYPE_UNUSED_ENTRY(164),
288 HNS3_RX_PTYPE_UNUSED_ENTRY(165),
289 HNS3_RX_PTYPE_UNUSED_ENTRY(166),
290 HNS3_RX_PTYPE_UNUSED_ENTRY(167),
291 HNS3_RX_PTYPE_UNUSED_ENTRY(168),
292 HNS3_RX_PTYPE_UNUSED_ENTRY(169),
293 HNS3_RX_PTYPE_UNUSED_ENTRY(170),
294 HNS3_RX_PTYPE_UNUSED_ENTRY(171),
295 HNS3_RX_PTYPE_UNUSED_ENTRY(172),
296 HNS3_RX_PTYPE_UNUSED_ENTRY(173),
297 HNS3_RX_PTYPE_UNUSED_ENTRY(174),
298 HNS3_RX_PTYPE_UNUSED_ENTRY(175),
299 HNS3_RX_PTYPE_UNUSED_ENTRY(176),
300 HNS3_RX_PTYPE_UNUSED_ENTRY(177),
301 HNS3_RX_PTYPE_UNUSED_ENTRY(178),
302 HNS3_RX_PTYPE_UNUSED_ENTRY(179),
303 HNS3_RX_PTYPE_UNUSED_ENTRY(180),
304 HNS3_RX_PTYPE_UNUSED_ENTRY(181),
305 HNS3_RX_PTYPE_UNUSED_ENTRY(182),
306 HNS3_RX_PTYPE_UNUSED_ENTRY(183),
307 HNS3_RX_PTYPE_UNUSED_ENTRY(184),
308 HNS3_RX_PTYPE_UNUSED_ENTRY(185),
309 HNS3_RX_PTYPE_UNUSED_ENTRY(186),
310 HNS3_RX_PTYPE_UNUSED_ENTRY(187),
311 HNS3_RX_PTYPE_UNUSED_ENTRY(188),
312 HNS3_RX_PTYPE_UNUSED_ENTRY(189),
313 HNS3_RX_PTYPE_UNUSED_ENTRY(190),
314 HNS3_RX_PTYPE_UNUSED_ENTRY(191),
315 HNS3_RX_PTYPE_UNUSED_ENTRY(192),
316 HNS3_RX_PTYPE_UNUSED_ENTRY(193),
317 HNS3_RX_PTYPE_UNUSED_ENTRY(194),
318 HNS3_RX_PTYPE_UNUSED_ENTRY(195),
319 HNS3_RX_PTYPE_UNUSED_ENTRY(196),
320 HNS3_RX_PTYPE_UNUSED_ENTRY(197),
321 HNS3_RX_PTYPE_UNUSED_ENTRY(198),
322 HNS3_RX_PTYPE_UNUSED_ENTRY(199),
323 HNS3_RX_PTYPE_UNUSED_ENTRY(200),
324 HNS3_RX_PTYPE_UNUSED_ENTRY(201),
325 HNS3_RX_PTYPE_UNUSED_ENTRY(202),
326 HNS3_RX_PTYPE_UNUSED_ENTRY(203),
327 HNS3_RX_PTYPE_UNUSED_ENTRY(204),
328 HNS3_RX_PTYPE_UNUSED_ENTRY(205),
329 HNS3_RX_PTYPE_UNUSED_ENTRY(206),
330 HNS3_RX_PTYPE_UNUSED_ENTRY(207),
331 HNS3_RX_PTYPE_UNUSED_ENTRY(208),
332 HNS3_RX_PTYPE_UNUSED_ENTRY(209),
333 HNS3_RX_PTYPE_UNUSED_ENTRY(210),
334 HNS3_RX_PTYPE_UNUSED_ENTRY(211),
335 HNS3_RX_PTYPE_UNUSED_ENTRY(212),
336 HNS3_RX_PTYPE_UNUSED_ENTRY(213),
337 HNS3_RX_PTYPE_UNUSED_ENTRY(214),
338 HNS3_RX_PTYPE_UNUSED_ENTRY(215),
339 HNS3_RX_PTYPE_UNUSED_ENTRY(216),
340 HNS3_RX_PTYPE_UNUSED_ENTRY(217),
341 HNS3_RX_PTYPE_UNUSED_ENTRY(218),
342 HNS3_RX_PTYPE_UNUSED_ENTRY(219),
343 HNS3_RX_PTYPE_UNUSED_ENTRY(220),
344 HNS3_RX_PTYPE_UNUSED_ENTRY(221),
345 HNS3_RX_PTYPE_UNUSED_ENTRY(222),
346 HNS3_RX_PTYPE_UNUSED_ENTRY(223),
347 HNS3_RX_PTYPE_UNUSED_ENTRY(224),
348 HNS3_RX_PTYPE_UNUSED_ENTRY(225),
349 HNS3_RX_PTYPE_UNUSED_ENTRY(226),
350 HNS3_RX_PTYPE_UNUSED_ENTRY(227),
351 HNS3_RX_PTYPE_UNUSED_ENTRY(228),
352 HNS3_RX_PTYPE_UNUSED_ENTRY(229),
353 HNS3_RX_PTYPE_UNUSED_ENTRY(230),
354 HNS3_RX_PTYPE_UNUSED_ENTRY(231),
355 HNS3_RX_PTYPE_UNUSED_ENTRY(232),
356 HNS3_RX_PTYPE_UNUSED_ENTRY(233),
357 HNS3_RX_PTYPE_UNUSED_ENTRY(234),
358 HNS3_RX_PTYPE_UNUSED_ENTRY(235),
359 HNS3_RX_PTYPE_UNUSED_ENTRY(236),
360 HNS3_RX_PTYPE_UNUSED_ENTRY(237),
361 HNS3_RX_PTYPE_UNUSED_ENTRY(238),
362 HNS3_RX_PTYPE_UNUSED_ENTRY(239),
363 HNS3_RX_PTYPE_UNUSED_ENTRY(240),
364 HNS3_RX_PTYPE_UNUSED_ENTRY(241),
365 HNS3_RX_PTYPE_UNUSED_ENTRY(242),
366 HNS3_RX_PTYPE_UNUSED_ENTRY(243),
367 HNS3_RX_PTYPE_UNUSED_ENTRY(244),
368 HNS3_RX_PTYPE_UNUSED_ENTRY(245),
369 HNS3_RX_PTYPE_UNUSED_ENTRY(246),
370 HNS3_RX_PTYPE_UNUSED_ENTRY(247),
371 HNS3_RX_PTYPE_UNUSED_ENTRY(248),
372 HNS3_RX_PTYPE_UNUSED_ENTRY(249),
373 HNS3_RX_PTYPE_UNUSED_ENTRY(250),
374 HNS3_RX_PTYPE_UNUSED_ENTRY(251),
375 HNS3_RX_PTYPE_UNUSED_ENTRY(252),
376 HNS3_RX_PTYPE_UNUSED_ENTRY(253),
377 HNS3_RX_PTYPE_UNUSED_ENTRY(254),
378 HNS3_RX_PTYPE_UNUSED_ENTRY(255),
379};
380
381#define HNS3_INVALID_PTYPE \
382 ARRAY_SIZE(hns3_rx_ptype_tbl)
383
ef0c5009 384static irqreturn_t hns3_irq_handle(int irq, void *vector)
76ad4f0e 385{
ef0c5009 386 struct hns3_enet_tqp_vector *tqp_vector = vector;
76ad4f0e 387
fb00331b 388 napi_schedule_irqoff(&tqp_vector->napi);
307ea4ce 389 tqp_vector->event_cnt++;
76ad4f0e
S		 390
391 return IRQ_HANDLED;
392}
393
394static void hns3_nic_uninit_irq(struct hns3_nic_priv *priv)
395{
396 struct hns3_enet_tqp_vector *tqp_vectors;
397 unsigned int i;
398
399 for (i = 0; i < priv->vector_num; i++) {
400 tqp_vectors = &priv->tqp_vector[i];
401
402 if (tqp_vectors->irq_init_flag != HNS3_VECTOR_INITED)
403 continue;
404
ffab9691 405 /* clear the affinity mask */
874bff0b
PL		 406 irq_set_affinity_hint(tqp_vectors->vector_irq, NULL);
407
76ad4f0e
S		 408 /* release the irq resource */
409 free_irq(tqp_vectors->vector_irq, tqp_vectors);
410 tqp_vectors->irq_init_flag = HNS3_VECTOR_NOT_INITED;
411 }
412}
413
414static int hns3_nic_init_irq(struct hns3_nic_priv *priv)
415{
416 struct hns3_enet_tqp_vector *tqp_vectors;
417 int txrx_int_idx = 0;
418 int rx_int_idx = 0;
419 int tx_int_idx = 0;
420 unsigned int i;
421 int ret;
422
423 for (i = 0; i < priv->vector_num; i++) {
424 tqp_vectors = &priv->tqp_vector[i];
425
426 if (tqp_vectors->irq_init_flag == HNS3_VECTOR_INITED)
427 continue;
428
429 if (tqp_vectors->tx_group.ring && tqp_vectors->rx_group.ring) {
5bffde62
YL		 430 snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN,
431 "%s-%s-%s-%d", hns3_driver_name,
432 pci_name(priv->ae_handle->pdev),
433 "TxRx", txrx_int_idx++);
76ad4f0e
S		 434 txrx_int_idx++;
435 } else if (tqp_vectors->rx_group.ring) {
5bffde62
YL		 436 snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN,
437 "%s-%s-%s-%d", hns3_driver_name,
438 pci_name(priv->ae_handle->pdev),
439 "Rx", rx_int_idx++);
76ad4f0e 440 } else if (tqp_vectors->tx_group.ring) {
5bffde62
YL		 441 snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN,
442 "%s-%s-%s-%d", hns3_driver_name,
443 pci_name(priv->ae_handle->pdev),
444 "Tx", tx_int_idx++);
76ad4f0e
S		 445 } else {
446 /* Skip this unused q_vector */
447 continue;
448 }
449
450 tqp_vectors->name[HNAE3_INT_NAME_LEN - 1] = '\0';
451
e99a308d 452 irq_set_status_flags(tqp_vectors->vector_irq, IRQ_NOAUTOEN);
76ad4f0e 453 ret = request_irq(tqp_vectors->vector_irq, hns3_irq_handle, 0,
9b2f3477 454 tqp_vectors->name, tqp_vectors);
76ad4f0e
S		 455 if (ret) {
456 netdev_err(priv->netdev, "request irq(%d) fail\n",
457 tqp_vectors->vector_irq);
d547ecdc 458 hns3_nic_uninit_irq(priv);
76ad4f0e
S		 459 return ret;
460 }
461
874bff0b
PL		 462 irq_set_affinity_hint(tqp_vectors->vector_irq,
463 &tqp_vectors->affinity_mask);
464
76ad4f0e
S		 465 tqp_vectors->irq_init_flag = HNS3_VECTOR_INITED;
466 }
467
468 return 0;
469}
470
471static void hns3_mask_vector_irq(struct hns3_enet_tqp_vector *tqp_vector,
472 u32 mask_en)
473{
474 writel(mask_en, tqp_vector->mask_addr);
475}
476
477static void hns3_vector_enable(struct hns3_enet_tqp_vector *tqp_vector)
478{
479 napi_enable(&tqp_vector->napi);
08a10068 480 enable_irq(tqp_vector->vector_irq);
76ad4f0e
S		 481
482 /* enable vector */
483 hns3_mask_vector_irq(tqp_vector, 1);
484}
485
486static void hns3_vector_disable(struct hns3_enet_tqp_vector *tqp_vector)
487{
488 /* disable vector */
489 hns3_mask_vector_irq(tqp_vector, 0);
490
491 disable_irq(tqp_vector->vector_irq);
492 napi_disable(&tqp_vector->napi);
307ea4ce
HT		 493 cancel_work_sync(&tqp_vector->rx_group.dim.work);
494 cancel_work_sync(&tqp_vector->tx_group.dim.work);
76ad4f0e
S		 495}
496
434776a5
FL		 497void hns3_set_vector_coalesce_rl(struct hns3_enet_tqp_vector *tqp_vector,
498 u32 rl_value)
76ad4f0e 499{
434776a5
FL		 500 u32 rl_reg = hns3_rl_usec_to_reg(rl_value);
501
76ad4f0e
S		 502 /* this defines the configuration for RL (Interrupt Rate Limiter).
503 * Rl defines rate of interrupts i.e. number of interrupts-per-second
504 * GL and RL(Rate Limiter) are 2 ways to acheive interrupt coalescing
505 */
de25bcc4
HT		 506 if (rl_reg > 0 && !tqp_vector->tx_group.coal.adapt_enable &&
507 !tqp_vector->rx_group.coal.adapt_enable)
434776a5
FL		 508 /* According to the hardware, the range of rl_reg is
509 * 0-59 and the unit is 4.
510 */
511 rl_reg |= HNS3_INT_RL_ENABLE_MASK;
512
513 writel(rl_reg, tqp_vector->mask_addr + HNS3_VECTOR_RL_OFFSET);
514}
515
516void hns3_set_vector_coalesce_rx_gl(struct hns3_enet_tqp_vector *tqp_vector,
517 u32 gl_value)
518{
5ac84b02 519 u32 new_val;
434776a5 520
5ac84b02
HT		 521 if (tqp_vector->rx_group.coal.unit_1us)
522 new_val = gl_value | HNS3_INT_GL_1US;
523 else
524 new_val = hns3_gl_usec_to_reg(gl_value);
525
526 writel(new_val, tqp_vector->mask_addr + HNS3_VECTOR_GL0_OFFSET);
434776a5
FL		 527}
528
529void hns3_set_vector_coalesce_tx_gl(struct hns3_enet_tqp_vector *tqp_vector,
530 u32 gl_value)
531{
5ac84b02
HT		 532 u32 new_val;
533
534 if (tqp_vector->tx_group.coal.unit_1us)
535 new_val = gl_value | HNS3_INT_GL_1US;
536 else
537 new_val = hns3_gl_usec_to_reg(gl_value);
434776a5 538
5ac84b02 539 writel(new_val, tqp_vector->mask_addr + HNS3_VECTOR_GL1_OFFSET);
76ad4f0e
S		 540}
541
91bfae25
HT		 542void hns3_set_vector_coalesce_tx_ql(struct hns3_enet_tqp_vector *tqp_vector,
543 u32 ql_value)
76ad4f0e 544{
91bfae25
HT		 545 writel(ql_value, tqp_vector->mask_addr + HNS3_VECTOR_TX_QL_OFFSET);
546}
547
548void hns3_set_vector_coalesce_rx_ql(struct hns3_enet_tqp_vector *tqp_vector,
549 u32 ql_value)
550{
551 writel(ql_value, tqp_vector->mask_addr + HNS3_VECTOR_RX_QL_OFFSET);
552}
553
554static void hns3_vector_coalesce_init(struct hns3_enet_tqp_vector *tqp_vector,
555 struct hns3_nic_priv *priv)
556{
557 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(priv->ae_handle->pdev);
558 struct hns3_enet_coalesce *tx_coal = &tqp_vector->tx_group.coal;
559 struct hns3_enet_coalesce *rx_coal = &tqp_vector->rx_group.coal;
73a13d8d
HT		 560 struct hns3_enet_coalesce *ptx_coal = &priv->tx_coal;
561 struct hns3_enet_coalesce *prx_coal = &priv->rx_coal;
91bfae25 562
73a13d8d
HT		 563 tx_coal->adapt_enable = ptx_coal->adapt_enable;
564 rx_coal->adapt_enable = prx_coal->adapt_enable;
91bfae25 565
73a13d8d
HT		 566 tx_coal->int_gl = ptx_coal->int_gl;
567 rx_coal->int_gl = prx_coal->int_gl;
5fd4789a 568
73a13d8d
HT		 569 rx_coal->flow_level = prx_coal->flow_level;
570 tx_coal->flow_level = ptx_coal->flow_level;
5fd4789a 571
5ac84b02
HT		 572 /* device version above V3(include V3), GL can configure 1us
573 * unit, so uses 1us unit.
574 */
575 if (ae_dev->dev_version >= HNAE3_DEVICE_VERSION_V3) {
576 tx_coal->unit_1us = 1;
577 rx_coal->unit_1us = 1;
578 }
579
91bfae25
HT		 580 if (ae_dev->dev_specs.int_ql_max) {
581 tx_coal->ql_enable = 1;
582 rx_coal->ql_enable = 1;
583 tx_coal->int_ql_max = ae_dev->dev_specs.int_ql_max;
584 rx_coal->int_ql_max = ae_dev->dev_specs.int_ql_max;
73a13d8d
HT		 585 tx_coal->int_ql = ptx_coal->int_ql;
586 rx_coal->int_ql = prx_coal->int_ql;
91bfae25 587 }
76ad4f0e
S		 588}
589
91bfae25
HT		 590static void
591hns3_vector_coalesce_init_hw(struct hns3_enet_tqp_vector *tqp_vector,
592 struct hns3_nic_priv *priv)
dd38c726 593{
91bfae25
HT		 594 struct hns3_enet_coalesce *tx_coal = &tqp_vector->tx_group.coal;
595 struct hns3_enet_coalesce *rx_coal = &tqp_vector->rx_group.coal;
dd38c726
YL		 596 struct hnae3_handle *h = priv->ae_handle;
597
91bfae25
HT		 598 hns3_set_vector_coalesce_tx_gl(tqp_vector, tx_coal->int_gl);
599 hns3_set_vector_coalesce_rx_gl(tqp_vector, rx_coal->int_gl);
dd38c726 600 hns3_set_vector_coalesce_rl(tqp_vector, h->kinfo.int_rl_setting);
91bfae25
HT		 601
602 if (tx_coal->ql_enable)
603 hns3_set_vector_coalesce_tx_ql(tqp_vector, tx_coal->int_ql);
604
605 if (rx_coal->ql_enable)
606 hns3_set_vector_coalesce_rx_ql(tqp_vector, rx_coal->int_ql);
dd38c726
YL		 607}
608
9df8f79a
YL		 609static int hns3_nic_set_real_num_queue(struct net_device *netdev)
610{
9780cb97 611 struct hnae3_handle *h = hns3_get_handle(netdev);
9df8f79a 612 struct hnae3_knic_private_info *kinfo = &h->kinfo;
35244430 613 struct hnae3_tc_info *tc_info = &kinfo->tc_info;
5a5c9091 614 unsigned int queue_size = kinfo->num_tqps;
a75a8efa
YL		 615 int i, ret;
616
5a5c9091 617 if (tc_info->num_tc <= 1 && !tc_info->mqprio_active) {
a75a8efa
YL		 618 netdev_reset_tc(netdev);
619 } else {
35244430 620 ret = netdev_set_num_tc(netdev, tc_info->num_tc);
a75a8efa
YL		 621 if (ret) {
622 netdev_err(netdev,
623 "netdev_set_num_tc fail, ret=%d!\n", ret);
624 return ret;
625 }
626
a8e76fef 627 for (i = 0; i < tc_info->num_tc; i++)
35244430
JS		 628 netdev_set_tc_queue(netdev, i, tc_info->tqp_count[i],
629 tc_info->tqp_offset[i]);
a75a8efa 630 }
9df8f79a
YL		 631
632 ret = netif_set_real_num_tx_queues(netdev, queue_size);
633 if (ret) {
634 netdev_err(netdev,
9b2f3477 635 "netif_set_real_num_tx_queues fail, ret=%d!\n", ret);
9df8f79a
YL		 636 return ret;
637 }
638
639 ret = netif_set_real_num_rx_queues(netdev, queue_size);
640 if (ret) {
641 netdev_err(netdev,
642 "netif_set_real_num_rx_queues fail, ret=%d!\n", ret);
643 return ret;
644 }
645
646 return 0;
647}
648
77e91848 649u16 hns3_get_max_available_channels(struct hnae3_handle *h)
678335a1 650{
0d43bf45 651 u16 alloc_tqps, max_rss_size, rss_size;
678335a1 652
0d43bf45 653 h->ae_algo->ops->get_tqps_and_rss_info(h, &alloc_tqps, &max_rss_size);
35244430 654 rss_size = alloc_tqps / h->kinfo.tc_info.num_tc;
678335a1 655
0d43bf45 656 return min_t(u16, rss_size, max_rss_size);
678335a1
PL		 657}
658
8df0fa91
HT		 659static void hns3_tqp_enable(struct hnae3_queue *tqp)
660{
661 u32 rcb_reg;
662
663 rcb_reg = hns3_read_dev(tqp, HNS3_RING_EN_REG);
664 rcb_reg |= BIT(HNS3_RING_EN_B);
665 hns3_write_dev(tqp, HNS3_RING_EN_REG, rcb_reg);
666}
667
668static void hns3_tqp_disable(struct hnae3_queue *tqp)
669{
670 u32 rcb_reg;
671
672 rcb_reg = hns3_read_dev(tqp, HNS3_RING_EN_REG);
673 rcb_reg &= ~BIT(HNS3_RING_EN_B);
674 hns3_write_dev(tqp, HNS3_RING_EN_REG, rcb_reg);
675}
676
ffab9691
JS		 677static void hns3_free_rx_cpu_rmap(struct net_device *netdev)
678{
679#ifdef CONFIG_RFS_ACCEL
680 free_irq_cpu_rmap(netdev->rx_cpu_rmap);
681 netdev->rx_cpu_rmap = NULL;
682#endif
683}
684
685static int hns3_set_rx_cpu_rmap(struct net_device *netdev)
686{
687#ifdef CONFIG_RFS_ACCEL
688 struct hns3_nic_priv *priv = netdev_priv(netdev);
689 struct hns3_enet_tqp_vector *tqp_vector;
690 int i, ret;
691
692 if (!netdev->rx_cpu_rmap) {
693 netdev->rx_cpu_rmap = alloc_irq_cpu_rmap(priv->vector_num);
694 if (!netdev->rx_cpu_rmap)
695 return -ENOMEM;
696 }
697
698 for (i = 0; i < priv->vector_num; i++) {
699 tqp_vector = &priv->tqp_vector[i];
700 ret = irq_cpu_rmap_add(netdev->rx_cpu_rmap,
701 tqp_vector->vector_irq);
702 if (ret) {
703 hns3_free_rx_cpu_rmap(netdev);
704 return ret;
705 }
706 }
707#endif
708 return 0;
709}
710
76ad4f0e
S		 711static int hns3_nic_net_up(struct net_device *netdev)
712{
713 struct hns3_nic_priv *priv = netdev_priv(netdev);
714 struct hnae3_handle *h = priv->ae_handle;
715 int i, j;
716 int ret;
717
7b763f3f
FL		 718 ret = hns3_nic_reset_all_ring(h);
719 if (ret)
720 return ret;
721
b7b585c2
JS		 722 clear_bit(HNS3_NIC_STATE_DOWN, &priv->state);
723
76ad4f0e
S		 724 /* enable the vectors */
725 for (i = 0; i < priv->vector_num; i++)
726 hns3_vector_enable(&priv->tqp_vector[i]);
727
8df0fa91
HT		 728 /* enable rcb */
729 for (j = 0; j < h->kinfo.num_tqps; j++)
730 hns3_tqp_enable(h->kinfo.tqp[j]);
731
76ad4f0e
S		 732 /* start the ae_dev */
733 ret = h->ae_algo->ops->start ? h->ae_algo->ops->start(h) : 0;
08a10068
YL		 734 if (ret) {
735 set_bit(HNS3_NIC_STATE_DOWN, &priv->state);
736 while (j--)
737 hns3_tqp_disable(h->kinfo.tqp[j]);
8df0fa91 738
08a10068
YL		 739 for (j = i - 1; j >= 0; j--)
740 hns3_vector_disable(&priv->tqp_vector[j]);
741 }
76ad4f0e 742
76ad4f0e
S		 743 return ret;
744}
745
2a73ac3e
YL		 746static void hns3_config_xps(struct hns3_nic_priv *priv)
747{
748 int i;
749
750 for (i = 0; i < priv->vector_num; i++) {
751 struct hns3_enet_tqp_vector *tqp_vector = &priv->tqp_vector[i];
752 struct hns3_enet_ring *ring = tqp_vector->tx_group.ring;
753
754 while (ring) {
755 int ret;
756
757 ret = netif_set_xps_queue(priv->netdev,
758 &tqp_vector->affinity_mask,
759 ring->tqp->tqp_index);
760 if (ret)
761 netdev_warn(priv->netdev,
762 "set xps queue failed: %d", ret);
763
764 ring = ring->next;
765 }
766 }
767}
768
76ad4f0e
S		 769static int hns3_nic_net_open(struct net_device *netdev)
770{
8cdb992f 771 struct hns3_nic_priv *priv = netdev_priv(netdev);
a75a8efa
YL		 772 struct hnae3_handle *h = hns3_get_handle(netdev);
773 struct hnae3_knic_private_info *kinfo;
774 int i, ret;
76ad4f0e 775
257e4f29
HT		 776 if (hns3_nic_resetting(netdev))
777 return -EBUSY;
778
5b09e88e
JS		 779 if (!test_bit(HNS3_NIC_STATE_DOWN, &priv->state)) {
780 netdev_warn(netdev, "net open repeatedly!\n");
781 return 0;
782 }
783
76ad4f0e
S		 784 netif_carrier_off(netdev);
785
9df8f79a
YL		 786 ret = hns3_nic_set_real_num_queue(netdev);
787 if (ret)
76ad4f0e 788 return ret;
76ad4f0e
S		 789
790 ret = hns3_nic_net_up(netdev);
791 if (ret) {
9b2f3477 792 netdev_err(netdev, "net up fail, ret=%d!\n", ret);
76ad4f0e
S		 793 return ret;
794 }
795
a75a8efa 796 kinfo = &h->kinfo;
9b2f3477 797 for (i = 0; i < HNAE3_MAX_USER_PRIO; i++)
35244430 798 netdev_set_prio_tc_map(netdev, i, kinfo->tc_info.prio_tc[i]);
a75a8efa 799
8cdb992f
JS		 800 if (h->ae_algo->ops->set_timer_task)
801 h->ae_algo->ops->set_timer_task(priv->ae_handle, true);
802
2a73ac3e 803 hns3_config_xps(priv);
1c822948
YL		 804
805 netif_dbg(h, drv, netdev, "net open\n");
806
76ad4f0e
S		 807 return 0;
808}
809
f96315f2
HT		 810static void hns3_reset_tx_queue(struct hnae3_handle *h)
811{
812 struct net_device *ndev = h->kinfo.netdev;
813 struct hns3_nic_priv *priv = netdev_priv(ndev);
814 struct netdev_queue *dev_queue;
815 u32 i;
816
817 for (i = 0; i < h->kinfo.num_tqps; i++) {
818 dev_queue = netdev_get_tx_queue(ndev,
5f06b903 819 priv->ring[i].queue_index);
f96315f2
HT		 820 netdev_tx_reset_queue(dev_queue);
821 }
822}
823
76ad4f0e
S		 824static void hns3_nic_net_down(struct net_device *netdev)
825{
826 struct hns3_nic_priv *priv = netdev_priv(netdev);
8df0fa91 827 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 828 const struct hnae3_ae_ops *ops;
829 int i;
830
7b763f3f
FL		 831 /* disable vectors */
832 for (i = 0; i < priv->vector_num; i++)
833 hns3_vector_disable(&priv->tqp_vector[i]);
8df0fa91
HT		 834
835 /* disable rcb */
836 for (i = 0; i < h->kinfo.num_tqps; i++)
837 hns3_tqp_disable(h->kinfo.tqp[i]);
7b763f3f 838
76ad4f0e
S		 839 /* stop ae_dev */
840 ops = priv->ae_handle->ae_algo->ops;
841 if (ops->stop)
842 ops->stop(priv->ae_handle);
843
3a30964a
YL		 844 /* delay ring buffer clearing to hns3_reset_notify_uninit_enet
845 * during reset process, because driver may not be able
846 * to disable the ring through firmware when downing the netdev.
847 */
848 if (!hns3_nic_resetting(netdev))
f96315f2
HT		 849 hns3_clear_all_ring(priv->ae_handle, false);
850
851 hns3_reset_tx_queue(priv->ae_handle);
76ad4f0e
S		 852}
853
854static int hns3_nic_net_stop(struct net_device *netdev)
855{
ff0699e0 856 struct hns3_nic_priv *priv = netdev_priv(netdev);
8cdb992f 857 struct hnae3_handle *h = hns3_get_handle(netdev);
ff0699e0
HT		 858
859 if (test_and_set_bit(HNS3_NIC_STATE_DOWN, &priv->state))
860 return 0;
861
1c822948
YL		 862 netif_dbg(h, drv, netdev, "net stop\n");
863
8cdb992f
JS		 864 if (h->ae_algo->ops->set_timer_task)
865 h->ae_algo->ops->set_timer_task(priv->ae_handle, false);
866
76ad4f0e 867 netif_carrier_off(netdev);
b416e872 868 netif_tx_disable(netdev);
76ad4f0e
S		 869
870 hns3_nic_net_down(netdev);
871
872 return 0;
873}
874
76ad4f0e
S		 875static int hns3_nic_uc_sync(struct net_device *netdev,
876 const unsigned char *addr)
877{
9780cb97 878 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 879
880 if (h->ae_algo->ops->add_uc_addr)
881 return h->ae_algo->ops->add_uc_addr(h, addr);
882
883 return 0;
884}
885
886static int hns3_nic_uc_unsync(struct net_device *netdev,
887 const unsigned char *addr)
888{
9780cb97 889 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e 890
ee4bcd3b
JS		 891 /* need ignore the request of removing device address, because
892 * we store the device address and other addresses of uc list
893 * in the function's mac filter list.
894 */
895 if (ether_addr_equal(addr, netdev->dev_addr))
896 return 0;
897
76ad4f0e
S		 898 if (h->ae_algo->ops->rm_uc_addr)
899 return h->ae_algo->ops->rm_uc_addr(h, addr);
900
901 return 0;
902}
903
904static int hns3_nic_mc_sync(struct net_device *netdev,
905 const unsigned char *addr)
906{
9780cb97 907 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e 908
720a8478 909 if (h->ae_algo->ops->add_mc_addr)
76ad4f0e
S		 910 return h->ae_algo->ops->add_mc_addr(h, addr);
911
912 return 0;
913}
914
915static int hns3_nic_mc_unsync(struct net_device *netdev,
916 const unsigned char *addr)
917{
9780cb97 918 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e 919
720a8478 920 if (h->ae_algo->ops->rm_mc_addr)
76ad4f0e
S		 921 return h->ae_algo->ops->rm_mc_addr(h, addr);
922
923 return 0;
924}
925
c60edc17
JS		 926static u8 hns3_get_netdev_flags(struct net_device *netdev)
927{
928 u8 flags = 0;
929
2ba30662 930 if (netdev->flags & IFF_PROMISC)
28673b33 931 flags = HNAE3_USER_UPE | HNAE3_USER_MPE | HNAE3_BPE;
2ba30662
JS		 932 else if (netdev->flags & IFF_ALLMULTI)
933 flags = HNAE3_USER_MPE;
c60edc17
JS		 934
935 return flags;
936}
937
1db9b1bf 938static void hns3_nic_set_rx_mode(struct net_device *netdev)
76ad4f0e 939{
9780cb97 940 struct hnae3_handle *h = hns3_get_handle(netdev);
c60edc17 941 u8 new_flags;
76ad4f0e 942
c60edc17
JS		 943 new_flags = hns3_get_netdev_flags(netdev);
944
c631c696
JS		 945 __dev_uc_sync(netdev, hns3_nic_uc_sync, hns3_nic_uc_unsync);
946 __dev_mc_sync(netdev, hns3_nic_mc_sync, hns3_nic_mc_unsync);
c60edc17 947
c60edc17 948 /* User mode Promisc mode enable and vlan filtering is disabled to
c631c696 949 * let all packets in.
c60edc17 950 */
c60edc17 951 h->netdev_flags = new_flags;
c631c696
JS		 952 hns3_request_update_promisc_mode(h);
953}
954
955void hns3_request_update_promisc_mode(struct hnae3_handle *handle)
956{
957 const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
958
959 if (ops->request_update_promisc_mode)
960 ops->request_update_promisc_mode(handle);
c60edc17
JS		 961}
962
907676b1
YL		 963static u32 hns3_tx_spare_space(struct hns3_enet_ring *ring)
964{
965 struct hns3_tx_spare *tx_spare = ring->tx_spare;
966 u32 ntc, ntu;
967
968 /* This smp_load_acquire() pairs with smp_store_release() in
969 * hns3_tx_spare_update() called in tx desc cleaning process.
970 */
971 ntc = smp_load_acquire(&tx_spare->last_to_clean);
972 ntu = tx_spare->next_to_use;
973
974 if (ntc > ntu)
975 return ntc - ntu - 1;
976
977 /* The free tx buffer is divided into two part, so pick the
978 * larger one.
979 */
38b99e1e 980 return max(ntc, tx_spare->len - ntu) - 1;
907676b1
YL		 981}
982
983static void hns3_tx_spare_update(struct hns3_enet_ring *ring)
984{
985 struct hns3_tx_spare *tx_spare = ring->tx_spare;
986
987 if (!tx_spare ||
988 tx_spare->last_to_clean == tx_spare->next_to_clean)
989 return;
990
991 /* This smp_store_release() pairs with smp_load_acquire() in
992 * hns3_tx_spare_space() called in xmit process.
993 */
994 smp_store_release(&tx_spare->last_to_clean,
995 tx_spare->next_to_clean);
996}
997
998static bool hns3_can_use_tx_bounce(struct hns3_enet_ring *ring,
999 struct sk_buff *skb,
1000 u32 space)
1001{
1002 u32 len = skb->len <= ring->tx_copybreak ? skb->len :
1003 skb_headlen(skb);
1004
1005 if (len > ring->tx_copybreak)
1006 return false;
1007
1008 if (ALIGN(len, dma_get_cache_alignment()) > space) {
1009 u64_stats_update_begin(&ring->syncp);
1010 ring->stats.tx_spare_full++;
1011 u64_stats_update_end(&ring->syncp);
1012 return false;
1013 }
1014
1015 return true;
1016}
1017
7459775e
YL		 1018static bool hns3_can_use_tx_sgl(struct hns3_enet_ring *ring,
1019 struct sk_buff *skb,
1020 u32 space)
1021{
1022 if (skb->len <= ring->tx_copybreak || !tx_sgl ||
1023 (!skb_has_frag_list(skb) &&
1024 skb_shinfo(skb)->nr_frags < tx_sgl))
1025 return false;
1026
1027 if (space < HNS3_MAX_SGL_SIZE) {
1028 u64_stats_update_begin(&ring->syncp);
1029 ring->stats.tx_spare_full++;
1030 u64_stats_update_end(&ring->syncp);
1031 return false;
1032 }
1033
1034 return true;
1035}
1036
907676b1
YL		 1037static void hns3_init_tx_spare_buffer(struct hns3_enet_ring *ring)
1038{
1039 struct hns3_tx_spare *tx_spare;
1040 struct page *page;
1a00197b 1041 u32 alloc_size;
907676b1
YL		 1042 dma_addr_t dma;
1043 int order;
1044
1a00197b
HT		 1045 alloc_size = tx_spare_buf_size ? tx_spare_buf_size :
1046 ring->tqp->handle->kinfo.tx_spare_buf_size;
1047 if (!alloc_size)
907676b1
YL		 1048 return;
1049
1a00197b 1050 order = get_order(alloc_size);
907676b1
YL		 1051 tx_spare = devm_kzalloc(ring_to_dev(ring), sizeof(*tx_spare),
1052 GFP_KERNEL);
1053 if (!tx_spare) {
1054 /* The driver still work without the tx spare buffer */
1055 dev_warn(ring_to_dev(ring), "failed to allocate hns3_tx_spare\n");
1056 return;
1057 }
1058
1059 page = alloc_pages_node(dev_to_node(ring_to_dev(ring)),
1060 GFP_KERNEL, order);
1061 if (!page) {
1062 dev_warn(ring_to_dev(ring), "failed to allocate tx spare pages\n");
1063 devm_kfree(ring_to_dev(ring), tx_spare);
1064 return;
1065 }
1066
1067 dma = dma_map_page(ring_to_dev(ring), page, 0,
1068 PAGE_SIZE << order, DMA_TO_DEVICE);
1069 if (dma_mapping_error(ring_to_dev(ring), dma)) {
1070 dev_warn(ring_to_dev(ring), "failed to map pages for tx spare\n");
1071 put_page(page);
1072 devm_kfree(ring_to_dev(ring), tx_spare);
1073 return;
1074 }
1075
1076 tx_spare->dma = dma;
1077 tx_spare->buf = page_address(page);
1078 tx_spare->len = PAGE_SIZE << order;
1079 ring->tx_spare = tx_spare;
1080}
1081
1082/* Use hns3_tx_spare_space() to make sure there is enough buffer
1083 * before calling below function to allocate tx buffer.
1084 */
1085static void *hns3_tx_spare_alloc(struct hns3_enet_ring *ring,
1086 unsigned int size, dma_addr_t *dma,
1087 u32 *cb_len)
1088{
1089 struct hns3_tx_spare *tx_spare = ring->tx_spare;
1090 u32 ntu = tx_spare->next_to_use;
1091
1092 size = ALIGN(size, dma_get_cache_alignment());
1093 *cb_len = size;
1094
1095 /* Tx spare buffer wraps back here because the end of
1096 * freed tx buffer is not enough.
1097 */
1098 if (ntu + size > tx_spare->len) {
1099 *cb_len += (tx_spare->len - ntu);
1100 ntu = 0;
1101 }
1102
1103 tx_spare->next_to_use = ntu + size;
1104 if (tx_spare->next_to_use == tx_spare->len)
1105 tx_spare->next_to_use = 0;
1106
1107 *dma = tx_spare->dma + ntu;
1108
1109 return tx_spare->buf + ntu;
1110}
1111
1112static void hns3_tx_spare_rollback(struct hns3_enet_ring *ring, u32 len)
1113{
1114 struct hns3_tx_spare *tx_spare = ring->tx_spare;
1115
1116 if (len > tx_spare->next_to_use) {
1117 len -= tx_spare->next_to_use;
1118 tx_spare->next_to_use = tx_spare->len - len;
1119 } else {
1120 tx_spare->next_to_use -= len;
1121 }
1122}
1123
1124static void hns3_tx_spare_reclaim_cb(struct hns3_enet_ring *ring,
1125 struct hns3_desc_cb *cb)
1126{
1127 struct hns3_tx_spare *tx_spare = ring->tx_spare;
1128 u32 ntc = tx_spare->next_to_clean;
1129 u32 len = cb->length;
1130
1131 tx_spare->next_to_clean += len;
1132
1133 if (tx_spare->next_to_clean >= tx_spare->len) {
1134 tx_spare->next_to_clean -= tx_spare->len;
1135
1136 if (tx_spare->next_to_clean) {
1137 ntc = 0;
1138 len = tx_spare->next_to_clean;
1139 }
1140 }
1141
1142 /* This tx spare buffer is only really reclaimed after calling
1143 * hns3_tx_spare_update(), so it is still safe to use the info in
7459775e
YL		 1144 * the tx buffer to do the dma sync or sg unmapping after
1145 * tx_spare->next_to_clean is moved forword.
907676b1
YL		 1146 */
1147 if (cb->type & (DESC_TYPE_BOUNCE_HEAD | DESC_TYPE_BOUNCE_ALL)) {
1148 dma_addr_t dma = tx_spare->dma + ntc;
1149
1150 dma_sync_single_for_cpu(ring_to_dev(ring), dma, len,
1151 DMA_TO_DEVICE);
7459775e
YL		 1152 } else {
1153 struct sg_table *sgt = tx_spare->buf + ntc;
1154
1155 dma_unmap_sg(ring_to_dev(ring), sgt->sgl, sgt->orig_nents,
1156 DMA_TO_DEVICE);
907676b1
YL		 1157 }
1158}
1159
3e281621 1160static int hns3_set_tso(struct sk_buff *skb, u32 *paylen_fdop_ol4cs,
811c0830 1161 u16 *mss, u32 *type_cs_vlan_tso, u32 *send_bytes)
76ad4f0e
S		 1162{
1163 u32 l4_offset, hdr_len;
1164 union l3_hdr_info l3;
1165 union l4_hdr_info l4;
1166 u32 l4_paylen;
1167 int ret;
1168
1169 if (!skb_is_gso(skb))
1170 return 0;
1171
1172 ret = skb_cow_head(skb, 0);
8ae10cfb 1173 if (unlikely(ret < 0))
76ad4f0e
S		 1174 return ret;
1175
1176 l3.hdr = skb_network_header(skb);
1177 l4.hdr = skb_transport_header(skb);
1178
1179 /* Software should clear the IPv4's checksum field when tso is
1180 * needed.
1181 */
1182 if (l3.v4->version == 4)
1183 l3.v4->check = 0;
1184
9b2f3477 1185 /* tunnel packet */
76ad4f0e
S		 1186 if (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE |
1187 SKB_GSO_GRE_CSUM |
3e281621
HT		 1188 SKB_GSO_UDP_TUNNEL |
1189 SKB_GSO_UDP_TUNNEL_CSUM)) {
76ad4f0e
S		 1190 /* reset l3&l4 pointers from outer to inner headers */
1191 l3.hdr = skb_inner_network_header(skb);
1192 l4.hdr = skb_inner_transport_header(skb);
1193
1194 /* Software should clear the IPv4's checksum field when
1195 * tso is needed.
1196 */
1197 if (l3.v4->version == 4)
1198 l3.v4->check = 0;
1199 }
1200
9b2f3477 1201 /* normal or tunnel packet */
76ad4f0e 1202 l4_offset = l4.hdr - skb->data;
76ad4f0e 1203
9b2f3477 1204 /* remove payload length from inner pseudo checksum when tso */
76ad4f0e 1205 l4_paylen = skb->len - l4_offset;
0692cfe9
HT		 1206
1207 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) {
1208 hdr_len = sizeof(*l4.udp) + l4_offset;
1209 csum_replace_by_diff(&l4.udp->check,
1210 (__force __wsum)htonl(l4_paylen));
1211 } else {
1212 hdr_len = (l4.tcp->doff << 2) + l4_offset;
1213 csum_replace_by_diff(&l4.tcp->check,
1214 (__force __wsum)htonl(l4_paylen));
1215 }
76ad4f0e 1216
811c0830
YL		 1217 *send_bytes = (skb_shinfo(skb)->gso_segs - 1) * hdr_len + skb->len;
1218
76ad4f0e 1219 /* find the txbd field values */
3e281621 1220 *paylen_fdop_ol4cs = skb->len - hdr_len;
cde4ffad 1221 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_TSO_B, 1);
76ad4f0e 1222
3e281621
HT		 1223 /* offload outer UDP header checksum */
1224 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM)
1225 hns3_set_field(*paylen_fdop_ol4cs, HNS3_TXD_OL4CS_B, 1);
1226
76ad4f0e
S		 1227 /* get MSS for TSO */
1228 *mss = skb_shinfo(skb)->gso_size;
1229
698a8954
YL		 1230 trace_hns3_tso(skb);
1231
76ad4f0e
S		 1232 return 0;
1233}
1234
1898d4e4
S		 1235static int hns3_get_l4_protocol(struct sk_buff *skb, u8 *ol4_proto,
1236 u8 *il4_proto)
76ad4f0e 1237{
1a6e552d 1238 union l3_hdr_info l3;
76ad4f0e
S		 1239 unsigned char *l4_hdr;
1240 unsigned char *exthdr;
1241 u8 l4_proto_tmp;
1242 __be16 frag_off;
1243
1244 /* find outer header point */
1245 l3.hdr = skb_network_header(skb);
35f58fd7 1246 l4_hdr = skb_transport_header(skb);
76ad4f0e
S		 1247
1248 if (skb->protocol == htons(ETH_P_IPV6)) {
1249 exthdr = l3.hdr + sizeof(*l3.v6);
1250 l4_proto_tmp = l3.v6->nexthdr;
1251 if (l4_hdr != exthdr)
1252 ipv6_skip_exthdr(skb, exthdr - skb->data,
1253 &l4_proto_tmp, &frag_off);
1254 } else if (skb->protocol == htons(ETH_P_IP)) {
1255 l4_proto_tmp = l3.v4->protocol;
1898d4e4
S		 1256 } else {
1257 return -EINVAL;
76ad4f0e
S		 1258 }
1259
1260 *ol4_proto = l4_proto_tmp;
1261
1262 /* tunnel packet */
1263 if (!skb->encapsulation) {
1264 *il4_proto = 0;
1898d4e4 1265 return 0;
76ad4f0e
S		 1266 }
1267
1268 /* find inner header point */
1269 l3.hdr = skb_inner_network_header(skb);
1270 l4_hdr = skb_inner_transport_header(skb);
1271
1272 if (l3.v6->version == 6) {
1273 exthdr = l3.hdr + sizeof(*l3.v6);
1274 l4_proto_tmp = l3.v6->nexthdr;
1275 if (l4_hdr != exthdr)
1276 ipv6_skip_exthdr(skb, exthdr - skb->data,
1277 &l4_proto_tmp, &frag_off);
1278 } else if (l3.v4->version == 4) {
1279 l4_proto_tmp = l3.v4->protocol;
1280 }
1281
1282 *il4_proto = l4_proto_tmp;
1898d4e4
S		 1283
1284 return 0;
76ad4f0e
S		 1285}
1286
3db084d2
YL		 1287/* when skb->encapsulation is 0, skb->ip_summed is CHECKSUM_PARTIAL
1288 * and it is udp packet, which has a dest port as the IANA assigned.
1289 * the hardware is expected to do the checksum offload, but the
1290 * hardware will not do the checksum offload when udp dest port is
905416f1 1291 * 4789, 4790 or 6081.
3db084d2
YL		 1292 */
1293static bool hns3_tunnel_csum_bug(struct sk_buff *skb)
1294{
ade36cce
HT		 1295 struct hns3_nic_priv *priv = netdev_priv(skb->dev);
1296 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(priv->ae_handle->pdev);
1a6e552d 1297 union l4_hdr_info l4;
3db084d2 1298
ade36cce
HT		 1299 /* device version above V3(include V3), the hardware can
1300 * do this checksum offload.
1301 */
1302 if (ae_dev->dev_version >= HNAE3_DEVICE_VERSION_V3)
1303 return false;
1304
3db084d2
YL		 1305 l4.hdr = skb_transport_header(skb);
1306
bea96410 1307 if (!(!skb->encapsulation &&
a156998f 1308 (l4.udp->dest == htons(IANA_VXLAN_UDP_PORT) ||
905416f1
HC		 1309 l4.udp->dest == htons(GENEVE_UDP_PORT) ||
1310 l4.udp->dest == htons(4790))))
3db084d2
YL		 1311 return false;
1312
3db084d2
YL		 1313 return true;
1314}
1315
757cd1e4
YL		 1316static void hns3_set_outer_l2l3l4(struct sk_buff *skb, u8 ol4_proto,
1317 u32 *ol_type_vlan_len_msec)
76ad4f0e 1318{
757cd1e4
YL		 1319 u32 l2_len, l3_len, l4_len;
1320 unsigned char *il2_hdr;
1a6e552d 1321 union l3_hdr_info l3;
757cd1e4 1322 union l4_hdr_info l4;
76ad4f0e
S		 1323
1324 l3.hdr = skb_network_header(skb);
757cd1e4 1325 l4.hdr = skb_transport_header(skb);
76ad4f0e 1326
757cd1e4
YL		 1327 /* compute OL2 header size, defined in 2 Bytes */
1328 l2_len = l3.hdr - skb->data;
1329 hns3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L2LEN_S, l2_len >> 1);
1330
1331 /* compute OL3 header size, defined in 4 Bytes */
1332 l3_len = l4.hdr - l3.hdr;
1333 hns3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L3LEN_S, l3_len >> 2);
76ad4f0e 1334
757cd1e4 1335 il2_hdr = skb_inner_mac_header(skb);
9b2f3477 1336 /* compute OL4 header size, defined in 4 Bytes */
757cd1e4
YL		 1337 l4_len = il2_hdr - l4.hdr;
1338 hns3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L4LEN_S, l4_len >> 2);
1339
1340 /* define outer network header type */
1341 if (skb->protocol == htons(ETH_P_IP)) {
1342 if (skb_is_gso(skb))
cde4ffad 1343 hns3_set_field(*ol_type_vlan_len_msec,
757cd1e4
YL		 1344 HNS3_TXD_OL3T_S,
1345 HNS3_OL3T_IPV4_CSUM);
1346 else
cde4ffad 1347 hns3_set_field(*ol_type_vlan_len_msec,
757cd1e4
YL		 1348 HNS3_TXD_OL3T_S,
1349 HNS3_OL3T_IPV4_NO_CSUM);
757cd1e4
YL		 1350 } else if (skb->protocol == htons(ETH_P_IPV6)) {
1351 hns3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_OL3T_S,
1352 HNS3_OL3T_IPV6);
1353 }
1354
1355 if (ol4_proto == IPPROTO_UDP)
1356 hns3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_TUNTYPE_S,
1357 HNS3_TUN_MAC_IN_UDP);
1358 else if (ol4_proto == IPPROTO_GRE)
1359 hns3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_TUNTYPE_S,
1360 HNS3_TUN_NVGRE);
1361}
1362
1363static int hns3_set_l2l3l4(struct sk_buff *skb, u8 ol4_proto,
1364 u8 il4_proto, u32 *type_cs_vlan_tso,
1365 u32 *ol_type_vlan_len_msec)
1366{
c264ed44 1367 unsigned char *l2_hdr = skb->data;
757cd1e4
YL		 1368 u32 l4_proto = ol4_proto;
1369 union l4_hdr_info l4;
1370 union l3_hdr_info l3;
1371 u32 l2_len, l3_len;
1372
1373 l4.hdr = skb_transport_header(skb);
1374 l3.hdr = skb_network_header(skb);
1375
1376 /* handle encapsulation skb */
1377 if (skb->encapsulation) {
1378 /* If this is a not UDP/GRE encapsulation skb */
1379 if (!(ol4_proto == IPPROTO_UDP || ol4_proto == IPPROTO_GRE)) {
76ad4f0e
S		 1380 /* drop the skb tunnel packet if hardware don't support,
1381 * because hardware can't calculate csum when TSO.
1382 */
1383 if (skb_is_gso(skb))
1384 return -EDOM;
1385
1386 /* the stack computes the IP header already,
1387 * driver calculate l4 checksum when not TSO.
1388 */
9bb5a495 1389 return skb_checksum_help(skb);
76ad4f0e
S		 1390 }
1391
757cd1e4
YL		 1392 hns3_set_outer_l2l3l4(skb, ol4_proto, ol_type_vlan_len_msec);
1393
1394 /* switch to inner header */
1395 l2_hdr = skb_inner_mac_header(skb);
76ad4f0e 1396 l3.hdr = skb_inner_network_header(skb);
757cd1e4 1397 l4.hdr = skb_inner_transport_header(skb);
76ad4f0e
S		 1398 l4_proto = il4_proto;
1399 }
1400
1401 if (l3.v4->version == 4) {
cde4ffad
YL		 1402 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_S,
1403 HNS3_L3T_IPV4);
76ad4f0e
S		 1404
1405 /* the stack computes the IP header already, the only time we
1406 * need the hardware to recompute it is in the case of TSO.
1407 */
1408 if (skb_is_gso(skb))
cde4ffad 1409 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3CS_B, 1);
76ad4f0e 1410 } else if (l3.v6->version == 6) {
cde4ffad
YL		 1411 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_S,
1412 HNS3_L3T_IPV6);
76ad4f0e
S		 1413 }
1414
757cd1e4
YL		 1415 /* compute inner(/normal) L2 header size, defined in 2 Bytes */
1416 l2_len = l3.hdr - l2_hdr;
1417 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L2LEN_S, l2_len >> 1);
1418
1419 /* compute inner(/normal) L3 header size, defined in 4 Bytes */
1420 l3_len = l4.hdr - l3.hdr;
1421 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3LEN_S, l3_len >> 2);
1422
1423 /* compute inner(/normal) L4 header size, defined in 4 Bytes */
76ad4f0e
S		 1424 switch (l4_proto) {
1425 case IPPROTO_TCP:
cde4ffad
YL		 1426 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
1427 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4T_S,
1428 HNS3_L4T_TCP);
757cd1e4
YL		 1429 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_S,
1430 l4.tcp->doff);
76ad4f0e
S		 1431 break;
1432 case IPPROTO_UDP:
d18e8118
YM		 1433 if (hns3_tunnel_csum_bug(skb)) {
1434 int ret = skb_put_padto(skb, HNS3_MIN_TUN_PKT_LEN);
1435
1436 return ret ? ret : skb_checksum_help(skb);
1437 }
3db084d2 1438
cde4ffad
YL		 1439 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
1440 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4T_S,
1441 HNS3_L4T_UDP);
757cd1e4
YL		 1442 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_S,
1443 (sizeof(struct udphdr) >> 2));
76ad4f0e
S		 1444 break;
1445 case IPPROTO_SCTP:
cde4ffad
YL		 1446 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
1447 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4T_S,
1448 HNS3_L4T_SCTP);
757cd1e4
YL		 1449 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_S,
1450 (sizeof(struct sctphdr) >> 2));
76ad4f0e
S		 1451 break;
1452 default:
1453 /* drop the skb tunnel packet if hardware don't support,
1454 * because hardware can't calculate csum when TSO.
1455 */
1456 if (skb_is_gso(skb))
1457 return -EDOM;
1458
1459 /* the stack computes the IP header already,
1460 * driver calculate l4 checksum when not TSO.
1461 */
9bb5a495 1462 return skb_checksum_help(skb);
76ad4f0e
S		 1463 }
1464
1465 return 0;
1466}
1467
eb977d99
YL		 1468static int hns3_handle_vtags(struct hns3_enet_ring *tx_ring,
1469 struct sk_buff *skb)
9699cffe 1470{
44e626f7 1471 struct hnae3_handle *handle = tx_ring->tqp->handle;
592b0179 1472 struct hnae3_ae_dev *ae_dev;
eb977d99
YL		 1473 struct vlan_ethhdr *vhdr;
1474 int rc;
1475
1476 if (!(skb->protocol == htons(ETH_P_8021Q) ||
1477 skb_vlan_tag_present(skb)))
1478 return 0;
44e626f7 1479
592b0179
GL		 1480 /* For HW limitation on HNAE3_DEVICE_VERSION_V2, if port based insert
1481 * VLAN enabled, only one VLAN header is allowed in skb, otherwise it
1482 * will cause RAS error.
44e626f7 1483 */
592b0179 1484 ae_dev = pci_get_drvdata(handle->pdev);
44e626f7 1485 if (unlikely(skb_vlan_tagged_multi(skb) &&
592b0179 1486 ae_dev->dev_version <= HNAE3_DEVICE_VERSION_V2 &&
44e626f7
JS		 1487 handle->port_base_vlan_state ==
1488 HNAE3_PORT_BASE_VLAN_ENABLE))
1489 return -EINVAL;
1490
9699cffe 1491 if (skb->protocol == htons(ETH_P_8021Q) &&
eb977d99 1492 !(handle->kinfo.netdev->features & NETIF_F_HW_VLAN_CTAG_TX)) {
9699cffe
PL		 1493 /* When HW VLAN acceleration is turned off, and the stack
1494 * sets the protocol to 802.1q, the driver just need to
1495 * set the protocol to the encapsulated ethertype.
1496 */
1497 skb->protocol = vlan_get_protocol(skb);
1498 return 0;
1499 }
1500
1501 if (skb_vlan_tag_present(skb)) {
9699cffe
PL		 1502 /* Based on hw strategy, use out_vtag in two layer tag case,
1503 * and use inner_vtag in one tag case.
1504 */
eb977d99
YL		 1505 if (skb->protocol == htons(ETH_P_8021Q) &&
1506 handle->port_base_vlan_state ==
1507 HNAE3_PORT_BASE_VLAN_DISABLE)
1508 rc = HNS3_OUTER_VLAN_TAG;
1509 else
1510 rc = HNS3_INNER_VLAN_TAG;
1511
1512 skb->protocol = vlan_get_protocol(skb);
1513 return rc;
9699cffe
PL		 1514 }
1515
eb977d99
YL		 1516 rc = skb_cow_head(skb, 0);
1517 if (unlikely(rc < 0))
1518 return rc;
1519
1520 vhdr = (struct vlan_ethhdr *)skb->data;
1521 vhdr->h_vlan_TCI |= cpu_to_be16((skb->priority << VLAN_PRIO_SHIFT)
1522 & VLAN_PRIO_MASK);
1523
9699cffe
PL		 1524 skb->protocol = vlan_get_protocol(skb);
1525 return 0;
1526}
1527
66d52f3b
HT		 1528/* check if the hardware is capable of checksum offloading */
1529static bool hns3_check_hw_tx_csum(struct sk_buff *skb)
1530{
1531 struct hns3_nic_priv *priv = netdev_priv(skb->dev);
1532
1533 /* Kindly note, due to backward compatibility of the TX descriptor,
1534 * HW checksum of the non-IP packets and GSO packets is handled at
1535 * different place in the following code
1536 */
b9046e88 1537 if (skb_csum_is_sctp(skb) || skb_is_gso(skb) ||
66d52f3b
HT		 1538 !test_bit(HNS3_NIC_STATE_HW_TX_CSUM_ENABLE, &priv->state))
1539 return false;
1540
1541 return true;
1542}
1543
eb977d99 1544static int hns3_fill_skb_desc(struct hns3_enet_ring *ring,
811c0830
YL		 1545 struct sk_buff *skb, struct hns3_desc *desc,
1546 struct hns3_desc_cb *desc_cb)
eb977d99
YL		 1547{
1548 u32 ol_type_vlan_len_msec = 0;
3e281621 1549 u32 paylen_ol4cs = skb->len;
eb977d99 1550 u32 type_cs_vlan_tso = 0;
66d52f3b 1551 u16 mss_hw_csum = 0;
eb977d99
YL		 1552 u16 inner_vtag = 0;
1553 u16 out_vtag = 0;
eb977d99
YL		 1554 int ret;
1555
1556 ret = hns3_handle_vtags(ring, skb);
1557 if (unlikely(ret < 0)) {
b20d7fe5
YL		 1558 u64_stats_update_begin(&ring->syncp);
1559 ring->stats.tx_vlan_err++;
1560 u64_stats_update_end(&ring->syncp);
eb977d99
YL		 1561 return ret;
1562 } else if (ret == HNS3_INNER_VLAN_TAG) {
1563 inner_vtag = skb_vlan_tag_get(skb);
1564 inner_vtag |= (skb->priority << VLAN_PRIO_SHIFT) &
1565 VLAN_PRIO_MASK;
1566 hns3_set_field(type_cs_vlan_tso, HNS3_TXD_VLAN_B, 1);
1567 } else if (ret == HNS3_OUTER_VLAN_TAG) {
1568 out_vtag = skb_vlan_tag_get(skb);
1569 out_vtag |= (skb->priority << VLAN_PRIO_SHIFT) &
1570 VLAN_PRIO_MASK;
1571 hns3_set_field(ol_type_vlan_len_msec, HNS3_TXD_OVLAN_B,
1572 1);
1573 }
1574
811c0830
YL		 1575 desc_cb->send_bytes = skb->len;
1576
eb977d99
YL		 1577 if (skb->ip_summed == CHECKSUM_PARTIAL) {
1578 u8 ol4_proto, il4_proto;
1579
66d52f3b
HT		 1580 if (hns3_check_hw_tx_csum(skb)) {
1581 /* set checksum start and offset, defined in 2 Bytes */
1582 hns3_set_field(type_cs_vlan_tso, HNS3_TXD_CSUM_START_S,
1583 skb_checksum_start_offset(skb) >> 1);
1584 hns3_set_field(ol_type_vlan_len_msec,
1585 HNS3_TXD_CSUM_OFFSET_S,
1586 skb->csum_offset >> 1);
1587 mss_hw_csum |= BIT(HNS3_TXD_HW_CS_B);
1588 goto out_hw_tx_csum;
1589 }
1590
eb977d99
YL		 1591 skb_reset_mac_len(skb);
1592
1593 ret = hns3_get_l4_protocol(skb, &ol4_proto, &il4_proto);
8ae10cfb 1594 if (unlikely(ret < 0)) {
b20d7fe5
YL		 1595 u64_stats_update_begin(&ring->syncp);
1596 ring->stats.tx_l4_proto_err++;
1597 u64_stats_update_end(&ring->syncp);
eb977d99 1598 return ret;
b20d7fe5 1599 }
eb977d99
YL		 1600
1601 ret = hns3_set_l2l3l4(skb, ol4_proto, il4_proto,
1602 &type_cs_vlan_tso,
1603 &ol_type_vlan_len_msec);
8ae10cfb 1604 if (unlikely(ret < 0)) {
b20d7fe5
YL		 1605 u64_stats_update_begin(&ring->syncp);
1606 ring->stats.tx_l2l3l4_err++;
1607 u64_stats_update_end(&ring->syncp);
eb977d99 1608 return ret;
b20d7fe5 1609 }
eb977d99 1610
3e281621 1611 ret = hns3_set_tso(skb, &paylen_ol4cs, &mss_hw_csum,
811c0830 1612 &type_cs_vlan_tso, &desc_cb->send_bytes);
8ae10cfb 1613 if (unlikely(ret < 0)) {
b20d7fe5
YL		 1614 u64_stats_update_begin(&ring->syncp);
1615 ring->stats.tx_tso_err++;
1616 u64_stats_update_end(&ring->syncp);
eb977d99 1617 return ret;
b20d7fe5 1618 }
eb977d99
YL		 1619 }
1620
66d52f3b 1621out_hw_tx_csum:
eb977d99
YL		 1622 /* Set txbd */
1623 desc->tx.ol_type_vlan_len_msec =
1624 cpu_to_le32(ol_type_vlan_len_msec);
1625 desc->tx.type_cs_vlan_tso_len = cpu_to_le32(type_cs_vlan_tso);
3e281621 1626 desc->tx.paylen_ol4cs = cpu_to_le32(paylen_ol4cs);
66d52f3b 1627 desc->tx.mss_hw_csum = cpu_to_le16(mss_hw_csum);
eb977d99
YL		 1628 desc->tx.vlan_tag = cpu_to_le16(inner_vtag);
1629 desc->tx.outer_vlan_tag = cpu_to_le16(out_vtag);
1630
1631 return 0;
1632}
1633
8677d78c
YL		 1634static int hns3_fill_desc(struct hns3_enet_ring *ring, dma_addr_t dma,
1635 unsigned int size)
76ad4f0e 1636{
8ae10cfb
YL		 1637#define HNS3_LIKELY_BD_NUM 1
1638
76ad4f0e 1639 struct hns3_desc *desc = &ring->desc[ring->next_to_use];
1e8a7977 1640 unsigned int frag_buf_num;
47e7b13b 1641 int k, sizeoflast;
bcdb12b7 1642
ceca4a5e 1643 if (likely(size <= HNS3_MAX_BD_SIZE)) {
ceca4a5e
YL		 1644 desc->addr = cpu_to_le64(dma);
1645 desc->tx.send_size = cpu_to_le16(size);
ceca4a5e 1646 desc->tx.bdtp_fe_sc_vld_ra_ri =
8ae10cfb 1647 cpu_to_le16(BIT(HNS3_TXD_VLD_B));
ceca4a5e 1648
698a8954 1649 trace_hns3_tx_desc(ring, ring->next_to_use);
ceca4a5e 1650 ring_ptr_move_fw(ring, next_to_use);
8ae10cfb 1651 return HNS3_LIKELY_BD_NUM;
ceca4a5e
YL		 1652 }
1653
5f543a54 1654 frag_buf_num = hns3_tx_bd_count(size);
48ae74c9 1655 sizeoflast = size % HNS3_MAX_BD_SIZE;
1e8a7977
FL		 1656 sizeoflast = sizeoflast ? sizeoflast : HNS3_MAX_BD_SIZE;
1657
1658 /* When frag size is bigger than hardware limit, split this frag */
1659 for (k = 0; k < frag_buf_num; k++) {
1e8a7977
FL		 1660 /* now, fill the descriptor */
1661 desc->addr = cpu_to_le64(dma + HNS3_MAX_BD_SIZE * k);
bcdb12b7 1662 desc->tx.send_size = cpu_to_le16((k == frag_buf_num - 1) ?
9b2f3477 1663 (u16)sizeoflast : (u16)HNS3_MAX_BD_SIZE);
1e8a7977 1664 desc->tx.bdtp_fe_sc_vld_ra_ri =
8ae10cfb 1665 cpu_to_le16(BIT(HNS3_TXD_VLD_B));
1e8a7977 1666
698a8954 1667 trace_hns3_tx_desc(ring, ring->next_to_use);
9b2f3477 1668 /* move ring pointer to next */
1e8a7977
FL		 1669 ring_ptr_move_fw(ring, next_to_use);
1670
1e8a7977
FL		 1671 desc = &ring->desc[ring->next_to_use];
1672 }
76ad4f0e 1673
8ae10cfb 1674 return frag_buf_num;
76ad4f0e
S		 1675}
1676
8677d78c
YL		 1677static int hns3_map_and_fill_desc(struct hns3_enet_ring *ring, void *priv,
1678 unsigned int type)
1679{
1680 struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use];
1681 struct device *dev = ring_to_dev(ring);
1682 unsigned int size;
1683 dma_addr_t dma;
1684
1685 if (type & (DESC_TYPE_FRAGLIST_SKB | DESC_TYPE_SKB)) {
1686 struct sk_buff *skb = (struct sk_buff *)priv;
1687
1688 size = skb_headlen(skb);
1689 if (!size)
1690 return 0;
1691
1692 dma = dma_map_single(dev, skb->data, size, DMA_TO_DEVICE);
907676b1
YL		 1693 } else if (type & DESC_TYPE_BOUNCE_HEAD) {
1694 /* Head data has been filled in hns3_handle_tx_bounce(),
1695 * just return 0 here.
1696 */
1697 return 0;
8677d78c
YL		 1698 } else {
1699 skb_frag_t *frag = (skb_frag_t *)priv;
1700
1701 size = skb_frag_size(frag);
1702 if (!size)
1703 return 0;
1704
1705 dma = skb_frag_dma_map(dev, frag, 0, size, DMA_TO_DEVICE);
1706 }
1707
1708 if (unlikely(dma_mapping_error(dev, dma))) {
1709 u64_stats_update_begin(&ring->syncp);
1710 ring->stats.sw_err_cnt++;
1711 u64_stats_update_end(&ring->syncp);
1712 return -ENOMEM;
1713 }
1714
1715 desc_cb->priv = priv;
1716 desc_cb->length = size;
1717 desc_cb->dma = dma;
1718 desc_cb->type = type;
1719
1720 return hns3_fill_desc(ring, dma, size);
1721}
1722
8ae10cfb
YL		 1723static unsigned int hns3_skb_bd_num(struct sk_buff *skb, unsigned int *bd_size,
1724 unsigned int bd_num)
76ad4f0e 1725{
8ae10cfb 1726 unsigned int size;
42611b70 1727 int i;
76ad4f0e 1728
8ae10cfb
YL		 1729 size = skb_headlen(skb);
1730 while (size > HNS3_MAX_BD_SIZE) {
1731 bd_size[bd_num++] = HNS3_MAX_BD_SIZE;
1732 size -= HNS3_MAX_BD_SIZE;
1733
1734 if (bd_num > HNS3_MAX_TSO_BD_NUM)
1735 return bd_num;
1736 }
76ad4f0e 1737
8ae10cfb
YL		 1738 if (size) {
1739 bd_size[bd_num++] = size;
1740 if (bd_num > HNS3_MAX_TSO_BD_NUM)
1741 return bd_num;
1742 }
76ad4f0e 1743
3d5f3741 1744 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
d7840976 1745 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
8ae10cfb
YL		 1746 size = skb_frag_size(frag);
1747 if (!size)
1748 continue;
1749
1750 while (size > HNS3_MAX_BD_SIZE) {
1751 bd_size[bd_num++] = HNS3_MAX_BD_SIZE;
1752 size -= HNS3_MAX_BD_SIZE;
1753
1754 if (bd_num > HNS3_MAX_TSO_BD_NUM)
1755 return bd_num;
1756 }
1757
1758 bd_size[bd_num++] = size;
1759 if (bd_num > HNS3_MAX_TSO_BD_NUM)
1760 return bd_num;
1761 }
1762
1763 return bd_num;
1764}
1765
fd665b3d 1766static unsigned int hns3_tx_bd_num(struct sk_buff *skb, unsigned int *bd_size,
d5d5e019
YL		 1767 u8 max_non_tso_bd_num, unsigned int bd_num,
1768 unsigned int recursion_level)
8ae10cfb 1769{
d5d5e019
YL		 1770#define HNS3_MAX_RECURSION_LEVEL 24
1771
8ae10cfb 1772 struct sk_buff *frag_skb;
8ae10cfb
YL		 1773
1774 /* If the total len is within the max bd limit */
d5d5e019
YL		 1775 if (likely(skb->len <= HNS3_MAX_BD_SIZE && !recursion_level &&
1776 !skb_has_frag_list(skb) &&
fd665b3d 1777 skb_shinfo(skb)->nr_frags < max_non_tso_bd_num))
8ae10cfb
YL		 1778 return skb_shinfo(skb)->nr_frags + 1U;
1779
d5d5e019
YL		 1780 if (unlikely(recursion_level >= HNS3_MAX_RECURSION_LEVEL))
1781 return UINT_MAX;
8ae10cfb
YL		 1782
1783 bd_num = hns3_skb_bd_num(skb, bd_size, bd_num);
8ae10cfb
YL		 1784 if (!skb_has_frag_list(skb) || bd_num > HNS3_MAX_TSO_BD_NUM)
1785 return bd_num;
1786
1787 skb_walk_frags(skb, frag_skb) {
d5d5e019
YL		 1788 bd_num = hns3_tx_bd_num(frag_skb, bd_size, max_non_tso_bd_num,
1789 bd_num, recursion_level + 1);
8ae10cfb
YL		 1790 if (bd_num > HNS3_MAX_TSO_BD_NUM)
1791 return bd_num;
3d5f3741 1792 }
76ad4f0e 1793
3d5f3741 1794 return bd_num;
76ad4f0e
S		 1795}
1796
db4970aa
YL		 1797static unsigned int hns3_gso_hdr_len(struct sk_buff *skb)
1798{
1799 if (!skb->encapsulation)
1800 return skb_transport_offset(skb) + tcp_hdrlen(skb);
1801
1802 return skb_inner_transport_offset(skb) + inner_tcp_hdrlen(skb);
1803}
1804
fd665b3d
HT		 1805/* HW need every continuous max_non_tso_bd_num buffer data to be larger
1806 * than MSS, we simplify it by ensuring skb_headlen + the first continuous
1807 * max_non_tso_bd_num - 1 frags to be larger than gso header len + mss,
1808 * and the remaining continuous max_non_tso_bd_num - 1 frags to be larger
1809 * than MSS except the last max_non_tso_bd_num - 1 frags.
db4970aa 1810 */
8ae10cfb 1811static bool hns3_skb_need_linearized(struct sk_buff *skb, unsigned int *bd_size,
fd665b3d 1812 unsigned int bd_num, u8 max_non_tso_bd_num)
db4970aa 1813{
db4970aa
YL		 1814 unsigned int tot_len = 0;
1815 int i;
1816
fd665b3d 1817 for (i = 0; i < max_non_tso_bd_num - 1U; i++)
8ae10cfb 1818 tot_len += bd_size[i];
db4970aa 1819
fd665b3d
HT		 1820 /* ensure the first max_non_tso_bd_num frags is greater than
1821 * mss + header
1822 */
1823 if (tot_len + bd_size[max_non_tso_bd_num - 1U] <
8ae10cfb 1824 skb_shinfo(skb)->gso_size + hns3_gso_hdr_len(skb))
db4970aa
YL		 1825 return true;
1826
fd665b3d
HT		 1827 /* ensure every continuous max_non_tso_bd_num - 1 buffer is greater
1828 * than mss except the last one.
8ae10cfb 1829 */
fd665b3d 1830 for (i = 0; i < bd_num - max_non_tso_bd_num; i++) {
8ae10cfb 1831 tot_len -= bd_size[i];
fd665b3d 1832 tot_len += bd_size[i + max_non_tso_bd_num - 1U];
db4970aa
YL		 1833
1834 if (tot_len < skb_shinfo(skb)->gso_size)
1835 return true;
1836 }
1837
1838 return false;
1839}
1840
698a8954
YL		 1841void hns3_shinfo_pack(struct skb_shared_info *shinfo, __u32 *size)
1842{
9d8d5a36 1843 int i;
698a8954
YL		 1844
1845 for (i = 0; i < MAX_SKB_FRAGS; i++)
1846 size[i] = skb_frag_size(&shinfo->frags[i]);
1847}
1848
d5d5e019
YL		 1849static int hns3_skb_linearize(struct hns3_enet_ring *ring,
1850 struct sk_buff *skb,
d5d5e019
YL		 1851 unsigned int bd_num)
1852{
1853 /* 'bd_num == UINT_MAX' means the skb' fraglist has a
1854 * recursion level of over HNS3_MAX_RECURSION_LEVEL.
1855 */
1856 if (bd_num == UINT_MAX) {
1857 u64_stats_update_begin(&ring->syncp);
1858 ring->stats.over_max_recursion++;
1859 u64_stats_update_end(&ring->syncp);
1860 return -ENOMEM;
1861 }
1862
1863 /* The skb->len has exceeded the hw limitation, linearization
1864 * will not help.
1865 */
1866 if (skb->len > HNS3_MAX_TSO_SIZE ||
adfb7b49 1867 (!skb_is_gso(skb) && skb->len > HNS3_MAX_NON_TSO_SIZE)) {
d5d5e019
YL		 1868 u64_stats_update_begin(&ring->syncp);
1869 ring->stats.hw_limitation++;
1870 u64_stats_update_end(&ring->syncp);
1871 return -ENOMEM;
1872 }
1873
1874 if (__skb_linearize(skb)) {
1875 u64_stats_update_begin(&ring->syncp);
1876 ring->stats.sw_err_cnt++;
1877 u64_stats_update_end(&ring->syncp);
1878 return -ENOMEM;
1879 }
1880
1881 return 0;
1882}
1883
3d5f3741 1884static int hns3_nic_maybe_stop_tx(struct hns3_enet_ring *ring,
2a597eff 1885 struct net_device *netdev,
d1a37ded 1886 struct sk_buff *skb)
76ad4f0e 1887{
2a597eff 1888 struct hns3_nic_priv *priv = netdev_priv(netdev);
fd665b3d 1889 u8 max_non_tso_bd_num = priv->max_non_tso_bd_num;
8ae10cfb 1890 unsigned int bd_size[HNS3_MAX_TSO_BD_NUM + 1U];
42611b70 1891 unsigned int bd_num;
76ad4f0e 1892
d5d5e019 1893 bd_num = hns3_tx_bd_num(skb, bd_size, max_non_tso_bd_num, 0, 0);
fd665b3d 1894 if (unlikely(bd_num > max_non_tso_bd_num)) {
8ae10cfb 1895 if (bd_num <= HNS3_MAX_TSO_BD_NUM && skb_is_gso(skb) &&
fd665b3d
HT		 1896 !hns3_skb_need_linearized(skb, bd_size, bd_num,
1897 max_non_tso_bd_num)) {
6ad595bc 1898 trace_hns3_over_max_bd(skb);
db4970aa 1899 goto out;
698a8954 1900 }
db4970aa 1901
adfb7b49 1902 if (hns3_skb_linearize(ring, skb, bd_num))
51e8439f 1903 return -ENOMEM;
3d5f3741 1904
d1a37ded 1905 bd_num = hns3_tx_bd_count(skb->len);
42611b70 1906
3d5f3741
YL		 1907 u64_stats_update_begin(&ring->syncp);
1908 ring->stats.tx_copy++;
1909 u64_stats_update_end(&ring->syncp);
51e8439f
PL		 1910 }
1911
db4970aa 1912out:
2a597eff
YL		 1913 if (likely(ring_space(ring) >= bd_num))
1914 return bd_num;
76ad4f0e 1915
2a597eff
YL		 1916 netif_stop_subqueue(netdev, ring->queue_index);
1917 smp_mb(); /* Memory barrier before checking ring_space */
1918
1919 /* Start queue in case hns3_clean_tx_ring has just made room
1920 * available and has not seen the queue stopped state performed
1921 * by netif_stop_subqueue above.
1922 */
1923 if (ring_space(ring) >= bd_num && netif_carrier_ok(netdev) &&
1924 !test_bit(HNS3_NIC_STATE_DOWN, &priv->state)) {
1925 netif_start_subqueue(netdev, ring->queue_index);
1926 return bd_num;
1927 }
1928
d5d5e019
YL		 1929 u64_stats_update_begin(&ring->syncp);
1930 ring->stats.tx_busy++;
1931 u64_stats_update_end(&ring->syncp);
1932
2a597eff 1933 return -EBUSY;
76ad4f0e
S		 1934}
1935
ba3f808f 1936static void hns3_clear_desc(struct hns3_enet_ring *ring, int next_to_use_orig)
76ad4f0e
S		 1937{
1938 struct device *dev = ring_to_dev(ring);
1939 unsigned int i;
1940
1941 for (i = 0; i < ring->desc_num; i++) {
8ceca59f 1942 struct hns3_desc *desc = &ring->desc[ring->next_to_use];
26f1ccdf 1943 struct hns3_desc_cb *desc_cb;
8ceca59f
YL		 1944
1945 memset(desc, 0, sizeof(*desc));
1946
76ad4f0e
S		 1947 /* check if this is where we started */
1948 if (ring->next_to_use == next_to_use_orig)
1949 break;
1950
aa9d22dd
YL		 1951 /* rollback one */
1952 ring_ptr_move_bw(ring, next_to_use);
1953
26f1ccdf
YL		 1954 desc_cb = &ring->desc_cb[ring->next_to_use];
1955
1956 if (!desc_cb->dma)
8ceca59f
YL		 1957 continue;
1958
76ad4f0e 1959 /* unmap the descriptor dma address */
26f1ccdf
YL		 1960 if (desc_cb->type & (DESC_TYPE_SKB | DESC_TYPE_FRAGLIST_SKB))
1961 dma_unmap_single(dev, desc_cb->dma, desc_cb->length,
1962 DMA_TO_DEVICE);
907676b1
YL		 1963 else if (desc_cb->type &
1964 (DESC_TYPE_BOUNCE_HEAD | DESC_TYPE_BOUNCE_ALL))
1965 hns3_tx_spare_rollback(ring, desc_cb->length);
26f1ccdf
YL		 1966 else if (desc_cb->length)
1967 dma_unmap_page(dev, desc_cb->dma, desc_cb->length,
76ad4f0e
S		 1968 DMA_TO_DEVICE);
1969
26f1ccdf
YL		 1970 desc_cb->length = 0;
1971 desc_cb->dma = 0;
1972 desc_cb->type = DESC_TYPE_UNKNOWN;
76ad4f0e
S		 1973 }
1974}
1975
8ae10cfb 1976static int hns3_fill_skb_to_desc(struct hns3_enet_ring *ring,
26f1ccdf 1977 struct sk_buff *skb, unsigned int type)
8ae10cfb 1978{
d5d5e019 1979 struct sk_buff *frag_skb;
8ae10cfb
YL		 1980 int i, ret, bd_num = 0;
1981
8677d78c
YL		 1982 ret = hns3_map_and_fill_desc(ring, skb, type);
1983 if (unlikely(ret < 0))
1984 return ret;
8ae10cfb 1985
8677d78c 1986 bd_num += ret;
8ae10cfb
YL		 1987
1988 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1989 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1990
8677d78c 1991 ret = hns3_map_and_fill_desc(ring, frag, DESC_TYPE_PAGE);
8ae10cfb
YL		 1992 if (unlikely(ret < 0))
1993 return ret;
1994
1995 bd_num += ret;
1996 }
1997
d5d5e019
YL		 1998 skb_walk_frags(skb, frag_skb) {
1999 ret = hns3_fill_skb_to_desc(ring, frag_skb,
2000 DESC_TYPE_FRAGLIST_SKB);
2001 if (unlikely(ret < 0))
2002 return ret;
2003
2004 bd_num += ret;
2005 }
2006
8ae10cfb
YL		 2007 return bd_num;
2008}
2009
f6061a05
YL		 2010static void hns3_tx_doorbell(struct hns3_enet_ring *ring, int num,
2011 bool doorbell)
2012{
2013 ring->pending_buf += num;
2014
2015 if (!doorbell) {
2016 u64_stats_update_begin(&ring->syncp);
2017 ring->stats.tx_more++;
2018 u64_stats_update_end(&ring->syncp);
2019 return;
2020 }
2021
2022 if (!ring->pending_buf)
2023 return;
2024
48ee56fd
YL		 2025 writel(ring->pending_buf,
2026 ring->tqp->io_base + HNS3_RING_TX_RING_TAIL_REG);
f6061a05 2027 ring->pending_buf = 0;
20d06ca2 2028 WRITE_ONCE(ring->last_to_use, ring->next_to_use);
f6061a05
YL		 2029}
2030
0bf5eb78
HT		 2031static void hns3_tsyn(struct net_device *netdev, struct sk_buff *skb,
2032 struct hns3_desc *desc)
2033{
2034 struct hnae3_handle *h = hns3_get_handle(netdev);
2035
2036 if (!(h->ae_algo->ops->set_tx_hwts_info &&
2037 h->ae_algo->ops->set_tx_hwts_info(h, skb)))
2038 return;
2039
2040 desc->tx.bdtp_fe_sc_vld_ra_ri |= cpu_to_le16(BIT(HNS3_TXD_TSYN_B));
2041}
2042
907676b1
YL		 2043static int hns3_handle_tx_bounce(struct hns3_enet_ring *ring,
2044 struct sk_buff *skb)
2045{
2046 struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use];
2047 unsigned int type = DESC_TYPE_BOUNCE_HEAD;
2048 unsigned int size = skb_headlen(skb);
2049 dma_addr_t dma;
2050 int bd_num = 0;
2051 u32 cb_len;
2052 void *buf;
2053 int ret;
2054
2055 if (skb->len <= ring->tx_copybreak) {
2056 size = skb->len;
2057 type = DESC_TYPE_BOUNCE_ALL;
2058 }
2059
2060 /* hns3_can_use_tx_bounce() is called to ensure the below
2061 * function can always return the tx buffer.
2062 */
2063 buf = hns3_tx_spare_alloc(ring, size, &dma, &cb_len);
2064
2065 ret = skb_copy_bits(skb, 0, buf, size);
2066 if (unlikely(ret < 0)) {
2067 hns3_tx_spare_rollback(ring, cb_len);
2068 u64_stats_update_begin(&ring->syncp);
2069 ring->stats.copy_bits_err++;
2070 u64_stats_update_end(&ring->syncp);
2071 return ret;
2072 }
2073
2074 desc_cb->priv = skb;
2075 desc_cb->length = cb_len;
2076 desc_cb->dma = dma;
2077 desc_cb->type = type;
2078
2079 bd_num += hns3_fill_desc(ring, dma, size);
2080
2081 if (type == DESC_TYPE_BOUNCE_HEAD) {
2082 ret = hns3_fill_skb_to_desc(ring, skb,
2083 DESC_TYPE_BOUNCE_HEAD);
2084 if (unlikely(ret < 0))
2085 return ret;
2086
2087 bd_num += ret;
2088 }
2089
2090 dma_sync_single_for_device(ring_to_dev(ring), dma, size,
2091 DMA_TO_DEVICE);
2092
2093 u64_stats_update_begin(&ring->syncp);
2094 ring->stats.tx_bounce++;
2095 u64_stats_update_end(&ring->syncp);
2096 return bd_num;
2097}
2098
7459775e
YL		 2099static int hns3_handle_tx_sgl(struct hns3_enet_ring *ring,
2100 struct sk_buff *skb)
2101{
2102 struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use];
2103 u32 nfrag = skb_shinfo(skb)->nr_frags + 1;
2104 struct sg_table *sgt;
2105 int i, bd_num = 0;
2106 dma_addr_t dma;
2107 u32 cb_len;
2108 int nents;
2109
2110 if (skb_has_frag_list(skb))
2111 nfrag = HNS3_MAX_TSO_BD_NUM;
2112
2113 /* hns3_can_use_tx_sgl() is called to ensure the below
2114 * function can always return the tx buffer.
2115 */
2116 sgt = hns3_tx_spare_alloc(ring, HNS3_SGL_SIZE(nfrag),
2117 &dma, &cb_len);
2118
2119 /* scatterlist follows by the sg table */
2120 sgt->sgl = (struct scatterlist *)(sgt + 1);
2121 sg_init_table(sgt->sgl, nfrag);
2122 nents = skb_to_sgvec(skb, sgt->sgl, 0, skb->len);
2123 if (unlikely(nents < 0)) {
2124 hns3_tx_spare_rollback(ring, cb_len);
2125 u64_stats_update_begin(&ring->syncp);
2126 ring->stats.skb2sgl_err++;
2127 u64_stats_update_end(&ring->syncp);
2128 return -ENOMEM;
2129 }
2130
2131 sgt->orig_nents = nents;
2132 sgt->nents = dma_map_sg(ring_to_dev(ring), sgt->sgl, sgt->orig_nents,
2133 DMA_TO_DEVICE);
2134 if (unlikely(!sgt->nents)) {
2135 hns3_tx_spare_rollback(ring, cb_len);
2136 u64_stats_update_begin(&ring->syncp);
2137 ring->stats.map_sg_err++;
2138 u64_stats_update_end(&ring->syncp);
2139 return -ENOMEM;
2140 }
2141
2142 desc_cb->priv = skb;
2143 desc_cb->length = cb_len;
2144 desc_cb->dma = dma;
2145 desc_cb->type = DESC_TYPE_SGL_SKB;
2146
2147 for (i = 0; i < sgt->nents; i++)
2148 bd_num += hns3_fill_desc(ring, sg_dma_address(sgt->sgl + i),
2149 sg_dma_len(sgt->sgl + i));
2150
2151 u64_stats_update_begin(&ring->syncp);
2152 ring->stats.tx_sgl++;
2153 u64_stats_update_end(&ring->syncp);
2154
2155 return bd_num;
2156}
2157
907676b1
YL		 2158static int hns3_handle_desc_filling(struct hns3_enet_ring *ring,
2159 struct sk_buff *skb)
2160{
2161 u32 space;
2162
2163 if (!ring->tx_spare)
2164 goto out;
2165
2166 space = hns3_tx_spare_space(ring);
2167
7459775e
YL		 2168 if (hns3_can_use_tx_sgl(ring, skb, space))
2169 return hns3_handle_tx_sgl(ring, skb);
2170
907676b1
YL		 2171 if (hns3_can_use_tx_bounce(ring, skb, space))
2172 return hns3_handle_tx_bounce(ring, skb);
2173
2174out:
2175 return hns3_fill_skb_to_desc(ring, skb, DESC_TYPE_SKB);
2176}
2177
d43e5aca 2178netdev_tx_t hns3_nic_net_xmit(struct sk_buff *skb, struct net_device *netdev)
76ad4f0e
S		 2179{
2180 struct hns3_nic_priv *priv = netdev_priv(netdev);
5f06b903 2181 struct hns3_enet_ring *ring = &priv->ring[skb->queue_mapping];
811c0830 2182 struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use];
76ad4f0e 2183 struct netdev_queue *dev_queue;
8ae10cfb 2184 int pre_ntu, next_to_use_head;
f6061a05 2185 bool doorbell;
76ad4f0e 2186 int ret;
76ad4f0e 2187
36c67349 2188 /* Hardware can only handle short frames above 32 bytes */
f6061a05
YL		 2189 if (skb_put_padto(skb, HNS3_MIN_TX_LEN)) {
2190 hns3_tx_doorbell(ring, 0, !netdev_xmit_more());
97b9e5c1
YL		 2191
2192 u64_stats_update_begin(&ring->syncp);
2193 ring->stats.sw_err_cnt++;
2194 u64_stats_update_end(&ring->syncp);
2195
36c67349 2196 return NETDEV_TX_OK;
f6061a05 2197 }
36c67349 2198
76ad4f0e
S		 2199 /* Prefetch the data used later */
2200 prefetch(skb->data);
2201
d1a37ded 2202 ret = hns3_nic_maybe_stop_tx(ring, netdev, skb);
8ae10cfb
YL		 2203 if (unlikely(ret <= 0)) {
2204 if (ret == -EBUSY) {
f6061a05 2205 hns3_tx_doorbell(ring, 0, true);
2a597eff 2206 return NETDEV_TX_BUSY;
3d5f3741 2207 }
76ad4f0e 2208
8ae10cfb 2209 hns3_rl_err(netdev, "xmit error: %d!\n", ret);
76ad4f0e 2210 goto out_err_tx_ok;
76ad4f0e
S		 2211 }
2212
76ad4f0e
S		 2213 next_to_use_head = ring->next_to_use;
2214
811c0830
YL		 2215 ret = hns3_fill_skb_desc(ring, skb, &ring->desc[ring->next_to_use],
2216 desc_cb);
cfdaeba5
YL		 2217 if (unlikely(ret < 0))
2218 goto fill_err;
2219
d5d5e019
YL		 2220 /* 'ret < 0' means filling error, 'ret == 0' means skb->len is
2221 * zero, which is unlikely, and 'ret > 0' means how many tx desc
2222 * need to be notified to the hw.
2223 */
907676b1 2224 ret = hns3_handle_desc_filling(ring, skb);
d5d5e019 2225 if (unlikely(ret <= 0))
aa9d22dd 2226 goto fill_err;
76ad4f0e 2227
8ae10cfb
YL		 2228 pre_ntu = ring->next_to_use ? (ring->next_to_use - 1) :
2229 (ring->desc_num - 1);
0bf5eb78
HT		 2230
2231 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))
2232 hns3_tsyn(netdev, skb, &ring->desc[pre_ntu]);
2233
8ae10cfb
YL		 2234 ring->desc[pre_ntu].tx.bdtp_fe_sc_vld_ra_ri |=
2235 cpu_to_le16(BIT(HNS3_TXD_FE_B));
698a8954 2236 trace_hns3_tx_desc(ring, pre_ntu);
76ad4f0e 2237
0bf5eb78
HT		 2238 skb_tx_timestamp(skb);
2239
76ad4f0e 2240 /* Complete translate all packets */
5f06b903 2241 dev_queue = netdev_get_tx_queue(netdev, ring->queue_index);
811c0830 2242 doorbell = __netdev_tx_sent_queue(dev_queue, desc_cb->send_bytes,
f6061a05 2243 netdev_xmit_more());
d5d5e019 2244 hns3_tx_doorbell(ring, ret, doorbell);
76ad4f0e
S		 2245
2246 return NETDEV_TX_OK;
2247
aa9d22dd 2248fill_err:
ba3f808f 2249 hns3_clear_desc(ring, next_to_use_head);
76ad4f0e
S		 2250
2251out_err_tx_ok:
2252 dev_kfree_skb_any(skb);
f6061a05 2253 hns3_tx_doorbell(ring, 0, !netdev_xmit_more());
76ad4f0e 2254 return NETDEV_TX_OK;
76ad4f0e
S		 2255}
2256
2257static int hns3_nic_net_set_mac_address(struct net_device *netdev, void *p)
2258{
9780cb97 2259 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 2260 struct sockaddr *mac_addr = p;
2261 int ret;
2262
2263 if (!mac_addr || !is_valid_ether_addr((const u8 *)mac_addr->sa_data))
2264 return -EADDRNOTAVAIL;
2265
5ec2a51e
JS		 2266 if (ether_addr_equal(netdev->dev_addr, mac_addr->sa_data)) {
2267 netdev_info(netdev, "already using mac address %pM\n",
2268 mac_addr->sa_data);
2269 return 0;
2270 }
2271
8e6de441
HT		 2272 /* For VF device, if there is a perm_addr, then the user will not
2273 * be allowed to change the address.
2274 */
2275 if (!hns3_is_phys_func(h->pdev) &&
2276 !is_zero_ether_addr(netdev->perm_addr)) {
2277 netdev_err(netdev, "has permanent MAC %pM, user MAC %pM not allow\n",
2278 netdev->perm_addr, mac_addr->sa_data);
2279 return -EPERM;
2280 }
2281
59098055 2282 ret = h->ae_algo->ops->set_mac_addr(h, mac_addr->sa_data, false);
76ad4f0e
S		 2283 if (ret) {
2284 netdev_err(netdev, "set_mac_address fail, ret=%d!\n", ret);
2285 return ret;
2286 }
2287
f3956ebb 2288 eth_hw_addr_set(netdev, mac_addr->sa_data);
76ad4f0e
S		 2289
2290 return 0;
2291}
2292
26483246
XW		 2293static int hns3_nic_do_ioctl(struct net_device *netdev,
2294 struct ifreq *ifr, int cmd)
2295{
2296 struct hnae3_handle *h = hns3_get_handle(netdev);
2297
2298 if (!netif_running(netdev))
2299 return -EINVAL;
2300
2301 if (!h->ae_algo->ops->do_ioctl)
2302 return -EOPNOTSUPP;
2303
2304 return h->ae_algo->ops->do_ioctl(h, ifr, cmd);
2305}
2306
76ad4f0e
S		 2307static int hns3_nic_set_features(struct net_device *netdev,
2308 netdev_features_t features)
2309{
181d454b 2310 netdev_features_t changed = netdev->features ^ features;
76ad4f0e 2311 struct hns3_nic_priv *priv = netdev_priv(netdev);
052ece6d 2312 struct hnae3_handle *h = priv->ae_handle;
1731be4c 2313 bool enable;
052ece6d 2314 int ret;
76ad4f0e 2315
5c9f6b39 2316 if (changed & (NETIF_F_GRO_HW) && h->ae_algo->ops->set_gro_en) {
1731be4c
YL		 2317 enable = !!(features & NETIF_F_GRO_HW);
2318 ret = h->ae_algo->ops->set_gro_en(h, enable);
5c9f6b39
PL		 2319 if (ret)
2320 return ret;
2321 }
2322
bd368416
JS		 2323 if ((changed & NETIF_F_HW_VLAN_CTAG_RX) &&
2324 h->ae_algo->ops->enable_hw_strip_rxvtag) {
1731be4c
YL		 2325 enable = !!(features & NETIF_F_HW_VLAN_CTAG_RX);
2326 ret = h->ae_algo->ops->enable_hw_strip_rxvtag(h, enable);
052ece6d
PL		 2327 if (ret)
2328 return ret;
2329 }
2330
c17852a8 2331 if ((changed & NETIF_F_NTUPLE) && h->ae_algo->ops->enable_fd) {
1731be4c
YL		 2332 enable = !!(features & NETIF_F_NTUPLE);
2333 h->ae_algo->ops->enable_fd(h, enable);
c17852a8
JS		 2334 }
2335
0205ec04
JS		 2336 if ((netdev->features & NETIF_F_HW_TC) > (features & NETIF_F_HW_TC) &&
2337 h->ae_algo->ops->cls_flower_active(h)) {
2338 netdev_err(netdev,
2339 "there are offloaded TC filters active, cannot disable HW TC offload");
2340 return -EINVAL;
2341 }
2342
2ba30662
JS		 2343 if ((changed & NETIF_F_HW_VLAN_CTAG_FILTER) &&
2344 h->ae_algo->ops->enable_vlan_filter) {
2345 enable = !!(features & NETIF_F_HW_VLAN_CTAG_FILTER);
2346 ret = h->ae_algo->ops->enable_vlan_filter(h, enable);
2347 if (ret)
2348 return ret;
2349 }
2350
76ad4f0e
S		 2351 netdev->features = features;
2352 return 0;
2353}
2354
2a7556bb
YL		 2355static netdev_features_t hns3_features_check(struct sk_buff *skb,
2356 struct net_device *dev,
2357 netdev_features_t features)
2358{
2359#define HNS3_MAX_HDR_LEN 480U
2360#define HNS3_MAX_L4_HDR_LEN 60U
2361
2362 size_t len;
2363
2364 if (skb->ip_summed != CHECKSUM_PARTIAL)
2365 return features;
2366
2367 if (skb->encapsulation)
2368 len = skb_inner_transport_header(skb) - skb->data;
2369 else
2370 len = skb_transport_header(skb) - skb->data;
2371
2372 /* Assume L4 is 60 byte as TCP is the only protocol with a
2373 * a flexible value, and it's max len is 60 bytes.
2374 */
2375 len += HNS3_MAX_L4_HDR_LEN;
2376
2377 /* Hardware only supports checksum on the skb with a max header
2378 * len of 480 bytes.
2379 */
2380 if (len > HNS3_MAX_HDR_LEN)
2381 features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
2382
2383 return features;
2384}
2385
6c88d9d7
PL		 2386static void hns3_nic_get_stats64(struct net_device *netdev,
2387 struct rtnl_link_stats64 *stats)
76ad4f0e
S		 2388{
2389 struct hns3_nic_priv *priv = netdev_priv(netdev);
2390 int queue_num = priv->ae_handle->kinfo.num_tqps;
c5f65480 2391 struct hnae3_handle *handle = priv->ae_handle;
76ad4f0e 2392 struct hns3_enet_ring *ring;
d3ec4ef6
JS		 2393 u64 rx_length_errors = 0;
2394 u64 rx_crc_errors = 0;
2395 u64 rx_multicast = 0;
76ad4f0e 2396 unsigned int start;
d3ec4ef6
JS		 2397 u64 tx_errors = 0;
2398 u64 rx_errors = 0;
76ad4f0e
S		 2399 unsigned int idx;
2400 u64 tx_bytes = 0;
2401 u64 rx_bytes = 0;
2402 u64 tx_pkts = 0;
2403 u64 rx_pkts = 0;
d2a5dca8
JS		 2404 u64 tx_drop = 0;
2405 u64 rx_drop = 0;
76ad4f0e 2406
b875cc37
JS		 2407 if (test_bit(HNS3_NIC_STATE_DOWN, &priv->state))
2408 return;
2409
c5f65480
JS		 2410 handle->ae_algo->ops->update_stats(handle, &netdev->stats);
2411
76ad4f0e
S		 2412 for (idx = 0; idx < queue_num; idx++) {
2413 /* fetch the tx stats */
5f06b903 2414 ring = &priv->ring[idx];
76ad4f0e 2415 do {
d36d36ce 2416 start = u64_stats_fetch_begin_irq(&ring->syncp);
76ad4f0e
S		 2417 tx_bytes += ring->stats.tx_bytes;
2418 tx_pkts += ring->stats.tx_pkts;
d2a5dca8 2419 tx_drop += ring->stats.sw_err_cnt;
b20d7fe5
YL		 2420 tx_drop += ring->stats.tx_vlan_err;
2421 tx_drop += ring->stats.tx_l4_proto_err;
2422 tx_drop += ring->stats.tx_l2l3l4_err;
2423 tx_drop += ring->stats.tx_tso_err;
d5d5e019
YL		 2424 tx_drop += ring->stats.over_max_recursion;
2425 tx_drop += ring->stats.hw_limitation;
907676b1 2426 tx_drop += ring->stats.copy_bits_err;
7459775e
YL		 2427 tx_drop += ring->stats.skb2sgl_err;
2428 tx_drop += ring->stats.map_sg_err;
d3ec4ef6 2429 tx_errors += ring->stats.sw_err_cnt;
b20d7fe5
YL		 2430 tx_errors += ring->stats.tx_vlan_err;
2431 tx_errors += ring->stats.tx_l4_proto_err;
2432 tx_errors += ring->stats.tx_l2l3l4_err;
2433 tx_errors += ring->stats.tx_tso_err;
d5d5e019
YL		 2434 tx_errors += ring->stats.over_max_recursion;
2435 tx_errors += ring->stats.hw_limitation;
907676b1 2436 tx_errors += ring->stats.copy_bits_err;
7459775e
YL		 2437 tx_errors += ring->stats.skb2sgl_err;
2438 tx_errors += ring->stats.map_sg_err;
76ad4f0e
S		 2439 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
2440
2441 /* fetch the rx stats */
5f06b903 2442 ring = &priv->ring[idx + queue_num];
76ad4f0e 2443 do {
d36d36ce 2444 start = u64_stats_fetch_begin_irq(&ring->syncp);
76ad4f0e
S		 2445 rx_bytes += ring->stats.rx_bytes;
2446 rx_pkts += ring->stats.rx_pkts;
d2a5dca8 2447 rx_drop += ring->stats.l2_err;
d3ec4ef6 2448 rx_errors += ring->stats.l2_err;
8b552079 2449 rx_errors += ring->stats.l3l4_csum_err;
d3ec4ef6 2450 rx_crc_errors += ring->stats.l2_err;
d3ec4ef6
JS		 2451 rx_multicast += ring->stats.rx_multicast;
2452 rx_length_errors += ring->stats.err_pkt_len;
76ad4f0e
S		 2453 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
2454 }
2455
2456 stats->tx_bytes = tx_bytes;
2457 stats->tx_packets = tx_pkts;
2458 stats->rx_bytes = rx_bytes;
2459 stats->rx_packets = rx_pkts;
2460
d3ec4ef6
JS		 2461 stats->rx_errors = rx_errors;
2462 stats->multicast = rx_multicast;
2463 stats->rx_length_errors = rx_length_errors;
2464 stats->rx_crc_errors = rx_crc_errors;
76ad4f0e
S		 2465 stats->rx_missed_errors = netdev->stats.rx_missed_errors;
2466
d3ec4ef6
JS		 2467 stats->tx_errors = tx_errors;
2468 stats->rx_dropped = rx_drop;
2469 stats->tx_dropped = tx_drop;
76ad4f0e
S		 2470 stats->collisions = netdev->stats.collisions;
2471 stats->rx_over_errors = netdev->stats.rx_over_errors;
2472 stats->rx_frame_errors = netdev->stats.rx_frame_errors;
2473 stats->rx_fifo_errors = netdev->stats.rx_fifo_errors;
2474 stats->tx_aborted_errors = netdev->stats.tx_aborted_errors;
2475 stats->tx_carrier_errors = netdev->stats.tx_carrier_errors;
2476 stats->tx_fifo_errors = netdev->stats.tx_fifo_errors;
2477 stats->tx_heartbeat_errors = netdev->stats.tx_heartbeat_errors;
2478 stats->tx_window_errors = netdev->stats.tx_window_errors;
2479 stats->rx_compressed = netdev->stats.rx_compressed;
2480 stats->tx_compressed = netdev->stats.tx_compressed;
2481}
2482
30d240df 2483static int hns3_setup_tc(struct net_device *netdev, void *type_data)
76ad4f0e 2484{
30d240df 2485 struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
75718800 2486 struct hnae3_knic_private_info *kinfo;
30d240df
YL		 2487 u8 tc = mqprio_qopt->qopt.num_tc;
2488 u16 mode = mqprio_qopt->mode;
2489 u8 hw = mqprio_qopt->qopt.hw;
75718800 2490 struct hnae3_handle *h;
76ad4f0e 2491
30d240df
YL		 2492 if (!((hw == TC_MQPRIO_HW_OFFLOAD_TCS &&
2493 mode == TC_MQPRIO_MODE_CHANNEL) || (!hw && tc == 0)))
2494 return -EOPNOTSUPP;
2495
76ad4f0e
S		 2496 if (tc > HNAE3_MAX_TC)
2497 return -EINVAL;
2498
76ad4f0e
S		 2499 if (!netdev)
2500 return -EINVAL;
2501
75718800
YL		 2502 h = hns3_get_handle(netdev);
2503 kinfo = &h->kinfo;
2504
1c822948
YL		 2505 netif_dbg(h, drv, netdev, "setup tc: num_tc=%u\n", tc);
2506
1cce5eb6 2507 return (kinfo->dcb_ops && kinfo->dcb_ops->setup_tc) ?
5a5c9091 2508 kinfo->dcb_ops->setup_tc(h, mqprio_qopt) : -EOPNOTSUPP;
76ad4f0e
S		 2509}
2510
0205ec04
JS		 2511static int hns3_setup_tc_cls_flower(struct hns3_nic_priv *priv,
2512 struct flow_cls_offload *flow)
2513{
2514 int tc = tc_classid_to_hwtc(priv->netdev, flow->classid);
2515 struct hnae3_handle *h = hns3_get_handle(priv->netdev);
2516
2517 switch (flow->command) {
2518 case FLOW_CLS_REPLACE:
2519 if (h->ae_algo->ops->add_cls_flower)
2520 return h->ae_algo->ops->add_cls_flower(h, flow, tc);
2521 break;
2522 case FLOW_CLS_DESTROY:
2523 if (h->ae_algo->ops->del_cls_flower)
2524 return h->ae_algo->ops->del_cls_flower(h, flow);
2525 break;
2526 default:
2527 break;
2528 }
2529
2530 return -EOPNOTSUPP;
2531}
2532
2533static int hns3_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
2534 void *cb_priv)
2535{
2536 struct hns3_nic_priv *priv = cb_priv;
2537
2538 if (!tc_cls_can_offload_and_chain0(priv->netdev, type_data))
2539 return -EOPNOTSUPP;
2540
2541 switch (type) {
2542 case TC_SETUP_CLSFLOWER:
2543 return hns3_setup_tc_cls_flower(priv, type_data);
2544 default:
2545 return -EOPNOTSUPP;
2546 }
2547}
2548
2549static LIST_HEAD(hns3_block_cb_list);
2550
2572ac53 2551static int hns3_nic_setup_tc(struct net_device *dev, enum tc_setup_type type,
de4784ca 2552 void *type_data)
76ad4f0e 2553{
0205ec04
JS		 2554 struct hns3_nic_priv *priv = netdev_priv(dev);
2555 int ret;
2556
2557 switch (type) {
2558 case TC_SETUP_QDISC_MQPRIO:
2559 ret = hns3_setup_tc(dev, type_data);
2560 break;
2561 case TC_SETUP_BLOCK:
2562 ret = flow_block_cb_setup_simple(type_data,
2563 &hns3_block_cb_list,
2564 hns3_setup_tc_block_cb,
2565 priv, priv, true);
2566 break;
2567 default:
38cf0426 2568 return -EOPNOTSUPP;
0205ec04 2569 }
76ad4f0e 2570
0205ec04 2571 return ret;
76ad4f0e
S		 2572}
2573
2574static int hns3_vlan_rx_add_vid(struct net_device *netdev,
2575 __be16 proto, u16 vid)
2576{
9780cb97 2577 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 2578 int ret = -EIO;
2579
2580 if (h->ae_algo->ops->set_vlan_filter)
2581 ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, false);
2582
2583 return ret;
2584}
2585
2586static int hns3_vlan_rx_kill_vid(struct net_device *netdev,
2587 __be16 proto, u16 vid)
2588{
9780cb97 2589 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 2590 int ret = -EIO;
2591
2592 if (h->ae_algo->ops->set_vlan_filter)
2593 ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, true);
2594
7fa6be4f 2595 return ret;
681ec399
YL		 2596}
2597
76ad4f0e
S		 2598static int hns3_ndo_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan,
2599 u8 qos, __be16 vlan_proto)
2600{
9780cb97 2601 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 2602 int ret = -EIO;
2603
1c822948 2604 netif_dbg(h, drv, netdev,
39edaf24
GL		 2605 "set vf vlan: vf=%d, vlan=%u, qos=%u, vlan_proto=0x%x\n",
2606 vf, vlan, qos, ntohs(vlan_proto));
1c822948 2607
76ad4f0e
S		 2608 if (h->ae_algo->ops->set_vf_vlan_filter)
2609 ret = h->ae_algo->ops->set_vf_vlan_filter(h, vf, vlan,
9b2f3477 2610 qos, vlan_proto);
76ad4f0e
S		 2611
2612 return ret;
2613}
2614
22044f95
JS		 2615static int hns3_set_vf_spoofchk(struct net_device *netdev, int vf, bool enable)
2616{
2617 struct hnae3_handle *handle = hns3_get_handle(netdev);
2618
2619 if (hns3_nic_resetting(netdev))
2620 return -EBUSY;
2621
2622 if (!handle->ae_algo->ops->set_vf_spoofchk)
2623 return -EOPNOTSUPP;
2624
2625 return handle->ae_algo->ops->set_vf_spoofchk(handle, vf, enable);
2626}
2627
e196ec75
JS		 2628static int hns3_set_vf_trust(struct net_device *netdev, int vf, bool enable)
2629{
2630 struct hnae3_handle *handle = hns3_get_handle(netdev);
2631
2632 if (!handle->ae_algo->ops->set_vf_trust)
2633 return -EOPNOTSUPP;
2634
2635 return handle->ae_algo->ops->set_vf_trust(handle, vf, enable);
2636}
2637
a8e8b7ff
S		 2638static int hns3_nic_change_mtu(struct net_device *netdev, int new_mtu)
2639{
9780cb97 2640 struct hnae3_handle *h = hns3_get_handle(netdev);
a8e8b7ff
S		 2641 int ret;
2642
6ff7ed80
HT		 2643 if (hns3_nic_resetting(netdev))
2644 return -EBUSY;
2645
a8e8b7ff
S		 2646 if (!h->ae_algo->ops->set_mtu)
2647 return -EOPNOTSUPP;
2648
1c822948
YL		 2649 netif_dbg(h, drv, netdev,
2650 "change mtu from %u to %d\n", netdev->mtu, new_mtu);
2651
a8e8b7ff 2652 ret = h->ae_algo->ops->set_mtu(h, new_mtu);
93d8daf4 2653 if (ret)
a8e8b7ff
S		 2654 netdev_err(netdev, "failed to change MTU in hardware %d\n",
2655 ret);
93d8daf4
YL		 2656 else
2657 netdev->mtu = new_mtu;
5bad95a1 2658
a8e8b7ff
S		 2659 return ret;
2660}
2661
f8fa222c
L		 2662static bool hns3_get_tx_timeo_queue_info(struct net_device *ndev)
2663{
2664 struct hns3_nic_priv *priv = netdev_priv(ndev);
e511c97d 2665 struct hnae3_handle *h = hns3_get_handle(ndev);
0bfdf286 2666 struct hns3_enet_ring *tx_ring;
e511c97d 2667 struct napi_struct *napi;
f8fa222c
L		 2668 int timeout_queue = 0;
2669 int hw_head, hw_tail;
e511c97d
JS		 2670 int fbd_num, fbd_oft;
2671 int ebd_num, ebd_oft;
2672 int bd_num, bd_err;
2673 int ring_en, tc;
f8fa222c
L		 2674 int i;
2675
2676 /* Find the stopped queue the same way the stack does */
fa6c4084 2677 for (i = 0; i < ndev->num_tx_queues; i++) {
f8fa222c
L		 2678 struct netdev_queue *q;
2679 unsigned long trans_start;
2680
2681 q = netdev_get_tx_queue(ndev, i);
5337824f 2682 trans_start = READ_ONCE(q->trans_start);
f8fa222c
L		 2683 if (netif_xmit_stopped(q) &&
2684 time_after(jiffies,
2685 (trans_start + ndev->watchdog_timeo))) {
2686 timeout_queue = i;
647522a5
YL		 2687 netdev_info(ndev, "queue state: 0x%lx, delta msecs: %u\n",
2688 q->state,
2689 jiffies_to_msecs(jiffies - trans_start));
f8fa222c
L		 2690 break;
2691 }
2692 }
2693
2694 if (i == ndev->num_tx_queues) {
2695 netdev_info(ndev,
2696 "no netdev TX timeout queue found, timeout count: %llu\n",
2697 priv->tx_timeout_count);
2698 return false;
2699 }
2700
beab694a
JS		 2701 priv->tx_timeout_count++;
2702
5f06b903 2703 tx_ring = &priv->ring[timeout_queue];
e511c97d
JS		 2704 napi = &tx_ring->tqp_vector->napi;
2705
2706 netdev_info(ndev,
2707 "tx_timeout count: %llu, queue id: %d, SW_NTU: 0x%x, SW_NTC: 0x%x, napi state: %lu\n",
2708 priv->tx_timeout_count, timeout_queue, tx_ring->next_to_use,
2709 tx_ring->next_to_clean, napi->state);
2710
2711 netdev_info(ndev,
20d06ca2 2712 "tx_pkts: %llu, tx_bytes: %llu, sw_err_cnt: %llu, tx_pending: %d\n",
e511c97d 2713 tx_ring->stats.tx_pkts, tx_ring->stats.tx_bytes,
20d06ca2 2714 tx_ring->stats.sw_err_cnt, tx_ring->pending_buf);
e511c97d
JS		 2715
2716 netdev_info(ndev,
f6061a05
YL		 2717 "seg_pkt_cnt: %llu, tx_more: %llu, restart_queue: %llu, tx_busy: %llu\n",
2718 tx_ring->stats.seg_pkt_cnt, tx_ring->stats.tx_more,
e511c97d
JS		 2719 tx_ring->stats.restart_queue, tx_ring->stats.tx_busy);
2720
2721 /* When mac received many pause frames continuous, it's unable to send
2722 * packets, which may cause tx timeout
2723 */
615466ce
YM		 2724 if (h->ae_algo->ops->get_mac_stats) {
2725 struct hns3_mac_stats mac_stats;
e511c97d 2726
615466ce 2727 h->ae_algo->ops->get_mac_stats(h, &mac_stats);
e511c97d 2728 netdev_info(ndev, "tx_pause_cnt: %llu, rx_pause_cnt: %llu\n",
615466ce 2729 mac_stats.tx_pause_cnt, mac_stats.rx_pause_cnt);
e511c97d 2730 }
f8fa222c
L		 2731
2732 hw_head = readl_relaxed(tx_ring->tqp->io_base +
2733 HNS3_RING_TX_RING_HEAD_REG);
2734 hw_tail = readl_relaxed(tx_ring->tqp->io_base +
2735 HNS3_RING_TX_RING_TAIL_REG);
e511c97d
JS		 2736 fbd_num = readl_relaxed(tx_ring->tqp->io_base +
2737 HNS3_RING_TX_RING_FBDNUM_REG);
2738 fbd_oft = readl_relaxed(tx_ring->tqp->io_base +
2739 HNS3_RING_TX_RING_OFFSET_REG);
2740 ebd_num = readl_relaxed(tx_ring->tqp->io_base +
2741 HNS3_RING_TX_RING_EBDNUM_REG);
2742 ebd_oft = readl_relaxed(tx_ring->tqp->io_base +
2743 HNS3_RING_TX_RING_EBD_OFFSET_REG);
2744 bd_num = readl_relaxed(tx_ring->tqp->io_base +
2745 HNS3_RING_TX_RING_BD_NUM_REG);
2746 bd_err = readl_relaxed(tx_ring->tqp->io_base +
2747 HNS3_RING_TX_RING_BD_ERR_REG);
2748 ring_en = readl_relaxed(tx_ring->tqp->io_base + HNS3_RING_EN_REG);
2749 tc = readl_relaxed(tx_ring->tqp->io_base + HNS3_RING_TX_RING_TC_REG);
2750
f8fa222c 2751 netdev_info(ndev,
e511c97d
JS		 2752 "BD_NUM: 0x%x HW_HEAD: 0x%x, HW_TAIL: 0x%x, BD_ERR: 0x%x, INT: 0x%x\n",
2753 bd_num, hw_head, hw_tail, bd_err,
f8fa222c 2754 readl(tx_ring->tqp_vector->mask_addr));
e511c97d
JS		 2755 netdev_info(ndev,
2756 "RING_EN: 0x%x, TC: 0x%x, FBD_NUM: 0x%x FBD_OFT: 0x%x, EBD_NUM: 0x%x, EBD_OFT: 0x%x\n",
2757 ring_en, tc, fbd_num, fbd_oft, ebd_num, ebd_oft);
f8fa222c
L		 2758
2759 return true;
2760}
2761
0290bd29 2762static void hns3_nic_net_timeout(struct net_device *ndev, unsigned int txqueue)
f8fa222c
L		 2763{
2764 struct hns3_nic_priv *priv = netdev_priv(ndev);
f8fa222c
L		 2765 struct hnae3_handle *h = priv->ae_handle;
2766
2767 if (!hns3_get_tx_timeo_queue_info(ndev))
2768 return;
2769
0742ed7c
HT		 2770 /* request the reset, and let the hclge to determine
2771 * which reset level should be done
2772 */
f8fa222c 2773 if (h->ae_algo->ops->reset_event)
6ae4e733 2774 h->ae_algo->ops->reset_event(h->pdev, h);
f8fa222c
L		 2775}
2776
d93ed94f
JS		 2777#ifdef CONFIG_RFS_ACCEL
2778static int hns3_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
2779 u16 rxq_index, u32 flow_id)
2780{
2781 struct hnae3_handle *h = hns3_get_handle(dev);
2782 struct flow_keys fkeys;
2783
2784 if (!h->ae_algo->ops->add_arfs_entry)
2785 return -EOPNOTSUPP;
2786
2787 if (skb->encapsulation)
2788 return -EPROTONOSUPPORT;
2789
2790 if (!skb_flow_dissect_flow_keys(skb, &fkeys, 0))
2791 return -EPROTONOSUPPORT;
2792
2793 if ((fkeys.basic.n_proto != htons(ETH_P_IP) &&
2794 fkeys.basic.n_proto != htons(ETH_P_IPV6)) ||
2795 (fkeys.basic.ip_proto != IPPROTO_TCP &&
2796 fkeys.basic.ip_proto != IPPROTO_UDP))
2797 return -EPROTONOSUPPORT;
2798
2799 return h->ae_algo->ops->add_arfs_entry(h, rxq_index, flow_id, &fkeys);
2800}
2801#endif
2802
6430f744
YM		 2803static int hns3_nic_get_vf_config(struct net_device *ndev, int vf,
2804 struct ifla_vf_info *ivf)
2805{
2806 struct hnae3_handle *h = hns3_get_handle(ndev);
2807
2808 if (!h->ae_algo->ops->get_vf_config)
2809 return -EOPNOTSUPP;
2810
2811 return h->ae_algo->ops->get_vf_config(h, vf, ivf);
2812}
2813
2814static int hns3_nic_set_vf_link_state(struct net_device *ndev, int vf,
2815 int link_state)
2816{
2817 struct hnae3_handle *h = hns3_get_handle(ndev);
2818
2819 if (!h->ae_algo->ops->set_vf_link_state)
2820 return -EOPNOTSUPP;
2821
2822 return h->ae_algo->ops->set_vf_link_state(h, vf, link_state);
2823}
2824
ee9e4424
YL		 2825static int hns3_nic_set_vf_rate(struct net_device *ndev, int vf,
2826 int min_tx_rate, int max_tx_rate)
2827{
2828 struct hnae3_handle *h = hns3_get_handle(ndev);
2829
2830 if (!h->ae_algo->ops->set_vf_rate)
2831 return -EOPNOTSUPP;
2832
2833 return h->ae_algo->ops->set_vf_rate(h, vf, min_tx_rate, max_tx_rate,
2834 false);
2835}
2836
8e6de441
HT		 2837static int hns3_nic_set_vf_mac(struct net_device *netdev, int vf_id, u8 *mac)
2838{
2839 struct hnae3_handle *h = hns3_get_handle(netdev);
2840
2841 if (!h->ae_algo->ops->set_vf_mac)
2842 return -EOPNOTSUPP;
2843
2844 if (is_multicast_ether_addr(mac)) {
2845 netdev_err(netdev,
2846 "Invalid MAC:%pM specified. Could not set MAC\n",
2847 mac);
2848 return -EINVAL;
2849 }
2850
2851 return h->ae_algo->ops->set_vf_mac(h, vf_id, mac);
2852}
2853
76ad4f0e
S		 2854static const struct net_device_ops hns3_nic_netdev_ops = {
2855 .ndo_open = hns3_nic_net_open,
2856 .ndo_stop = hns3_nic_net_stop,
2857 .ndo_start_xmit = hns3_nic_net_xmit,
f8fa222c 2858 .ndo_tx_timeout = hns3_nic_net_timeout,
76ad4f0e 2859 .ndo_set_mac_address = hns3_nic_net_set_mac_address,
a7605370 2860 .ndo_eth_ioctl = hns3_nic_do_ioctl,
a8e8b7ff 2861 .ndo_change_mtu = hns3_nic_change_mtu,
76ad4f0e 2862 .ndo_set_features = hns3_nic_set_features,
2a7556bb 2863 .ndo_features_check = hns3_features_check,
76ad4f0e
S		 2864 .ndo_get_stats64 = hns3_nic_get_stats64,
2865 .ndo_setup_tc = hns3_nic_setup_tc,
2866 .ndo_set_rx_mode = hns3_nic_set_rx_mode,
76ad4f0e
S		 2867 .ndo_vlan_rx_add_vid = hns3_vlan_rx_add_vid,
2868 .ndo_vlan_rx_kill_vid = hns3_vlan_rx_kill_vid,
2869 .ndo_set_vf_vlan = hns3_ndo_set_vf_vlan,
22044f95 2870 .ndo_set_vf_spoofchk = hns3_set_vf_spoofchk,
e196ec75 2871 .ndo_set_vf_trust = hns3_set_vf_trust,
d93ed94f
JS		 2872#ifdef CONFIG_RFS_ACCEL
2873 .ndo_rx_flow_steer = hns3_rx_flow_steer,
2874#endif
6430f744
YM		 2875 .ndo_get_vf_config = hns3_nic_get_vf_config,
2876 .ndo_set_vf_link_state = hns3_nic_set_vf_link_state,
ee9e4424 2877 .ndo_set_vf_rate = hns3_nic_set_vf_rate,
8e6de441 2878 .ndo_set_vf_mac = hns3_nic_set_vf_mac,
76ad4f0e
S		 2879};
2880
97afd47b 2881bool hns3_is_phys_func(struct pci_dev *pdev)
2312e050
FL		 2882{
2883 u32 dev_id = pdev->device;
2884
2885 switch (dev_id) {
2886 case HNAE3_DEV_ID_GE:
2887 case HNAE3_DEV_ID_25GE:
2888 case HNAE3_DEV_ID_25GE_RDMA:
2889 case HNAE3_DEV_ID_25GE_RDMA_MACSEC:
2890 case HNAE3_DEV_ID_50GE_RDMA:
2891 case HNAE3_DEV_ID_50GE_RDMA_MACSEC:
2892 case HNAE3_DEV_ID_100G_RDMA_MACSEC:
ae6f010c 2893 case HNAE3_DEV_ID_200G_RDMA:
2312e050 2894 return true;
c155e22b
GH		 2895 case HNAE3_DEV_ID_VF:
2896 case HNAE3_DEV_ID_RDMA_DCB_PFC_VF:
2312e050
FL		 2897 return false;
2898 default:
adcf738b 2899 dev_warn(&pdev->dev, "un-recognized pci device-id %u",
2312e050
FL		 2900 dev_id);
2901 }
2902
2903 return false;
2904}
2905
2312e050
FL		 2906static void hns3_disable_sriov(struct pci_dev *pdev)
2907{
2908 /* If our VFs are assigned we cannot shut down SR-IOV
2909 * without causing issues, so just leave the hardware
2910 * available but disabled
2911 */
2912 if (pci_vfs_assigned(pdev)) {
2913 dev_warn(&pdev->dev,
2914 "disabling driver while VFs are assigned\n");
2915 return;
2916 }
2917
2918 pci_disable_sriov(pdev);
2919}
2920
76ad4f0e
S		 2921/* hns3_probe - Device initialization routine
2922 * @pdev: PCI device information struct
2923 * @ent: entry in hns3_pci_tbl
2924 *
2925 * hns3_probe initializes a PF identified by a pci_dev structure.
2926 * The OS initialization, configuring of the PF private structure,
2927 * and a hardware reset occur.
2928 *
2929 * Returns 0 on success, negative on failure
2930 */
2931static int hns3_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
2932{
2933 struct hnae3_ae_dev *ae_dev;
2934 int ret;
2935
9b2f3477 2936 ae_dev = devm_kzalloc(&pdev->dev, sizeof(*ae_dev), GFP_KERNEL);
60df7e91
HT		 2937 if (!ae_dev)
2938 return -ENOMEM;
76ad4f0e
S		 2939
2940 ae_dev->pdev = pdev;
e92a0843 2941 ae_dev->flag = ent->driver_data;
76ad4f0e
S		 2942 pci_set_drvdata(pdev, ae_dev);
2943
74354140 2944 ret = hnae3_register_ae_dev(ae_dev);
674a1357 2945 if (ret)
74354140 2946 pci_set_drvdata(pdev, NULL);
2312e050 2947
74354140 2948 return ret;
76ad4f0e
S		 2949}
2950
2951/* hns3_remove - Device removal routine
2952 * @pdev: PCI device information struct
2953 */
2954static void hns3_remove(struct pci_dev *pdev)
2955{
2956 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
2957
2312e050
FL		 2958 if (hns3_is_phys_func(pdev) && IS_ENABLED(CONFIG_PCI_IOV))
2959 hns3_disable_sriov(pdev);
2960
76ad4f0e 2961 hnae3_unregister_ae_dev(ae_dev);
ac864c23 2962 pci_set_drvdata(pdev, NULL);
76ad4f0e
S		 2963}
2964
fa8d82e8
PL		 2965/**
2966 * hns3_pci_sriov_configure
2967 * @pdev: pointer to a pci_dev structure
2968 * @num_vfs: number of VFs to allocate
2969 *
2970 * Enable or change the number of VFs. Called when the user updates the number
2971 * of VFs in sysfs.
2972 **/
743e1a84 2973static int hns3_pci_sriov_configure(struct pci_dev *pdev, int num_vfs)
fa8d82e8
PL		 2974{
2975 int ret;
2976
2977 if (!(hns3_is_phys_func(pdev) && IS_ENABLED(CONFIG_PCI_IOV))) {
2978 dev_warn(&pdev->dev, "Can not config SRIOV\n");
2979 return -EINVAL;
2980 }
2981
2982 if (num_vfs) {
2983 ret = pci_enable_sriov(pdev, num_vfs);
2984 if (ret)
2985 dev_err(&pdev->dev, "SRIOV enable failed %d\n", ret);
743e1a84
SM		 2986 else
2987 return num_vfs;
fa8d82e8
PL		 2988 } else if (!pci_vfs_assigned(pdev)) {
2989 pci_disable_sriov(pdev);
2990 } else {
2991 dev_warn(&pdev->dev,
2992 "Unable to free VFs because some are assigned to VMs.\n");
2993 }
2994
2995 return 0;
2996}
2997
ce2c1d2e
YL		 2998static void hns3_shutdown(struct pci_dev *pdev)
2999{
3000 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
3001
3002 hnae3_unregister_ae_dev(ae_dev);
ce2c1d2e
YL		 3003 pci_set_drvdata(pdev, NULL);
3004
3005 if (system_state == SYSTEM_POWER_OFF)
3006 pci_set_power_state(pdev, PCI_D3hot);
3007}
3008
715c58e9
JZ		 3009static int __maybe_unused hns3_suspend(struct device *dev)
3010{
3011 struct hnae3_ae_dev *ae_dev = dev_get_drvdata(dev);
3012
d0494135 3013 if (ae_dev && hns3_is_phys_func(ae_dev->pdev)) {
715c58e9 3014 dev_info(dev, "Begin to suspend.\n");
d0494135 3015 if (ae_dev->ops && ae_dev->ops->reset_prepare)
715c58e9
JZ		 3016 ae_dev->ops->reset_prepare(ae_dev, HNAE3_FUNC_RESET);
3017 }
3018
3019 return 0;
3020}
3021
3022static int __maybe_unused hns3_resume(struct device *dev)
3023{
3024 struct hnae3_ae_dev *ae_dev = dev_get_drvdata(dev);
3025
d0494135 3026 if (ae_dev && hns3_is_phys_func(ae_dev->pdev)) {
715c58e9 3027 dev_info(dev, "Begin to resume.\n");
d0494135 3028 if (ae_dev->ops && ae_dev->ops->reset_done)
715c58e9
JZ		 3029 ae_dev->ops->reset_done(ae_dev);
3030 }
3031
3032 return 0;
3033}
3034
5a9f0eac
SJ		 3035static pci_ers_result_t hns3_error_detected(struct pci_dev *pdev,
3036 pci_channel_state_t state)
3037{
3038 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
3039 pci_ers_result_t ret;
3040
c5aaf176 3041 dev_info(&pdev->dev, "PCI error detected, state(=%u)!!\n", state);
5a9f0eac
SJ		 3042
3043 if (state == pci_channel_io_perm_failure)
3044 return PCI_ERS_RESULT_DISCONNECT;
3045
661262bc 3046 if (!ae_dev || !ae_dev->ops) {
5a9f0eac 3047 dev_err(&pdev->dev,
661262bc 3048 "Can't recover - error happened before device initialized\n");
5a9f0eac
SJ		 3049 return PCI_ERS_RESULT_NONE;
3050 }
3051
381c356e
SJ		 3052 if (ae_dev->ops->handle_hw_ras_error)
3053 ret = ae_dev->ops->handle_hw_ras_error(ae_dev);
5a9f0eac
SJ		 3054 else
3055 return PCI_ERS_RESULT_NONE;
3056
3057 return ret;
3058}
3059
6ae4e733
SJ		 3060static pci_ers_result_t hns3_slot_reset(struct pci_dev *pdev)
3061{
3062 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
ad9bf545 3063 const struct hnae3_ae_ops *ops;
123297b7 3064 enum hnae3_reset_type reset_type;
6ae4e733
SJ		 3065 struct device *dev = &pdev->dev;
3066
661262bc
WL		 3067 if (!ae_dev || !ae_dev->ops)
3068 return PCI_ERS_RESULT_NONE;
3069
ad9bf545 3070 ops = ae_dev->ops;
6ae4e733 3071 /* request the reset */
fa17c708
GH		 3072 if (ops->reset_event && ops->get_reset_level &&
3073 ops->set_default_reset_request) {
9d5e67d1 3074 if (ae_dev->hw_err_reset_req) {
123297b7
SJ		 3075 reset_type = ops->get_reset_level(ae_dev,
3076 &ae_dev->hw_err_reset_req);
3077 ops->set_default_reset_request(ae_dev, reset_type);
3078 dev_info(dev, "requesting reset due to PCI error\n");
3079 ops->reset_event(pdev, NULL);
3080 }
69b51bbb 3081
6ae4e733
SJ		 3082 return PCI_ERS_RESULT_RECOVERED;
3083 }
3084
3085 return PCI_ERS_RESULT_DISCONNECT;
3086}
3087
6b9a97ee
HT		 3088static void hns3_reset_prepare(struct pci_dev *pdev)
3089{
3090 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
3091
8de91e92 3092 dev_info(&pdev->dev, "FLR prepare\n");
bb1890d5
JZ		 3093 if (ae_dev && ae_dev->ops && ae_dev->ops->reset_prepare)
3094 ae_dev->ops->reset_prepare(ae_dev, HNAE3_FLR_RESET);
6b9a97ee
HT		 3095}
3096
3097static void hns3_reset_done(struct pci_dev *pdev)
3098{
3099 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
3100
8de91e92 3101 dev_info(&pdev->dev, "FLR done\n");
bb1890d5
JZ		 3102 if (ae_dev && ae_dev->ops && ae_dev->ops->reset_done)
3103 ae_dev->ops->reset_done(ae_dev);
6b9a97ee
HT		 3104}
3105
5a9f0eac
SJ		 3106static const struct pci_error_handlers hns3_err_handler = {
3107 .error_detected = hns3_error_detected,
6ae4e733 3108 .slot_reset = hns3_slot_reset,
6b9a97ee
HT		 3109 .reset_prepare = hns3_reset_prepare,
3110 .reset_done = hns3_reset_done,
5a9f0eac
SJ		 3111};
3112
715c58e9
JZ		 3113static SIMPLE_DEV_PM_OPS(hns3_pm_ops, hns3_suspend, hns3_resume);
3114
76ad4f0e
S		 3115static struct pci_driver hns3_driver = {
3116 .name = hns3_driver_name,
3117 .id_table = hns3_pci_tbl,
3118 .probe = hns3_probe,
3119 .remove = hns3_remove,
ce2c1d2e 3120 .shutdown = hns3_shutdown,
715c58e9 3121 .driver.pm = &hns3_pm_ops,
fa8d82e8 3122 .sriov_configure = hns3_pci_sriov_configure,
5a9f0eac 3123 .err_handler = &hns3_err_handler,
76ad4f0e
S		 3124};
3125
3126/* set default feature to hns3 */
3127static void hns3_set_default_feature(struct net_device *netdev)
3128{
3e85af6a
PL		 3129 struct hnae3_handle *h = hns3_get_handle(netdev);
3130 struct pci_dev *pdev = h->pdev;
295ba232 3131 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
3e85af6a 3132
76ad4f0e
S		 3133 netdev->priv_flags |= IFF_UNICAST_FLT;
3134
76ad4f0e
S		 3135 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
3136
66d52f3b 3137 netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER |
052ece6d 3138 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
76ad4f0e
S		 3139 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
3140 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
3141 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
57e72c12 3142 NETIF_F_SCTP_CRC | NETIF_F_FRAGLIST;
76ad4f0e 3143
295ba232 3144 if (ae_dev->dev_version >= HNAE3_DEVICE_VERSION_V2) {
5c9f6b39 3145 netdev->features |= NETIF_F_GRO_HW;
c17852a8 3146
dc9b5ce0 3147 if (!(h->flags & HNAE3_SUPPORT_VF))
c17852a8 3148 netdev->features |= NETIF_F_NTUPLE;
c17852a8 3149 }
0692cfe9 3150
dc9b5ce0 3151 if (test_bit(HNAE3_DEV_SUPPORT_UDP_GSO_B, ae_dev->caps))
0692cfe9 3152 netdev->features |= NETIF_F_GSO_UDP_L4;
66d52f3b 3153
dc9b5ce0 3154 if (test_bit(HNAE3_DEV_SUPPORT_HW_TX_CSUM_B, ae_dev->caps))
66d52f3b 3155 netdev->features |= NETIF_F_HW_CSUM;
dc9b5ce0 3156 else
66d52f3b 3157 netdev->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
3e281621 3158
dc9b5ce0 3159 if (test_bit(HNAE3_DEV_SUPPORT_UDP_TUNNEL_CSUM_B, ae_dev->caps))
3e281621 3160 netdev->features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
0205ec04 3161
dc9b5ce0 3162 if (test_bit(HNAE3_DEV_SUPPORT_FD_FORWARD_TC_B, ae_dev->caps))
0205ec04 3163 netdev->features |= NETIF_F_HW_TC;
2ba30662 3164
dc9b5ce0
JS		 3165 netdev->hw_features |= netdev->features;
3166 if (!test_bit(HNAE3_DEV_SUPPORT_VLAN_FLTR_MDF_B, ae_dev->caps))
3167 netdev->hw_features &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
3168
3169 netdev->vlan_features |= netdev->features &
3170 ~(NETIF_F_HW_VLAN_CTAG_FILTER | NETIF_F_HW_VLAN_CTAG_TX |
3171 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_GRO_HW | NETIF_F_NTUPLE |
3172 NETIF_F_HW_TC);
3173
3174 netdev->hw_enc_features |= netdev->vlan_features | NETIF_F_TSO_MANGLEID;
76ad4f0e
S		 3175}
3176
3177static int hns3_alloc_buffer(struct hns3_enet_ring *ring,
3178 struct hns3_desc_cb *cb)
3179{
dbba6da0 3180 unsigned int order = hns3_page_order(ring);
76ad4f0e
S		 3181 struct page *p;
3182
93188e96
YL		 3183 if (ring->page_pool) {
3184 p = page_pool_dev_alloc_frag(ring->page_pool,
3185 &cb->page_offset,
3186 hns3_buf_size(ring));
3187 if (unlikely(!p))
3188 return -ENOMEM;
3189
3190 cb->priv = p;
3191 cb->buf = page_address(p);
3192 cb->dma = page_pool_get_dma_addr(p);
3193 cb->type = DESC_TYPE_PP_FRAG;
3194 cb->reuse_flag = 0;
3195 return 0;
3196 }
3197
76ad4f0e
S		 3198 p = dev_alloc_pages(order);
3199 if (!p)
3200 return -ENOMEM;
3201
3202 cb->priv = p;
3203 cb->page_offset = 0;
3204 cb->reuse_flag = 0;
3205 cb->buf = page_address(p);
dbba6da0 3206 cb->length = hns3_page_size(ring);
76ad4f0e 3207 cb->type = DESC_TYPE_PAGE;
aeda9bf8
YL		 3208 page_ref_add(p, USHRT_MAX - 1);
3209 cb->pagecnt_bias = USHRT_MAX;
76ad4f0e 3210
76ad4f0e
S		 3211 return 0;
3212}
3213
3214static void hns3_free_buffer(struct hns3_enet_ring *ring,
619ae331 3215 struct hns3_desc_cb *cb, int budget)
76ad4f0e 3216{
907676b1 3217 if (cb->type & (DESC_TYPE_SKB | DESC_TYPE_BOUNCE_HEAD |
7459775e 3218 DESC_TYPE_BOUNCE_ALL | DESC_TYPE_SGL_SKB))
619ae331 3219 napi_consume_skb(cb->priv, budget);
93188e96
YL		 3220 else if (!HNAE3_IS_TX_RING(ring)) {
3221 if (cb->type & DESC_TYPE_PAGE && cb->pagecnt_bias)
3222 __page_frag_cache_drain(cb->priv, cb->pagecnt_bias);
3223 else if (cb->type & DESC_TYPE_PP_FRAG)
3224 page_pool_put_full_page(ring->page_pool, cb->priv,
3225 false);
3226 }
76ad4f0e
S		 3227 memset(cb, 0, sizeof(*cb));
3228}
3229
3230static int hns3_map_buffer(struct hns3_enet_ring *ring, struct hns3_desc_cb *cb)
3231{
3232 cb->dma = dma_map_page(ring_to_dev(ring), cb->priv, 0,
3233 cb->length, ring_to_dma_dir(ring));
3234
2211f4e1 3235 if (unlikely(dma_mapping_error(ring_to_dev(ring), cb->dma)))
76ad4f0e
S		 3236 return -EIO;
3237
3238 return 0;
3239}
3240
3241static void hns3_unmap_buffer(struct hns3_enet_ring *ring,
3242 struct hns3_desc_cb *cb)
3243{
26f1ccdf 3244 if (cb->type & (DESC_TYPE_SKB | DESC_TYPE_FRAGLIST_SKB))
76ad4f0e
S		 3245 dma_unmap_single(ring_to_dev(ring), cb->dma, cb->length,
3246 ring_to_dma_dir(ring));
907676b1 3247 else if ((cb->type & DESC_TYPE_PAGE) && cb->length)
76ad4f0e
S		 3248 dma_unmap_page(ring_to_dev(ring), cb->dma, cb->length,
3249 ring_to_dma_dir(ring));
7459775e
YL		 3250 else if (cb->type & (DESC_TYPE_BOUNCE_ALL | DESC_TYPE_BOUNCE_HEAD |
3251 DESC_TYPE_SGL_SKB))
907676b1 3252 hns3_tx_spare_reclaim_cb(ring, cb);
76ad4f0e
S		 3253}
3254
3255static void hns3_buffer_detach(struct hns3_enet_ring *ring, int i)
3256{
3257 hns3_unmap_buffer(ring, &ring->desc_cb[i]);
3258 ring->desc[i].addr = 0;
9f9f0f19 3259 ring->desc_cb[i].refill = 0;
76ad4f0e
S		 3260}
3261
619ae331
YL		 3262static void hns3_free_buffer_detach(struct hns3_enet_ring *ring, int i,
3263 int budget)
76ad4f0e
S		 3264{
3265 struct hns3_desc_cb *cb = &ring->desc_cb[i];
3266
3267 if (!ring->desc_cb[i].dma)
3268 return;
3269
3270 hns3_buffer_detach(ring, i);
619ae331 3271 hns3_free_buffer(ring, cb, budget);
76ad4f0e
S		 3272}
3273
3274static void hns3_free_buffers(struct hns3_enet_ring *ring)
3275{
3276 int i;
3277
3278 for (i = 0; i < ring->desc_num; i++)
619ae331 3279 hns3_free_buffer_detach(ring, i, 0);
76ad4f0e
S		 3280}
3281
3282/* free desc along with its attached buffer */
3283static void hns3_free_desc(struct hns3_enet_ring *ring)
3284{
024cc792
HT		 3285 int size = ring->desc_num * sizeof(ring->desc[0]);
3286
76ad4f0e
S		 3287 hns3_free_buffers(ring);
3288
024cc792
HT		 3289 if (ring->desc) {
3290 dma_free_coherent(ring_to_dev(ring), size,
3291 ring->desc, ring->desc_dma_addr);
3292 ring->desc = NULL;
3293 }
76ad4f0e
S		 3294}
3295
3296static int hns3_alloc_desc(struct hns3_enet_ring *ring)
3297{
3298 int size = ring->desc_num * sizeof(ring->desc[0]);
3299
750afb08
LC		 3300 ring->desc = dma_alloc_coherent(ring_to_dev(ring), size,
3301 &ring->desc_dma_addr, GFP_KERNEL);
76ad4f0e
S		 3302 if (!ring->desc)
3303 return -ENOMEM;
3304
76ad4f0e
S		 3305 return 0;
3306}
3307
4d2cad32 3308static int hns3_alloc_and_map_buffer(struct hns3_enet_ring *ring,
76ad4f0e
S		 3309 struct hns3_desc_cb *cb)
3310{
3311 int ret;
3312
3313 ret = hns3_alloc_buffer(ring, cb);
93188e96 3314 if (ret || ring->page_pool)
76ad4f0e
S		 3315 goto out;
3316
3317 ret = hns3_map_buffer(ring, cb);
3318 if (ret)
3319 goto out_with_buf;
3320
3321 return 0;
3322
3323out_with_buf:
619ae331 3324 hns3_free_buffer(ring, cb, 0);
76ad4f0e
S		 3325out:
3326 return ret;
3327}
3328
4d2cad32 3329static int hns3_alloc_and_attach_buffer(struct hns3_enet_ring *ring, int i)
76ad4f0e 3330{
4d2cad32 3331 int ret = hns3_alloc_and_map_buffer(ring, &ring->desc_cb[i]);
76ad4f0e
S		 3332
3333 if (ret)
3334 return ret;
3335
93188e96
YL		 3336 ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma +
3337 ring->desc_cb[i].page_offset);
9f9f0f19 3338 ring->desc_cb[i].refill = 1;
76ad4f0e
S		 3339
3340 return 0;
3341}
3342
3343/* Allocate memory for raw pkg, and map with dma */
3344static int hns3_alloc_ring_buffers(struct hns3_enet_ring *ring)
3345{
3346 int i, j, ret;
3347
3348 for (i = 0; i < ring->desc_num; i++) {
4d2cad32 3349 ret = hns3_alloc_and_attach_buffer(ring, i);
76ad4f0e
S		 3350 if (ret)
3351 goto out_buffer_fail;
3352 }
3353
3354 return 0;
3355
3356out_buffer_fail:
3357 for (j = i - 1; j >= 0; j--)
619ae331 3358 hns3_free_buffer_detach(ring, j, 0);
76ad4f0e
S		 3359 return ret;
3360}
3361
9b2f3477 3362/* detach a in-used buffer and replace with a reserved one */
76ad4f0e
S		 3363static void hns3_replace_buffer(struct hns3_enet_ring *ring, int i,
3364 struct hns3_desc_cb *res_cb)
3365{
b9077428 3366 hns3_unmap_buffer(ring, &ring->desc_cb[i]);
76ad4f0e 3367 ring->desc_cb[i] = *res_cb;
9f9f0f19 3368 ring->desc_cb[i].refill = 1;
93188e96
YL		 3369 ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma +
3370 ring->desc_cb[i].page_offset);
7d0b130c 3371 ring->desc[i].rx.bd_base_info = 0;
76ad4f0e
S		 3372}
3373
3374static void hns3_reuse_buffer(struct hns3_enet_ring *ring, int i)
3375{
3376 ring->desc_cb[i].reuse_flag = 0;
9f9f0f19 3377 ring->desc_cb[i].refill = 1;
9b2f3477
WL		 3378 ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma +
3379 ring->desc_cb[i].page_offset);
7d0b130c 3380 ring->desc[i].rx.bd_base_info = 0;
c2a2e127
BS		 3381
3382 dma_sync_single_for_device(ring_to_dev(ring),
3383 ring->desc_cb[i].dma + ring->desc_cb[i].page_offset,
3384 hns3_buf_size(ring),
3385 DMA_FROM_DEVICE);
76ad4f0e
S		 3386}
3387
20d06ca2 3388static bool hns3_nic_reclaim_desc(struct hns3_enet_ring *ring,
619ae331 3389 int *bytes, int *pkts, int budget)
76ad4f0e 3390{
20d06ca2
YL		 3391 /* pair with ring->last_to_use update in hns3_tx_doorbell(),
3392 * smp_store_release() is not used in hns3_tx_doorbell() because
3393 * the doorbell operation already have the needed barrier operation.
3394 */
3395 int ltu = smp_load_acquire(&ring->last_to_use);
26cda2f1
YL		 3396 int ntc = ring->next_to_clean;
3397 struct hns3_desc_cb *desc_cb;
20d06ca2
YL		 3398 bool reclaimed = false;
3399 struct hns3_desc *desc;
3400
3401 while (ltu != ntc) {
3402 desc = &ring->desc[ntc];
3403
3404 if (le16_to_cpu(desc->tx.bdtp_fe_sc_vld_ra_ri) &
3405 BIT(HNS3_TXD_VLD_B))
3406 break;
76ad4f0e 3407
ce74370c 3408 desc_cb = &ring->desc_cb[ntc];
811c0830 3409
907676b1 3410 if (desc_cb->type & (DESC_TYPE_SKB | DESC_TYPE_BOUNCE_ALL |
7459775e
YL		 3411 DESC_TYPE_BOUNCE_HEAD |
3412 DESC_TYPE_SGL_SKB)) {
811c0830
YL		 3413 (*pkts)++;
3414 (*bytes) += desc_cb->send_bytes;
3415 }
3416
ce74370c 3417 /* desc_cb will be cleaned, after hnae3_free_buffer_detach */
619ae331 3418 hns3_free_buffer_detach(ring, ntc, budget);
76ad4f0e 3419
ce74370c
YL		 3420 if (++ntc == ring->desc_num)
3421 ntc = 0;
3422
3423 /* Issue prefetch for next Tx descriptor */
3424 prefetch(&ring->desc_cb[ntc]);
20d06ca2 3425 reclaimed = true;
ce74370c 3426 }
26cda2f1 3427
20d06ca2
YL		 3428 if (unlikely(!reclaimed))
3429 return false;
3430
26cda2f1
YL		 3431 /* This smp_store_release() pairs with smp_load_acquire() in
3432 * ring_space called by hns3_nic_net_xmit.
3433 */
3434 smp_store_release(&ring->next_to_clean, ntc);
907676b1
YL		 3435
3436 hns3_tx_spare_update(ring);
3437
20d06ca2 3438 return true;
76ad4f0e
S		 3439}
3440
619ae331 3441void hns3_clean_tx_ring(struct hns3_enet_ring *ring, int budget)
76ad4f0e 3442{
c8711956 3443 struct net_device *netdev = ring_to_netdev(ring);
7a810110 3444 struct hns3_nic_priv *priv = netdev_priv(netdev);
76ad4f0e
S		 3445 struct netdev_queue *dev_queue;
3446 int bytes, pkts;
76ad4f0e
S		 3447
3448 bytes = 0;
3449 pkts = 0;
20d06ca2 3450
619ae331 3451 if (unlikely(!hns3_nic_reclaim_desc(ring, &bytes, &pkts, budget)))
20d06ca2 3452 return;
76ad4f0e
S		 3453
3454 ring->tqp_vector->tx_group.total_bytes += bytes;
3455 ring->tqp_vector->tx_group.total_packets += pkts;
3456
3457 u64_stats_update_begin(&ring->syncp);
3458 ring->stats.tx_bytes += bytes;
3459 ring->stats.tx_pkts += pkts;
3460 u64_stats_update_end(&ring->syncp);
3461
3462 dev_queue = netdev_get_tx_queue(netdev, ring->tqp->tqp_index);
3463 netdev_tx_completed_queue(dev_queue, pkts, bytes);
3464
2a597eff 3465 if (unlikely(netif_carrier_ok(netdev) &&
8ae10cfb 3466 ring_space(ring) > HNS3_MAX_TSO_BD_NUM)) {
76ad4f0e
S		 3467 /* Make sure that anybody stopping the queue after this
3468 * sees the new next_to_clean.
3469 */
3470 smp_mb();
7a810110
JS		 3471 if (netif_tx_queue_stopped(dev_queue) &&
3472 !test_bit(HNS3_NIC_STATE_DOWN, &priv->state)) {
76ad4f0e
S		 3473 netif_tx_wake_queue(dev_queue);
3474 ring->stats.restart_queue++;
3475 }
3476 }
76ad4f0e
S		 3477}
3478
3479static int hns3_desc_unused(struct hns3_enet_ring *ring)
3480{
3481 int ntc = ring->next_to_clean;
3482 int ntu = ring->next_to_use;
3483
9f9f0f19
YL		 3484 if (unlikely(ntc == ntu && !ring->desc_cb[ntc].refill))
3485 return ring->desc_num;
3486
76ad4f0e
S		 3487 return ((ntc >= ntu) ? 0 : ring->desc_num) + ntc - ntu;
3488}
3489
68752b24
YL		 3490/* Return true if there is any allocation failure */
3491static bool hns3_nic_alloc_rx_buffers(struct hns3_enet_ring *ring,
9b2f3477 3492 int cleand_count)
76ad4f0e
S		 3493{
3494 struct hns3_desc_cb *desc_cb;
3495 struct hns3_desc_cb res_cbs;
3496 int i, ret;
3497
3498 for (i = 0; i < cleand_count; i++) {
3499 desc_cb = &ring->desc_cb[ring->next_to_use];
3500 if (desc_cb->reuse_flag) {
3501 u64_stats_update_begin(&ring->syncp);
3502 ring->stats.reuse_pg_cnt++;
3503 u64_stats_update_end(&ring->syncp);
3504
3505 hns3_reuse_buffer(ring, ring->next_to_use);
3506 } else {
4d2cad32 3507 ret = hns3_alloc_and_map_buffer(ring, &res_cbs);
76ad4f0e
S		 3508 if (ret) {
3509 u64_stats_update_begin(&ring->syncp);
3510 ring->stats.sw_err_cnt++;
3511 u64_stats_update_end(&ring->syncp);
3512
c8711956 3513 hns3_rl_err(ring_to_netdev(ring),
b20d7fe5
YL		 3514 "alloc rx buffer failed: %d\n",
3515 ret);
68752b24
YL		 3516
3517 writel(i, ring->tqp->io_base +
3518 HNS3_RING_RX_RING_HEAD_REG);
3519 return true;
76ad4f0e
S		 3520 }
3521 hns3_replace_buffer(ring, ring->next_to_use, &res_cbs);
d21ff4f9
YL		 3522
3523 u64_stats_update_begin(&ring->syncp);
3524 ring->stats.non_reuse_pg++;
3525 u64_stats_update_end(&ring->syncp);
76ad4f0e
S		 3526 }
3527
3528 ring_ptr_move_fw(ring, next_to_use);
3529 }
3530
48ee56fd 3531 writel(i, ring->tqp->io_base + HNS3_RING_RX_RING_HEAD_REG);
68752b24 3532 return false;
76ad4f0e
S		 3533}
3534
aeda9bf8
YL		 3535static bool hns3_can_reuse_page(struct hns3_desc_cb *cb)
3536{
fa7711b8 3537 return page_count(cb->priv) == cb->pagecnt_bias;
aeda9bf8
YL		 3538}
3539
76ad4f0e
S		 3540static void hns3_nic_reuse_page(struct sk_buff *skb, int i,
3541 struct hns3_enet_ring *ring, int pull_len,
3542 struct hns3_desc_cb *desc_cb)
3543{
389ca146 3544 struct hns3_desc *desc = &ring->desc[ring->next_to_clean];
fa7711b8 3545 u32 frag_offset = desc_cb->page_offset + pull_len;
389ca146 3546 int size = le16_to_cpu(desc->rx.size);
dbba6da0 3547 u32 truesize = hns3_buf_size(ring);
fa7711b8 3548 u32 frag_size = size - pull_len;
96104500 3549 bool reused;
76ad4f0e 3550
93188e96
YL		 3551 if (ring->page_pool) {
3552 skb_add_rx_frag(skb, i, desc_cb->priv, frag_offset,
3553 frag_size, truesize);
3554 return;
3555 }
3556
fa7711b8
YL		 3557 /* Avoid re-using remote or pfmem page */
3558 if (unlikely(!dev_page_is_reusable(desc_cb->priv)))
3559 goto out;
76ad4f0e 3560
96104500
YL		 3561 reused = hns3_can_reuse_page(desc_cb);
3562
3563 /* Rx page can be reused when:
3564 * 1. Rx page is only owned by the driver when page_offset
3565 * is zero, which means 0 @ truesize will be used by
3566 * stack after skb_add_rx_frag() is called, and the rest
3567 * of rx page can be reused by driver.
3568 * Or
3569 * 2. Rx page is only owned by the driver when page_offset
3570 * is non-zero, which means page_offset @ truesize will
3571 * be used by stack after skb_add_rx_frag() is called,
3572 * and 0 @ truesize can be reused by driver.
389ca146 3573 */
96104500
YL		 3574 if ((!desc_cb->page_offset && reused) ||
3575 ((desc_cb->page_offset + truesize + truesize) <=
3576 hns3_page_size(ring) && desc_cb->page_offset)) {
fa7711b8 3577 desc_cb->page_offset += truesize;
389ca146 3578 desc_cb->reuse_flag = 1;
96104500
YL		 3579 } else if (desc_cb->page_offset && reused) {
3580 desc_cb->page_offset = 0;
3581 desc_cb->reuse_flag = 1;
99f6b5fb
YL		 3582 } else if (frag_size <= ring->rx_copybreak) {
3583 void *frag = napi_alloc_frag(frag_size);
3584
3585 if (unlikely(!frag)) {
3586 u64_stats_update_begin(&ring->syncp);
3587 ring->stats.frag_alloc_err++;
3588 u64_stats_update_end(&ring->syncp);
3589
3590 hns3_rl_err(ring_to_netdev(ring),
3591 "failed to allocate rx frag\n");
3592 goto out;
3593 }
3594
3595 desc_cb->reuse_flag = 1;
3596 memcpy(frag, desc_cb->buf + frag_offset, frag_size);
3597 skb_add_rx_frag(skb, i, virt_to_page(frag),
3598 offset_in_page(frag), frag_size, frag_size);
3599
3600 u64_stats_update_begin(&ring->syncp);
3601 ring->stats.frag_alloc++;
3602 u64_stats_update_end(&ring->syncp);
3603 return;
aeda9bf8
YL		 3604 }
3605
fa7711b8
YL		 3606out:
3607 desc_cb->pagecnt_bias--;
3608
aeda9bf8
YL		 3609 if (unlikely(!desc_cb->pagecnt_bias)) {
3610 page_ref_add(desc_cb->priv, USHRT_MAX);
3611 desc_cb->pagecnt_bias = USHRT_MAX;
76ad4f0e 3612 }
fa7711b8
YL		 3613
3614 skb_add_rx_frag(skb, i, desc_cb->priv, frag_offset,
3615 frag_size, truesize);
3616
3617 if (unlikely(!desc_cb->reuse_flag))
3618 __page_frag_cache_drain(desc_cb->priv, desc_cb->pagecnt_bias);
76ad4f0e
S		 3619}
3620
e2ee1c5a 3621static int hns3_gro_complete(struct sk_buff *skb, u32 l234info)
d474d88f
YL		 3622{
3623 __be16 type = skb->protocol;
3624 struct tcphdr *th;
3625 int depth = 0;
3626
e2ee1c5a 3627 while (eth_type_vlan(type)) {
d474d88f
YL		 3628 struct vlan_hdr *vh;
3629
3630 if ((depth + VLAN_HLEN) > skb_headlen(skb))
3631 return -EFAULT;
3632
3633 vh = (struct vlan_hdr *)(skb->data + depth);
3634 type = vh->h_vlan_encapsulated_proto;
3635 depth += VLAN_HLEN;
3636 }
3637
e2ee1c5a
YL		 3638 skb_set_network_header(skb, depth);
3639
d474d88f 3640 if (type == htons(ETH_P_IP)) {
e2ee1c5a
YL		 3641 const struct iphdr *iph = ip_hdr(skb);
3642
d474d88f 3643 depth += sizeof(struct iphdr);
e2ee1c5a
YL		 3644 skb_set_transport_header(skb, depth);
3645 th = tcp_hdr(skb);
3646 th->check = ~tcp_v4_check(skb->len - depth, iph->saddr,
3647 iph->daddr, 0);
d474d88f 3648 } else if (type == htons(ETH_P_IPV6)) {
e2ee1c5a
YL		 3649 const struct ipv6hdr *iph = ipv6_hdr(skb);
3650
d474d88f 3651 depth += sizeof(struct ipv6hdr);
e2ee1c5a
YL		 3652 skb_set_transport_header(skb, depth);
3653 th = tcp_hdr(skb);
3654 th->check = ~tcp_v6_check(skb->len - depth, &iph->saddr,
3655 &iph->daddr, 0);
d474d88f 3656 } else {
b20d7fe5
YL		 3657 hns3_rl_err(skb->dev,
3658 "Error: FW GRO supports only IPv4/IPv6, not 0x%04x, depth: %d\n",
3659 be16_to_cpu(type), depth);
d474d88f
YL		 3660 return -EFAULT;
3661 }
3662
d474d88f
YL		 3663 skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count;
3664 if (th->cwr)
3665 skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN;
3666
e2ee1c5a
YL		 3667 if (l234info & BIT(HNS3_RXD_GRO_FIXID_B))
3668 skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_FIXEDID;
d474d88f 3669
e2ee1c5a
YL		 3670 skb->csum_start = (unsigned char *)th - skb->head;
3671 skb->csum_offset = offsetof(struct tcphdr, check);
3672 skb->ip_summed = CHECKSUM_PARTIAL;
698a8954
YL		 3673
3674 trace_hns3_gro(skb);
3675
d474d88f
YL		 3676 return 0;
3677}
3678
1ddc028a
HT		 3679static bool hns3_checksum_complete(struct hns3_enet_ring *ring,
3680 struct sk_buff *skb, u32 ptype, u16 csum)
4b2fe769 3681{
1ddc028a
HT		 3682 if (ptype == HNS3_INVALID_PTYPE ||
3683 hns3_rx_ptype_tbl[ptype].ip_summed != CHECKSUM_COMPLETE)
3684 return false;
4b2fe769
HT		 3685
3686 u64_stats_update_begin(&ring->syncp);
3687 ring->stats.csum_complete++;
3688 u64_stats_update_end(&ring->syncp);
3689 skb->ip_summed = CHECKSUM_COMPLETE;
1ddc028a
HT		 3690 skb->csum = csum_unfold((__force __sum16)csum);
3691
3692 return true;
4b2fe769
HT		 3693}
3694
79664077
HT		 3695static void hns3_rx_handle_csum(struct sk_buff *skb, u32 l234info,
3696 u32 ol_info, u32 ptype)
76ad4f0e 3697{
76ad4f0e 3698 int l3_type, l4_type;
76ad4f0e 3699 int ol4_type;
76ad4f0e 3700
79664077
HT		 3701 if (ptype != HNS3_INVALID_PTYPE) {
3702 skb->csum_level = hns3_rx_ptype_tbl[ptype].csum_level;
3703 skb->ip_summed = hns3_rx_ptype_tbl[ptype].ip_summed;
76ad4f0e
S		 3704
3705 return;
3706 }
3707
39c38824 3708 ol4_type = hnae3_get_field(ol_info, HNS3_RXD_OL4ID_M,
e4e87715 3709 HNS3_RXD_OL4ID_S);
76ad4f0e
S		 3710 switch (ol4_type) {
3711 case HNS3_OL4_TYPE_MAC_IN_UDP:
3712 case HNS3_OL4_TYPE_NVGRE:
3713 skb->csum_level = 1;
df561f66 3714 fallthrough;
76ad4f0e 3715 case HNS3_OL4_TYPE_NO_TUN:
47e7b13b
YL		 3716 l3_type = hnae3_get_field(l234info, HNS3_RXD_L3ID_M,
3717 HNS3_RXD_L3ID_S);
3718 l4_type = hnae3_get_field(l234info, HNS3_RXD_L4ID_M,
3719 HNS3_RXD_L4ID_S);
76ad4f0e 3720 /* Can checksum ipv4 or ipv6 + UDP/TCP/SCTP packets */
94c5e532
PL		 3721 if ((l3_type == HNS3_L3_TYPE_IPV4 ||
3722 l3_type == HNS3_L3_TYPE_IPV6) &&
3723 (l4_type == HNS3_L4_TYPE_UDP ||
3724 l4_type == HNS3_L4_TYPE_TCP ||
3725 l4_type == HNS3_L4_TYPE_SCTP))
76ad4f0e
S		 3726 skb->ip_summed = CHECKSUM_UNNECESSARY;
3727 break;
fa7a4bd5
JS		 3728 default:
3729 break;
76ad4f0e
S		 3730 }
3731}
3732
79664077 3733static void hns3_rx_checksum(struct hns3_enet_ring *ring, struct sk_buff *skb,
1ddc028a
HT		 3734 u32 l234info, u32 bd_base_info, u32 ol_info,
3735 u16 csum)
79664077
HT		 3736{
3737 struct net_device *netdev = ring_to_netdev(ring);
3738 struct hns3_nic_priv *priv = netdev_priv(netdev);
3739 u32 ptype = HNS3_INVALID_PTYPE;
3740
3741 skb->ip_summed = CHECKSUM_NONE;
3742
3743 skb_checksum_none_assert(skb);
3744
3745 if (!(netdev->features & NETIF_F_RXCSUM))
3746 return;
3747
3748 if (test_bit(HNS3_NIC_STATE_RXD_ADV_LAYOUT_ENABLE, &priv->state))
3749 ptype = hnae3_get_field(ol_info, HNS3_RXD_PTYPE_M,
3750 HNS3_RXD_PTYPE_S);
3751
1ddc028a 3752 if (hns3_checksum_complete(ring, skb, ptype, csum))
79664077 3753 return;
79664077
HT		 3754
3755 /* check if hardware has done checksum */
3756 if (!(bd_base_info & BIT(HNS3_RXD_L3L4P_B)))
3757 return;
3758
3759 if (unlikely(l234info & (BIT(HNS3_RXD_L3E_B) | BIT(HNS3_RXD_L4E_B) |
3760 BIT(HNS3_RXD_OL3E_B) |
3761 BIT(HNS3_RXD_OL4E_B)))) {
3762 u64_stats_update_begin(&ring->syncp);
3763 ring->stats.l3l4_csum_err++;
3764 u64_stats_update_end(&ring->syncp);
3765
3766 return;
3767 }
3768
3769 hns3_rx_handle_csum(skb, l234info, ol_info, ptype);
3770}
3771
d43e5aca
YL		 3772static void hns3_rx_skb(struct hns3_enet_ring *ring, struct sk_buff *skb)
3773{
81ae0e04
PL		 3774 if (skb_has_frag_list(skb))
3775 napi_gro_flush(&ring->tqp_vector->napi, false);
3776
d43e5aca
YL		 3777 napi_gro_receive(&ring->tqp_vector->napi, skb);
3778}
3779
701a6d6a
JS		 3780static bool hns3_parse_vlan_tag(struct hns3_enet_ring *ring,
3781 struct hns3_desc *desc, u32 l234info,
3782 u16 *vlan_tag)
5b5455a9 3783{
44e626f7 3784 struct hnae3_handle *handle = ring->tqp->handle;
5b5455a9 3785 struct pci_dev *pdev = ring->tqp->handle->pdev;
295ba232 3786 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
5b5455a9 3787
295ba232 3788 if (unlikely(ae_dev->dev_version < HNAE3_DEVICE_VERSION_V2)) {
701a6d6a
JS		 3789 *vlan_tag = le16_to_cpu(desc->rx.ot_vlan_tag);
3790 if (!(*vlan_tag & VLAN_VID_MASK))
3791 *vlan_tag = le16_to_cpu(desc->rx.vlan_tag);
5b5455a9 3792
701a6d6a 3793 return (*vlan_tag != 0);
5b5455a9
PL		 3794 }
3795
3796#define HNS3_STRP_OUTER_VLAN 0x1
3797#define HNS3_STRP_INNER_VLAN 0x2
44e626f7 3798#define HNS3_STRP_BOTH 0x3
5b5455a9 3799
44e626f7
JS		 3800 /* Hardware always insert VLAN tag into RX descriptor when
3801 * remove the tag from packet, driver needs to determine
3802 * reporting which tag to stack.
3803 */
e4e87715
PL		 3804 switch (hnae3_get_field(l234info, HNS3_RXD_STRP_TAGP_M,
3805 HNS3_RXD_STRP_TAGP_S)) {
5b5455a9 3806 case HNS3_STRP_OUTER_VLAN:
44e626f7
JS		 3807 if (handle->port_base_vlan_state !=
3808 HNAE3_PORT_BASE_VLAN_DISABLE)
3809 return false;
3810
701a6d6a
JS		 3811 *vlan_tag = le16_to_cpu(desc->rx.ot_vlan_tag);
3812 return true;
5b5455a9 3813 case HNS3_STRP_INNER_VLAN:
44e626f7
JS		 3814 if (handle->port_base_vlan_state !=
3815 HNAE3_PORT_BASE_VLAN_DISABLE)
3816 return false;
3817
701a6d6a 3818 *vlan_tag = le16_to_cpu(desc->rx.vlan_tag);
44e626f7
JS		 3819 return true;
3820 case HNS3_STRP_BOTH:
3821 if (handle->port_base_vlan_state ==
3822 HNAE3_PORT_BASE_VLAN_DISABLE)
3823 *vlan_tag = le16_to_cpu(desc->rx.ot_vlan_tag);
3824 else
3825 *vlan_tag = le16_to_cpu(desc->rx.vlan_tag);
3826
701a6d6a 3827 return true;
5b5455a9 3828 default:
701a6d6a 3829 return false;
5b5455a9 3830 }
5b5455a9
PL		 3831}
3832
8c30e194
YL		 3833static void hns3_rx_ring_move_fw(struct hns3_enet_ring *ring)
3834{
3835 ring->desc[ring->next_to_clean].rx.bd_base_info &=
3836 cpu_to_le32(~BIT(HNS3_RXD_VLD_B));
9f9f0f19 3837 ring->desc_cb[ring->next_to_clean].refill = 0;
8c30e194
YL		 3838 ring->next_to_clean += 1;
3839
3840 if (unlikely(ring->next_to_clean == ring->desc_num))
3841 ring->next_to_clean = 0;
3842}
3843
b9a8f883 3844static int hns3_alloc_skb(struct hns3_enet_ring *ring, unsigned int length,
e5597095
PL		 3845 unsigned char *va)
3846{
e5597095 3847 struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_clean];
c8711956 3848 struct net_device *netdev = ring_to_netdev(ring);
e5597095
PL		 3849 struct sk_buff *skb;
3850
3851 ring->skb = napi_alloc_skb(&ring->tqp_vector->napi, HNS3_RX_HEAD_SIZE);
3852 skb = ring->skb;
3853 if (unlikely(!skb)) {
b20d7fe5 3854 hns3_rl_err(netdev, "alloc rx skb fail\n");
e5597095
PL		 3855
3856 u64_stats_update_begin(&ring->syncp);
3857 ring->stats.sw_err_cnt++;
3858 u64_stats_update_end(&ring->syncp);
3859
3860 return -ENOMEM;
3861 }
3862
698a8954 3863 trace_hns3_rx_desc(ring);
e5597095
PL		 3864 prefetchw(skb->data);
3865
3866 ring->pending_buf = 1;
81ae0e04
PL		 3867 ring->frag_num = 0;
3868 ring->tail_skb = NULL;
e5597095
PL		 3869 if (length <= HNS3_RX_HEAD_SIZE) {
3870 memcpy(__skb_put(skb, length), va, ALIGN(length, sizeof(long)));
3871
a79afa78
AL		 3872 /* We can reuse buffer as-is, just make sure it is reusable */
3873 if (dev_page_is_reusable(desc_cb->priv))
e5597095 3874 desc_cb->reuse_flag = 1;
93188e96
YL		 3875 else if (desc_cb->type & DESC_TYPE_PP_FRAG)
3876 page_pool_put_full_page(ring->page_pool, desc_cb->priv,
3877 false);
e5597095 3878 else /* This page cannot be reused so discard it */
aeda9bf8
YL		 3879 __page_frag_cache_drain(desc_cb->priv,
3880 desc_cb->pagecnt_bias);
e5597095 3881
8c30e194 3882 hns3_rx_ring_move_fw(ring);
e5597095
PL		 3883 return 0;
3884 }
93188e96
YL		 3885
3886 if (ring->page_pool)
3887 skb_mark_for_recycle(skb);
3888
e5597095
PL		 3889 u64_stats_update_begin(&ring->syncp);
3890 ring->stats.seg_pkt_cnt++;
3891 u64_stats_update_end(&ring->syncp);
3892
c43f1255 3893 ring->pull_len = eth_get_headlen(netdev, va, HNS3_RX_HEAD_SIZE);
e5597095 3894 __skb_put(skb, ring->pull_len);
81ae0e04 3895 hns3_nic_reuse_page(skb, ring->frag_num++, ring, ring->pull_len,
e5597095 3896 desc_cb);
8c30e194 3897 hns3_rx_ring_move_fw(ring);
e5597095 3898
b2598318 3899 return 0;
e5597095
PL		 3900}
3901
b2598318 3902static int hns3_add_frag(struct hns3_enet_ring *ring)
e5597095 3903{
d35bced8
YL		 3904 struct sk_buff *skb = ring->skb;
3905 struct sk_buff *head_skb = skb;
81ae0e04 3906 struct sk_buff *new_skb;
e5597095 3907 struct hns3_desc_cb *desc_cb;
b2598318 3908 struct hns3_desc *desc;
e5597095 3909 u32 bd_base_info;
e5597095 3910
b2598318 3911 do {
e5597095
PL		 3912 desc = &ring->desc[ring->next_to_clean];
3913 desc_cb = &ring->desc_cb[ring->next_to_clean];
3914 bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
d394d33b
JS		 3915 /* make sure HW write desc complete */
3916 dma_rmb();
e8149933 3917 if (!(bd_base_info & BIT(HNS3_RXD_VLD_B)))
e5597095
PL		 3918 return -ENXIO;
3919
81ae0e04 3920 if (unlikely(ring->frag_num >= MAX_SKB_FRAGS)) {
7fda3a93 3921 new_skb = napi_alloc_skb(&ring->tqp_vector->napi, 0);
81ae0e04 3922 if (unlikely(!new_skb)) {
c8711956 3923 hns3_rl_err(ring_to_netdev(ring),
b20d7fe5 3924 "alloc rx fraglist skb fail\n");
81ae0e04
PL		 3925 return -ENXIO;
3926 }
93188e96
YL		 3927
3928 if (ring->page_pool)
3929 skb_mark_for_recycle(new_skb);
3930
81ae0e04
PL		 3931 ring->frag_num = 0;
3932
3933 if (ring->tail_skb) {
3934 ring->tail_skb->next = new_skb;
3935 ring->tail_skb = new_skb;
3936 } else {
3937 skb_shinfo(skb)->frag_list = new_skb;
3938 ring->tail_skb = new_skb;
3939 }
3940 }
3941
3942 if (ring->tail_skb) {
dbba6da0 3943 head_skb->truesize += hns3_buf_size(ring);
81ae0e04
PL		 3944 head_skb->data_len += le16_to_cpu(desc->rx.size);
3945 head_skb->len += le16_to_cpu(desc->rx.size);
3946 skb = ring->tail_skb;
3947 }
3948
c2a2e127
BS		 3949 dma_sync_single_for_cpu(ring_to_dev(ring),
3950 desc_cb->dma + desc_cb->page_offset,
3951 hns3_buf_size(ring),
3952 DMA_FROM_DEVICE);
3953
81ae0e04 3954 hns3_nic_reuse_page(skb, ring->frag_num++, ring, 0, desc_cb);
698a8954 3955 trace_hns3_rx_desc(ring);
8c30e194 3956 hns3_rx_ring_move_fw(ring);
e5597095 3957 ring->pending_buf++;
b2598318 3958 } while (!(bd_base_info & BIT(HNS3_RXD_FE_B)));
e5597095
PL		 3959
3960 return 0;
3961}
3962
d474d88f
YL		 3963static int hns3_set_gro_and_checksum(struct hns3_enet_ring *ring,
3964 struct sk_buff *skb, u32 l234info,
1ddc028a 3965 u32 bd_base_info, u32 ol_info, u16 csum)
a6d53b97 3966{
79664077
HT		 3967 struct net_device *netdev = ring_to_netdev(ring);
3968 struct hns3_nic_priv *priv = netdev_priv(netdev);
a6d53b97
PL		 3969 u32 l3_type;
3970
e2ee1c5a
YL		 3971 skb_shinfo(skb)->gso_size = hnae3_get_field(bd_base_info,
3972 HNS3_RXD_GRO_SIZE_M,
3973 HNS3_RXD_GRO_SIZE_S);
a6d53b97 3974 /* if there is no HW GRO, do not set gro params */
e2ee1c5a 3975 if (!skb_shinfo(skb)->gso_size) {
1ddc028a
HT		 3976 hns3_rx_checksum(ring, skb, l234info, bd_base_info, ol_info,
3977 csum);
d474d88f
YL		 3978 return 0;
3979 }
a6d53b97 3980
e2ee1c5a
YL		 3981 NAPI_GRO_CB(skb)->count = hnae3_get_field(l234info,
3982 HNS3_RXD_GRO_COUNT_M,
3983 HNS3_RXD_GRO_COUNT_S);
a6d53b97 3984
79664077
HT		 3985 if (test_bit(HNS3_NIC_STATE_RXD_ADV_LAYOUT_ENABLE, &priv->state)) {
3986 u32 ptype = hnae3_get_field(ol_info, HNS3_RXD_PTYPE_M,
3987 HNS3_RXD_PTYPE_S);
3988
3989 l3_type = hns3_rx_ptype_tbl[ptype].l3_type;
3990 } else {
3991 l3_type = hnae3_get_field(l234info, HNS3_RXD_L3ID_M,
3992 HNS3_RXD_L3ID_S);
3993 }
3994
a6d53b97
PL		 3995 if (l3_type == HNS3_L3_TYPE_IPV4)
3996 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
3997 else if (l3_type == HNS3_L3_TYPE_IPV6)
3998 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
3999 else
d474d88f 4000 return -EFAULT;
a6d53b97 4001
e2ee1c5a 4002 return hns3_gro_complete(skb, l234info);
a6d53b97
PL		 4003}
4004
232fc64b 4005static void hns3_set_rx_skb_rss_type(struct hns3_enet_ring *ring,
ea485867 4006 struct sk_buff *skb, u32 rss_hash)
232fc64b 4007{
232fc64b
PL		 4008 struct hnae3_handle *handle = ring->tqp->handle;
4009 enum pkt_hash_types rss_type;
4010
ea485867 4011 if (rss_hash)
232fc64b
PL		 4012 rss_type = handle->kinfo.rss_type;
4013 else
4014 rss_type = PKT_HASH_TYPE_NONE;
4015
ea485867 4016 skb_set_hash(skb, rss_hash, rss_type);
232fc64b
PL		 4017}
4018
ea485867 4019static int hns3_handle_bdinfo(struct hns3_enet_ring *ring, struct sk_buff *skb)
76ad4f0e 4020{
c8711956 4021 struct net_device *netdev = ring_to_netdev(ring);
c376fa1a 4022 enum hns3_pkt_l2t_type l2_frame_type;
39c38824 4023 u32 bd_base_info, l234info, ol_info;
ea485867 4024 struct hns3_desc *desc;
d474d88f 4025 unsigned int len;
ea485867 4026 int pre_ntc, ret;
1ddc028a 4027 u16 csum;
ea485867
YL		 4028
4029 /* bdinfo handled below is only valid on the last BD of the
4030 * current packet, and ring->next_to_clean indicates the first
4031 * descriptor of next packet, so need - 1 below.
4032 */
4033 pre_ntc = ring->next_to_clean ? (ring->next_to_clean - 1) :
4034 (ring->desc_num - 1);
4035 desc = &ring->desc[pre_ntc];
4036 bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
4037 l234info = le32_to_cpu(desc->rx.l234_info);
39c38824 4038 ol_info = le32_to_cpu(desc->rx.ol_info);
1ddc028a 4039 csum = le16_to_cpu(desc->csum);
d474d88f 4040
0bf5eb78
HT		 4041 if (unlikely(bd_base_info & BIT(HNS3_RXD_TS_VLD_B))) {
4042 struct hnae3_handle *h = hns3_get_handle(netdev);
4043 u32 nsec = le32_to_cpu(desc->ts_nsec);
4044 u32 sec = le32_to_cpu(desc->ts_sec);
4045
4046 if (h->ae_algo->ops->get_rx_hwts)
4047 h->ae_algo->ops->get_rx_hwts(h, skb, nsec, sec);
4048 }
4049
d474d88f
YL		 4050 /* Based on hw strategy, the tag offloaded will be stored at
4051 * ot_vlan_tag in two layer tag case, and stored at vlan_tag
4052 * in one layer tag case.
4053 */
4054 if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX) {
4055 u16 vlan_tag;
4056
4057 if (hns3_parse_vlan_tag(ring, desc, l234info, &vlan_tag))
4058 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
4059 vlan_tag);
4060 }
4061
d474d88f
YL		 4062 if (unlikely(!desc->rx.pkt_len || (l234info & (BIT(HNS3_RXD_TRUNCAT_B) |
4063 BIT(HNS3_RXD_L2E_B))))) {
4064 u64_stats_update_begin(&ring->syncp);
4065 if (l234info & BIT(HNS3_RXD_L2E_B))
4066 ring->stats.l2_err++;
4067 else
4068 ring->stats.err_pkt_len++;
4069 u64_stats_update_end(&ring->syncp);
4070
4071 return -EFAULT;
4072 }
4073
4074 len = skb->len;
4075
4076 /* Do update ip stack process */
4077 skb->protocol = eth_type_trans(skb, netdev);
4078
4079 /* This is needed in order to enable forwarding support */
39c38824 4080 ret = hns3_set_gro_and_checksum(ring, skb, l234info,
1ddc028a 4081 bd_base_info, ol_info, csum);
d474d88f
YL		 4082 if (unlikely(ret)) {
4083 u64_stats_update_begin(&ring->syncp);
4084 ring->stats.rx_err_cnt++;
4085 u64_stats_update_end(&ring->syncp);
4086 return ret;
4087 }
4088
4089 l2_frame_type = hnae3_get_field(l234info, HNS3_RXD_DMAC_M,
4090 HNS3_RXD_DMAC_S);
4091
4092 u64_stats_update_begin(&ring->syncp);
4093 ring->stats.rx_pkts++;
4094 ring->stats.rx_bytes += len;
4095
4096 if (l2_frame_type == HNS3_L2_TYPE_MULTICAST)
4097 ring->stats.rx_multicast++;
4098
4099 u64_stats_update_end(&ring->syncp);
4100
4101 ring->tqp_vector->rx_group.total_bytes += len;
ea485867
YL		 4102
4103 hns3_set_rx_skb_rss_type(ring, skb, le32_to_cpu(desc->rx.rss_hash));
d474d88f
YL		 4104 return 0;
4105}
4106
d35bced8 4107static int hns3_handle_rx_bd(struct hns3_enet_ring *ring)
d474d88f 4108{
e5597095 4109 struct sk_buff *skb = ring->skb;
76ad4f0e
S		 4110 struct hns3_desc_cb *desc_cb;
4111 struct hns3_desc *desc;
b9a8f883 4112 unsigned int length;
76ad4f0e 4113 u32 bd_base_info;
e5597095 4114 int ret;
76ad4f0e
S		 4115
4116 desc = &ring->desc[ring->next_to_clean];
4117 desc_cb = &ring->desc_cb[ring->next_to_clean];
4118
4119 prefetch(desc);
4120
8c30e194
YL		 4121 if (!skb) {
4122 bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
8c30e194
YL		 4123 /* Check valid BD */
4124 if (unlikely(!(bd_base_info & BIT(HNS3_RXD_VLD_B))))
4125 return -ENXIO;
4126
4127 dma_rmb();
4128 length = le16_to_cpu(desc->rx.size);
76ad4f0e 4129
cb0e3e61 4130 ring->va = desc_cb->buf + desc_cb->page_offset;
76ad4f0e 4131
c2a2e127
BS		 4132 dma_sync_single_for_cpu(ring_to_dev(ring),
4133 desc_cb->dma + desc_cb->page_offset,
4134 hns3_buf_size(ring),
4135 DMA_FROM_DEVICE);
c2a2e127 4136
8c30e194
YL		 4137 /* Prefetch first cache line of first page.
4138 * Idea is to cache few bytes of the header of the packet.
4139 * Our L1 Cache line size is 64B so need to prefetch twice to make
4140 * it 128B. But in actual we can have greater size of caches with
4141 * 128B Level 1 cache lines. In such a case, single fetch would
4142 * suffice to cache in the relevant part of the header.
4143 */
4144 net_prefetch(ring->va);
76ad4f0e 4145
e5597095 4146 ret = hns3_alloc_skb(ring, length, ring->va);
d35bced8 4147 skb = ring->skb;
76ad4f0e 4148
e5597095
PL		 4149 if (ret < 0) /* alloc buffer fail */
4150 return ret;
b2598318
YL		 4151 if (!(bd_base_info & BIT(HNS3_RXD_FE_B))) { /* need add frag */
4152 ret = hns3_add_frag(ring);
e5597095
PL		 4153 if (ret)
4154 return ret;
e5597095 4155 }
76ad4f0e 4156 } else {
b2598318 4157 ret = hns3_add_frag(ring);
e5597095
PL		 4158 if (ret)
4159 return ret;
b2598318 4160 }
76ad4f0e 4161
b2598318
YL		 4162 /* As the head data may be changed when GRO enable, copy
4163 * the head data in after other data rx completed
4164 */
4165 if (skb->len > HNS3_RX_HEAD_SIZE)
e5597095
PL		 4166 memcpy(skb->data, ring->va,
4167 ALIGN(ring->pull_len, sizeof(long)));
76ad4f0e 4168
ea485867 4169 ret = hns3_handle_bdinfo(ring, skb);
d474d88f 4170 if (unlikely(ret)) {
76ad4f0e 4171 dev_kfree_skb_any(skb);
d474d88f 4172 return ret;
76ad4f0e
S		 4173 }
4174
d93ed94f 4175 skb_record_rx_queue(skb, ring->tqp->tqp_index);
76ad4f0e
S		 4176 return 0;
4177}
4178
9b2f3477
WL		 4179int hns3_clean_rx_ring(struct hns3_enet_ring *ring, int budget,
4180 void (*rx_fn)(struct hns3_enet_ring *, struct sk_buff *))
76ad4f0e
S		 4181{
4182#define RCB_NOF_ALLOC_RX_BUFF_ONCE 16
63380a1a 4183 int unused_count = hns3_desc_unused(ring);
68752b24 4184 bool failure = false;
a4ee7624 4185 int recv_pkts = 0;
8c30e194 4186 int err;
76ad4f0e 4187
63380a1a 4188 unused_count -= ring->pending_buf;
76ad4f0e 4189
8c30e194 4190 while (recv_pkts < budget) {
76ad4f0e 4191 /* Reuse or realloc buffers */
a4ee7624 4192 if (unused_count >= RCB_NOF_ALLOC_RX_BUFF_ONCE) {
68752b24
YL		 4193 failure = failure ||
4194 hns3_nic_alloc_rx_buffers(ring, unused_count);
4195 unused_count = 0;
76ad4f0e
S		 4196 }
4197
4198 /* Poll one pkt */
d35bced8
YL		 4199 err = hns3_handle_rx_bd(ring);
4200 /* Do not get FE for the packet or failed to alloc skb */
4201 if (unlikely(!ring->skb || err == -ENXIO)) {
76ad4f0e 4202 goto out;
d35bced8
YL		 4203 } else if (likely(!err)) {
4204 rx_fn(ring, ring->skb);
4205 recv_pkts++;
76ad4f0e
S		 4206 }
4207
a4ee7624 4208 unused_count += ring->pending_buf;
e5597095
PL		 4209 ring->skb = NULL;
4210 ring->pending_buf = 0;
76ad4f0e
S		 4211 }
4212
4213out:
3b6db4a0
YM		 4214 /* sync head pointer before exiting, since hardware will calculate
4215 * FBD number with head pointer
4216 */
4217 if (unused_count > 0)
4218 failure = failure ||
4219 hns3_nic_alloc_rx_buffers(ring, unused_count);
4220
68752b24 4221 return failure ? budget : recv_pkts;
76ad4f0e
S		 4222}
4223
307ea4ce 4224static void hns3_update_rx_int_coalesce(struct hns3_enet_tqp_vector *tqp_vector)
76ad4f0e 4225{
307ea4ce
HT		 4226 struct hns3_enet_ring_group *rx_group = &tqp_vector->rx_group;
4227 struct dim_sample sample = {};
4a43caf5 4228
307ea4ce
HT		 4229 if (!rx_group->coal.adapt_enable)
4230 return;
76ad4f0e 4231
307ea4ce
HT		 4232 dim_update_sample(tqp_vector->event_cnt, rx_group->total_packets,
4233 rx_group->total_bytes, &sample);
4234 net_dim(&rx_group->dim, sample);
76ad4f0e
S		 4235}
4236
307ea4ce 4237static void hns3_update_tx_int_coalesce(struct hns3_enet_tqp_vector *tqp_vector)
76ad4f0e 4238{
8b1ff1ea 4239 struct hns3_enet_ring_group *tx_group = &tqp_vector->tx_group;
307ea4ce 4240 struct dim_sample sample = {};
8b1ff1ea 4241
307ea4ce 4242 if (!tx_group->coal.adapt_enable)
cd9d187b 4243 return;
cd9d187b 4244
307ea4ce
HT		 4245 dim_update_sample(tqp_vector->event_cnt, tx_group->total_packets,
4246 tx_group->total_bytes, &sample);
4247 net_dim(&tx_group->dim, sample);
76ad4f0e
S		 4248}
4249
4250static int hns3_nic_common_poll(struct napi_struct *napi, int budget)
4251{
ff0699e0 4252 struct hns3_nic_priv *priv = netdev_priv(napi->dev);
76ad4f0e
S		 4253 struct hns3_enet_ring *ring;
4254 int rx_pkt_total = 0;
4255
4256 struct hns3_enet_tqp_vector *tqp_vector =
4257 container_of(napi, struct hns3_enet_tqp_vector, napi);
4258 bool clean_complete = true;
ceca4a5e 4259 int rx_budget = budget;
76ad4f0e 4260
ff0699e0
HT		 4261 if (unlikely(test_bit(HNS3_NIC_STATE_DOWN, &priv->state))) {
4262 napi_complete(napi);
4263 return 0;
4264 }
4265
76ad4f0e
S		 4266 /* Since the actual Tx work is minimal, we can give the Tx a larger
4267 * budget and be more aggressive about cleaning up the Tx descriptors.
4268 */
799997a3 4269 hns3_for_each_ring(ring, tqp_vector->tx_group)
619ae331 4270 hns3_clean_tx_ring(ring, budget);
76ad4f0e
S		 4271
4272 /* make sure rx ring budget not smaller than 1 */
ceca4a5e
YL		 4273 if (tqp_vector->num_tqps > 1)
4274 rx_budget = max(budget / tqp_vector->num_tqps, 1);
76ad4f0e
S		 4275
4276 hns3_for_each_ring(ring, tqp_vector->rx_group) {
d43e5aca
YL		 4277 int rx_cleaned = hns3_clean_rx_ring(ring, rx_budget,
4278 hns3_rx_skb);
76ad4f0e
S		 4279 if (rx_cleaned >= rx_budget)
4280 clean_complete = false;
4281
4282 rx_pkt_total += rx_cleaned;
4283 }
4284
4285 tqp_vector->rx_group.total_packets += rx_pkt_total;
4286
4287 if (!clean_complete)
4288 return budget;
4289
531eba0f
HT		 4290 if (napi_complete(napi) &&
4291 likely(!test_bit(HNS3_NIC_STATE_DOWN, &priv->state))) {
307ea4ce
HT		 4292 hns3_update_rx_int_coalesce(tqp_vector);
4293 hns3_update_tx_int_coalesce(tqp_vector);
4294
ff0699e0
HT		 4295 hns3_mask_vector_irq(tqp_vector, 1);
4296 }
76ad4f0e
S		 4297
4298 return rx_pkt_total;
4299}
4300
4301static int hns3_get_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,
4302 struct hnae3_ring_chain_node *head)
4303{
4304 struct pci_dev *pdev = tqp_vector->handle->pdev;
4305 struct hnae3_ring_chain_node *cur_chain = head;
4306 struct hnae3_ring_chain_node *chain;
4307 struct hns3_enet_ring *tx_ring;
4308 struct hns3_enet_ring *rx_ring;
4309
4310 tx_ring = tqp_vector->tx_group.ring;
4311 if (tx_ring) {
4312 cur_chain->tqp_index = tx_ring->tqp->tqp_index;
e4e87715
PL		 4313 hnae3_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B,
4314 HNAE3_RING_TYPE_TX);
4315 hnae3_set_field(cur_chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
4316 HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_TX);
76ad4f0e
S		 4317
4318 cur_chain->next = NULL;
4319
4320 while (tx_ring->next) {
4321 tx_ring = tx_ring->next;
4322
4323 chain = devm_kzalloc(&pdev->dev, sizeof(*chain),
4324 GFP_KERNEL);
4325 if (!chain)
73b907a0 4326 goto err_free_chain;
76ad4f0e
S		 4327
4328 cur_chain->next = chain;
4329 chain->tqp_index = tx_ring->tqp->tqp_index;
e4e87715
PL		 4330 hnae3_set_bit(chain->flag, HNAE3_RING_TYPE_B,
4331 HNAE3_RING_TYPE_TX);
4332 hnae3_set_field(chain->int_gl_idx,
4333 HNAE3_RING_GL_IDX_M,
4334 HNAE3_RING_GL_IDX_S,
4335 HNAE3_RING_GL_TX);
76ad4f0e
S		 4336
4337 cur_chain = chain;
4338 }
4339 }
4340
4341 rx_ring = tqp_vector->rx_group.ring;
4342 if (!tx_ring && rx_ring) {
4343 cur_chain->next = NULL;
4344 cur_chain->tqp_index = rx_ring->tqp->tqp_index;
e4e87715
PL		 4345 hnae3_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B,
4346 HNAE3_RING_TYPE_RX);
4347 hnae3_set_field(cur_chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
4348 HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_RX);
76ad4f0e
S		 4349
4350 rx_ring = rx_ring->next;
4351 }
4352
4353 while (rx_ring) {
4354 chain = devm_kzalloc(&pdev->dev, sizeof(*chain), GFP_KERNEL);
4355 if (!chain)
73b907a0 4356 goto err_free_chain;
76ad4f0e
S		 4357
4358 cur_chain->next = chain;
4359 chain->tqp_index = rx_ring->tqp->tqp_index;
e4e87715
PL		 4360 hnae3_set_bit(chain->flag, HNAE3_RING_TYPE_B,
4361 HNAE3_RING_TYPE_RX);
4362 hnae3_set_field(chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
4363 HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_RX);
11af96a4 4364
76ad4f0e
S		 4365 cur_chain = chain;
4366
4367 rx_ring = rx_ring->next;
4368 }
4369
4370 return 0;
73b907a0
HT		 4371
4372err_free_chain:
4373 cur_chain = head->next;
4374 while (cur_chain) {
4375 chain = cur_chain->next;
cda69d24 4376 devm_kfree(&pdev->dev, cur_chain);
73b907a0
HT		 4377 cur_chain = chain;
4378 }
cda69d24 4379 head->next = NULL;
73b907a0
HT		 4380
4381 return -ENOMEM;
76ad4f0e
S		 4382}
4383
4384static void hns3_free_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,
4385 struct hnae3_ring_chain_node *head)
4386{
4387 struct pci_dev *pdev = tqp_vector->handle->pdev;
4388 struct hnae3_ring_chain_node *chain_tmp, *chain;
4389
4390 chain = head->next;
4391
4392 while (chain) {
4393 chain_tmp = chain->next;
4394 devm_kfree(&pdev->dev, chain);
4395 chain = chain_tmp;
4396 }
4397}
4398
4399static void hns3_add_ring_to_group(struct hns3_enet_ring_group *group,
4400 struct hns3_enet_ring *ring)
4401{
4402 ring->next = group->ring;
4403 group->ring = ring;
4404
4405 group->count++;
4406}
4407
874bff0b
PL		 4408static void hns3_nic_set_cpumask(struct hns3_nic_priv *priv)
4409{
4410 struct pci_dev *pdev = priv->ae_handle->pdev;
4411 struct hns3_enet_tqp_vector *tqp_vector;
4412 int num_vectors = priv->vector_num;
4413 int numa_node;
4414 int vector_i;
4415
4416 numa_node = dev_to_node(&pdev->dev);
4417
4418 for (vector_i = 0; vector_i < num_vectors; vector_i++) {
4419 tqp_vector = &priv->tqp_vector[vector_i];
4420 cpumask_set_cpu(cpumask_local_spread(vector_i, numa_node),
4421 &tqp_vector->affinity_mask);
4422 }
4423}
4424
307ea4ce
HT		 4425static void hns3_rx_dim_work(struct work_struct *work)
4426{
4427 struct dim *dim = container_of(work, struct dim, work);
4428 struct hns3_enet_ring_group *group = container_of(dim,
4429 struct hns3_enet_ring_group, dim);
4430 struct hns3_enet_tqp_vector *tqp_vector = group->ring->tqp_vector;
4431 struct dim_cq_moder cur_moder =
4432 net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
4433
4434 hns3_set_vector_coalesce_rx_gl(group->ring->tqp_vector, cur_moder.usec);
4435 tqp_vector->rx_group.coal.int_gl = cur_moder.usec;
4436
4437 if (cur_moder.pkts < tqp_vector->rx_group.coal.int_ql_max) {
4438 hns3_set_vector_coalesce_rx_ql(tqp_vector, cur_moder.pkts);
4439 tqp_vector->rx_group.coal.int_ql = cur_moder.pkts;
4440 }
4441
4442 dim->state = DIM_START_MEASURE;
4443}
4444
4445static void hns3_tx_dim_work(struct work_struct *work)
4446{
4447 struct dim *dim = container_of(work, struct dim, work);
4448 struct hns3_enet_ring_group *group = container_of(dim,
4449 struct hns3_enet_ring_group, dim);
4450 struct hns3_enet_tqp_vector *tqp_vector = group->ring->tqp_vector;
4451 struct dim_cq_moder cur_moder =
4452 net_dim_get_tx_moderation(dim->mode, dim->profile_ix);
4453
4454 hns3_set_vector_coalesce_tx_gl(tqp_vector, cur_moder.usec);
4455 tqp_vector->tx_group.coal.int_gl = cur_moder.usec;
4456
4457 if (cur_moder.pkts < tqp_vector->tx_group.coal.int_ql_max) {
4458 hns3_set_vector_coalesce_tx_ql(tqp_vector, cur_moder.pkts);
4459 tqp_vector->tx_group.coal.int_ql = cur_moder.pkts;
4460 }
4461
4462 dim->state = DIM_START_MEASURE;
4463}
4464
4465static void hns3_nic_init_dim(struct hns3_enet_tqp_vector *tqp_vector)
4466{
4467 INIT_WORK(&tqp_vector->rx_group.dim.work, hns3_rx_dim_work);
307ea4ce 4468 INIT_WORK(&tqp_vector->tx_group.dim.work, hns3_tx_dim_work);
307ea4ce
HT		 4469}
4470
76ad4f0e
S		 4471static int hns3_nic_init_vector_data(struct hns3_nic_priv *priv)
4472{
76ad4f0e
S		 4473 struct hnae3_handle *h = priv->ae_handle;
4474 struct hns3_enet_tqp_vector *tqp_vector;
9d8d5a36 4475 int ret;
ece4bf46 4476 int i;
76ad4f0e 4477
874bff0b
PL		 4478 hns3_nic_set_cpumask(priv);
4479
dd38c726
YL		 4480 for (i = 0; i < priv->vector_num; i++) {
4481 tqp_vector = &priv->tqp_vector[i];
91bfae25 4482 hns3_vector_coalesce_init_hw(tqp_vector, priv);
dd38c726 4483 tqp_vector->num_tqps = 0;
307ea4ce 4484 hns3_nic_init_dim(tqp_vector);
dd38c726 4485 }
76ad4f0e 4486
dd38c726
YL		 4487 for (i = 0; i < h->kinfo.num_tqps; i++) {
4488 u16 vector_i = i % priv->vector_num;
4489 u16 tqp_num = h->kinfo.num_tqps;
76ad4f0e
S		 4490
4491 tqp_vector = &priv->tqp_vector[vector_i];
4492
4493 hns3_add_ring_to_group(&tqp_vector->tx_group,
5f06b903 4494 &priv->ring[i]);
76ad4f0e
S		 4495
4496 hns3_add_ring_to_group(&tqp_vector->rx_group,
5f06b903 4497 &priv->ring[i + tqp_num]);
76ad4f0e 4498
5f06b903
YL		 4499 priv->ring[i].tqp_vector = tqp_vector;
4500 priv->ring[i + tqp_num].tqp_vector = tqp_vector;
dd38c726 4501 tqp_vector->num_tqps++;
76ad4f0e
S		 4502 }
4503
dd38c726 4504 for (i = 0; i < priv->vector_num; i++) {
d392ecd1
SM		 4505 struct hnae3_ring_chain_node vector_ring_chain;
4506
76ad4f0e
S		 4507 tqp_vector = &priv->tqp_vector[i];
4508
4509 tqp_vector->rx_group.total_bytes = 0;
4510 tqp_vector->rx_group.total_packets = 0;
4511 tqp_vector->tx_group.total_bytes = 0;
4512 tqp_vector->tx_group.total_packets = 0;
76ad4f0e
S		 4513 tqp_vector->handle = h;
4514
4515 ret = hns3_get_vector_ring_chain(tqp_vector,
4516 &vector_ring_chain);
4517 if (ret)
cda69d24 4518 goto map_ring_fail;
76ad4f0e
S		 4519
4520 ret = h->ae_algo->ops->map_ring_to_vector(h,
4521 tqp_vector->vector_irq, &vector_ring_chain);
76ad4f0e
S		 4522
4523 hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain);
4524
dd38c726 4525 if (ret)
ece4bf46 4526 goto map_ring_fail;
dd38c726 4527
76ad4f0e
S		 4528 netif_napi_add(priv->netdev, &tqp_vector->napi,
4529 hns3_nic_common_poll, NAPI_POLL_WEIGHT);
4530 }
4531
dd38c726 4532 return 0;
ece4bf46
HT		 4533
4534map_ring_fail:
4535 while (i--)
4536 netif_napi_del(&priv->tqp_vector[i].napi);
4537
4538 return ret;
dd38c726
YL		 4539}
4540
73a13d8d
HT		 4541static void hns3_nic_init_coal_cfg(struct hns3_nic_priv *priv)
4542{
4543 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(priv->ae_handle->pdev);
4544 struct hns3_enet_coalesce *tx_coal = &priv->tx_coal;
4545 struct hns3_enet_coalesce *rx_coal = &priv->rx_coal;
4546
4547 /* initialize the configuration for interrupt coalescing.
4548 * 1. GL (Interrupt Gap Limiter)
4549 * 2. RL (Interrupt Rate Limiter)
4550 * 3. QL (Interrupt Quantity Limiter)
4551 *
4552 * Default: enable interrupt coalescing self-adaptive and GL
4553 */
4554 tx_coal->adapt_enable = 1;
4555 rx_coal->adapt_enable = 1;
4556
4557 tx_coal->int_gl = HNS3_INT_GL_50K;
4558 rx_coal->int_gl = HNS3_INT_GL_50K;
4559
4560 rx_coal->flow_level = HNS3_FLOW_LOW;
4561 tx_coal->flow_level = HNS3_FLOW_LOW;
4562
4563 if (ae_dev->dev_specs.int_ql_max) {
4564 tx_coal->int_ql = HNS3_INT_QL_DEFAULT_CFG;
4565 rx_coal->int_ql = HNS3_INT_QL_DEFAULT_CFG;
4566 }
4567}
4568
dd38c726
YL		 4569static int hns3_nic_alloc_vector_data(struct hns3_nic_priv *priv)
4570{
4571 struct hnae3_handle *h = priv->ae_handle;
4572 struct hns3_enet_tqp_vector *tqp_vector;
4573 struct hnae3_vector_info *vector;
4574 struct pci_dev *pdev = h->pdev;
4575 u16 tqp_num = h->kinfo.num_tqps;
4576 u16 vector_num;
4577 int ret = 0;
4578 u16 i;
4579
4580 /* RSS size, cpu online and vector_num should be the same */
4581 /* Should consider 2p/4p later */
4582 vector_num = min_t(u16, num_online_cpus(), tqp_num);
75edb610 4583
dd38c726
YL		 4584 vector = devm_kcalloc(&pdev->dev, vector_num, sizeof(*vector),
4585 GFP_KERNEL);
4586 if (!vector)
4587 return -ENOMEM;
4588
9b2f3477 4589 /* save the actual available vector number */
dd38c726
YL		 4590 vector_num = h->ae_algo->ops->get_vector(h, vector_num, vector);
4591
4592 priv->vector_num = vector_num;
4593 priv->tqp_vector = (struct hns3_enet_tqp_vector *)
4594 devm_kcalloc(&pdev->dev, vector_num, sizeof(*priv->tqp_vector),
4595 GFP_KERNEL);
4596 if (!priv->tqp_vector) {
4597 ret = -ENOMEM;
4598 goto out;
4599 }
4600
4601 for (i = 0; i < priv->vector_num; i++) {
4602 tqp_vector = &priv->tqp_vector[i];
4603 tqp_vector->idx = i;
4604 tqp_vector->mask_addr = vector[i].io_addr;
4605 tqp_vector->vector_irq = vector[i].vector;
91bfae25 4606 hns3_vector_coalesce_init(tqp_vector, priv);
dd38c726
YL		 4607 }
4608
76ad4f0e
S		 4609out:
4610 devm_kfree(&pdev->dev, vector);
4611 return ret;
4612}
4613
dd38c726
YL		 4614static void hns3_clear_ring_group(struct hns3_enet_ring_group *group)
4615{
4616 group->ring = NULL;
4617 group->count = 0;
4618}
4619
e2152785 4620static void hns3_nic_uninit_vector_data(struct hns3_nic_priv *priv)
76ad4f0e
S		 4621{
4622 struct hnae3_ring_chain_node vector_ring_chain;
4623 struct hnae3_handle *h = priv->ae_handle;
4624 struct hns3_enet_tqp_vector *tqp_vector;
e2152785 4625 int i;
76ad4f0e
S		 4626
4627 for (i = 0; i < priv->vector_num; i++) {
4628 tqp_vector = &priv->tqp_vector[i];
4629
2c9dd668
HT		 4630 if (!tqp_vector->rx_group.ring && !tqp_vector->tx_group.ring)
4631 continue;
4632
ff7dfcdd
HT		 4633 /* Since the mapping can be overwritten, when fail to get the
4634 * chain between vector and ring, we should go on to deal with
4635 * the remaining options.
4636 */
4637 if (hns3_get_vector_ring_chain(tqp_vector, &vector_ring_chain))
4638 dev_warn(priv->dev, "failed to get ring chain\n");
76ad4f0e 4639
e2152785 4640 h->ae_algo->ops->unmap_ring_from_vector(h,
76ad4f0e 4641 tqp_vector->vector_irq, &vector_ring_chain);
76ad4f0e
S		 4642
4643 hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain);
4644
dd38c726
YL		 4645 hns3_clear_ring_group(&tqp_vector->rx_group);
4646 hns3_clear_ring_group(&tqp_vector->tx_group);
76ad4f0e
S		 4647 netif_napi_del(&priv->tqp_vector[i].napi);
4648 }
dd38c726
YL		 4649}
4650
08a10068 4651static void hns3_nic_dealloc_vector_data(struct hns3_nic_priv *priv)
dd38c726
YL		 4652{
4653 struct hnae3_handle *h = priv->ae_handle;
4654 struct pci_dev *pdev = h->pdev;
4655 int i, ret;
4656
4657 for (i = 0; i < priv->vector_num; i++) {
4658 struct hns3_enet_tqp_vector *tqp_vector;
4659
4660 tqp_vector = &priv->tqp_vector[i];
4661 ret = h->ae_algo->ops->put_vector(h, tqp_vector->vector_irq);
4662 if (ret)
08a10068 4663 return;
dd38c726 4664 }
76ad4f0e 4665
dd38c726 4666 devm_kfree(&pdev->dev, priv->tqp_vector);
76ad4f0e
S		 4667}
4668
5f06b903
YL		 4669static void hns3_ring_get_cfg(struct hnae3_queue *q, struct hns3_nic_priv *priv,
4670 unsigned int ring_type)
76ad4f0e 4671{
76ad4f0e 4672 int queue_num = priv->ae_handle->kinfo.num_tqps;
76ad4f0e 4673 struct hns3_enet_ring *ring;
c0425944 4674 int desc_num;
76ad4f0e 4675
76ad4f0e 4676 if (ring_type == HNAE3_RING_TYPE_TX) {
5f06b903 4677 ring = &priv->ring[q->tqp_index];
c0425944 4678 desc_num = priv->ae_handle->kinfo.num_tx_desc;
5f06b903 4679 ring->queue_index = q->tqp_index;
907676b1
YL		 4680 ring->tx_copybreak = priv->tx_copybreak;
4681 ring->last_to_use = 0;
76ad4f0e 4682 } else {
5f06b903 4683 ring = &priv->ring[q->tqp_index + queue_num];
c0425944 4684 desc_num = priv->ae_handle->kinfo.num_rx_desc;
5f06b903 4685 ring->queue_index = q->tqp_index;
99f6b5fb 4686 ring->rx_copybreak = priv->rx_copybreak;
76ad4f0e
S		 4687 }
4688
e4e87715 4689 hnae3_set_bit(ring->flag, HNAE3_RING_TYPE_B, ring_type);
76ad4f0e 4690
76ad4f0e
S		 4691 ring->tqp = q;
4692 ring->desc = NULL;
4693 ring->desc_cb = NULL;
4694 ring->dev = priv->dev;
4695 ring->desc_dma_addr = 0;
4696 ring->buf_size = q->buf_size;
2c9dd668 4697 ring->desc_num = desc_num;
76ad4f0e
S		 4698 ring->next_to_use = 0;
4699 ring->next_to_clean = 0;
76ad4f0e
S		 4700}
4701
5f06b903
YL		 4702static void hns3_queue_to_ring(struct hnae3_queue *tqp,
4703 struct hns3_nic_priv *priv)
76ad4f0e 4704{
5f06b903
YL		 4705 hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_TX);
4706 hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_RX);
76ad4f0e
S		 4707}
4708
4709static int hns3_get_ring_config(struct hns3_nic_priv *priv)
4710{
4711 struct hnae3_handle *h = priv->ae_handle;
4712 struct pci_dev *pdev = h->pdev;
5f06b903 4713 int i;
76ad4f0e 4714
5f06b903
YL		 4715 priv->ring = devm_kzalloc(&pdev->dev,
4716 array3_size(h->kinfo.num_tqps,
4717 sizeof(*priv->ring), 2),
4718 GFP_KERNEL);
4719 if (!priv->ring)
76ad4f0e
S		 4720 return -ENOMEM;
4721
5f06b903
YL		 4722 for (i = 0; i < h->kinfo.num_tqps; i++)
4723 hns3_queue_to_ring(h->kinfo.tqp[i], priv);
76ad4f0e
S		 4724
4725 return 0;
76ad4f0e
S		 4726}
4727
09f2af64
PL		 4728static void hns3_put_ring_config(struct hns3_nic_priv *priv)
4729{
5f06b903 4730 if (!priv->ring)
7b8f622e
HT		 4731 return;
4732
5f06b903
YL		 4733 devm_kfree(priv->dev, priv->ring);
4734 priv->ring = NULL;
09f2af64
PL		 4735}
4736
93188e96
YL		 4737static void hns3_alloc_page_pool(struct hns3_enet_ring *ring)
4738{
4739 struct page_pool_params pp_params = {
4740 .flags = PP_FLAG_DMA_MAP | PP_FLAG_PAGE_FRAG |
4741 PP_FLAG_DMA_SYNC_DEV,
4742 .order = hns3_page_order(ring),
4743 .pool_size = ring->desc_num * hns3_buf_size(ring) /
4744 (PAGE_SIZE << hns3_page_order(ring)),
4745 .nid = dev_to_node(ring_to_dev(ring)),
4746 .dev = ring_to_dev(ring),
4747 .dma_dir = DMA_FROM_DEVICE,
4748 .offset = 0,
4749 .max_len = PAGE_SIZE << hns3_page_order(ring),
4750 };
4751
4752 ring->page_pool = page_pool_create(&pp_params);
4753 if (IS_ERR(ring->page_pool)) {
4754 dev_warn(ring_to_dev(ring), "page pool creation failed: %ld\n",
4755 PTR_ERR(ring->page_pool));
4756 ring->page_pool = NULL;
4757 }
4758}
4759
76ad4f0e
S		 4760static int hns3_alloc_ring_memory(struct hns3_enet_ring *ring)
4761{
4762 int ret;
4763
4764 if (ring->desc_num <= 0 || ring->buf_size <= 0)
4765 return -EINVAL;
4766
77296bf6
YL		 4767 ring->desc_cb = devm_kcalloc(ring_to_dev(ring), ring->desc_num,
4768 sizeof(ring->desc_cb[0]), GFP_KERNEL);
76ad4f0e
S		 4769 if (!ring->desc_cb) {
4770 ret = -ENOMEM;
4771 goto out;
4772 }
4773
4774 ret = hns3_alloc_desc(ring);
4775 if (ret)
4776 goto out_with_desc_cb;
4777
4778 if (!HNAE3_IS_TX_RING(ring)) {
f7ec554b
YL		 4779 if (page_pool_enabled)
4780 hns3_alloc_page_pool(ring);
93188e96 4781
76ad4f0e
S		 4782 ret = hns3_alloc_ring_buffers(ring);
4783 if (ret)
4784 goto out_with_desc;
907676b1
YL		 4785 } else {
4786 hns3_init_tx_spare_buffer(ring);
76ad4f0e
S		 4787 }
4788
4789 return 0;
4790
4791out_with_desc:
4792 hns3_free_desc(ring);
4793out_with_desc_cb:
77296bf6 4794 devm_kfree(ring_to_dev(ring), ring->desc_cb);
76ad4f0e
S		 4795 ring->desc_cb = NULL;
4796out:
4797 return ret;
4798}
4799
a723fb8e 4800void hns3_fini_ring(struct hns3_enet_ring *ring)
76ad4f0e
S		 4801{
4802 hns3_free_desc(ring);
77296bf6 4803 devm_kfree(ring_to_dev(ring), ring->desc_cb);
76ad4f0e
S		 4804 ring->desc_cb = NULL;
4805 ring->next_to_clean = 0;
4806 ring->next_to_use = 0;
20d06ca2 4807 ring->last_to_use = 0;
ac574b80 4808 ring->pending_buf = 0;
907676b1 4809 if (!HNAE3_IS_TX_RING(ring) && ring->skb) {
ac574b80
PL		 4810 dev_kfree_skb_any(ring->skb);
4811 ring->skb = NULL;
907676b1
YL		 4812 } else if (HNAE3_IS_TX_RING(ring) && ring->tx_spare) {
4813 struct hns3_tx_spare *tx_spare = ring->tx_spare;
4814
4815 dma_unmap_page(ring_to_dev(ring), tx_spare->dma, tx_spare->len,
4816 DMA_TO_DEVICE);
4817 free_pages((unsigned long)tx_spare->buf,
4818 get_order(tx_spare->len));
4819 devm_kfree(ring_to_dev(ring), tx_spare);
4820 ring->tx_spare = NULL;
ac574b80 4821 }
93188e96
YL		 4822
4823 if (!HNAE3_IS_TX_RING(ring) && ring->page_pool) {
4824 page_pool_destroy(ring->page_pool);
4825 ring->page_pool = NULL;
4826 }
76ad4f0e
S		 4827}
4828
1db9b1bf 4829static int hns3_buf_size2type(u32 buf_size)
76ad4f0e
S		 4830{
4831 int bd_size_type;
4832
4833 switch (buf_size) {
4834 case 512:
4835 bd_size_type = HNS3_BD_SIZE_512_TYPE;
4836 break;
4837 case 1024:
4838 bd_size_type = HNS3_BD_SIZE_1024_TYPE;
4839 break;
4840 case 2048:
4841 bd_size_type = HNS3_BD_SIZE_2048_TYPE;
4842 break;
4843 case 4096:
4844 bd_size_type = HNS3_BD_SIZE_4096_TYPE;
4845 break;
4846 default:
4847 bd_size_type = HNS3_BD_SIZE_2048_TYPE;
4848 }
4849
4850 return bd_size_type;
4851}
4852
4853static void hns3_init_ring_hw(struct hns3_enet_ring *ring)
4854{
4855 dma_addr_t dma = ring->desc_dma_addr;
4856 struct hnae3_queue *q = ring->tqp;
4857
4858 if (!HNAE3_IS_TX_RING(ring)) {
9b2f3477 4859 hns3_write_dev(q, HNS3_RING_RX_RING_BASEADDR_L_REG, (u32)dma);
76ad4f0e
S		 4860 hns3_write_dev(q, HNS3_RING_RX_RING_BASEADDR_H_REG,
4861 (u32)((dma >> 31) >> 1));
4862
4863 hns3_write_dev(q, HNS3_RING_RX_RING_BD_LEN_REG,
4864 hns3_buf_size2type(ring->buf_size));
4865 hns3_write_dev(q, HNS3_RING_RX_RING_BD_NUM_REG,
4866 ring->desc_num / 8 - 1);
76ad4f0e
S		 4867 } else {
4868 hns3_write_dev(q, HNS3_RING_TX_RING_BASEADDR_L_REG,
4869 (u32)dma);
4870 hns3_write_dev(q, HNS3_RING_TX_RING_BASEADDR_H_REG,
4871 (u32)((dma >> 31) >> 1));
4872
76ad4f0e
S		 4873 hns3_write_dev(q, HNS3_RING_TX_RING_BD_NUM_REG,
4874 ring->desc_num / 8 - 1);
4875 }
4876}
4877
1c772154
YL		 4878static void hns3_init_tx_ring_tc(struct hns3_nic_priv *priv)
4879{
4880 struct hnae3_knic_private_info *kinfo = &priv->ae_handle->kinfo;
35244430 4881 struct hnae3_tc_info *tc_info = &kinfo->tc_info;
1c772154
YL		 4882 int i;
4883
a8e76fef 4884 for (i = 0; i < tc_info->num_tc; i++) {
1c772154
YL		 4885 int j;
4886
35244430 4887 for (j = 0; j < tc_info->tqp_count[i]; j++) {
1c772154
YL		 4888 struct hnae3_queue *q;
4889
35244430
JS		 4890 q = priv->ring[tc_info->tqp_offset[i] + j].tqp;
4891 hns3_write_dev(q, HNS3_RING_TX_RING_TC_REG, i);
1c772154
YL		 4892 }
4893 }
4894}
4895
5668abda 4896int hns3_init_all_ring(struct hns3_nic_priv *priv)
76ad4f0e
S		 4897{
4898 struct hnae3_handle *h = priv->ae_handle;
4899 int ring_num = h->kinfo.num_tqps * 2;
4900 int i, j;
4901 int ret;
4902
4903 for (i = 0; i < ring_num; i++) {
5f06b903 4904 ret = hns3_alloc_ring_memory(&priv->ring[i]);
76ad4f0e
S		 4905 if (ret) {
4906 dev_err(priv->dev,
4907 "Alloc ring memory fail! ret=%d\n", ret);
4908 goto out_when_alloc_ring_memory;
4909 }
4910
5f06b903 4911 u64_stats_init(&priv->ring[i].syncp);
76ad4f0e
S		 4912 }
4913
4914 return 0;
4915
4916out_when_alloc_ring_memory:
4917 for (j = i - 1; j >= 0; j--)
5f06b903 4918 hns3_fini_ring(&priv->ring[j]);
76ad4f0e
S		 4919
4920 return -ENOMEM;
4921}
4922
64749c9c 4923static void hns3_uninit_all_ring(struct hns3_nic_priv *priv)
76ad4f0e
S		 4924{
4925 struct hnae3_handle *h = priv->ae_handle;
4926 int i;
4927
4928 for (i = 0; i < h->kinfo.num_tqps; i++) {
5f06b903
YL		 4929 hns3_fini_ring(&priv->ring[i]);
4930 hns3_fini_ring(&priv->ring[i + h->kinfo.num_tqps]);
76ad4f0e 4931 }
76ad4f0e
S		 4932}
4933
4934/* Set mac addr if it is configured. or leave it to the AE driver */
8e6de441 4935static int hns3_init_mac_addr(struct net_device *netdev)
76ad4f0e
S		 4936{
4937 struct hns3_nic_priv *priv = netdev_priv(netdev);
4938 struct hnae3_handle *h = priv->ae_handle;
4939 u8 mac_addr_temp[ETH_ALEN];
7fa6be4f 4940 int ret = 0;
76ad4f0e 4941
8e6de441 4942 if (h->ae_algo->ops->get_mac_addr)
76ad4f0e 4943 h->ae_algo->ops->get_mac_addr(h, mac_addr_temp);
76ad4f0e
S		 4944
4945 /* Check if the MAC address is valid, if not get a random one */
8e6de441 4946 if (!is_valid_ether_addr(mac_addr_temp)) {
76ad4f0e
S		 4947 eth_hw_addr_random(netdev);
4948 dev_warn(priv->dev, "using random MAC address %pM\n",
4949 netdev->dev_addr);
ee4bcd3b 4950 } else if (!ether_addr_equal(netdev->dev_addr, mac_addr_temp)) {
f3956ebb 4951 eth_hw_addr_set(netdev, mac_addr_temp);
8e6de441 4952 ether_addr_copy(netdev->perm_addr, mac_addr_temp);
ee4bcd3b
JS		 4953 } else {
4954 return 0;
76ad4f0e 4955 }
139e8792
L		 4956
4957 if (h->ae_algo->ops->set_mac_addr)
7fa6be4f 4958 ret = h->ae_algo->ops->set_mac_addr(h, netdev->dev_addr, true);
139e8792 4959
7fa6be4f 4960 return ret;
76ad4f0e
S		 4961}
4962
c8a8045b
HT		 4963static int hns3_init_phy(struct net_device *netdev)
4964{
4965 struct hnae3_handle *h = hns3_get_handle(netdev);
4966 int ret = 0;
4967
4968 if (h->ae_algo->ops->mac_connect_phy)
4969 ret = h->ae_algo->ops->mac_connect_phy(h);
4970
4971 return ret;
4972}
4973
4974static void hns3_uninit_phy(struct net_device *netdev)
4975{
4976 struct hnae3_handle *h = hns3_get_handle(netdev);
4977
4978 if (h->ae_algo->ops->mac_disconnect_phy)
4979 h->ae_algo->ops->mac_disconnect_phy(h);
4980}
4981
a6d818e3
YL		 4982static int hns3_client_start(struct hnae3_handle *handle)
4983{
4984 if (!handle->ae_algo->ops->client_start)
4985 return 0;
4986
4987 return handle->ae_algo->ops->client_start(handle);
4988}
4989
4990static void hns3_client_stop(struct hnae3_handle *handle)
4991{
4992 if (!handle->ae_algo->ops->client_stop)
4993 return;
4994
4995 handle->ae_algo->ops->client_stop(handle);
4996}
4997
bb87be87
YL		 4998static void hns3_info_show(struct hns3_nic_priv *priv)
4999{
5000 struct hnae3_knic_private_info *kinfo = &priv->ae_handle->kinfo;
5001
5002 dev_info(priv->dev, "MAC address: %pM\n", priv->netdev->dev_addr);
adcf738b
GL		 5003 dev_info(priv->dev, "Task queue pairs numbers: %u\n", kinfo->num_tqps);
5004 dev_info(priv->dev, "RSS size: %u\n", kinfo->rss_size);
5005 dev_info(priv->dev, "Allocated RSS size: %u\n", kinfo->req_rss_size);
5006 dev_info(priv->dev, "RX buffer length: %u\n", kinfo->rx_buf_len);
5007 dev_info(priv->dev, "Desc num per TX queue: %u\n", kinfo->num_tx_desc);
5008 dev_info(priv->dev, "Desc num per RX queue: %u\n", kinfo->num_rx_desc);
35244430
JS		 5009 dev_info(priv->dev, "Total number of enabled TCs: %u\n",
5010 kinfo->tc_info.num_tc);
adcf738b 5011 dev_info(priv->dev, "Max mtu size: %u\n", priv->netdev->max_mtu);
bb87be87
YL		 5012}
5013
9f0c6f4b
YM		 5014static void hns3_set_cq_period_mode(struct hns3_nic_priv *priv,
5015 enum dim_cq_period_mode mode, bool is_tx)
5016{
5017 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(priv->ae_handle->pdev);
5018 struct hnae3_handle *handle = priv->ae_handle;
5019 int i;
5020
5021 if (is_tx) {
5022 priv->tx_cqe_mode = mode;
5023
5024 for (i = 0; i < priv->vector_num; i++)
5025 priv->tqp_vector[i].tx_group.dim.mode = mode;
5026 } else {
5027 priv->rx_cqe_mode = mode;
5028
5029 for (i = 0; i < priv->vector_num; i++)
5030 priv->tqp_vector[i].rx_group.dim.mode = mode;
5031 }
5032
5033 /* only device version above V3(include V3), GL can switch CQ/EQ
5034 * period mode.
5035 */
5036 if (ae_dev->dev_version >= HNAE3_DEVICE_VERSION_V3) {
5037 u32 new_mode;
5038 u64 reg;
5039
5040 new_mode = (mode == DIM_CQ_PERIOD_MODE_START_FROM_CQE) ?
5041 HNS3_CQ_MODE_CQE : HNS3_CQ_MODE_EQE;
5042 reg = is_tx ? HNS3_GL1_CQ_MODE_REG : HNS3_GL0_CQ_MODE_REG;
5043
5044 writel(new_mode, handle->kinfo.io_base + reg);
5045 }
5046}
5047
cce1689e
YM		 5048void hns3_cq_period_mode_init(struct hns3_nic_priv *priv,
5049 enum dim_cq_period_mode tx_mode,
5050 enum dim_cq_period_mode rx_mode)
9f0c6f4b
YM		 5051{
5052 hns3_set_cq_period_mode(priv, tx_mode, true);
5053 hns3_set_cq_period_mode(priv, rx_mode, false);
5054}
5055
c511dfff
HT		 5056static void hns3_state_init(struct hnae3_handle *handle)
5057{
5058 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(handle->pdev);
5059 struct net_device *netdev = handle->kinfo.netdev;
5060 struct hns3_nic_priv *priv = netdev_priv(netdev);
5061
5062 set_bit(HNS3_NIC_STATE_INITED, &priv->state);
5063
5064 if (ae_dev->dev_version >= HNAE3_DEVICE_VERSION_V3)
5065 set_bit(HNAE3_PFLAG_LIMIT_PROMISC, &handle->supported_pflags);
5066
5067 if (test_bit(HNAE3_DEV_SUPPORT_HW_TX_CSUM_B, ae_dev->caps))
5068 set_bit(HNS3_NIC_STATE_HW_TX_CSUM_ENABLE, &priv->state);
5069
5070 if (hnae3_ae_dev_rxd_adv_layout_supported(ae_dev))
5071 set_bit(HNS3_NIC_STATE_RXD_ADV_LAYOUT_ENABLE, &priv->state);
5072}
5073
76ad4f0e
S		 5074static int hns3_client_init(struct hnae3_handle *handle)
5075{
5076 struct pci_dev *pdev = handle->pdev;
fd665b3d 5077 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
0d43bf45 5078 u16 alloc_tqps, max_rss_size;
76ad4f0e
S		 5079 struct hns3_nic_priv *priv;
5080 struct net_device *netdev;
5081 int ret;
5082
0d43bf45
HT		 5083 handle->ae_algo->ops->get_tqps_and_rss_info(handle, &alloc_tqps,
5084 &max_rss_size);
5085 netdev = alloc_etherdev_mq(sizeof(struct hns3_nic_priv), alloc_tqps);
76ad4f0e
S		 5086 if (!netdev)
5087 return -ENOMEM;
5088
5089 priv = netdev_priv(netdev);
5090 priv->dev = &pdev->dev;
5091 priv->netdev = netdev;
5092 priv->ae_handle = handle;
f8fa222c 5093 priv->tx_timeout_count = 0;
fd665b3d 5094 priv->max_non_tso_bd_num = ae_dev->dev_specs.max_non_tso_bd_num;
b7b585c2 5095 set_bit(HNS3_NIC_STATE_DOWN, &priv->state);
76ad4f0e 5096
bb87be87
YL		 5097 handle->msg_enable = netif_msg_init(debug, DEFAULT_MSG_LEVEL);
5098
76ad4f0e
S		 5099 handle->kinfo.netdev = netdev;
5100 handle->priv = (void *)priv;
5101
8e6de441 5102 hns3_init_mac_addr(netdev);
76ad4f0e
S		 5103
5104 hns3_set_default_feature(netdev);
5105
5106 netdev->watchdog_timeo = HNS3_TX_TIMEOUT;
5107 netdev->priv_flags |= IFF_UNICAST_FLT;
5108 netdev->netdev_ops = &hns3_nic_netdev_ops;
5109 SET_NETDEV_DEV(netdev, &pdev->dev);
5110 hns3_ethtool_set_ops(netdev);
76ad4f0e
S		 5111
5112 /* Carrier off reporting is important to ethtool even BEFORE open */
5113 netif_carrier_off(netdev);
5114
5115 ret = hns3_get_ring_config(priv);
5116 if (ret) {
5117 ret = -ENOMEM;
5118 goto out_get_ring_cfg;
5119 }
5120
73a13d8d
HT		 5121 hns3_nic_init_coal_cfg(priv);
5122
dd38c726
YL		 5123 ret = hns3_nic_alloc_vector_data(priv);
5124 if (ret) {
5125 ret = -ENOMEM;
5126 goto out_alloc_vector_data;
5127 }
5128
76ad4f0e
S		 5129 ret = hns3_nic_init_vector_data(priv);
5130 if (ret) {
5131 ret = -ENOMEM;
5132 goto out_init_vector_data;
5133 }
5134
5135 ret = hns3_init_all_ring(priv);
5136 if (ret) {
5137 ret = -ENOMEM;
5f06b903 5138 goto out_init_ring;
76ad4f0e
S		 5139 }
5140
9f0c6f4b
YM		 5141 hns3_cq_period_mode_init(priv, DIM_CQ_PERIOD_MODE_START_FROM_EQE,
5142 DIM_CQ_PERIOD_MODE_START_FROM_EQE);
5143
c8a8045b
HT		 5144 ret = hns3_init_phy(netdev);
5145 if (ret)
5146 goto out_init_phy;
5147
08a10068
YL		 5148 /* the device can work without cpu rmap, only aRFS needs it */
5149 ret = hns3_set_rx_cpu_rmap(netdev);
5150 if (ret)
5151 dev_warn(priv->dev, "set rx cpu rmap fail, ret=%d\n", ret);
5152
5153 ret = hns3_nic_init_irq(priv);
5154 if (ret) {
5155 dev_err(priv->dev, "init irq failed! ret=%d\n", ret);
5156 hns3_free_rx_cpu_rmap(netdev);
5157 goto out_init_irq_fail;
5158 }
5159
a6d818e3
YL		 5160 ret = hns3_client_start(handle);
5161 if (ret) {
5162 dev_err(priv->dev, "hns3_client_start fail! ret=%d\n", ret);
bf6de231 5163 goto out_client_start;
a6d818e3
YL		 5164 }
5165
986743db
YL		 5166 hns3_dcbnl_setup(handle);
5167
5e69ea7e
YM		 5168 ret = hns3_dbg_init(handle);
5169 if (ret) {
5170 dev_err(priv->dev, "failed to init debugfs, ret = %d\n",
5171 ret);
5172 goto out_client_start;
5173 }
b2292360 5174
e070c8b9 5175 netdev->max_mtu = HNS3_MAX_MTU(ae_dev->dev_specs.max_frm_size);
a8e8b7ff 5176
c511dfff 5177 hns3_state_init(handle);
5e7414cd 5178
a289a7e5
JS		 5179 ret = register_netdev(netdev);
5180 if (ret) {
5181 dev_err(priv->dev, "probe register netdev fail!\n");
5182 goto out_reg_netdev_fail;
5183 }
5184
bb87be87
YL		 5185 if (netif_msg_drv(handle))
5186 hns3_info_show(priv);
5187
76ad4f0e
S		 5188 return ret;
5189
a289a7e5
JS		 5190out_reg_netdev_fail:
5191 hns3_dbg_uninit(handle);
18655128 5192out_client_start:
08a10068
YL		 5193 hns3_free_rx_cpu_rmap(netdev);
5194 hns3_nic_uninit_irq(priv);
5195out_init_irq_fail:
c8a8045b
HT		 5196 hns3_uninit_phy(netdev);
5197out_init_phy:
5198 hns3_uninit_all_ring(priv);
5f06b903 5199out_init_ring:
e2152785 5200 hns3_nic_uninit_vector_data(priv);
76ad4f0e 5201out_init_vector_data:
dd38c726
YL		 5202 hns3_nic_dealloc_vector_data(priv);
5203out_alloc_vector_data:
5f06b903 5204 priv->ring = NULL;
76ad4f0e
S		 5205out_get_ring_cfg:
5206 priv->ae_handle = NULL;
5207 free_netdev(netdev);
5208 return ret;
5209}
5210
5211static void hns3_client_uninit(struct hnae3_handle *handle, bool reset)
5212{
5213 struct net_device *netdev = handle->kinfo.netdev;
5214 struct hns3_nic_priv *priv = netdev_priv(netdev);
76ad4f0e
S		 5215
5216 if (netdev->reg_state != NETREG_UNINITIALIZED)
5217 unregister_netdev(netdev);
5218
eb32c896
HT		 5219 hns3_client_stop(handle);
5220
0d2f68c7
HT		 5221 hns3_uninit_phy(netdev);
5222
814da63c
HT		 5223 if (!test_and_clear_bit(HNS3_NIC_STATE_INITED, &priv->state)) {
5224 netdev_warn(netdev, "already uninitialized\n");
5225 goto out_netdev_free;
5226 }
5227
08a10068
YL		 5228 hns3_free_rx_cpu_rmap(netdev);
5229
5230 hns3_nic_uninit_irq(priv);
5231
f96315f2 5232 hns3_clear_all_ring(handle, true);
7b763f3f 5233
e2152785 5234 hns3_nic_uninit_vector_data(priv);
76ad4f0e 5235
08a10068 5236 hns3_nic_dealloc_vector_data(priv);
dd38c726 5237
64749c9c 5238 hns3_uninit_all_ring(priv);
76ad4f0e 5239
ec777890
YL		 5240 hns3_put_ring_config(priv);
5241
814da63c 5242out_netdev_free:
e22b5e72 5243 hns3_dbg_uninit(handle);
76ad4f0e
S		 5244 free_netdev(netdev);
5245}
5246
5247static void hns3_link_status_change(struct hnae3_handle *handle, bool linkup)
5248{
5249 struct net_device *netdev = handle->kinfo.netdev;
5250
5251 if (!netdev)
5252 return;
5253
5254 if (linkup) {
76ad4f0e 5255 netif_tx_wake_all_queues(netdev);
a7e90ee5 5256 netif_carrier_on(netdev);
bb87be87
YL		 5257 if (netif_msg_link(handle))
5258 netdev_info(netdev, "link up\n");
76ad4f0e
S		 5259 } else {
5260 netif_carrier_off(netdev);
5261 netif_tx_stop_all_queues(netdev);
bb87be87
YL		 5262 if (netif_msg_link(handle))
5263 netdev_info(netdev, "link down\n");
76ad4f0e
S		 5264 }
5265}
5266
beebca3a 5267static void hns3_clear_tx_ring(struct hns3_enet_ring *ring)
bb6b94a8 5268{
beebca3a 5269 while (ring->next_to_clean != ring->next_to_use) {
7b763f3f 5270 ring->desc[ring->next_to_clean].tx.bdtp_fe_sc_vld_ra_ri = 0;
619ae331 5271 hns3_free_buffer_detach(ring, ring->next_to_clean, 0);
beebca3a
YL		 5272 ring_ptr_move_fw(ring, next_to_clean);
5273 }
f6061a05
YL		 5274
5275 ring->pending_buf = 0;
beebca3a
YL		 5276}
5277
7b763f3f
FL		 5278static int hns3_clear_rx_ring(struct hns3_enet_ring *ring)
5279{
5280 struct hns3_desc_cb res_cbs;
5281 int ret;
5282
5283 while (ring->next_to_use != ring->next_to_clean) {
5284 /* When a buffer is not reused, it's memory has been
5285 * freed in hns3_handle_rx_bd or will be freed by
5286 * stack, so we need to replace the buffer here.
5287 */
5288 if (!ring->desc_cb[ring->next_to_use].reuse_flag) {
4d2cad32 5289 ret = hns3_alloc_and_map_buffer(ring, &res_cbs);
7b763f3f
FL		 5290 if (ret) {
5291 u64_stats_update_begin(&ring->syncp);
5292 ring->stats.sw_err_cnt++;
5293 u64_stats_update_end(&ring->syncp);
5294 /* if alloc new buffer fail, exit directly
5295 * and reclear in up flow.
5296 */
c8711956 5297 netdev_warn(ring_to_netdev(ring),
7b763f3f
FL		 5298 "reserve buffer map failed, ret = %d\n",
5299 ret);
5300 return ret;
5301 }
9b2f3477 5302 hns3_replace_buffer(ring, ring->next_to_use, &res_cbs);
7b763f3f
FL		 5303 }
5304 ring_ptr_move_fw(ring, next_to_use);
5305 }
5306
cc5ff6e9
PL		 5307 /* Free the pending skb in rx ring */
5308 if (ring->skb) {
5309 dev_kfree_skb_any(ring->skb);
5310 ring->skb = NULL;
5311 ring->pending_buf = 0;
5312 }
5313
7b763f3f
FL		 5314 return 0;
5315}
5316
5317static void hns3_force_clear_rx_ring(struct hns3_enet_ring *ring)
beebca3a 5318{
beebca3a
YL		 5319 while (ring->next_to_use != ring->next_to_clean) {
5320 /* When a buffer is not reused, it's memory has been
5321 * freed in hns3_handle_rx_bd or will be freed by
5322 * stack, so only need to unmap the buffer here.
5323 */
5324 if (!ring->desc_cb[ring->next_to_use].reuse_flag) {
5325 hns3_unmap_buffer(ring,
5326 &ring->desc_cb[ring->next_to_use]);
5327 ring->desc_cb[ring->next_to_use].dma = 0;
5328 }
5329
5330 ring_ptr_move_fw(ring, next_to_use);
5331 }
bb6b94a8
L		 5332}
5333
f96315f2 5334static void hns3_clear_all_ring(struct hnae3_handle *h, bool force)
bb6b94a8
L		 5335{
5336 struct net_device *ndev = h->kinfo.netdev;
5337 struct hns3_nic_priv *priv = netdev_priv(ndev);
5338 u32 i;
5339
5340 for (i = 0; i < h->kinfo.num_tqps; i++) {
bb6b94a8
L		 5341 struct hns3_enet_ring *ring;
5342
5f06b903 5343 ring = &priv->ring[i];
beebca3a 5344 hns3_clear_tx_ring(ring);
bb6b94a8 5345
5f06b903 5346 ring = &priv->ring[i + h->kinfo.num_tqps];
7b763f3f
FL		 5347 /* Continue to clear other rings even if clearing some
5348 * rings failed.
5349 */
f96315f2
HT		 5350 if (force)
5351 hns3_force_clear_rx_ring(ring);
5352 else
5353 hns3_clear_rx_ring(ring);
bb6b94a8
L		 5354 }
5355}
5356
7b763f3f
FL		 5357int hns3_nic_reset_all_ring(struct hnae3_handle *h)
5358{
5359 struct net_device *ndev = h->kinfo.netdev;
5360 struct hns3_nic_priv *priv = netdev_priv(ndev);
5361 struct hns3_enet_ring *rx_ring;
5362 int i, j;
5363 int ret;
5364
8fa86551
YM		 5365 ret = h->ae_algo->ops->reset_queue(h);
5366 if (ret)
5367 return ret;
7fa6be4f 5368
8fa86551 5369 for (i = 0; i < h->kinfo.num_tqps; i++) {
5f06b903 5370 hns3_init_ring_hw(&priv->ring[i]);
7b763f3f
FL		 5371
5372 /* We need to clear tx ring here because self test will
5373 * use the ring and will not run down before up
5374 */
5f06b903
YL		 5375 hns3_clear_tx_ring(&priv->ring[i]);
5376 priv->ring[i].next_to_clean = 0;
5377 priv->ring[i].next_to_use = 0;
20d06ca2 5378 priv->ring[i].last_to_use = 0;
7b763f3f 5379
5f06b903 5380 rx_ring = &priv->ring[i + h->kinfo.num_tqps];
7b763f3f
FL		 5381 hns3_init_ring_hw(rx_ring);
5382 ret = hns3_clear_rx_ring(rx_ring);
5383 if (ret)
5384 return ret;
5385
5386 /* We can not know the hardware head and tail when this
5387 * function is called in reset flow, so we reuse all desc.
5388 */
5389 for (j = 0; j < rx_ring->desc_num; j++)
5390 hns3_reuse_buffer(rx_ring, j);
5391
5392 rx_ring->next_to_clean = 0;
5393 rx_ring->next_to_use = 0;
5394 }
5395
1c772154
YL		 5396 hns3_init_tx_ring_tc(priv);
5397
7b763f3f
FL		 5398 return 0;
5399}
5400
bb6b94a8
L		 5401static int hns3_reset_notify_down_enet(struct hnae3_handle *handle)
5402{
5403 struct hnae3_knic_private_info *kinfo = &handle->kinfo;
5404 struct net_device *ndev = kinfo->netdev;
257e4f29
HT		 5405 struct hns3_nic_priv *priv = netdev_priv(ndev);
5406
5407 if (test_and_set_bit(HNS3_NIC_STATE_RESETTING, &priv->state))
5408 return 0;
bb6b94a8
L		 5409
5410 if (!netif_running(ndev))
6b1385cc 5411 return 0;
bb6b94a8
L		 5412
5413 return hns3_nic_net_stop(ndev);
5414}
5415
5416static int hns3_reset_notify_up_enet(struct hnae3_handle *handle)
5417{
5418 struct hnae3_knic_private_info *kinfo = &handle->kinfo;
257e4f29 5419 struct hns3_nic_priv *priv = netdev_priv(kinfo->netdev);
bb6b94a8
L		 5420 int ret = 0;
5421
b4047aac
JS		 5422 if (!test_bit(HNS3_NIC_STATE_INITED, &priv->state)) {
5423 netdev_err(kinfo->netdev, "device is not initialized yet\n");
5424 return -EFAULT;
5425 }
5426
e8884027
HT		 5427 clear_bit(HNS3_NIC_STATE_RESETTING, &priv->state);
5428
bb6b94a8 5429 if (netif_running(kinfo->netdev)) {
e8884027 5430 ret = hns3_nic_net_open(kinfo->netdev);
bb6b94a8 5431 if (ret) {
e8884027 5432 set_bit(HNS3_NIC_STATE_RESETTING, &priv->state);
bb6b94a8 5433 netdev_err(kinfo->netdev,
9b2f3477 5434 "net up fail, ret=%d!\n", ret);
bb6b94a8
L		 5435 return ret;
5436 }
bb6b94a8
L		 5437 }
5438
5439 return ret;
5440}
5441
5442static int hns3_reset_notify_init_enet(struct hnae3_handle *handle)
5443{
5444 struct net_device *netdev = handle->kinfo.netdev;
5445 struct hns3_nic_priv *priv = netdev_priv(netdev);
5446 int ret;
5447
bb6b94a8
L		 5448 /* Carrier off reporting is important to ethtool even BEFORE open */
5449 netif_carrier_off(netdev);
5450
2c9dd668 5451 ret = hns3_get_ring_config(priv);
862d969a
HT		 5452 if (ret)
5453 return ret;
5454
2c9dd668
HT		 5455 ret = hns3_nic_alloc_vector_data(priv);
5456 if (ret)
5457 goto err_put_ring;
5458
bb6b94a8
L		 5459 ret = hns3_nic_init_vector_data(priv);
5460 if (ret)
862d969a 5461 goto err_dealloc_vector;
bb6b94a8
L		 5462
5463 ret = hns3_init_all_ring(priv);
862d969a
HT		 5464 if (ret)
5465 goto err_uninit_vector;
bb6b94a8 5466
9f0c6f4b
YM		 5467 hns3_cq_period_mode_init(priv, priv->tx_cqe_mode, priv->rx_cqe_mode);
5468
08a10068
YL		 5469 /* the device can work without cpu rmap, only aRFS needs it */
5470 ret = hns3_set_rx_cpu_rmap(netdev);
5471 if (ret)
5472 dev_warn(priv->dev, "set rx cpu rmap fail, ret=%d\n", ret);
5473
5474 ret = hns3_nic_init_irq(priv);
5475 if (ret) {
5476 dev_err(priv->dev, "init irq failed! ret=%d\n", ret);
5477 hns3_free_rx_cpu_rmap(netdev);
5478 goto err_init_irq_fail;
5479 }
5480
ee4bcd3b
JS		 5481 if (!hns3_is_phys_func(handle->pdev))
5482 hns3_init_mac_addr(netdev);
5483
cd513a69
HT		 5484 ret = hns3_client_start(handle);
5485 if (ret) {
5486 dev_err(priv->dev, "hns3_client_start fail! ret=%d\n", ret);
08a10068 5487 goto err_client_start_fail;
cd513a69
HT		 5488 }
5489
814da63c
HT		 5490 set_bit(HNS3_NIC_STATE_INITED, &priv->state);
5491
862d969a
HT		 5492 return ret;
5493
08a10068
YL		 5494err_client_start_fail:
5495 hns3_free_rx_cpu_rmap(netdev);
5496 hns3_nic_uninit_irq(priv);
5497err_init_irq_fail:
cd513a69 5498 hns3_uninit_all_ring(priv);
862d969a
HT		 5499err_uninit_vector:
5500 hns3_nic_uninit_vector_data(priv);
862d969a
HT		 5501err_dealloc_vector:
5502 hns3_nic_dealloc_vector_data(priv);
2c9dd668
HT		 5503err_put_ring:
5504 hns3_put_ring_config(priv);
862d969a 5505
bb6b94a8
L		 5506 return ret;
5507}
5508
5509static int hns3_reset_notify_uninit_enet(struct hnae3_handle *handle)
5510{
5511 struct net_device *netdev = handle->kinfo.netdev;
5512 struct hns3_nic_priv *priv = netdev_priv(netdev);
bb6b94a8 5513
1eeb3367 5514 if (!test_and_clear_bit(HNS3_NIC_STATE_INITED, &priv->state)) {
814da63c
HT		 5515 netdev_warn(netdev, "already uninitialized\n");
5516 return 0;
5517 }
5518
08a10068
YL		 5519 hns3_free_rx_cpu_rmap(netdev);
5520 hns3_nic_uninit_irq(priv);
f96315f2
HT		 5521 hns3_clear_all_ring(handle, true);
5522 hns3_reset_tx_queue(priv->ae_handle);
bb6b94a8 5523
e2152785 5524 hns3_nic_uninit_vector_data(priv);
bb6b94a8 5525
08a10068 5526 hns3_nic_dealloc_vector_data(priv);
862d969a 5527
64749c9c 5528 hns3_uninit_all_ring(priv);
bb6b94a8 5529
2c9dd668 5530 hns3_put_ring_config(priv);
2c9dd668 5531
205238f4 5532 return 0;
bb6b94a8
L		 5533}
5534
e445f08a
HC		 5535int hns3_reset_notify(struct hnae3_handle *handle,
5536 enum hnae3_reset_notify_type type)
bb6b94a8
L		 5537{
5538 int ret = 0;
5539
5540 switch (type) {
5541 case HNAE3_UP_CLIENT:
e1586241
SM		 5542 ret = hns3_reset_notify_up_enet(handle);
5543 break;
bb6b94a8
L		 5544 case HNAE3_DOWN_CLIENT:
5545 ret = hns3_reset_notify_down_enet(handle);
5546 break;
5547 case HNAE3_INIT_CLIENT:
5548 ret = hns3_reset_notify_init_enet(handle);
5549 break;
5550 case HNAE3_UNINIT_CLIENT:
5551 ret = hns3_reset_notify_uninit_enet(handle);
5552 break;
5553 default:
5554 break;
5555 }
5556
5557 return ret;
5558}
5559
3a5a5f06
PL		 5560static int hns3_change_channels(struct hnae3_handle *handle, u32 new_tqp_num,
5561 bool rxfh_configured)
5562{
5563 int ret;
5564
5565 ret = handle->ae_algo->ops->set_channels(handle, new_tqp_num,
5566 rxfh_configured);
5567 if (ret) {
5568 dev_err(&handle->pdev->dev,
5569 "Change tqp num(%u) fail.\n", new_tqp_num);
5570 return ret;
5571 }
5572
5573 ret = hns3_reset_notify(handle, HNAE3_INIT_CLIENT);
5574 if (ret)
5575 return ret;
5576
5577 ret = hns3_reset_notify(handle, HNAE3_UP_CLIENT);
5578 if (ret)
5579 hns3_reset_notify(handle, HNAE3_UNINIT_CLIENT);
5580
5581 return ret;
5582}
5583
09f2af64
PL		 5584int hns3_set_channels(struct net_device *netdev,
5585 struct ethtool_channels *ch)
5586{
09f2af64
PL		 5587 struct hnae3_handle *h = hns3_get_handle(netdev);
5588 struct hnae3_knic_private_info *kinfo = &h->kinfo;
90c68a41 5589 bool rxfh_configured = netif_is_rxfh_configured(netdev);
09f2af64
PL		 5590 u32 new_tqp_num = ch->combined_count;
5591 u16 org_tqp_num;
5592 int ret;
5593
44950d28
JS		 5594 if (hns3_nic_resetting(netdev))
5595 return -EBUSY;
5596
09f2af64
PL		 5597 if (ch->rx_count || ch->tx_count)
5598 return -EINVAL;
5599
5a5c9091
JS		 5600 if (kinfo->tc_info.mqprio_active) {
5601 dev_err(&netdev->dev,
5602 "it's not allowed to set channels via ethtool when MQPRIO mode is on\n");
5603 return -EINVAL;
5604 }
5605
678335a1 5606 if (new_tqp_num > hns3_get_max_available_channels(h) ||
c78b5b6c 5607 new_tqp_num < 1) {
09f2af64 5608 dev_err(&netdev->dev,
adcf738b 5609 "Change tqps fail, the tqp range is from 1 to %u",
678335a1 5610 hns3_get_max_available_channels(h));
09f2af64
PL		 5611 return -EINVAL;
5612 }
5613
c78b5b6c 5614 if (kinfo->rss_size == new_tqp_num)
09f2af64
PL		 5615 return 0;
5616
1c822948
YL		 5617 netif_dbg(h, drv, netdev,
5618 "set channels: tqp_num=%u, rxfh=%d\n",
5619 new_tqp_num, rxfh_configured);
5620
65749f73
HT		 5621 ret = hns3_reset_notify(h, HNAE3_DOWN_CLIENT);
5622 if (ret)
5623 return ret;
dd38c726 5624
65749f73
HT		 5625 ret = hns3_reset_notify(h, HNAE3_UNINIT_CLIENT);
5626 if (ret)
5627 return ret;
09f2af64
PL		 5628
5629 org_tqp_num = h->kinfo.num_tqps;
3a5a5f06 5630 ret = hns3_change_channels(h, new_tqp_num, rxfh_configured);
09f2af64 5631 if (ret) {
3a5a5f06
PL		 5632 int ret1;
5633
5634 netdev_warn(netdev,
5635 "Change channels fail, revert to old value\n");
5636 ret1 = hns3_change_channels(h, org_tqp_num, rxfh_configured);
5637 if (ret1) {
5638 netdev_err(netdev,
5639 "revert to old channel fail\n");
5640 return ret1;
09f2af64 5641 }
3a5a5f06 5642
65749f73 5643 return ret;
3a5a5f06 5644 }
09f2af64 5645
3a5a5f06 5646 return 0;
09f2af64
PL		 5647}
5648
a83d2961
WL		 5649static const struct hns3_hw_error_info hns3_hw_err[] = {
5650 { .type = HNAE3_PPU_POISON_ERROR,
5651 .msg = "PPU poison" },
5652 { .type = HNAE3_CMDQ_ECC_ERROR,
5653 .msg = "IMP CMDQ error" },
5654 { .type = HNAE3_IMP_RD_POISON_ERROR,
5655 .msg = "IMP RD poison" },
6cd131dd
YM		 5656 { .type = HNAE3_ROCEE_AXI_RESP_ERROR,
5657 .msg = "ROCEE AXI RESP error" },
a83d2961
WL		 5658};
5659
5660static void hns3_process_hw_error(struct hnae3_handle *handle,
5661 enum hnae3_hw_error_type type)
5662{
5663 int i;
5664
5665 for (i = 0; i < ARRAY_SIZE(hns3_hw_err); i++) {
5666 if (hns3_hw_err[i].type == type) {
5667 dev_err(&handle->pdev->dev, "Detected %s!\n",
5668 hns3_hw_err[i].msg);
5669 break;
5670 }
5671 }
5672}
5673
1db9b1bf 5674static const struct hnae3_client_ops client_ops = {
76ad4f0e
S		 5675 .init_instance = hns3_client_init,
5676 .uninit_instance = hns3_client_uninit,
5677 .link_status_change = hns3_link_status_change,
bb6b94a8 5678 .reset_notify = hns3_reset_notify,
a83d2961 5679 .process_hw_error = hns3_process_hw_error,
76ad4f0e
S		 5680};
5681
5682/* hns3_init_module - Driver registration routine
5683 * hns3_init_module is the first routine called when the driver is
5684 * loaded. All it does is register with the PCI subsystem.
5685 */
5686static int __init hns3_init_module(void)
5687{
5688 int ret;
5689
5690 pr_info("%s: %s - version\n", hns3_driver_name, hns3_driver_string);
5691 pr_info("%s: %s\n", hns3_driver_name, hns3_copyright);
5692
5693 client.type = HNAE3_CLIENT_KNIC;
cdc37385 5694 snprintf(client.name, HNAE3_CLIENT_NAME_LENGTH, "%s",
76ad4f0e
S		 5695 hns3_driver_name);
5696
5697 client.ops = &client_ops;
5698
13562d1f
XW		 5699 INIT_LIST_HEAD(&client.node);
5700
b2292360 5701 hns3_dbg_register_debugfs(hns3_driver_name);
5702
76ad4f0e
S		 5703 ret = hnae3_register_client(&client);
5704 if (ret)
b2292360 5705 goto err_reg_client;
76ad4f0e
S		 5706
5707 ret = pci_register_driver(&hns3_driver);
5708 if (ret)
b2292360 5709 goto err_reg_driver;
76ad4f0e
S		 5710
5711 return ret;
b2292360 5712
5713err_reg_driver:
5714 hnae3_unregister_client(&client);
5715err_reg_client:
5716 hns3_dbg_unregister_debugfs();
5717 return ret;
76ad4f0e
S		 5718}
5719module_init(hns3_init_module);
5720
5721/* hns3_exit_module - Driver exit cleanup routine
5722 * hns3_exit_module is called just before the driver is removed
5723 * from memory.
5724 */
5725static void __exit hns3_exit_module(void)
5726{
5727 pci_unregister_driver(&hns3_driver);
5728 hnae3_unregister_client(&client);
b2292360 5729 hns3_dbg_unregister_debugfs();
76ad4f0e
S		 5730}
5731module_exit(hns3_exit_module);
5732
5733MODULE_DESCRIPTION("HNS3: Hisilicon Ethernet Driver");
5734MODULE_AUTHOR("Huawei Tech. Co., Ltd.");
5735MODULE_LICENSE("GPL");
5736MODULE_ALIAS("pci:hns-nic");
Linux 6.x block layer and io_uring tree(s)