1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2018, Intel Corporation. */
4 /* Intel(R) Ethernet Connection E800 Series Linux Driver */
6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
10 #define DRV_VERSION "ice-0.7.0-k"
11 #define DRV_SUMMARY "Intel(R) Ethernet Connection E800 Series Linux Driver"
12 const char ice_drv_ver[] = DRV_VERSION;
13 static const char ice_driver_string[] = DRV_SUMMARY;
14 static const char ice_copyright[] = "Copyright (c) 2018, Intel Corporation.";
16 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
17 MODULE_DESCRIPTION(DRV_SUMMARY);
18 MODULE_LICENSE("GPL");
19 MODULE_VERSION(DRV_VERSION);
21 static int debug = -1;
22 module_param(debug, int, 0644);
23 #ifndef CONFIG_DYNAMIC_DEBUG
24 MODULE_PARM_DESC(debug, "netif level (0=none,...,16=all), hw debug_mask (0x8XXXXXXX)");
26 MODULE_PARM_DESC(debug, "netif level (0=none,...,16=all)");
27 #endif /* !CONFIG_DYNAMIC_DEBUG */
29 static struct workqueue_struct *ice_wq;
30 static const struct net_device_ops ice_netdev_ops;
32 static void ice_pf_dis_all_vsi(struct ice_pf *pf);
33 static void ice_rebuild(struct ice_pf *pf);
34 static int ice_vsi_release(struct ice_vsi *vsi);
35 static void ice_update_vsi_stats(struct ice_vsi *vsi);
36 static void ice_update_pf_stats(struct ice_pf *pf);
39 * ice_get_free_slot - get the next non-NULL location index in array
40 * @array: array to search
41 * @size: size of the array
42 * @curr: last known occupied index to be used as a search hint
44 * void * is being used to keep the functionality generic. This lets us use this
45 * function on any array of pointers.
47 static int ice_get_free_slot(void *array, int size, int curr)
49 int **tmp_array = (int **)array;
52 if (curr < (size - 1) && !tmp_array[curr + 1]) {
57 while ((i < size) && (tmp_array[i]))
68 * ice_search_res - Search the tracker for a block of resources
69 * @res: pointer to the resource
70 * @needed: size of the block needed
71 * @id: identifier to track owner
72 * Returns the base item index of the block, or -ENOMEM for error
74 static int ice_search_res(struct ice_res_tracker *res, u16 needed, u16 id)
76 int start = res->search_hint;
79 id |= ICE_RES_VALID_BIT;
82 /* skip already allocated entries */
83 if (res->list[end++] & ICE_RES_VALID_BIT) {
85 if ((start + needed) > res->num_entries)
89 if (end == (start + needed)) {
92 /* there was enough, so assign it to the requestor */
96 if (end == res->num_entries)
99 res->search_hint = end;
108 * ice_get_res - get a block of resources
109 * @pf: board private structure
110 * @res: pointer to the resource
111 * @needed: size of the block needed
112 * @id: identifier to track owner
114 * Returns the base item index of the block, or -ENOMEM for error
115 * The search_hint trick and lack of advanced fit-finding only works
116 * because we're highly likely to have all the same sized requests.
117 * Linear search time and any fragmentation should be minimal.
120 ice_get_res(struct ice_pf *pf, struct ice_res_tracker *res, u16 needed, u16 id)
127 if (!needed || needed > res->num_entries || id >= ICE_RES_VALID_BIT) {
128 dev_err(&pf->pdev->dev,
129 "param err: needed=%d, num_entries = %d id=0x%04x\n",
130 needed, res->num_entries, id);
134 /* search based on search_hint */
135 ret = ice_search_res(res, needed, id);
138 /* previous search failed. Reset search hint and try again */
139 res->search_hint = 0;
140 ret = ice_search_res(res, needed, id);
147 * ice_free_res - free a block of resources
148 * @res: pointer to the resource
149 * @index: starting index previously returned by ice_get_res
150 * @id: identifier to track owner
151 * Returns number of resources freed
153 static int ice_free_res(struct ice_res_tracker *res, u16 index, u16 id)
158 if (!res || index >= res->num_entries)
161 id |= ICE_RES_VALID_BIT;
162 for (i = index; i < res->num_entries && res->list[i] == id; i++) {
171 * ice_add_mac_to_list - Add a mac address filter entry to the list
172 * @vsi: the VSI to be forwarded to
173 * @add_list: pointer to the list which contains MAC filter entries
174 * @macaddr: the MAC address to be added.
176 * Adds mac address filter entry to the temp list
178 * Returns 0 on success or ENOMEM on failure.
180 static int ice_add_mac_to_list(struct ice_vsi *vsi, struct list_head *add_list,
183 struct ice_fltr_list_entry *tmp;
184 struct ice_pf *pf = vsi->back;
186 tmp = devm_kzalloc(&pf->pdev->dev, sizeof(*tmp), GFP_ATOMIC);
190 tmp->fltr_info.flag = ICE_FLTR_TX;
191 tmp->fltr_info.src = vsi->vsi_num;
192 tmp->fltr_info.lkup_type = ICE_SW_LKUP_MAC;
193 tmp->fltr_info.fltr_act = ICE_FWD_TO_VSI;
194 tmp->fltr_info.fwd_id.vsi_id = vsi->vsi_num;
195 ether_addr_copy(tmp->fltr_info.l_data.mac.mac_addr, macaddr);
197 INIT_LIST_HEAD(&tmp->list_entry);
198 list_add(&tmp->list_entry, add_list);
204 * ice_add_mac_to_sync_list - creates list of mac addresses to be synced
205 * @netdev: the net device on which the sync is happening
206 * @addr: mac address to sync
208 * This is a callback function which is called by the in kernel device sync
209 * functions (like __dev_uc_sync, __dev_mc_sync, etc). This function only
210 * populates the tmp_sync_list, which is later used by ice_add_mac to add the
211 * mac filters from the hardware.
213 static int ice_add_mac_to_sync_list(struct net_device *netdev, const u8 *addr)
215 struct ice_netdev_priv *np = netdev_priv(netdev);
216 struct ice_vsi *vsi = np->vsi;
218 if (ice_add_mac_to_list(vsi, &vsi->tmp_sync_list, addr))
225 * ice_add_mac_to_unsync_list - creates list of mac addresses to be unsynced
226 * @netdev: the net device on which the unsync is happening
227 * @addr: mac address to unsync
229 * This is a callback function which is called by the in kernel device unsync
230 * functions (like __dev_uc_unsync, __dev_mc_unsync, etc). This function only
231 * populates the tmp_unsync_list, which is later used by ice_remove_mac to
232 * delete the mac filters from the hardware.
234 static int ice_add_mac_to_unsync_list(struct net_device *netdev, const u8 *addr)
236 struct ice_netdev_priv *np = netdev_priv(netdev);
237 struct ice_vsi *vsi = np->vsi;
239 if (ice_add_mac_to_list(vsi, &vsi->tmp_unsync_list, addr))
246 * ice_free_fltr_list - free filter lists helper
247 * @dev: pointer to the device struct
248 * @h: pointer to the list head to be freed
250 * Helper function to free filter lists previously created using
251 * ice_add_mac_to_list
253 static void ice_free_fltr_list(struct device *dev, struct list_head *h)
255 struct ice_fltr_list_entry *e, *tmp;
257 list_for_each_entry_safe(e, tmp, h, list_entry) {
258 list_del(&e->list_entry);
264 * ice_vsi_fltr_changed - check if filter state changed
265 * @vsi: VSI to be checked
267 * returns true if filter state has changed, false otherwise.
269 static bool ice_vsi_fltr_changed(struct ice_vsi *vsi)
271 return test_bit(ICE_VSI_FLAG_UMAC_FLTR_CHANGED, vsi->flags) ||
272 test_bit(ICE_VSI_FLAG_MMAC_FLTR_CHANGED, vsi->flags) ||
273 test_bit(ICE_VSI_FLAG_VLAN_FLTR_CHANGED, vsi->flags);
277 * ice_vsi_sync_fltr - Update the VSI filter list to the HW
278 * @vsi: ptr to the VSI
280 * Push any outstanding VSI filter changes through the AdminQ.
282 static int ice_vsi_sync_fltr(struct ice_vsi *vsi)
284 struct device *dev = &vsi->back->pdev->dev;
285 struct net_device *netdev = vsi->netdev;
286 bool promisc_forced_on = false;
287 struct ice_pf *pf = vsi->back;
288 struct ice_hw *hw = &pf->hw;
289 enum ice_status status = 0;
290 u32 changed_flags = 0;
296 while (test_and_set_bit(__ICE_CFG_BUSY, vsi->state))
297 usleep_range(1000, 2000);
299 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags;
300 vsi->current_netdev_flags = vsi->netdev->flags;
302 INIT_LIST_HEAD(&vsi->tmp_sync_list);
303 INIT_LIST_HEAD(&vsi->tmp_unsync_list);
305 if (ice_vsi_fltr_changed(vsi)) {
306 clear_bit(ICE_VSI_FLAG_UMAC_FLTR_CHANGED, vsi->flags);
307 clear_bit(ICE_VSI_FLAG_MMAC_FLTR_CHANGED, vsi->flags);
308 clear_bit(ICE_VSI_FLAG_VLAN_FLTR_CHANGED, vsi->flags);
310 /* grab the netdev's addr_list_lock */
311 netif_addr_lock_bh(netdev);
312 __dev_uc_sync(netdev, ice_add_mac_to_sync_list,
313 ice_add_mac_to_unsync_list);
314 __dev_mc_sync(netdev, ice_add_mac_to_sync_list,
315 ice_add_mac_to_unsync_list);
316 /* our temp lists are populated. release lock */
317 netif_addr_unlock_bh(netdev);
320 /* Remove mac addresses in the unsync list */
321 status = ice_remove_mac(hw, &vsi->tmp_unsync_list);
322 ice_free_fltr_list(dev, &vsi->tmp_unsync_list);
324 netdev_err(netdev, "Failed to delete MAC filters\n");
325 /* if we failed because of alloc failures, just bail */
326 if (status == ICE_ERR_NO_MEMORY) {
332 /* Add mac addresses in the sync list */
333 status = ice_add_mac(hw, &vsi->tmp_sync_list);
334 ice_free_fltr_list(dev, &vsi->tmp_sync_list);
336 netdev_err(netdev, "Failed to add MAC filters\n");
337 /* If there is no more space for new umac filters, vsi
338 * should go into promiscuous mode. There should be some
339 * space reserved for promiscuous filters.
341 if (hw->adminq.sq_last_status == ICE_AQ_RC_ENOSPC &&
342 !test_and_set_bit(__ICE_FLTR_OVERFLOW_PROMISC,
344 promisc_forced_on = true;
346 "Reached MAC filter limit, forcing promisc mode on VSI %d\n",
353 /* check for changes in promiscuous modes */
354 if (changed_flags & IFF_ALLMULTI)
355 netdev_warn(netdev, "Unsupported configuration\n");
357 if (((changed_flags & IFF_PROMISC) || promisc_forced_on) ||
358 test_bit(ICE_VSI_FLAG_PROMISC_CHANGED, vsi->flags)) {
359 clear_bit(ICE_VSI_FLAG_PROMISC_CHANGED, vsi->flags);
360 if (vsi->current_netdev_flags & IFF_PROMISC) {
361 /* Apply TX filter rule to get traffic from VMs */
362 status = ice_cfg_dflt_vsi(hw, vsi->vsi_num, true,
365 netdev_err(netdev, "Error setting default VSI %i tx rule\n",
367 vsi->current_netdev_flags &= ~IFF_PROMISC;
371 /* Apply RX filter rule to get traffic from wire */
372 status = ice_cfg_dflt_vsi(hw, vsi->vsi_num, true,
375 netdev_err(netdev, "Error setting default VSI %i rx rule\n",
377 vsi->current_netdev_flags &= ~IFF_PROMISC;
382 /* Clear TX filter rule to stop traffic from VMs */
383 status = ice_cfg_dflt_vsi(hw, vsi->vsi_num, false,
386 netdev_err(netdev, "Error clearing default VSI %i tx rule\n",
388 vsi->current_netdev_flags |= IFF_PROMISC;
392 /* Clear filter RX to remove traffic from wire */
393 status = ice_cfg_dflt_vsi(hw, vsi->vsi_num, false,
396 netdev_err(netdev, "Error clearing default VSI %i rx rule\n",
398 vsi->current_netdev_flags |= IFF_PROMISC;
407 set_bit(ICE_VSI_FLAG_PROMISC_CHANGED, vsi->flags);
410 /* if something went wrong then set the changed flag so we try again */
411 set_bit(ICE_VSI_FLAG_UMAC_FLTR_CHANGED, vsi->flags);
412 set_bit(ICE_VSI_FLAG_MMAC_FLTR_CHANGED, vsi->flags);
414 clear_bit(__ICE_CFG_BUSY, vsi->state);
419 * ice_sync_fltr_subtask - Sync the VSI filter list with HW
420 * @pf: board private structure
422 static void ice_sync_fltr_subtask(struct ice_pf *pf)
426 if (!pf || !(test_bit(ICE_FLAG_FLTR_SYNC, pf->flags)))
429 clear_bit(ICE_FLAG_FLTR_SYNC, pf->flags);
431 for (v = 0; v < pf->num_alloc_vsi; v++)
432 if (pf->vsi[v] && ice_vsi_fltr_changed(pf->vsi[v]) &&
433 ice_vsi_sync_fltr(pf->vsi[v])) {
434 /* come back and try again later */
435 set_bit(ICE_FLAG_FLTR_SYNC, pf->flags);
441 * ice_is_reset_recovery_pending - schedule a reset
442 * @state: pf state field
444 static bool ice_is_reset_recovery_pending(unsigned long int *state)
446 return test_bit(__ICE_RESET_RECOVERY_PENDING, state);
450 * ice_prepare_for_reset - prep for the core to reset
451 * @pf: board private structure
453 * Inform or close all dependent features in prep for reset.
456 ice_prepare_for_reset(struct ice_pf *pf)
458 struct ice_hw *hw = &pf->hw;
461 ice_for_each_vsi(pf, v)
463 ice_remove_vsi_fltr(hw, pf->vsi[v]->vsi_num);
465 dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n");
467 /* disable the VSIs and their queues that are not already DOWN */
468 /* pf_dis_all_vsi modifies netdev structures -rtnl_lock needed */
469 ice_pf_dis_all_vsi(pf);
471 ice_for_each_vsi(pf, v)
473 pf->vsi[v]->vsi_num = 0;
475 ice_shutdown_all_ctrlq(hw);
479 * ice_do_reset - Initiate one of many types of resets
480 * @pf: board private structure
481 * @reset_type: reset type requested
482 * before this function was called.
484 static void ice_do_reset(struct ice_pf *pf, enum ice_reset_req reset_type)
486 struct device *dev = &pf->pdev->dev;
487 struct ice_hw *hw = &pf->hw;
489 dev_dbg(dev, "reset_type 0x%x requested\n", reset_type);
490 WARN_ON(in_interrupt());
492 /* PFR is a bit of a special case because it doesn't result in an OICR
493 * interrupt. So for PFR, we prepare for reset, issue the reset and
494 * rebuild sequentially.
496 if (reset_type == ICE_RESET_PFR) {
497 set_bit(__ICE_RESET_RECOVERY_PENDING, pf->state);
498 ice_prepare_for_reset(pf);
501 /* trigger the reset */
502 if (ice_reset(hw, reset_type)) {
503 dev_err(dev, "reset %d failed\n", reset_type);
504 set_bit(__ICE_RESET_FAILED, pf->state);
505 clear_bit(__ICE_RESET_RECOVERY_PENDING, pf->state);
509 if (reset_type == ICE_RESET_PFR) {
512 clear_bit(__ICE_RESET_RECOVERY_PENDING, pf->state);
517 * ice_reset_subtask - Set up for resetting the device and driver
518 * @pf: board private structure
520 static void ice_reset_subtask(struct ice_pf *pf)
522 enum ice_reset_req reset_type;
526 /* When a CORER/GLOBR/EMPR is about to happen, the hardware triggers an
527 * OICR interrupt. The OICR handler (ice_misc_intr) determines what
528 * type of reset happened and sets __ICE_RESET_RECOVERY_PENDING bit in
529 * pf->state. So if reset/recovery is pending (as indicated by this bit)
530 * we do a rebuild and return.
532 if (ice_is_reset_recovery_pending(pf->state)) {
533 clear_bit(__ICE_GLOBR_RECV, pf->state);
534 clear_bit(__ICE_CORER_RECV, pf->state);
535 ice_prepare_for_reset(pf);
537 /* make sure we are ready to rebuild */
538 if (ice_check_reset(&pf->hw))
539 set_bit(__ICE_RESET_FAILED, pf->state);
542 clear_bit(__ICE_RESET_RECOVERY_PENDING, pf->state);
546 /* No pending resets to finish processing. Check for new resets */
547 if (test_and_clear_bit(__ICE_GLOBR_REQ, pf->state))
548 reset_type = ICE_RESET_GLOBR;
549 else if (test_and_clear_bit(__ICE_CORER_REQ, pf->state))
550 reset_type = ICE_RESET_CORER;
551 else if (test_and_clear_bit(__ICE_PFR_REQ, pf->state))
552 reset_type = ICE_RESET_PFR;
556 /* reset if not already down or resetting */
557 if (!test_bit(__ICE_DOWN, pf->state) &&
558 !test_bit(__ICE_CFG_BUSY, pf->state)) {
559 ice_do_reset(pf, reset_type);
567 * ice_watchdog_subtask - periodic tasks not using event driven scheduling
568 * @pf: board private structure
570 static void ice_watchdog_subtask(struct ice_pf *pf)
574 /* if interface is down do nothing */
575 if (test_bit(__ICE_DOWN, pf->state) ||
576 test_bit(__ICE_CFG_BUSY, pf->state))
579 /* make sure we don't do these things too often */
580 if (time_before(jiffies,
581 pf->serv_tmr_prev + pf->serv_tmr_period))
584 pf->serv_tmr_prev = jiffies;
586 /* Update the stats for active netdevs so the network stack
587 * can look at updated numbers whenever it cares to
589 ice_update_pf_stats(pf);
590 for (i = 0; i < pf->num_alloc_vsi; i++)
591 if (pf->vsi[i] && pf->vsi[i]->netdev)
592 ice_update_vsi_stats(pf->vsi[i]);
596 * ice_print_link_msg - print link up or down message
597 * @vsi: the VSI whose link status is being queried
598 * @isup: boolean for if the link is now up or down
600 void ice_print_link_msg(struct ice_vsi *vsi, bool isup)
605 if (vsi->current_isup == isup)
608 vsi->current_isup = isup;
611 netdev_info(vsi->netdev, "NIC Link is Down\n");
615 switch (vsi->port_info->phy.link_info.link_speed) {
616 case ICE_AQ_LINK_SPEED_40GB:
619 case ICE_AQ_LINK_SPEED_25GB:
622 case ICE_AQ_LINK_SPEED_20GB:
625 case ICE_AQ_LINK_SPEED_10GB:
628 case ICE_AQ_LINK_SPEED_5GB:
631 case ICE_AQ_LINK_SPEED_2500MB:
634 case ICE_AQ_LINK_SPEED_1000MB:
637 case ICE_AQ_LINK_SPEED_100MB:
645 switch (vsi->port_info->fc.current_mode) {
649 case ICE_FC_TX_PAUSE:
652 case ICE_FC_RX_PAUSE:
660 netdev_info(vsi->netdev, "NIC Link is up %sbps, Flow Control: %s\n",
665 * ice_init_link_events - enable/initialize link events
666 * @pi: pointer to the port_info instance
668 * Returns -EIO on failure, 0 on success
670 static int ice_init_link_events(struct ice_port_info *pi)
674 mask = ~((u16)(ICE_AQ_LINK_EVENT_UPDOWN | ICE_AQ_LINK_EVENT_MEDIA_NA |
675 ICE_AQ_LINK_EVENT_MODULE_QUAL_FAIL));
677 if (ice_aq_set_event_mask(pi->hw, pi->lport, mask, NULL)) {
678 dev_dbg(ice_hw_to_dev(pi->hw),
679 "Failed to set link event mask for port %d\n",
684 if (ice_aq_get_link_info(pi, true, NULL, NULL)) {
685 dev_dbg(ice_hw_to_dev(pi->hw),
686 "Failed to enable link events for port %d\n",
695 * ice_vsi_link_event - update the vsi's netdev
696 * @vsi: the vsi on which the link event occurred
697 * @link_up: whether or not the vsi needs to be set up or down
699 static void ice_vsi_link_event(struct ice_vsi *vsi, bool link_up)
701 if (!vsi || test_bit(__ICE_DOWN, vsi->state))
704 if (vsi->type == ICE_VSI_PF) {
706 dev_dbg(&vsi->back->pdev->dev,
707 "vsi->netdev is not initialized!\n");
711 netif_carrier_on(vsi->netdev);
712 netif_tx_wake_all_queues(vsi->netdev);
714 netif_carrier_off(vsi->netdev);
715 netif_tx_stop_all_queues(vsi->netdev);
721 * ice_link_event - process the link event
722 * @pf: pf that the link event is associated with
723 * @pi: port_info for the port that the link event is associated with
725 * Returns -EIO if ice_get_link_status() fails
726 * Returns 0 on success
729 ice_link_event(struct ice_pf *pf, struct ice_port_info *pi)
731 u8 new_link_speed, old_link_speed;
732 struct ice_phy_info *phy_info;
733 bool new_link_same_as_old;
734 bool new_link, old_link;
739 phy_info->link_info_old = phy_info->link_info;
740 /* Force ice_get_link_status() to update link info */
741 phy_info->get_link_info = true;
743 old_link = (phy_info->link_info_old.link_info & ICE_AQ_LINK_UP);
744 old_link_speed = phy_info->link_info_old.link_speed;
747 if (ice_get_link_status(pi, &new_link)) {
748 dev_dbg(&pf->pdev->dev,
749 "Could not get link status for port %d\n", lport);
753 new_link_speed = phy_info->link_info.link_speed;
755 new_link_same_as_old = (new_link == old_link &&
756 new_link_speed == old_link_speed);
758 ice_for_each_vsi(pf, v) {
759 struct ice_vsi *vsi = pf->vsi[v];
761 if (!vsi || !vsi->port_info)
764 if (new_link_same_as_old &&
765 (test_bit(__ICE_DOWN, vsi->state) ||
766 new_link == netif_carrier_ok(vsi->netdev)))
769 if (vsi->port_info->lport == lport) {
770 ice_print_link_msg(vsi, new_link);
771 ice_vsi_link_event(vsi, new_link);
779 * ice_handle_link_event - handle link event via ARQ
780 * @pf: pf that the link event is associated with
782 * Return -EINVAL if port_info is null
783 * Return status on succes
785 static int ice_handle_link_event(struct ice_pf *pf)
787 struct ice_port_info *port_info;
790 port_info = pf->hw.port_info;
794 status = ice_link_event(pf, port_info);
796 dev_dbg(&pf->pdev->dev,
797 "Could not process link event, error %d\n", status);
803 * __ice_clean_ctrlq - helper function to clean controlq rings
804 * @pf: ptr to struct ice_pf
805 * @q_type: specific Control queue type
807 static int __ice_clean_ctrlq(struct ice_pf *pf, enum ice_ctl_q q_type)
809 struct ice_rq_event_info event;
810 struct ice_hw *hw = &pf->hw;
811 struct ice_ctl_q_info *cq;
816 /* Do not clean control queue if/when PF reset fails */
817 if (test_bit(__ICE_RESET_FAILED, pf->state))
821 case ICE_CTL_Q_ADMIN:
826 dev_warn(&pf->pdev->dev, "Unknown control queue type 0x%x\n",
831 /* check for error indications - PF_xx_AxQLEN register layout for
832 * FW/MBX/SB are identical so just use defines for PF_FW_AxQLEN.
834 val = rd32(hw, cq->rq.len);
835 if (val & (PF_FW_ARQLEN_ARQVFE_M | PF_FW_ARQLEN_ARQOVFL_M |
836 PF_FW_ARQLEN_ARQCRIT_M)) {
838 if (val & PF_FW_ARQLEN_ARQVFE_M)
839 dev_dbg(&pf->pdev->dev,
840 "%s Receive Queue VF Error detected\n", qtype);
841 if (val & PF_FW_ARQLEN_ARQOVFL_M) {
842 dev_dbg(&pf->pdev->dev,
843 "%s Receive Queue Overflow Error detected\n",
846 if (val & PF_FW_ARQLEN_ARQCRIT_M)
847 dev_dbg(&pf->pdev->dev,
848 "%s Receive Queue Critical Error detected\n",
850 val &= ~(PF_FW_ARQLEN_ARQVFE_M | PF_FW_ARQLEN_ARQOVFL_M |
851 PF_FW_ARQLEN_ARQCRIT_M);
853 wr32(hw, cq->rq.len, val);
856 val = rd32(hw, cq->sq.len);
857 if (val & (PF_FW_ATQLEN_ATQVFE_M | PF_FW_ATQLEN_ATQOVFL_M |
858 PF_FW_ATQLEN_ATQCRIT_M)) {
860 if (val & PF_FW_ATQLEN_ATQVFE_M)
861 dev_dbg(&pf->pdev->dev,
862 "%s Send Queue VF Error detected\n", qtype);
863 if (val & PF_FW_ATQLEN_ATQOVFL_M) {
864 dev_dbg(&pf->pdev->dev,
865 "%s Send Queue Overflow Error detected\n",
868 if (val & PF_FW_ATQLEN_ATQCRIT_M)
869 dev_dbg(&pf->pdev->dev,
870 "%s Send Queue Critical Error detected\n",
872 val &= ~(PF_FW_ATQLEN_ATQVFE_M | PF_FW_ATQLEN_ATQOVFL_M |
873 PF_FW_ATQLEN_ATQCRIT_M);
875 wr32(hw, cq->sq.len, val);
878 event.buf_len = cq->rq_buf_size;
879 event.msg_buf = devm_kzalloc(&pf->pdev->dev, event.buf_len,
888 ret = ice_clean_rq_elem(hw, cq, &event, &pending);
889 if (ret == ICE_ERR_AQ_NO_WORK)
892 dev_err(&pf->pdev->dev,
893 "%s Receive Queue event error %d\n", qtype,
898 opcode = le16_to_cpu(event.desc.opcode);
901 case ice_aqc_opc_get_link_status:
902 if (ice_handle_link_event(pf))
903 dev_err(&pf->pdev->dev,
904 "Could not handle link event\n");
907 dev_dbg(&pf->pdev->dev,
908 "%s Receive Queue unknown event 0x%04x ignored\n",
912 } while (pending && (i++ < ICE_DFLT_IRQ_WORK));
914 devm_kfree(&pf->pdev->dev, event.msg_buf);
916 return pending && (i == ICE_DFLT_IRQ_WORK);
920 * ice_ctrlq_pending - check if there is a difference between ntc and ntu
921 * @hw: pointer to hardware info
922 * @cq: control queue information
924 * returns true if there are pending messages in a queue, false if there aren't
926 static bool ice_ctrlq_pending(struct ice_hw *hw, struct ice_ctl_q_info *cq)
930 ntu = (u16)(rd32(hw, cq->rq.head) & cq->rq.head_mask);
931 return cq->rq.next_to_clean != ntu;
935 * ice_clean_adminq_subtask - clean the AdminQ rings
936 * @pf: board private structure
938 static void ice_clean_adminq_subtask(struct ice_pf *pf)
940 struct ice_hw *hw = &pf->hw;
942 if (!test_bit(__ICE_ADMINQ_EVENT_PENDING, pf->state))
945 if (__ice_clean_ctrlq(pf, ICE_CTL_Q_ADMIN))
948 clear_bit(__ICE_ADMINQ_EVENT_PENDING, pf->state);
950 /* There might be a situation where new messages arrive to a control
951 * queue between processing the last message and clearing the
952 * EVENT_PENDING bit. So before exiting, check queue head again (using
953 * ice_ctrlq_pending) and process new messages if any.
955 if (ice_ctrlq_pending(hw, &hw->adminq))
956 __ice_clean_ctrlq(pf, ICE_CTL_Q_ADMIN);
962 * ice_service_task_schedule - schedule the service task to wake up
963 * @pf: board private structure
965 * If not already scheduled, this puts the task into the work queue.
967 static void ice_service_task_schedule(struct ice_pf *pf)
969 if (!test_bit(__ICE_DOWN, pf->state) &&
970 !test_and_set_bit(__ICE_SERVICE_SCHED, pf->state))
971 queue_work(ice_wq, &pf->serv_task);
975 * ice_service_task_complete - finish up the service task
976 * @pf: board private structure
978 static void ice_service_task_complete(struct ice_pf *pf)
980 WARN_ON(!test_bit(__ICE_SERVICE_SCHED, pf->state));
982 /* force memory (pf->state) to sync before next service task */
983 smp_mb__before_atomic();
984 clear_bit(__ICE_SERVICE_SCHED, pf->state);
988 * ice_service_timer - timer callback to schedule service task
989 * @t: pointer to timer_list
991 static void ice_service_timer(struct timer_list *t)
993 struct ice_pf *pf = from_timer(pf, t, serv_tmr);
995 mod_timer(&pf->serv_tmr, round_jiffies(pf->serv_tmr_period + jiffies));
996 ice_service_task_schedule(pf);
1000 * ice_service_task - manage and run subtasks
1001 * @work: pointer to work_struct contained by the PF struct
1003 static void ice_service_task(struct work_struct *work)
1005 struct ice_pf *pf = container_of(work, struct ice_pf, serv_task);
1006 unsigned long start_time = jiffies;
1010 /* process reset requests first */
1011 ice_reset_subtask(pf);
1013 /* bail if a reset/recovery cycle is pending */
1014 if (ice_is_reset_recovery_pending(pf->state) ||
1015 test_bit(__ICE_SUSPENDED, pf->state)) {
1016 ice_service_task_complete(pf);
1020 ice_sync_fltr_subtask(pf);
1021 ice_watchdog_subtask(pf);
1022 ice_clean_adminq_subtask(pf);
1024 /* Clear __ICE_SERVICE_SCHED flag to allow scheduling next event */
1025 ice_service_task_complete(pf);
1027 /* If the tasks have taken longer than one service timer period
1028 * or there is more work to be done, reset the service timer to
1029 * schedule the service task now.
1031 if (time_after(jiffies, (start_time + pf->serv_tmr_period)) ||
1032 test_bit(__ICE_ADMINQ_EVENT_PENDING, pf->state))
1033 mod_timer(&pf->serv_tmr, jiffies);
1037 * ice_set_ctrlq_len - helper function to set controlq length
1038 * @hw: pointer to the hw instance
1040 static void ice_set_ctrlq_len(struct ice_hw *hw)
1042 hw->adminq.num_rq_entries = ICE_AQ_LEN;
1043 hw->adminq.num_sq_entries = ICE_AQ_LEN;
1044 hw->adminq.rq_buf_size = ICE_AQ_MAX_BUF_LEN;
1045 hw->adminq.sq_buf_size = ICE_AQ_MAX_BUF_LEN;
1049 * ice_irq_affinity_notify - Callback for affinity changes
1050 * @notify: context as to what irq was changed
1051 * @mask: the new affinity mask
1053 * This is a callback function used by the irq_set_affinity_notifier function
1054 * so that we may register to receive changes to the irq affinity masks.
1056 static void ice_irq_affinity_notify(struct irq_affinity_notify *notify,
1057 const cpumask_t *mask)
1059 struct ice_q_vector *q_vector =
1060 container_of(notify, struct ice_q_vector, affinity_notify);
1062 cpumask_copy(&q_vector->affinity_mask, mask);
1066 * ice_irq_affinity_release - Callback for affinity notifier release
1067 * @ref: internal core kernel usage
1069 * This is a callback function used by the irq_set_affinity_notifier function
1070 * to inform the current notification subscriber that they will no longer
1071 * receive notifications.
1073 static void ice_irq_affinity_release(struct kref __always_unused *ref) {}
1076 * ice_vsi_dis_irq - Mask off queue interrupt generation on the VSI
1077 * @vsi: the VSI being un-configured
1079 static void ice_vsi_dis_irq(struct ice_vsi *vsi)
1081 struct ice_pf *pf = vsi->back;
1082 struct ice_hw *hw = &pf->hw;
1083 int base = vsi->base_vector;
1087 /* disable interrupt causation from each queue */
1088 if (vsi->tx_rings) {
1089 ice_for_each_txq(vsi, i) {
1090 if (vsi->tx_rings[i]) {
1093 reg = vsi->tx_rings[i]->reg_idx;
1094 val = rd32(hw, QINT_TQCTL(reg));
1095 val &= ~QINT_TQCTL_CAUSE_ENA_M;
1096 wr32(hw, QINT_TQCTL(reg), val);
1101 if (vsi->rx_rings) {
1102 ice_for_each_rxq(vsi, i) {
1103 if (vsi->rx_rings[i]) {
1106 reg = vsi->rx_rings[i]->reg_idx;
1107 val = rd32(hw, QINT_RQCTL(reg));
1108 val &= ~QINT_RQCTL_CAUSE_ENA_M;
1109 wr32(hw, QINT_RQCTL(reg), val);
1114 /* disable each interrupt */
1115 if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags)) {
1116 for (i = vsi->base_vector;
1117 i < (vsi->num_q_vectors + vsi->base_vector); i++)
1118 wr32(hw, GLINT_DYN_CTL(i), 0);
1121 for (i = 0; i < vsi->num_q_vectors; i++)
1122 synchronize_irq(pf->msix_entries[i + base].vector);
1127 * ice_vsi_ena_irq - Enable IRQ for the given VSI
1128 * @vsi: the VSI being configured
1130 static int ice_vsi_ena_irq(struct ice_vsi *vsi)
1132 struct ice_pf *pf = vsi->back;
1133 struct ice_hw *hw = &pf->hw;
1135 if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags)) {
1138 for (i = 0; i < vsi->num_q_vectors; i++)
1139 ice_irq_dynamic_ena(hw, vsi, vsi->q_vectors[i]);
1147 * ice_vsi_delete - delete a VSI from the switch
1148 * @vsi: pointer to VSI being removed
1150 static void ice_vsi_delete(struct ice_vsi *vsi)
1152 struct ice_pf *pf = vsi->back;
1153 struct ice_vsi_ctx ctxt;
1154 enum ice_status status;
1156 ctxt.vsi_num = vsi->vsi_num;
1158 memcpy(&ctxt.info, &vsi->info, sizeof(struct ice_aqc_vsi_props));
1160 status = ice_aq_free_vsi(&pf->hw, &ctxt, false, NULL);
1162 dev_err(&pf->pdev->dev, "Failed to delete VSI %i in FW\n",
1167 * ice_vsi_req_irq_msix - get MSI-X vectors from the OS for the VSI
1168 * @vsi: the VSI being configured
1169 * @basename: name for the vector
1171 static int ice_vsi_req_irq_msix(struct ice_vsi *vsi, char *basename)
1173 int q_vectors = vsi->num_q_vectors;
1174 struct ice_pf *pf = vsi->back;
1175 int base = vsi->base_vector;
1181 for (vector = 0; vector < q_vectors; vector++) {
1182 struct ice_q_vector *q_vector = vsi->q_vectors[vector];
1184 irq_num = pf->msix_entries[base + vector].vector;
1186 if (q_vector->tx.ring && q_vector->rx.ring) {
1187 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
1188 "%s-%s-%d", basename, "TxRx", rx_int_idx++);
1190 } else if (q_vector->rx.ring) {
1191 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
1192 "%s-%s-%d", basename, "rx", rx_int_idx++);
1193 } else if (q_vector->tx.ring) {
1194 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
1195 "%s-%s-%d", basename, "tx", tx_int_idx++);
1197 /* skip this unused q_vector */
1200 err = devm_request_irq(&pf->pdev->dev,
1201 pf->msix_entries[base + vector].vector,
1202 vsi->irq_handler, 0, q_vector->name,
1205 netdev_err(vsi->netdev,
1206 "MSIX request_irq failed, error: %d\n", err);
1210 /* register for affinity change notifications */
1211 q_vector->affinity_notify.notify = ice_irq_affinity_notify;
1212 q_vector->affinity_notify.release = ice_irq_affinity_release;
1213 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify);
1215 /* assign the mask for this irq */
1216 irq_set_affinity_hint(irq_num, &q_vector->affinity_mask);
1219 vsi->irqs_ready = true;
1225 irq_num = pf->msix_entries[base + vector].vector,
1226 irq_set_affinity_notifier(irq_num, NULL);
1227 irq_set_affinity_hint(irq_num, NULL);
1228 devm_free_irq(&pf->pdev->dev, irq_num, &vsi->q_vectors[vector]);
1234 * ice_vsi_set_rss_params - Setup RSS capabilities per VSI type
1235 * @vsi: the VSI being configured
1237 static void ice_vsi_set_rss_params(struct ice_vsi *vsi)
1239 struct ice_hw_common_caps *cap;
1240 struct ice_pf *pf = vsi->back;
1242 if (!test_bit(ICE_FLAG_RSS_ENA, pf->flags)) {
1247 cap = &pf->hw.func_caps.common_cap;
1248 switch (vsi->type) {
1250 /* PF VSI will inherit RSS instance of PF */
1251 vsi->rss_table_size = cap->rss_table_size;
1252 vsi->rss_size = min_t(int, num_online_cpus(),
1253 BIT(cap->rss_table_entry_width));
1254 vsi->rss_lut_type = ICE_AQC_GSET_RSS_LUT_TABLE_TYPE_PF;
1257 dev_warn(&pf->pdev->dev, "Unknown VSI type %d\n", vsi->type);
1263 * ice_vsi_setup_q_map - Setup a VSI queue map
1264 * @vsi: the VSI being configured
1265 * @ctxt: VSI context structure
1267 static void ice_vsi_setup_q_map(struct ice_vsi *vsi, struct ice_vsi_ctx *ctxt)
1269 u16 offset = 0, qmap = 0, numq_tc;
1270 u16 pow = 0, max_rss = 0, qcount;
1271 u16 qcount_tx = vsi->alloc_txq;
1272 u16 qcount_rx = vsi->alloc_rxq;
1273 bool ena_tc0 = false;
1276 /* at least TC0 should be enabled by default */
1277 if (vsi->tc_cfg.numtc) {
1278 if (!(vsi->tc_cfg.ena_tc & BIT(0)))
1285 vsi->tc_cfg.numtc++;
1286 vsi->tc_cfg.ena_tc |= 1;
1289 numq_tc = qcount_rx / vsi->tc_cfg.numtc;
1291 /* TC mapping is a function of the number of Rx queues assigned to the
1292 * VSI for each traffic class and the offset of these queues.
1293 * The first 10 bits are for queue offset for TC0, next 4 bits for no:of
1294 * queues allocated to TC0. No:of queues is a power-of-2.
1296 * If TC is not enabled, the queue offset is set to 0, and allocate one
1297 * queue, this way, traffic for the given TC will be sent to the default
1300 * Setup number and offset of Rx queues for all TCs for the VSI
1303 /* qcount will change if RSS is enabled */
1304 if (test_bit(ICE_FLAG_RSS_ENA, vsi->back->flags)) {
1305 if (vsi->type == ICE_VSI_PF)
1306 max_rss = ICE_MAX_LG_RSS_QS;
1308 max_rss = ICE_MAX_SMALL_RSS_QS;
1310 qcount = min_t(int, numq_tc, max_rss);
1311 qcount = min_t(int, qcount, vsi->rss_size);
1316 /* find the (rounded up) power-of-2 of qcount */
1317 pow = order_base_2(qcount);
1319 for (i = 0; i < ICE_MAX_TRAFFIC_CLASS; i++) {
1320 if (!(vsi->tc_cfg.ena_tc & BIT(i))) {
1321 /* TC is not enabled */
1322 vsi->tc_cfg.tc_info[i].qoffset = 0;
1323 vsi->tc_cfg.tc_info[i].qcount = 1;
1324 ctxt->info.tc_mapping[i] = 0;
1329 vsi->tc_cfg.tc_info[i].qoffset = offset;
1330 vsi->tc_cfg.tc_info[i].qcount = qcount;
1332 qmap = ((offset << ICE_AQ_VSI_TC_Q_OFFSET_S) &
1333 ICE_AQ_VSI_TC_Q_OFFSET_M) |
1334 ((pow << ICE_AQ_VSI_TC_Q_NUM_S) &
1335 ICE_AQ_VSI_TC_Q_NUM_M);
1337 ctxt->info.tc_mapping[i] = cpu_to_le16(qmap);
1340 vsi->num_txq = qcount_tx;
1341 vsi->num_rxq = offset;
1343 /* Rx queue mapping */
1344 ctxt->info.mapping_flags |= cpu_to_le16(ICE_AQ_VSI_Q_MAP_CONTIG);
1345 /* q_mapping buffer holds the info for the first queue allocated for
1346 * this VSI in the PF space and also the number of queues associated
1349 ctxt->info.q_mapping[0] = cpu_to_le16(vsi->rxq_map[0]);
1350 ctxt->info.q_mapping[1] = cpu_to_le16(vsi->num_rxq);
1354 * ice_set_dflt_vsi_ctx - Set default VSI context before adding a VSI
1355 * @ctxt: the VSI context being set
1357 * This initializes a default VSI context for all sections except the Queues.
1359 static void ice_set_dflt_vsi_ctx(struct ice_vsi_ctx *ctxt)
1363 memset(&ctxt->info, 0, sizeof(ctxt->info));
1364 /* VSI's should be allocated from shared pool */
1365 ctxt->alloc_from_pool = true;
1366 /* Src pruning enabled by default */
1367 ctxt->info.sw_flags = ICE_AQ_VSI_SW_FLAG_SRC_PRUNE;
1368 /* Traffic from VSI can be sent to LAN */
1369 ctxt->info.sw_flags2 = ICE_AQ_VSI_SW_FLAG_LAN_ENA;
1371 /* By default bits 3 and 4 in vlan_flags are 0's which results in legacy
1372 * behavior (show VLAN, DEI, and UP) in descriptor. Also, allow all
1373 * packets untagged/tagged.
1375 ctxt->info.vlan_flags = ((ICE_AQ_VSI_VLAN_MODE_ALL &
1376 ICE_AQ_VSI_VLAN_MODE_M) >>
1377 ICE_AQ_VSI_VLAN_MODE_S);
1379 /* Have 1:1 UP mapping for both ingress/egress tables */
1380 table |= ICE_UP_TABLE_TRANSLATE(0, 0);
1381 table |= ICE_UP_TABLE_TRANSLATE(1, 1);
1382 table |= ICE_UP_TABLE_TRANSLATE(2, 2);
1383 table |= ICE_UP_TABLE_TRANSLATE(3, 3);
1384 table |= ICE_UP_TABLE_TRANSLATE(4, 4);
1385 table |= ICE_UP_TABLE_TRANSLATE(5, 5);
1386 table |= ICE_UP_TABLE_TRANSLATE(6, 6);
1387 table |= ICE_UP_TABLE_TRANSLATE(7, 7);
1388 ctxt->info.ingress_table = cpu_to_le32(table);
1389 ctxt->info.egress_table = cpu_to_le32(table);
1390 /* Have 1:1 UP mapping for outer to inner UP table */
1391 ctxt->info.outer_up_table = cpu_to_le32(table);
1392 /* No Outer tag support outer_tag_flags remains to zero */
1396 * ice_set_rss_vsi_ctx - Set RSS VSI context before adding a VSI
1397 * @ctxt: the VSI context being set
1398 * @vsi: the VSI being configured
1400 static void ice_set_rss_vsi_ctx(struct ice_vsi_ctx *ctxt, struct ice_vsi *vsi)
1402 u8 lut_type, hash_type;
1404 switch (vsi->type) {
1406 /* PF VSI will inherit RSS instance of PF */
1407 lut_type = ICE_AQ_VSI_Q_OPT_RSS_LUT_PF;
1408 hash_type = ICE_AQ_VSI_Q_OPT_RSS_TPLZ;
1411 dev_warn(&vsi->back->pdev->dev, "Unknown VSI type %d\n",
1416 ctxt->info.q_opt_rss = ((lut_type << ICE_AQ_VSI_Q_OPT_RSS_LUT_S) &
1417 ICE_AQ_VSI_Q_OPT_RSS_LUT_M) |
1418 ((hash_type << ICE_AQ_VSI_Q_OPT_RSS_HASH_S) &
1419 ICE_AQ_VSI_Q_OPT_RSS_HASH_M);
1423 * ice_vsi_add - Create a new VSI or fetch preallocated VSI
1424 * @vsi: the VSI being configured
1426 * This initializes a VSI context depending on the VSI type to be added and
1427 * passes it down to the add_vsi aq command to create a new VSI.
1429 static int ice_vsi_add(struct ice_vsi *vsi)
1431 struct ice_vsi_ctx ctxt = { 0 };
1432 struct ice_pf *pf = vsi->back;
1433 struct ice_hw *hw = &pf->hw;
1436 switch (vsi->type) {
1438 ctxt.flags = ICE_AQ_VSI_TYPE_PF;
1444 ice_set_dflt_vsi_ctx(&ctxt);
1445 /* if the switch is in VEB mode, allow VSI loopback */
1446 if (vsi->vsw->bridge_mode == BRIDGE_MODE_VEB)
1447 ctxt.info.sw_flags |= ICE_AQ_VSI_SW_FLAG_ALLOW_LB;
1449 /* Set LUT type and HASH type if RSS is enabled */
1450 if (test_bit(ICE_FLAG_RSS_ENA, pf->flags))
1451 ice_set_rss_vsi_ctx(&ctxt, vsi);
1453 ctxt.info.sw_id = vsi->port_info->sw_id;
1454 ice_vsi_setup_q_map(vsi, &ctxt);
1456 ret = ice_aq_add_vsi(hw, &ctxt, NULL);
1458 dev_err(&vsi->back->pdev->dev,
1459 "Add VSI AQ call failed, err %d\n", ret);
1462 vsi->info = ctxt.info;
1463 vsi->vsi_num = ctxt.vsi_num;
1469 * ice_vsi_release_msix - Clear the queue to Interrupt mapping in HW
1470 * @vsi: the VSI being cleaned up
1472 static void ice_vsi_release_msix(struct ice_vsi *vsi)
1474 struct ice_pf *pf = vsi->back;
1475 u16 vector = vsi->base_vector;
1476 struct ice_hw *hw = &pf->hw;
1481 for (i = 0; i < vsi->num_q_vectors; i++, vector++) {
1482 struct ice_q_vector *q_vector = vsi->q_vectors[i];
1484 wr32(hw, GLINT_ITR(ICE_RX_ITR, vector), 0);
1485 wr32(hw, GLINT_ITR(ICE_TX_ITR, vector), 0);
1486 for (q = 0; q < q_vector->num_ring_tx; q++) {
1487 wr32(hw, QINT_TQCTL(vsi->txq_map[txq]), 0);
1491 for (q = 0; q < q_vector->num_ring_rx; q++) {
1492 wr32(hw, QINT_RQCTL(vsi->rxq_map[rxq]), 0);
1501 * ice_vsi_clear_rings - Deallocates the Tx and Rx rings for VSI
1502 * @vsi: the VSI having rings deallocated
1504 static void ice_vsi_clear_rings(struct ice_vsi *vsi)
1508 if (vsi->tx_rings) {
1509 for (i = 0; i < vsi->alloc_txq; i++) {
1510 if (vsi->tx_rings[i]) {
1511 kfree_rcu(vsi->tx_rings[i], rcu);
1512 vsi->tx_rings[i] = NULL;
1516 if (vsi->rx_rings) {
1517 for (i = 0; i < vsi->alloc_rxq; i++) {
1518 if (vsi->rx_rings[i]) {
1519 kfree_rcu(vsi->rx_rings[i], rcu);
1520 vsi->rx_rings[i] = NULL;
1527 * ice_vsi_alloc_rings - Allocates Tx and Rx rings for the VSI
1528 * @vsi: VSI which is having rings allocated
1530 static int ice_vsi_alloc_rings(struct ice_vsi *vsi)
1532 struct ice_pf *pf = vsi->back;
1535 /* Allocate tx_rings */
1536 for (i = 0; i < vsi->alloc_txq; i++) {
1537 struct ice_ring *ring;
1539 /* allocate with kzalloc(), free with kfree_rcu() */
1540 ring = kzalloc(sizeof(*ring), GFP_KERNEL);
1546 ring->reg_idx = vsi->txq_map[i];
1547 ring->ring_active = false;
1549 ring->netdev = vsi->netdev;
1550 ring->dev = &pf->pdev->dev;
1551 ring->count = vsi->num_desc;
1553 vsi->tx_rings[i] = ring;
1556 /* Allocate rx_rings */
1557 for (i = 0; i < vsi->alloc_rxq; i++) {
1558 struct ice_ring *ring;
1560 /* allocate with kzalloc(), free with kfree_rcu() */
1561 ring = kzalloc(sizeof(*ring), GFP_KERNEL);
1566 ring->reg_idx = vsi->rxq_map[i];
1567 ring->ring_active = false;
1569 ring->netdev = vsi->netdev;
1570 ring->dev = &pf->pdev->dev;
1571 ring->count = vsi->num_desc;
1572 vsi->rx_rings[i] = ring;
1578 ice_vsi_clear_rings(vsi);
1583 * ice_vsi_free_irq - Free the irq association with the OS
1584 * @vsi: the VSI being configured
1586 static void ice_vsi_free_irq(struct ice_vsi *vsi)
1588 struct ice_pf *pf = vsi->back;
1589 int base = vsi->base_vector;
1591 if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags)) {
1594 if (!vsi->q_vectors || !vsi->irqs_ready)
1597 vsi->irqs_ready = false;
1598 for (i = 0; i < vsi->num_q_vectors; i++) {
1599 u16 vector = i + base;
1602 irq_num = pf->msix_entries[vector].vector;
1604 /* free only the irqs that were actually requested */
1605 if (!vsi->q_vectors[i] ||
1606 !(vsi->q_vectors[i]->num_ring_tx ||
1607 vsi->q_vectors[i]->num_ring_rx))
1610 /* clear the affinity notifier in the IRQ descriptor */
1611 irq_set_affinity_notifier(irq_num, NULL);
1613 /* clear the affinity_mask in the IRQ descriptor */
1614 irq_set_affinity_hint(irq_num, NULL);
1615 synchronize_irq(irq_num);
1616 devm_free_irq(&pf->pdev->dev, irq_num,
1619 ice_vsi_release_msix(vsi);
1624 * ice_vsi_cfg_msix - MSIX mode Interrupt Config in the HW
1625 * @vsi: the VSI being configured
1627 static void ice_vsi_cfg_msix(struct ice_vsi *vsi)
1629 struct ice_pf *pf = vsi->back;
1630 u16 vector = vsi->base_vector;
1631 struct ice_hw *hw = &pf->hw;
1632 u32 txq = 0, rxq = 0;
1636 for (i = 0; i < vsi->num_q_vectors; i++, vector++) {
1637 struct ice_q_vector *q_vector = vsi->q_vectors[i];
1639 itr_gran = hw->itr_gran_200;
1641 if (q_vector->num_ring_rx) {
1643 ITR_TO_REG(vsi->rx_rings[rxq]->rx_itr_setting,
1645 q_vector->rx.latency_range = ICE_LOW_LATENCY;
1648 if (q_vector->num_ring_tx) {
1650 ITR_TO_REG(vsi->tx_rings[txq]->tx_itr_setting,
1652 q_vector->tx.latency_range = ICE_LOW_LATENCY;
1654 wr32(hw, GLINT_ITR(ICE_RX_ITR, vector), q_vector->rx.itr);
1655 wr32(hw, GLINT_ITR(ICE_TX_ITR, vector), q_vector->tx.itr);
1657 /* Both Transmit Queue Interrupt Cause Control register
1658 * and Receive Queue Interrupt Cause control register
1659 * expects MSIX_INDX field to be the vector index
1660 * within the function space and not the absolute
1661 * vector index across PF or across device.
1662 * For SR-IOV VF VSIs queue vector index always starts
1663 * with 1 since first vector index(0) is used for OICR
1664 * in VF space. Since VMDq and other PF VSIs are withtin
1665 * the PF function space, use the vector index thats
1666 * tracked for this PF.
1668 for (q = 0; q < q_vector->num_ring_tx; q++) {
1672 val = QINT_TQCTL_CAUSE_ENA_M |
1673 (itr << QINT_TQCTL_ITR_INDX_S) |
1674 (vector << QINT_TQCTL_MSIX_INDX_S);
1675 wr32(hw, QINT_TQCTL(vsi->txq_map[txq]), val);
1679 for (q = 0; q < q_vector->num_ring_rx; q++) {
1683 val = QINT_RQCTL_CAUSE_ENA_M |
1684 (itr << QINT_RQCTL_ITR_INDX_S) |
1685 (vector << QINT_RQCTL_MSIX_INDX_S);
1686 wr32(hw, QINT_RQCTL(vsi->rxq_map[rxq]), val);
1695 * ice_ena_misc_vector - enable the non-queue interrupts
1696 * @pf: board private structure
1698 static void ice_ena_misc_vector(struct ice_pf *pf)
1700 struct ice_hw *hw = &pf->hw;
1703 /* clear things first */
1704 wr32(hw, PFINT_OICR_ENA, 0); /* disable all */
1705 rd32(hw, PFINT_OICR); /* read to clear */
1707 val = (PFINT_OICR_ECC_ERR_M |
1708 PFINT_OICR_MAL_DETECT_M |
1710 PFINT_OICR_PCI_EXCEPTION_M |
1711 PFINT_OICR_HMC_ERR_M |
1712 PFINT_OICR_PE_CRITERR_M);
1714 wr32(hw, PFINT_OICR_ENA, val);
1716 /* SW_ITR_IDX = 0, but don't change INTENA */
1717 wr32(hw, GLINT_DYN_CTL(pf->oicr_idx),
1718 GLINT_DYN_CTL_SW_ITR_INDX_M | GLINT_DYN_CTL_INTENA_MSK_M);
1722 * ice_misc_intr - misc interrupt handler
1723 * @irq: interrupt number
1724 * @data: pointer to a q_vector
1726 static irqreturn_t ice_misc_intr(int __always_unused irq, void *data)
1728 struct ice_pf *pf = (struct ice_pf *)data;
1729 struct ice_hw *hw = &pf->hw;
1730 irqreturn_t ret = IRQ_NONE;
1733 set_bit(__ICE_ADMINQ_EVENT_PENDING, pf->state);
1735 oicr = rd32(hw, PFINT_OICR);
1736 ena_mask = rd32(hw, PFINT_OICR_ENA);
1738 if (oicr & PFINT_OICR_GRST_M) {
1740 /* we have a reset warning */
1741 ena_mask &= ~PFINT_OICR_GRST_M;
1742 reset = (rd32(hw, GLGEN_RSTAT) & GLGEN_RSTAT_RESET_TYPE_M) >>
1743 GLGEN_RSTAT_RESET_TYPE_S;
1745 if (reset == ICE_RESET_CORER)
1747 else if (reset == ICE_RESET_GLOBR)
1752 /* If a reset cycle isn't already in progress, we set a bit in
1753 * pf->state so that the service task can start a reset/rebuild.
1754 * We also make note of which reset happened so that peer
1755 * devices/drivers can be informed.
1757 if (!test_bit(__ICE_RESET_RECOVERY_PENDING, pf->state)) {
1758 if (reset == ICE_RESET_CORER)
1759 set_bit(__ICE_CORER_RECV, pf->state);
1760 else if (reset == ICE_RESET_GLOBR)
1761 set_bit(__ICE_GLOBR_RECV, pf->state);
1763 set_bit(__ICE_EMPR_RECV, pf->state);
1765 set_bit(__ICE_RESET_RECOVERY_PENDING, pf->state);
1769 if (oicr & PFINT_OICR_HMC_ERR_M) {
1770 ena_mask &= ~PFINT_OICR_HMC_ERR_M;
1771 dev_dbg(&pf->pdev->dev,
1772 "HMC Error interrupt - info 0x%x, data 0x%x\n",
1773 rd32(hw, PFHMC_ERRORINFO),
1774 rd32(hw, PFHMC_ERRORDATA));
1777 /* Report and mask off any remaining unexpected interrupts */
1780 dev_dbg(&pf->pdev->dev, "unhandled interrupt oicr=0x%08x\n",
1782 /* If a critical error is pending there is no choice but to
1785 if (oicr & (PFINT_OICR_PE_CRITERR_M |
1786 PFINT_OICR_PCI_EXCEPTION_M |
1787 PFINT_OICR_ECC_ERR_M)) {
1788 set_bit(__ICE_PFR_REQ, pf->state);
1789 ice_service_task_schedule(pf);
1795 /* re-enable interrupt causes that are not handled during this pass */
1796 wr32(hw, PFINT_OICR_ENA, ena_mask);
1797 if (!test_bit(__ICE_DOWN, pf->state)) {
1798 ice_service_task_schedule(pf);
1799 ice_irq_dynamic_ena(hw, NULL, NULL);
1806 * ice_vsi_map_rings_to_vectors - Map VSI rings to interrupt vectors
1807 * @vsi: the VSI being configured
1809 * This function maps descriptor rings to the queue-specific vectors allotted
1810 * through the MSI-X enabling code. On a constrained vector budget, we map Tx
1811 * and Rx rings to the vector as "efficiently" as possible.
1813 static void ice_vsi_map_rings_to_vectors(struct ice_vsi *vsi)
1815 int q_vectors = vsi->num_q_vectors;
1816 int tx_rings_rem, rx_rings_rem;
1819 /* initially assigning remaining rings count to VSIs num queue value */
1820 tx_rings_rem = vsi->num_txq;
1821 rx_rings_rem = vsi->num_rxq;
1823 for (v_id = 0; v_id < q_vectors; v_id++) {
1824 struct ice_q_vector *q_vector = vsi->q_vectors[v_id];
1825 int tx_rings_per_v, rx_rings_per_v, q_id, q_base;
1827 /* Tx rings mapping to vector */
1828 tx_rings_per_v = DIV_ROUND_UP(tx_rings_rem, q_vectors - v_id);
1829 q_vector->num_ring_tx = tx_rings_per_v;
1830 q_vector->tx.ring = NULL;
1831 q_base = vsi->num_txq - tx_rings_rem;
1833 for (q_id = q_base; q_id < (q_base + tx_rings_per_v); q_id++) {
1834 struct ice_ring *tx_ring = vsi->tx_rings[q_id];
1836 tx_ring->q_vector = q_vector;
1837 tx_ring->next = q_vector->tx.ring;
1838 q_vector->tx.ring = tx_ring;
1840 tx_rings_rem -= tx_rings_per_v;
1842 /* Rx rings mapping to vector */
1843 rx_rings_per_v = DIV_ROUND_UP(rx_rings_rem, q_vectors - v_id);
1844 q_vector->num_ring_rx = rx_rings_per_v;
1845 q_vector->rx.ring = NULL;
1846 q_base = vsi->num_rxq - rx_rings_rem;
1848 for (q_id = q_base; q_id < (q_base + rx_rings_per_v); q_id++) {
1849 struct ice_ring *rx_ring = vsi->rx_rings[q_id];
1851 rx_ring->q_vector = q_vector;
1852 rx_ring->next = q_vector->rx.ring;
1853 q_vector->rx.ring = rx_ring;
1855 rx_rings_rem -= rx_rings_per_v;
1860 * ice_vsi_set_num_qs - Set num queues, descriptors and vectors for a VSI
1861 * @vsi: the VSI being configured
1863 * Return 0 on success and a negative value on error
1865 static void ice_vsi_set_num_qs(struct ice_vsi *vsi)
1867 struct ice_pf *pf = vsi->back;
1869 switch (vsi->type) {
1871 vsi->alloc_txq = pf->num_lan_tx;
1872 vsi->alloc_rxq = pf->num_lan_rx;
1873 vsi->num_desc = ALIGN(ICE_DFLT_NUM_DESC, ICE_REQ_DESC_MULTIPLE);
1874 vsi->num_q_vectors = max_t(int, pf->num_lan_rx, pf->num_lan_tx);
1877 dev_warn(&vsi->back->pdev->dev, "Unknown VSI type %d\n",
1884 * ice_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi
1886 * @alloc_qvectors: a bool to specify if q_vectors need to be allocated.
1888 * On error: returns error code (negative)
1889 * On success: returns 0
1891 static int ice_vsi_alloc_arrays(struct ice_vsi *vsi, bool alloc_qvectors)
1893 struct ice_pf *pf = vsi->back;
1895 /* allocate memory for both Tx and Rx ring pointers */
1896 vsi->tx_rings = devm_kcalloc(&pf->pdev->dev, vsi->alloc_txq,
1897 sizeof(struct ice_ring *), GFP_KERNEL);
1901 vsi->rx_rings = devm_kcalloc(&pf->pdev->dev, vsi->alloc_rxq,
1902 sizeof(struct ice_ring *), GFP_KERNEL);
1906 if (alloc_qvectors) {
1907 /* allocate memory for q_vector pointers */
1908 vsi->q_vectors = devm_kcalloc(&pf->pdev->dev,
1910 sizeof(struct ice_q_vector *),
1912 if (!vsi->q_vectors)
1919 devm_kfree(&pf->pdev->dev, vsi->rx_rings);
1921 devm_kfree(&pf->pdev->dev, vsi->tx_rings);
1927 * ice_msix_clean_rings - MSIX mode Interrupt Handler
1928 * @irq: interrupt number
1929 * @data: pointer to a q_vector
1931 static irqreturn_t ice_msix_clean_rings(int __always_unused irq, void *data)
1933 struct ice_q_vector *q_vector = (struct ice_q_vector *)data;
1935 if (!q_vector->tx.ring && !q_vector->rx.ring)
1938 napi_schedule(&q_vector->napi);
1944 * ice_vsi_alloc - Allocates the next available struct vsi in the PF
1945 * @pf: board private structure
1946 * @type: type of VSI
1948 * returns a pointer to a VSI on success, NULL on failure.
1950 static struct ice_vsi *ice_vsi_alloc(struct ice_pf *pf, enum ice_vsi_type type)
1952 struct ice_vsi *vsi = NULL;
1954 /* Need to protect the allocation of the VSIs at the PF level */
1955 mutex_lock(&pf->sw_mutex);
1957 /* If we have already allocated our maximum number of VSIs,
1958 * pf->next_vsi will be ICE_NO_VSI. If not, pf->next_vsi index
1959 * is available to be populated
1961 if (pf->next_vsi == ICE_NO_VSI) {
1962 dev_dbg(&pf->pdev->dev, "out of VSI slots!\n");
1966 vsi = devm_kzalloc(&pf->pdev->dev, sizeof(*vsi), GFP_KERNEL);
1972 set_bit(__ICE_DOWN, vsi->state);
1973 vsi->idx = pf->next_vsi;
1974 vsi->work_lmt = ICE_DFLT_IRQ_WORK;
1976 ice_vsi_set_num_qs(vsi);
1978 switch (vsi->type) {
1980 if (ice_vsi_alloc_arrays(vsi, true))
1983 /* Setup default MSIX irq handler for VSI */
1984 vsi->irq_handler = ice_msix_clean_rings;
1987 dev_warn(&pf->pdev->dev, "Unknown VSI type %d\n", vsi->type);
1991 /* fill VSI slot in the PF struct */
1992 pf->vsi[pf->next_vsi] = vsi;
1994 /* prepare pf->next_vsi for next use */
1995 pf->next_vsi = ice_get_free_slot(pf->vsi, pf->num_alloc_vsi,
2000 devm_kfree(&pf->pdev->dev, vsi);
2003 mutex_unlock(&pf->sw_mutex);
2008 * ice_free_irq_msix_misc - Unroll misc vector setup
2009 * @pf: board private structure
2011 static void ice_free_irq_msix_misc(struct ice_pf *pf)
2013 /* disable OICR interrupt */
2014 wr32(&pf->hw, PFINT_OICR_ENA, 0);
2017 if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags) && pf->msix_entries) {
2018 synchronize_irq(pf->msix_entries[pf->oicr_idx].vector);
2019 devm_free_irq(&pf->pdev->dev,
2020 pf->msix_entries[pf->oicr_idx].vector, pf);
2023 ice_free_res(pf->irq_tracker, pf->oicr_idx, ICE_RES_MISC_VEC_ID);
2027 * ice_req_irq_msix_misc - Setup the misc vector to handle non queue events
2028 * @pf: board private structure
2030 * This sets up the handler for MSIX 0, which is used to manage the
2031 * non-queue interrupts, e.g. AdminQ and errors. This is not used
2032 * when in MSI or Legacy interrupt mode.
2034 static int ice_req_irq_msix_misc(struct ice_pf *pf)
2036 struct ice_hw *hw = &pf->hw;
2037 int oicr_idx, err = 0;
2041 if (!pf->int_name[0])
2042 snprintf(pf->int_name, sizeof(pf->int_name) - 1, "%s-%s:misc",
2043 dev_driver_string(&pf->pdev->dev),
2044 dev_name(&pf->pdev->dev));
2046 /* Do not request IRQ but do enable OICR interrupt since settings are
2047 * lost during reset. Note that this function is called only during
2048 * rebuild path and not while reset is in progress.
2050 if (ice_is_reset_recovery_pending(pf->state))
2053 /* reserve one vector in irq_tracker for misc interrupts */
2054 oicr_idx = ice_get_res(pf, pf->irq_tracker, 1, ICE_RES_MISC_VEC_ID);
2058 pf->oicr_idx = oicr_idx;
2060 err = devm_request_irq(&pf->pdev->dev,
2061 pf->msix_entries[pf->oicr_idx].vector,
2062 ice_misc_intr, 0, pf->int_name, pf);
2064 dev_err(&pf->pdev->dev,
2065 "devm_request_irq for %s failed: %d\n",
2067 ice_free_res(pf->irq_tracker, 1, ICE_RES_MISC_VEC_ID);
2072 ice_ena_misc_vector(pf);
2074 val = ((pf->oicr_idx & PFINT_OICR_CTL_MSIX_INDX_M) |
2075 PFINT_OICR_CTL_CAUSE_ENA_M);
2076 wr32(hw, PFINT_OICR_CTL, val);
2078 /* This enables Admin queue Interrupt causes */
2079 val = ((pf->oicr_idx & PFINT_FW_CTL_MSIX_INDX_M) |
2080 PFINT_FW_CTL_CAUSE_ENA_M);
2081 wr32(hw, PFINT_FW_CTL, val);
2083 itr_gran = hw->itr_gran_200;
2085 wr32(hw, GLINT_ITR(ICE_RX_ITR, pf->oicr_idx),
2086 ITR_TO_REG(ICE_ITR_8K, itr_gran));
2089 ice_irq_dynamic_ena(hw, NULL, NULL);
2095 * ice_vsi_get_qs_contig - Assign a contiguous chunk of queues to VSI
2096 * @vsi: the VSI getting queues
2098 * Return 0 on success and a negative value on error
2100 static int ice_vsi_get_qs_contig(struct ice_vsi *vsi)
2102 struct ice_pf *pf = vsi->back;
2103 int offset, ret = 0;
2105 mutex_lock(&pf->avail_q_mutex);
2106 /* look for contiguous block of queues for tx */
2107 offset = bitmap_find_next_zero_area(pf->avail_txqs, ICE_MAX_TXQS,
2108 0, vsi->alloc_txq, 0);
2109 if (offset < ICE_MAX_TXQS) {
2112 bitmap_set(pf->avail_txqs, offset, vsi->alloc_txq);
2113 for (i = 0; i < vsi->alloc_txq; i++)
2114 vsi->txq_map[i] = i + offset;
2117 vsi->tx_mapping_mode = ICE_VSI_MAP_SCATTER;
2120 /* look for contiguous block of queues for rx */
2121 offset = bitmap_find_next_zero_area(pf->avail_rxqs, ICE_MAX_RXQS,
2122 0, vsi->alloc_rxq, 0);
2123 if (offset < ICE_MAX_RXQS) {
2126 bitmap_set(pf->avail_rxqs, offset, vsi->alloc_rxq);
2127 for (i = 0; i < vsi->alloc_rxq; i++)
2128 vsi->rxq_map[i] = i + offset;
2131 vsi->rx_mapping_mode = ICE_VSI_MAP_SCATTER;
2133 mutex_unlock(&pf->avail_q_mutex);
2139 * ice_vsi_get_qs_scatter - Assign a scattered queues to VSI
2140 * @vsi: the VSI getting queues
2142 * Return 0 on success and a negative value on error
2144 static int ice_vsi_get_qs_scatter(struct ice_vsi *vsi)
2146 struct ice_pf *pf = vsi->back;
2149 mutex_lock(&pf->avail_q_mutex);
2151 if (vsi->tx_mapping_mode == ICE_VSI_MAP_SCATTER) {
2152 for (i = 0; i < vsi->alloc_txq; i++) {
2153 index = find_next_zero_bit(pf->avail_txqs,
2154 ICE_MAX_TXQS, index);
2155 if (index < ICE_MAX_TXQS) {
2156 set_bit(index, pf->avail_txqs);
2157 vsi->txq_map[i] = index;
2159 goto err_scatter_tx;
2164 if (vsi->rx_mapping_mode == ICE_VSI_MAP_SCATTER) {
2165 for (i = 0; i < vsi->alloc_rxq; i++) {
2166 index = find_next_zero_bit(pf->avail_rxqs,
2167 ICE_MAX_RXQS, index);
2168 if (index < ICE_MAX_RXQS) {
2169 set_bit(index, pf->avail_rxqs);
2170 vsi->rxq_map[i] = index;
2172 goto err_scatter_rx;
2177 mutex_unlock(&pf->avail_q_mutex);
2181 /* unflag any queues we have grabbed (i is failed position) */
2182 for (index = 0; index < i; index++) {
2183 clear_bit(vsi->rxq_map[index], pf->avail_rxqs);
2184 vsi->rxq_map[index] = 0;
2188 /* i is either position of failed attempt or vsi->alloc_txq */
2189 for (index = 0; index < i; index++) {
2190 clear_bit(vsi->txq_map[index], pf->avail_txqs);
2191 vsi->txq_map[index] = 0;
2194 mutex_unlock(&pf->avail_q_mutex);
2199 * ice_vsi_get_qs - Assign queues from PF to VSI
2200 * @vsi: the VSI to assign queues to
2202 * Returns 0 on success and a negative value on error
2204 static int ice_vsi_get_qs(struct ice_vsi *vsi)
2208 vsi->tx_mapping_mode = ICE_VSI_MAP_CONTIG;
2209 vsi->rx_mapping_mode = ICE_VSI_MAP_CONTIG;
2211 /* NOTE: ice_vsi_get_qs_contig() will set the rx/tx mapping
2212 * modes individually to scatter if assigning contiguous queues
2215 ret = ice_vsi_get_qs_contig(vsi);
2217 if (vsi->tx_mapping_mode == ICE_VSI_MAP_SCATTER)
2218 vsi->alloc_txq = max_t(u16, vsi->alloc_txq,
2219 ICE_MAX_SCATTER_TXQS);
2220 if (vsi->rx_mapping_mode == ICE_VSI_MAP_SCATTER)
2221 vsi->alloc_rxq = max_t(u16, vsi->alloc_rxq,
2222 ICE_MAX_SCATTER_RXQS);
2223 ret = ice_vsi_get_qs_scatter(vsi);
2230 * ice_vsi_put_qs - Release queues from VSI to PF
2231 * @vsi: the VSI thats going to release queues
2233 static void ice_vsi_put_qs(struct ice_vsi *vsi)
2235 struct ice_pf *pf = vsi->back;
2238 mutex_lock(&pf->avail_q_mutex);
2240 for (i = 0; i < vsi->alloc_txq; i++) {
2241 clear_bit(vsi->txq_map[i], pf->avail_txqs);
2242 vsi->txq_map[i] = ICE_INVAL_Q_INDEX;
2245 for (i = 0; i < vsi->alloc_rxq; i++) {
2246 clear_bit(vsi->rxq_map[i], pf->avail_rxqs);
2247 vsi->rxq_map[i] = ICE_INVAL_Q_INDEX;
2250 mutex_unlock(&pf->avail_q_mutex);
2254 * ice_free_q_vector - Free memory allocated for a specific interrupt vector
2255 * @vsi: VSI having the memory freed
2256 * @v_idx: index of the vector to be freed
2258 static void ice_free_q_vector(struct ice_vsi *vsi, int v_idx)
2260 struct ice_q_vector *q_vector;
2261 struct ice_ring *ring;
2263 if (!vsi->q_vectors[v_idx]) {
2264 dev_dbg(&vsi->back->pdev->dev, "Queue vector at index %d not found\n",
2268 q_vector = vsi->q_vectors[v_idx];
2270 ice_for_each_ring(ring, q_vector->tx)
2271 ring->q_vector = NULL;
2272 ice_for_each_ring(ring, q_vector->rx)
2273 ring->q_vector = NULL;
2275 /* only VSI with an associated netdev is set up with NAPI */
2277 netif_napi_del(&q_vector->napi);
2279 devm_kfree(&vsi->back->pdev->dev, q_vector);
2280 vsi->q_vectors[v_idx] = NULL;
2284 * ice_vsi_free_q_vectors - Free memory allocated for interrupt vectors
2285 * @vsi: the VSI having memory freed
2287 static void ice_vsi_free_q_vectors(struct ice_vsi *vsi)
2291 for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++)
2292 ice_free_q_vector(vsi, v_idx);
2296 * ice_cfg_netdev - Setup the netdev flags
2297 * @vsi: the VSI being configured
2299 * Returns 0 on success, negative value on failure
2301 static int ice_cfg_netdev(struct ice_vsi *vsi)
2303 netdev_features_t csumo_features;
2304 netdev_features_t vlano_features;
2305 netdev_features_t dflt_features;
2306 netdev_features_t tso_features;
2307 struct ice_netdev_priv *np;
2308 struct net_device *netdev;
2309 u8 mac_addr[ETH_ALEN];
2311 netdev = alloc_etherdev_mqs(sizeof(struct ice_netdev_priv),
2312 vsi->alloc_txq, vsi->alloc_rxq);
2316 vsi->netdev = netdev;
2317 np = netdev_priv(netdev);
2320 dflt_features = NETIF_F_SG |
2324 csumo_features = NETIF_F_RXCSUM |
2328 vlano_features = NETIF_F_HW_VLAN_CTAG_FILTER |
2329 NETIF_F_HW_VLAN_CTAG_TX |
2330 NETIF_F_HW_VLAN_CTAG_RX;
2332 tso_features = NETIF_F_TSO;
2334 /* set features that user can change */
2335 netdev->hw_features = dflt_features | csumo_features |
2336 vlano_features | tso_features;
2338 /* enable features */
2339 netdev->features |= netdev->hw_features;
2340 /* encap and VLAN devices inherit default, csumo and tso features */
2341 netdev->hw_enc_features |= dflt_features | csumo_features |
2343 netdev->vlan_features |= dflt_features | csumo_features |
2346 if (vsi->type == ICE_VSI_PF) {
2347 SET_NETDEV_DEV(netdev, &vsi->back->pdev->dev);
2348 ether_addr_copy(mac_addr, vsi->port_info->mac.perm_addr);
2350 ether_addr_copy(netdev->dev_addr, mac_addr);
2351 ether_addr_copy(netdev->perm_addr, mac_addr);
2354 netdev->priv_flags |= IFF_UNICAST_FLT;
2356 /* assign netdev_ops */
2357 netdev->netdev_ops = &ice_netdev_ops;
2359 /* setup watchdog timeout value to be 5 second */
2360 netdev->watchdog_timeo = 5 * HZ;
2362 ice_set_ethtool_ops(netdev);
2364 netdev->min_mtu = ETH_MIN_MTU;
2365 netdev->max_mtu = ICE_MAX_MTU;
2371 * ice_vsi_free_arrays - clean up vsi resources
2372 * @vsi: pointer to VSI being cleared
2373 * @free_qvectors: bool to specify if q_vectors should be deallocated
2375 static void ice_vsi_free_arrays(struct ice_vsi *vsi, bool free_qvectors)
2377 struct ice_pf *pf = vsi->back;
2379 /* free the ring and vector containers */
2380 if (free_qvectors && vsi->q_vectors) {
2381 devm_kfree(&pf->pdev->dev, vsi->q_vectors);
2382 vsi->q_vectors = NULL;
2384 if (vsi->tx_rings) {
2385 devm_kfree(&pf->pdev->dev, vsi->tx_rings);
2386 vsi->tx_rings = NULL;
2388 if (vsi->rx_rings) {
2389 devm_kfree(&pf->pdev->dev, vsi->rx_rings);
2390 vsi->rx_rings = NULL;
2395 * ice_vsi_clear - clean up and deallocate the provided vsi
2396 * @vsi: pointer to VSI being cleared
2398 * This deallocates the vsi's queue resources, removes it from the PF's
2399 * VSI array if necessary, and deallocates the VSI
2401 * Returns 0 on success, negative on failure
2403 static int ice_vsi_clear(struct ice_vsi *vsi)
2405 struct ice_pf *pf = NULL;
2415 if (!pf->vsi[vsi->idx] || pf->vsi[vsi->idx] != vsi) {
2416 dev_dbg(&pf->pdev->dev, "vsi does not exist at pf->vsi[%d]\n",
2421 mutex_lock(&pf->sw_mutex);
2422 /* updates the PF for this cleared vsi */
2424 pf->vsi[vsi->idx] = NULL;
2425 if (vsi->idx < pf->next_vsi)
2426 pf->next_vsi = vsi->idx;
2428 ice_vsi_free_arrays(vsi, true);
2429 mutex_unlock(&pf->sw_mutex);
2430 devm_kfree(&pf->pdev->dev, vsi);
2436 * ice_vsi_alloc_q_vector - Allocate memory for a single interrupt vector
2437 * @vsi: the VSI being configured
2438 * @v_idx: index of the vector in the vsi struct
2440 * We allocate one q_vector. If allocation fails we return -ENOMEM.
2442 static int ice_vsi_alloc_q_vector(struct ice_vsi *vsi, int v_idx)
2444 struct ice_pf *pf = vsi->back;
2445 struct ice_q_vector *q_vector;
2447 /* allocate q_vector */
2448 q_vector = devm_kzalloc(&pf->pdev->dev, sizeof(*q_vector), GFP_KERNEL);
2452 q_vector->vsi = vsi;
2453 q_vector->v_idx = v_idx;
2454 /* only set affinity_mask if the CPU is online */
2455 if (cpu_online(v_idx))
2456 cpumask_set_cpu(v_idx, &q_vector->affinity_mask);
2459 netif_napi_add(vsi->netdev, &q_vector->napi, ice_napi_poll,
2461 /* tie q_vector and vsi together */
2462 vsi->q_vectors[v_idx] = q_vector;
2468 * ice_vsi_alloc_q_vectors - Allocate memory for interrupt vectors
2469 * @vsi: the VSI being configured
2471 * We allocate one q_vector per queue interrupt. If allocation fails we
2474 static int ice_vsi_alloc_q_vectors(struct ice_vsi *vsi)
2476 struct ice_pf *pf = vsi->back;
2477 int v_idx = 0, num_q_vectors;
2480 if (vsi->q_vectors[0]) {
2481 dev_dbg(&pf->pdev->dev, "VSI %d has existing q_vectors\n",
2486 if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags)) {
2487 num_q_vectors = vsi->num_q_vectors;
2493 for (v_idx = 0; v_idx < num_q_vectors; v_idx++) {
2494 err = ice_vsi_alloc_q_vector(vsi, v_idx);
2503 ice_free_q_vector(vsi, v_idx);
2505 dev_err(&pf->pdev->dev,
2506 "Failed to allocate %d q_vector for VSI %d, ret=%d\n",
2507 vsi->num_q_vectors, vsi->vsi_num, err);
2508 vsi->num_q_vectors = 0;
2513 * ice_vsi_setup_vector_base - Set up the base vector for the given VSI
2514 * @vsi: ptr to the VSI
2516 * This should only be called after ice_vsi_alloc() which allocates the
2517 * corresponding SW VSI structure and initializes num_queue_pairs for the
2518 * newly allocated VSI.
2520 * Returns 0 on success or negative on failure
2522 static int ice_vsi_setup_vector_base(struct ice_vsi *vsi)
2524 struct ice_pf *pf = vsi->back;
2525 int num_q_vectors = 0;
2527 if (vsi->base_vector) {
2528 dev_dbg(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n",
2529 vsi->vsi_num, vsi->base_vector);
2533 if (!test_bit(ICE_FLAG_MSIX_ENA, pf->flags))
2536 switch (vsi->type) {
2538 num_q_vectors = vsi->num_q_vectors;
2541 dev_warn(&vsi->back->pdev->dev, "Unknown VSI type %d\n",
2547 vsi->base_vector = ice_get_res(pf, pf->irq_tracker,
2548 num_q_vectors, vsi->idx);
2550 if (vsi->base_vector < 0) {
2551 dev_err(&pf->pdev->dev,
2552 "Failed to get tracking for %d vectors for VSI %d, err=%d\n",
2553 num_q_vectors, vsi->vsi_num, vsi->base_vector);
2561 * ice_fill_rss_lut - Fill the RSS lookup table with default values
2562 * @lut: Lookup table
2563 * @rss_table_size: Lookup table size
2564 * @rss_size: Range of queue number for hashing
2566 void ice_fill_rss_lut(u8 *lut, u16 rss_table_size, u16 rss_size)
2570 for (i = 0; i < rss_table_size; i++)
2571 lut[i] = i % rss_size;
2575 * ice_vsi_cfg_rss - Configure RSS params for a VSI
2576 * @vsi: VSI to be configured
2578 static int ice_vsi_cfg_rss(struct ice_vsi *vsi)
2580 u8 seed[ICE_AQC_GET_SET_RSS_KEY_DATA_RSS_KEY_SIZE];
2581 struct ice_aqc_get_set_rss_keys *key;
2582 struct ice_pf *pf = vsi->back;
2583 enum ice_status status;
2587 vsi->rss_size = min_t(int, vsi->rss_size, vsi->num_rxq);
2589 lut = devm_kzalloc(&pf->pdev->dev, vsi->rss_table_size, GFP_KERNEL);
2593 if (vsi->rss_lut_user)
2594 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
2596 ice_fill_rss_lut(lut, vsi->rss_table_size, vsi->rss_size);
2598 status = ice_aq_set_rss_lut(&pf->hw, vsi->vsi_num, vsi->rss_lut_type,
2599 lut, vsi->rss_table_size);
2602 dev_err(&vsi->back->pdev->dev,
2603 "set_rss_lut failed, error %d\n", status);
2605 goto ice_vsi_cfg_rss_exit;
2608 key = devm_kzalloc(&vsi->back->pdev->dev, sizeof(*key), GFP_KERNEL);
2611 goto ice_vsi_cfg_rss_exit;
2614 if (vsi->rss_hkey_user)
2615 memcpy(seed, vsi->rss_hkey_user,
2616 ICE_AQC_GET_SET_RSS_KEY_DATA_RSS_KEY_SIZE);
2618 netdev_rss_key_fill((void *)seed,
2619 ICE_AQC_GET_SET_RSS_KEY_DATA_RSS_KEY_SIZE);
2620 memcpy(&key->standard_rss_key, seed,
2621 ICE_AQC_GET_SET_RSS_KEY_DATA_RSS_KEY_SIZE);
2623 status = ice_aq_set_rss_key(&pf->hw, vsi->vsi_num, key);
2626 dev_err(&vsi->back->pdev->dev, "set_rss_key failed, error %d\n",
2631 devm_kfree(&pf->pdev->dev, key);
2632 ice_vsi_cfg_rss_exit:
2633 devm_kfree(&pf->pdev->dev, lut);
2638 * ice_vsi_reinit_setup - return resource and reallocate resource for a VSI
2639 * @vsi: pointer to the ice_vsi
2641 * This reallocates the VSIs queue resources
2643 * Returns 0 on success and negative value on failure
2645 static int ice_vsi_reinit_setup(struct ice_vsi *vsi)
2647 u16 max_txqs[ICE_MAX_TRAFFIC_CLASS] = { 0 };
2653 ice_vsi_free_q_vectors(vsi);
2654 ice_free_res(vsi->back->irq_tracker, vsi->base_vector, vsi->idx);
2655 vsi->base_vector = 0;
2656 ice_vsi_clear_rings(vsi);
2657 ice_vsi_free_arrays(vsi, false);
2658 ice_vsi_set_num_qs(vsi);
2660 /* Initialize VSI struct elements and create VSI in FW */
2661 ret = ice_vsi_add(vsi);
2665 ret = ice_vsi_alloc_arrays(vsi, false);
2669 switch (vsi->type) {
2672 ret = ice_cfg_netdev(vsi);
2676 ret = register_netdev(vsi->netdev);
2680 netif_carrier_off(vsi->netdev);
2681 netif_tx_stop_all_queues(vsi->netdev);
2684 ret = ice_vsi_alloc_q_vectors(vsi);
2688 ret = ice_vsi_setup_vector_base(vsi);
2692 ret = ice_vsi_alloc_rings(vsi);
2696 ice_vsi_map_rings_to_vectors(vsi);
2702 ice_vsi_set_tc_cfg(vsi);
2704 /* configure VSI nodes based on number of queues and TC's */
2705 for (i = 0; i < vsi->tc_cfg.numtc; i++)
2706 max_txqs[i] = vsi->num_txq;
2708 ret = ice_cfg_vsi_lan(vsi->port_info, vsi->vsi_num,
2709 vsi->tc_cfg.ena_tc, max_txqs);
2711 dev_info(&vsi->back->pdev->dev,
2712 "Failed VSI lan queue config\n");
2718 ice_vsi_free_q_vectors(vsi);
2721 vsi->current_netdev_flags = 0;
2722 unregister_netdev(vsi->netdev);
2723 free_netdev(vsi->netdev);
2728 set_bit(__ICE_RESET_FAILED, vsi->back->state);
2733 * ice_vsi_setup - Set up a VSI by a given type
2734 * @pf: board private structure
2736 * @pi: pointer to the port_info instance
2738 * This allocates the sw VSI structure and its queue resources.
2740 * Returns pointer to the successfully allocated and configure VSI sw struct on
2741 * success, otherwise returns NULL on failure.
2743 static struct ice_vsi *
2744 ice_vsi_setup(struct ice_pf *pf, enum ice_vsi_type type,
2745 struct ice_port_info *pi)
2747 u16 max_txqs[ICE_MAX_TRAFFIC_CLASS] = { 0 };
2748 struct device *dev = &pf->pdev->dev;
2749 struct ice_vsi_ctx ctxt = { 0 };
2750 struct ice_vsi *vsi;
2753 vsi = ice_vsi_alloc(pf, type);
2755 dev_err(dev, "could not allocate VSI\n");
2759 vsi->port_info = pi;
2760 vsi->vsw = pf->first_sw;
2762 if (ice_vsi_get_qs(vsi)) {
2763 dev_err(dev, "Failed to allocate queues. vsi->idx = %d\n",
2768 /* set RSS capabilities */
2769 ice_vsi_set_rss_params(vsi);
2771 /* create the VSI */
2772 ret = ice_vsi_add(vsi);
2776 ctxt.vsi_num = vsi->vsi_num;
2778 switch (vsi->type) {
2780 ret = ice_cfg_netdev(vsi);
2782 goto err_cfg_netdev;
2784 ret = register_netdev(vsi->netdev);
2786 goto err_register_netdev;
2788 netif_carrier_off(vsi->netdev);
2790 /* make sure transmit queues start off as stopped */
2791 netif_tx_stop_all_queues(vsi->netdev);
2792 ret = ice_vsi_alloc_q_vectors(vsi);
2796 ret = ice_vsi_setup_vector_base(vsi);
2800 ret = ice_vsi_alloc_rings(vsi);
2804 ice_vsi_map_rings_to_vectors(vsi);
2806 /* Do not exit if configuring RSS had an issue, at least
2807 * receive traffic on first queue. Hence no need to capture
2810 if (test_bit(ICE_FLAG_RSS_ENA, pf->flags))
2811 ice_vsi_cfg_rss(vsi);
2814 /* if vsi type is not recognized, clean up the resources and
2820 ice_vsi_set_tc_cfg(vsi);
2822 /* configure VSI nodes based on number of queues and TC's */
2823 for (i = 0; i < vsi->tc_cfg.numtc; i++)
2824 max_txqs[i] = vsi->num_txq;
2826 ret = ice_cfg_vsi_lan(vsi->port_info, vsi->vsi_num,
2827 vsi->tc_cfg.ena_tc, max_txqs);
2829 dev_info(&pf->pdev->dev, "Failed VSI lan queue config\n");
2836 ice_vsi_free_q_vectors(vsi);
2838 if (vsi->netdev && vsi->netdev->reg_state == NETREG_REGISTERED)
2839 unregister_netdev(vsi->netdev);
2840 err_register_netdev:
2842 free_netdev(vsi->netdev);
2846 ret = ice_aq_free_vsi(&pf->hw, &ctxt, false, NULL);
2848 dev_err(&vsi->back->pdev->dev,
2849 "Free VSI AQ call failed, err %d\n", ret);
2851 ice_vsi_put_qs(vsi);
2853 pf->q_left_tx += vsi->alloc_txq;
2854 pf->q_left_rx += vsi->alloc_rxq;
2861 * ice_vsi_add_vlan - Add vsi membership for given vlan
2862 * @vsi: the vsi being configured
2863 * @vid: vlan id to be added
2865 static int ice_vsi_add_vlan(struct ice_vsi *vsi, u16 vid)
2867 struct ice_fltr_list_entry *tmp;
2868 struct ice_pf *pf = vsi->back;
2869 LIST_HEAD(tmp_add_list);
2870 enum ice_status status;
2873 tmp = devm_kzalloc(&pf->pdev->dev, sizeof(*tmp), GFP_KERNEL);
2877 tmp->fltr_info.lkup_type = ICE_SW_LKUP_VLAN;
2878 tmp->fltr_info.fltr_act = ICE_FWD_TO_VSI;
2879 tmp->fltr_info.flag = ICE_FLTR_TX;
2880 tmp->fltr_info.src = vsi->vsi_num;
2881 tmp->fltr_info.fwd_id.vsi_id = vsi->vsi_num;
2882 tmp->fltr_info.l_data.vlan.vlan_id = vid;
2884 INIT_LIST_HEAD(&tmp->list_entry);
2885 list_add(&tmp->list_entry, &tmp_add_list);
2887 status = ice_add_vlan(&pf->hw, &tmp_add_list);
2890 dev_err(&pf->pdev->dev, "Failure Adding VLAN %d on VSI %i\n",
2894 ice_free_fltr_list(&pf->pdev->dev, &tmp_add_list);
2899 * ice_vlan_rx_add_vid - Add a vlan id filter to HW offload
2900 * @netdev: network interface to be adjusted
2901 * @proto: unused protocol
2902 * @vid: vlan id to be added
2904 * net_device_ops implementation for adding vlan ids
2906 static int ice_vlan_rx_add_vid(struct net_device *netdev,
2907 __always_unused __be16 proto, u16 vid)
2909 struct ice_netdev_priv *np = netdev_priv(netdev);
2910 struct ice_vsi *vsi = np->vsi;
2913 if (vid >= VLAN_N_VID) {
2914 netdev_err(netdev, "VLAN id requested %d is out of range %d\n",
2922 /* Add all VLAN ids including 0 to the switch filter. VLAN id 0 is
2923 * needed to continue allowing all untagged packets since VLAN prune
2924 * list is applied to all packets by the switch
2926 ret = ice_vsi_add_vlan(vsi, vid);
2929 set_bit(vid, vsi->active_vlans);
2935 * ice_vsi_kill_vlan - Remove VSI membership for a given VLAN
2936 * @vsi: the VSI being configured
2937 * @vid: VLAN id to be removed
2939 static void ice_vsi_kill_vlan(struct ice_vsi *vsi, u16 vid)
2941 struct ice_fltr_list_entry *list;
2942 struct ice_pf *pf = vsi->back;
2943 LIST_HEAD(tmp_add_list);
2945 list = devm_kzalloc(&pf->pdev->dev, sizeof(*list), GFP_KERNEL);
2949 list->fltr_info.lkup_type = ICE_SW_LKUP_VLAN;
2950 list->fltr_info.fwd_id.vsi_id = vsi->vsi_num;
2951 list->fltr_info.fltr_act = ICE_FWD_TO_VSI;
2952 list->fltr_info.l_data.vlan.vlan_id = vid;
2953 list->fltr_info.flag = ICE_FLTR_TX;
2954 list->fltr_info.src = vsi->vsi_num;
2956 INIT_LIST_HEAD(&list->list_entry);
2957 list_add(&list->list_entry, &tmp_add_list);
2959 if (ice_remove_vlan(&pf->hw, &tmp_add_list))
2960 dev_err(&pf->pdev->dev, "Error removing VLAN %d on vsi %i\n",
2963 ice_free_fltr_list(&pf->pdev->dev, &tmp_add_list);
2967 * ice_vlan_rx_kill_vid - Remove a vlan id filter from HW offload
2968 * @netdev: network interface to be adjusted
2969 * @proto: unused protocol
2970 * @vid: vlan id to be removed
2972 * net_device_ops implementation for removing vlan ids
2974 static int ice_vlan_rx_kill_vid(struct net_device *netdev,
2975 __always_unused __be16 proto, u16 vid)
2977 struct ice_netdev_priv *np = netdev_priv(netdev);
2978 struct ice_vsi *vsi = np->vsi;
2983 /* return code is ignored as there is nothing a user
2984 * can do about failure to remove and a log message was
2985 * already printed from the other function
2987 ice_vsi_kill_vlan(vsi, vid);
2989 clear_bit(vid, vsi->active_vlans);
2995 * ice_setup_pf_sw - Setup the HW switch on startup or after reset
2996 * @pf: board private structure
2998 * Returns 0 on success, negative value on failure
3000 static int ice_setup_pf_sw(struct ice_pf *pf)
3002 LIST_HEAD(tmp_add_list);
3003 u8 broadcast[ETH_ALEN];
3004 struct ice_vsi *vsi;
3007 if (!ice_is_reset_recovery_pending(pf->state)) {
3008 vsi = ice_vsi_setup(pf, ICE_VSI_PF, pf->hw.port_info);
3015 status = ice_vsi_reinit_setup(vsi);
3020 /* tmp_add_list contains a list of MAC addresses for which MAC
3021 * filters need to be programmed. Add the VSI's unicast MAC to
3024 status = ice_add_mac_to_list(vsi, &tmp_add_list,
3025 vsi->port_info->mac.perm_addr);
3029 /* VSI needs to receive broadcast traffic, so add the broadcast
3030 * MAC address to the list.
3032 eth_broadcast_addr(broadcast);
3033 status = ice_add_mac_to_list(vsi, &tmp_add_list, broadcast);
3037 /* program MAC filters for entries in tmp_add_list */
3038 status = ice_add_mac(&pf->hw, &tmp_add_list);
3040 dev_err(&pf->pdev->dev, "Could not add MAC filters\n");
3045 ice_free_fltr_list(&pf->pdev->dev, &tmp_add_list);
3049 ice_free_fltr_list(&pf->pdev->dev, &tmp_add_list);
3052 ice_vsi_free_q_vectors(vsi);
3053 if (vsi->netdev && vsi->netdev->reg_state == NETREG_REGISTERED)
3054 unregister_netdev(vsi->netdev);
3056 free_netdev(vsi->netdev);
3060 ice_vsi_delete(vsi);
3061 ice_vsi_put_qs(vsi);
3062 pf->q_left_tx += vsi->alloc_txq;
3063 pf->q_left_rx += vsi->alloc_rxq;
3070 * ice_determine_q_usage - Calculate queue distribution
3071 * @pf: board private structure
3073 * Return -ENOMEM if we don't get enough queues for all ports
3075 static void ice_determine_q_usage(struct ice_pf *pf)
3077 u16 q_left_tx, q_left_rx;
3079 q_left_tx = pf->hw.func_caps.common_cap.num_txq;
3080 q_left_rx = pf->hw.func_caps.common_cap.num_rxq;
3082 pf->num_lan_tx = min_t(int, q_left_tx, num_online_cpus());
3084 /* only 1 rx queue unless RSS is enabled */
3085 if (!test_bit(ICE_FLAG_RSS_ENA, pf->flags))
3088 pf->num_lan_rx = min_t(int, q_left_rx, num_online_cpus());
3090 pf->q_left_tx = q_left_tx - pf->num_lan_tx;
3091 pf->q_left_rx = q_left_rx - pf->num_lan_rx;
3095 * ice_deinit_pf - Unrolls initialziations done by ice_init_pf
3096 * @pf: board private structure to initialize
3098 static void ice_deinit_pf(struct ice_pf *pf)
3100 if (pf->serv_tmr.function)
3101 del_timer_sync(&pf->serv_tmr);
3102 if (pf->serv_task.func)
3103 cancel_work_sync(&pf->serv_task);
3104 mutex_destroy(&pf->sw_mutex);
3105 mutex_destroy(&pf->avail_q_mutex);
3109 * ice_init_pf - Initialize general software structures (struct ice_pf)
3110 * @pf: board private structure to initialize
3112 static void ice_init_pf(struct ice_pf *pf)
3114 bitmap_zero(pf->flags, ICE_PF_FLAGS_NBITS);
3115 set_bit(ICE_FLAG_MSIX_ENA, pf->flags);
3117 mutex_init(&pf->sw_mutex);
3118 mutex_init(&pf->avail_q_mutex);
3120 /* Clear avail_[t|r]x_qs bitmaps (set all to avail) */
3121 mutex_lock(&pf->avail_q_mutex);
3122 bitmap_zero(pf->avail_txqs, ICE_MAX_TXQS);
3123 bitmap_zero(pf->avail_rxqs, ICE_MAX_RXQS);
3124 mutex_unlock(&pf->avail_q_mutex);
3126 if (pf->hw.func_caps.common_cap.rss_table_size)
3127 set_bit(ICE_FLAG_RSS_ENA, pf->flags);
3129 /* setup service timer and periodic service task */
3130 timer_setup(&pf->serv_tmr, ice_service_timer, 0);
3131 pf->serv_tmr_period = HZ;
3132 INIT_WORK(&pf->serv_task, ice_service_task);
3133 clear_bit(__ICE_SERVICE_SCHED, pf->state);
3137 * ice_ena_msix_range - Request a range of MSIX vectors from the OS
3138 * @pf: board private structure
3140 * compute the number of MSIX vectors required (v_budget) and request from
3141 * the OS. Return the number of vectors reserved or negative on failure
3143 static int ice_ena_msix_range(struct ice_pf *pf)
3145 int v_left, v_actual, v_budget = 0;
3148 v_left = pf->hw.func_caps.common_cap.num_msix_vectors;
3150 /* reserve one vector for miscellaneous handler */
3155 /* reserve vectors for LAN traffic */
3156 pf->num_lan_msix = min_t(int, num_online_cpus(), v_left);
3157 v_budget += pf->num_lan_msix;
3159 pf->msix_entries = devm_kcalloc(&pf->pdev->dev, v_budget,
3160 sizeof(struct msix_entry), GFP_KERNEL);
3162 if (!pf->msix_entries) {
3167 for (i = 0; i < v_budget; i++)
3168 pf->msix_entries[i].entry = i;
3170 /* actually reserve the vectors */
3171 v_actual = pci_enable_msix_range(pf->pdev, pf->msix_entries,
3172 ICE_MIN_MSIX, v_budget);
3175 dev_err(&pf->pdev->dev, "unable to reserve MSI-X vectors\n");
3180 if (v_actual < v_budget) {
3181 dev_warn(&pf->pdev->dev,
3182 "not enough vectors. requested = %d, obtained = %d\n",
3183 v_budget, v_actual);
3184 if (v_actual >= (pf->num_lan_msix + 1)) {
3185 pf->num_avail_msix = v_actual - (pf->num_lan_msix + 1);
3186 } else if (v_actual >= 2) {
3187 pf->num_lan_msix = 1;
3188 pf->num_avail_msix = v_actual - 2;
3190 pci_disable_msix(pf->pdev);
3199 devm_kfree(&pf->pdev->dev, pf->msix_entries);
3203 pf->num_lan_msix = 0;
3204 clear_bit(ICE_FLAG_MSIX_ENA, pf->flags);
3209 * ice_dis_msix - Disable MSI-X interrupt setup in OS
3210 * @pf: board private structure
3212 static void ice_dis_msix(struct ice_pf *pf)
3214 pci_disable_msix(pf->pdev);
3215 devm_kfree(&pf->pdev->dev, pf->msix_entries);
3216 pf->msix_entries = NULL;
3217 clear_bit(ICE_FLAG_MSIX_ENA, pf->flags);
3221 * ice_init_interrupt_scheme - Determine proper interrupt scheme
3222 * @pf: board private structure to initialize
3224 static int ice_init_interrupt_scheme(struct ice_pf *pf)
3229 if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags))
3230 vectors = ice_ena_msix_range(pf);
3237 /* set up vector assignment tracking */
3238 size = sizeof(struct ice_res_tracker) + (sizeof(u16) * vectors);
3240 pf->irq_tracker = devm_kzalloc(&pf->pdev->dev, size, GFP_KERNEL);
3241 if (!pf->irq_tracker) {
3246 pf->irq_tracker->num_entries = vectors;
3252 * ice_clear_interrupt_scheme - Undo things done by ice_init_interrupt_scheme
3253 * @pf: board private structure
3255 static void ice_clear_interrupt_scheme(struct ice_pf *pf)
3257 if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags))
3260 if (pf->irq_tracker) {
3261 devm_kfree(&pf->pdev->dev, pf->irq_tracker);
3262 pf->irq_tracker = NULL;
3267 * ice_probe - Device initialization routine
3268 * @pdev: PCI device information struct
3269 * @ent: entry in ice_pci_tbl
3271 * Returns 0 on success, negative on failure
3273 static int ice_probe(struct pci_dev *pdev,
3274 const struct pci_device_id __always_unused *ent)
3280 /* this driver uses devres, see Documentation/driver-model/devres.txt */
3281 err = pcim_enable_device(pdev);
3285 err = pcim_iomap_regions(pdev, BIT(ICE_BAR0), pci_name(pdev));
3287 dev_err(&pdev->dev, "BAR0 I/O map error %d\n", err);
3291 pf = devm_kzalloc(&pdev->dev, sizeof(*pf), GFP_KERNEL);
3295 /* set up for high or low dma */
3296 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
3298 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
3300 dev_err(&pdev->dev, "DMA configuration failed: 0x%x\n", err);
3304 pci_enable_pcie_error_reporting(pdev);
3305 pci_set_master(pdev);
3308 pci_set_drvdata(pdev, pf);
3309 set_bit(__ICE_DOWN, pf->state);
3312 hw->hw_addr = pcim_iomap_table(pdev)[ICE_BAR0];
3314 hw->vendor_id = pdev->vendor;
3315 hw->device_id = pdev->device;
3316 pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
3317 hw->subsystem_vendor_id = pdev->subsystem_vendor;
3318 hw->subsystem_device_id = pdev->subsystem_device;
3319 hw->bus.device = PCI_SLOT(pdev->devfn);
3320 hw->bus.func = PCI_FUNC(pdev->devfn);
3321 ice_set_ctrlq_len(hw);
3323 pf->msg_enable = netif_msg_init(debug, ICE_DFLT_NETIF_M);
3325 #ifndef CONFIG_DYNAMIC_DEBUG
3327 hw->debug_mask = debug;
3330 err = ice_init_hw(hw);
3332 dev_err(&pdev->dev, "ice_init_hw failed: %d\n", err);
3334 goto err_exit_unroll;
3337 dev_info(&pdev->dev, "firmware %d.%d.%05d api %d.%d\n",
3338 hw->fw_maj_ver, hw->fw_min_ver, hw->fw_build,
3339 hw->api_maj_ver, hw->api_min_ver);
3343 ice_determine_q_usage(pf);
3345 pf->num_alloc_vsi = min_t(u16, ICE_MAX_VSI_ALLOC,
3346 hw->func_caps.guaranteed_num_vsi);
3347 if (!pf->num_alloc_vsi) {
3349 goto err_init_pf_unroll;
3352 pf->vsi = devm_kcalloc(&pdev->dev, pf->num_alloc_vsi,
3353 sizeof(struct ice_vsi *), GFP_KERNEL);
3356 goto err_init_pf_unroll;
3359 err = ice_init_interrupt_scheme(pf);
3362 "ice_init_interrupt_scheme failed: %d\n", err);
3364 goto err_init_interrupt_unroll;
3367 /* In case of MSIX we are going to setup the misc vector right here
3368 * to handle admin queue events etc. In case of legacy and MSI
3369 * the misc functionality and queue processing is combined in
3370 * the same vector and that gets setup at open.
3372 if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags)) {
3373 err = ice_req_irq_msix_misc(pf);
3376 "setup of misc vector failed: %d\n", err);
3377 goto err_init_interrupt_unroll;
3381 /* create switch struct for the switch element created by FW on boot */
3382 pf->first_sw = devm_kzalloc(&pdev->dev, sizeof(struct ice_sw),
3384 if (!pf->first_sw) {
3386 goto err_msix_misc_unroll;
3389 pf->first_sw->bridge_mode = BRIDGE_MODE_VEB;
3390 pf->first_sw->pf = pf;
3392 /* record the sw_id available for later use */
3393 pf->first_sw->sw_id = hw->port_info->sw_id;
3395 err = ice_setup_pf_sw(pf);
3398 "probe failed due to setup pf switch:%d\n", err);
3399 goto err_alloc_sw_unroll;
3402 /* Driver is mostly up */
3403 clear_bit(__ICE_DOWN, pf->state);
3405 /* since everything is good, start the service timer */
3406 mod_timer(&pf->serv_tmr, round_jiffies(jiffies + pf->serv_tmr_period));
3408 err = ice_init_link_events(pf->hw.port_info);
3410 dev_err(&pdev->dev, "ice_init_link_events failed: %d\n", err);
3411 goto err_alloc_sw_unroll;
3416 err_alloc_sw_unroll:
3417 set_bit(__ICE_DOWN, pf->state);
3418 devm_kfree(&pf->pdev->dev, pf->first_sw);
3419 err_msix_misc_unroll:
3420 ice_free_irq_msix_misc(pf);
3421 err_init_interrupt_unroll:
3422 ice_clear_interrupt_scheme(pf);
3423 devm_kfree(&pdev->dev, pf->vsi);
3428 pci_disable_pcie_error_reporting(pdev);
3433 * ice_remove - Device removal routine
3434 * @pdev: PCI device information struct
3436 static void ice_remove(struct pci_dev *pdev)
3438 struct ice_pf *pf = pci_get_drvdata(pdev);
3445 set_bit(__ICE_DOWN, pf->state);
3447 for (i = 0; i < pf->num_alloc_vsi; i++) {
3451 err = ice_vsi_release(pf->vsi[i]);
3453 dev_dbg(&pf->pdev->dev, "Failed to release VSI index %d (err %d)\n",
3457 ice_free_irq_msix_misc(pf);
3458 ice_clear_interrupt_scheme(pf);
3460 ice_deinit_hw(&pf->hw);
3461 pci_disable_pcie_error_reporting(pdev);
3464 /* ice_pci_tbl - PCI Device ID Table
3466 * Wildcard entries (PCI_ANY_ID) should come last
3467 * Last entry must be all 0s
3469 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
3470 * Class, Class Mask, private data (not used) }
3472 static const struct pci_device_id ice_pci_tbl[] = {
3473 { PCI_VDEVICE(INTEL, ICE_DEV_ID_C810_BACKPLANE), 0 },
3474 { PCI_VDEVICE(INTEL, ICE_DEV_ID_C810_QSFP), 0 },
3475 { PCI_VDEVICE(INTEL, ICE_DEV_ID_C810_SFP), 0 },
3476 { PCI_VDEVICE(INTEL, ICE_DEV_ID_C810_10G_BASE_T), 0 },
3477 { PCI_VDEVICE(INTEL, ICE_DEV_ID_C810_SGMII), 0 },
3478 /* required last entry */
3481 MODULE_DEVICE_TABLE(pci, ice_pci_tbl);
3483 static struct pci_driver ice_driver = {
3484 .name = KBUILD_MODNAME,
3485 .id_table = ice_pci_tbl,
3487 .remove = ice_remove,
3491 * ice_module_init - Driver registration routine
3493 * ice_module_init is the first routine called when the driver is
3494 * loaded. All it does is register with the PCI subsystem.
3496 static int __init ice_module_init(void)
3500 pr_info("%s - version %s\n", ice_driver_string, ice_drv_ver);
3501 pr_info("%s\n", ice_copyright);
3503 ice_wq = alloc_ordered_workqueue("%s", WQ_MEM_RECLAIM, KBUILD_MODNAME);
3505 pr_err("Failed to create workqueue\n");
3509 status = pci_register_driver(&ice_driver);
3511 pr_err("failed to register pci driver, err %d\n", status);
3512 destroy_workqueue(ice_wq);
3517 module_init(ice_module_init);
3520 * ice_module_exit - Driver exit cleanup routine
3522 * ice_module_exit is called just before the driver is removed
3525 static void __exit ice_module_exit(void)
3527 pci_unregister_driver(&ice_driver);
3528 destroy_workqueue(ice_wq);
3529 pr_info("module unloaded\n");
3531 module_exit(ice_module_exit);
3534 * ice_set_mac_address - NDO callback to set mac address
3535 * @netdev: network interface device structure
3536 * @pi: pointer to an address structure
3538 * Returns 0 on success, negative on failure
3540 static int ice_set_mac_address(struct net_device *netdev, void *pi)
3542 struct ice_netdev_priv *np = netdev_priv(netdev);
3543 struct ice_vsi *vsi = np->vsi;
3544 struct ice_pf *pf = vsi->back;
3545 struct ice_hw *hw = &pf->hw;
3546 struct sockaddr *addr = pi;
3547 enum ice_status status;
3548 LIST_HEAD(a_mac_list);
3549 LIST_HEAD(r_mac_list);
3554 mac = (u8 *)addr->sa_data;
3556 if (!is_valid_ether_addr(mac))
3557 return -EADDRNOTAVAIL;
3559 if (ether_addr_equal(netdev->dev_addr, mac)) {
3560 netdev_warn(netdev, "already using mac %pM\n", mac);
3564 if (test_bit(__ICE_DOWN, pf->state) ||
3565 ice_is_reset_recovery_pending(pf->state)) {
3566 netdev_err(netdev, "can't set mac %pM. device not ready\n",
3571 /* When we change the mac address we also have to change the mac address
3572 * based filter rules that were created previously for the old mac
3573 * address. So first, we remove the old filter rule using ice_remove_mac
3574 * and then create a new filter rule using ice_add_mac. Note that for
3575 * both these operations, we first need to form a "list" of mac
3576 * addresses (even though in this case, we have only 1 mac address to be
3577 * added/removed) and this done using ice_add_mac_to_list. Depending on
3578 * the ensuing operation this "list" of mac addresses is either to be
3579 * added or removed from the filter.
3581 err = ice_add_mac_to_list(vsi, &r_mac_list, netdev->dev_addr);
3583 err = -EADDRNOTAVAIL;
3587 status = ice_remove_mac(hw, &r_mac_list);
3589 err = -EADDRNOTAVAIL;
3593 err = ice_add_mac_to_list(vsi, &a_mac_list, mac);
3595 err = -EADDRNOTAVAIL;
3599 status = ice_add_mac(hw, &a_mac_list);
3601 err = -EADDRNOTAVAIL;
3606 /* free list entries */
3607 ice_free_fltr_list(&pf->pdev->dev, &r_mac_list);
3608 ice_free_fltr_list(&pf->pdev->dev, &a_mac_list);
3611 netdev_err(netdev, "can't set mac %pM. filter update failed\n",
3616 /* change the netdev's mac address */
3617 memcpy(netdev->dev_addr, mac, netdev->addr_len);
3618 netdev_dbg(vsi->netdev, "updated mac address to %pM\n",
3621 /* write new mac address to the firmware */
3622 flags = ICE_AQC_MAN_MAC_UPDATE_LAA_WOL;
3623 status = ice_aq_manage_mac_write(hw, mac, flags, NULL);
3625 netdev_err(netdev, "can't set mac %pM. write to firmware failed.\n",
3632 * ice_set_rx_mode - NDO callback to set the netdev filters
3633 * @netdev: network interface device structure
3635 static void ice_set_rx_mode(struct net_device *netdev)
3637 struct ice_netdev_priv *np = netdev_priv(netdev);
3638 struct ice_vsi *vsi = np->vsi;
3643 /* Set the flags to synchronize filters
3644 * ndo_set_rx_mode may be triggered even without a change in netdev
3647 set_bit(ICE_VSI_FLAG_UMAC_FLTR_CHANGED, vsi->flags);
3648 set_bit(ICE_VSI_FLAG_MMAC_FLTR_CHANGED, vsi->flags);
3649 set_bit(ICE_FLAG_FLTR_SYNC, vsi->back->flags);
3651 /* schedule our worker thread which will take care of
3652 * applying the new filter changes
3654 ice_service_task_schedule(vsi->back);
3658 * ice_fdb_add - add an entry to the hardware database
3659 * @ndm: the input from the stack
3660 * @tb: pointer to array of nladdr (unused)
3661 * @dev: the net device pointer
3662 * @addr: the MAC address entry being added
3664 * @flags: instructions from stack about fdb operation
3666 static int ice_fdb_add(struct ndmsg *ndm, struct nlattr __always_unused *tb[],
3667 struct net_device *dev, const unsigned char *addr,
3673 netdev_err(dev, "VLANs aren't supported yet for dev_uc|mc_add()\n");
3676 if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) {
3677 netdev_err(dev, "FDB only supports static addresses\n");
3681 if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr))
3682 err = dev_uc_add_excl(dev, addr);
3683 else if (is_multicast_ether_addr(addr))
3684 err = dev_mc_add_excl(dev, addr);
3688 /* Only return duplicate errors if NLM_F_EXCL is set */
3689 if (err == -EEXIST && !(flags & NLM_F_EXCL))
3696 * ice_fdb_del - delete an entry from the hardware database
3697 * @ndm: the input from the stack
3698 * @tb: pointer to array of nladdr (unused)
3699 * @dev: the net device pointer
3700 * @addr: the MAC address entry being added
3703 static int ice_fdb_del(struct ndmsg *ndm, __always_unused struct nlattr *tb[],
3704 struct net_device *dev, const unsigned char *addr,
3705 __always_unused u16 vid)
3709 if (ndm->ndm_state & NUD_PERMANENT) {
3710 netdev_err(dev, "FDB only supports static addresses\n");
3714 if (is_unicast_ether_addr(addr))
3715 err = dev_uc_del(dev, addr);
3716 else if (is_multicast_ether_addr(addr))
3717 err = dev_mc_del(dev, addr);
3725 * ice_vsi_manage_vlan_insertion - Manage VLAN insertion for the VSI for Tx
3726 * @vsi: the vsi being changed
3728 static int ice_vsi_manage_vlan_insertion(struct ice_vsi *vsi)
3730 struct device *dev = &vsi->back->pdev->dev;
3731 struct ice_hw *hw = &vsi->back->hw;
3732 struct ice_vsi_ctx ctxt = { 0 };
3733 enum ice_status status;
3735 /* Here we are configuring the VSI to let the driver add VLAN tags by
3736 * setting vlan_flags to ICE_AQ_VSI_VLAN_MODE_ALL. The actual VLAN tag
3737 * insertion happens in the Tx hot path, in ice_tx_map.
3739 ctxt.info.vlan_flags = ICE_AQ_VSI_VLAN_MODE_ALL;
3741 ctxt.info.valid_sections = cpu_to_le16(ICE_AQ_VSI_PROP_VLAN_VALID);
3742 ctxt.vsi_num = vsi->vsi_num;
3744 status = ice_aq_update_vsi(hw, &ctxt, NULL);
3746 dev_err(dev, "update VSI for VLAN insert failed, err %d aq_err %d\n",
3747 status, hw->adminq.sq_last_status);
3751 vsi->info.vlan_flags = ctxt.info.vlan_flags;
3756 * ice_vsi_manage_vlan_stripping - Manage VLAN stripping for the VSI for Rx
3757 * @vsi: the vsi being changed
3758 * @ena: boolean value indicating if this is a enable or disable request
3760 static int ice_vsi_manage_vlan_stripping(struct ice_vsi *vsi, bool ena)
3762 struct device *dev = &vsi->back->pdev->dev;
3763 struct ice_hw *hw = &vsi->back->hw;
3764 struct ice_vsi_ctx ctxt = { 0 };
3765 enum ice_status status;
3767 /* Here we are configuring what the VSI should do with the VLAN tag in
3768 * the Rx packet. We can either leave the tag in the packet or put it in
3769 * the Rx descriptor.
3772 /* Strip VLAN tag from Rx packet and put it in the desc */
3773 ctxt.info.vlan_flags = ICE_AQ_VSI_VLAN_EMOD_STR_BOTH;
3775 /* Disable stripping. Leave tag in packet */
3776 ctxt.info.vlan_flags = ICE_AQ_VSI_VLAN_EMOD_NOTHING;
3779 /* Allow all packets untagged/tagged */
3780 ctxt.info.vlan_flags |= ICE_AQ_VSI_VLAN_MODE_ALL;
3782 ctxt.info.valid_sections = cpu_to_le16(ICE_AQ_VSI_PROP_VLAN_VALID);
3783 ctxt.vsi_num = vsi->vsi_num;
3785 status = ice_aq_update_vsi(hw, &ctxt, NULL);
3787 dev_err(dev, "update VSI for VALN strip failed, ena = %d err %d aq_err %d\n",
3788 ena, status, hw->adminq.sq_last_status);
3792 vsi->info.vlan_flags = ctxt.info.vlan_flags;
3797 * ice_set_features - set the netdev feature flags
3798 * @netdev: ptr to the netdev being adjusted
3799 * @features: the feature set that the stack is suggesting
3801 static int ice_set_features(struct net_device *netdev,
3802 netdev_features_t features)
3804 struct ice_netdev_priv *np = netdev_priv(netdev);
3805 struct ice_vsi *vsi = np->vsi;
3808 if ((features & NETIF_F_HW_VLAN_CTAG_RX) &&
3809 !(netdev->features & NETIF_F_HW_VLAN_CTAG_RX))
3810 ret = ice_vsi_manage_vlan_stripping(vsi, true);
3811 else if (!(features & NETIF_F_HW_VLAN_CTAG_RX) &&
3812 (netdev->features & NETIF_F_HW_VLAN_CTAG_RX))
3813 ret = ice_vsi_manage_vlan_stripping(vsi, false);
3814 else if ((features & NETIF_F_HW_VLAN_CTAG_TX) &&
3815 !(netdev->features & NETIF_F_HW_VLAN_CTAG_TX))
3816 ret = ice_vsi_manage_vlan_insertion(vsi);
3817 else if (!(features & NETIF_F_HW_VLAN_CTAG_TX) &&
3818 (netdev->features & NETIF_F_HW_VLAN_CTAG_TX))
3819 ret = ice_vsi_manage_vlan_insertion(vsi);
3825 * ice_vsi_vlan_setup - Setup vlan offload properties on a VSI
3826 * @vsi: VSI to setup vlan properties for
3828 static int ice_vsi_vlan_setup(struct ice_vsi *vsi)
3832 if (vsi->netdev->features & NETIF_F_HW_VLAN_CTAG_RX)
3833 ret = ice_vsi_manage_vlan_stripping(vsi, true);
3834 if (vsi->netdev->features & NETIF_F_HW_VLAN_CTAG_TX)
3835 ret = ice_vsi_manage_vlan_insertion(vsi);
3841 * ice_restore_vlan - Reinstate VLANs when vsi/netdev comes back up
3842 * @vsi: the VSI being brought back up
3844 static int ice_restore_vlan(struct ice_vsi *vsi)
3852 err = ice_vsi_vlan_setup(vsi);
3856 for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID) {
3857 err = ice_vlan_rx_add_vid(vsi->netdev, htons(ETH_P_8021Q), vid);
3866 * ice_setup_tx_ctx - setup a struct ice_tlan_ctx instance
3867 * @ring: The Tx ring to configure
3868 * @tlan_ctx: Pointer to the Tx LAN queue context structure to be initialized
3869 * @pf_q: queue index in the PF space
3871 * Configure the Tx descriptor ring in TLAN context.
3874 ice_setup_tx_ctx(struct ice_ring *ring, struct ice_tlan_ctx *tlan_ctx, u16 pf_q)
3876 struct ice_vsi *vsi = ring->vsi;
3877 struct ice_hw *hw = &vsi->back->hw;
3879 tlan_ctx->base = ring->dma >> ICE_TLAN_CTX_BASE_S;
3881 tlan_ctx->port_num = vsi->port_info->lport;
3883 /* Transmit Queue Length */
3884 tlan_ctx->qlen = ring->count;
3887 tlan_ctx->pf_num = hw->pf_id;
3889 /* queue belongs to a specific VSI type
3890 * VF / VM index should be programmed per vmvf_type setting:
3891 * for vmvf_type = VF, it is VF number between 0-256
3892 * for vmvf_type = VM, it is VM number between 0-767
3893 * for PF or EMP this field should be set to zero
3895 switch (vsi->type) {
3897 tlan_ctx->vmvf_type = ICE_TLAN_CTX_VMVF_TYPE_PF;
3903 /* make sure the context is associated with the right VSI */
3904 tlan_ctx->src_vsi = vsi->vsi_num;
3906 tlan_ctx->tso_ena = ICE_TX_LEGACY;
3907 tlan_ctx->tso_qnum = pf_q;
3909 /* Legacy or Advanced Host Interface:
3910 * 0: Advanced Host Interface
3911 * 1: Legacy Host Interface
3913 tlan_ctx->legacy_int = ICE_TX_LEGACY;
3917 * ice_vsi_cfg_txqs - Configure the VSI for Tx
3918 * @vsi: the VSI being configured
3920 * Return 0 on success and a negative value on error
3921 * Configure the Tx VSI for operation.
3923 static int ice_vsi_cfg_txqs(struct ice_vsi *vsi)
3925 struct ice_aqc_add_tx_qgrp *qg_buf;
3926 struct ice_aqc_add_txqs_perq *txq;
3927 struct ice_pf *pf = vsi->back;
3928 enum ice_status status;
3929 u16 buf_len, i, pf_q;
3930 int err = 0, tc = 0;
3933 buf_len = sizeof(struct ice_aqc_add_tx_qgrp);
3934 qg_buf = devm_kzalloc(&pf->pdev->dev, buf_len, GFP_KERNEL);
3938 if (vsi->num_txq > ICE_MAX_TXQ_PER_TXQG) {
3942 qg_buf->num_txqs = 1;
3945 /* set up and configure the tx queues */
3946 ice_for_each_txq(vsi, i) {
3947 struct ice_tlan_ctx tlan_ctx = { 0 };
3949 pf_q = vsi->txq_map[i];
3950 ice_setup_tx_ctx(vsi->tx_rings[i], &tlan_ctx, pf_q);
3951 /* copy context contents into the qg_buf */
3952 qg_buf->txqs[0].txq_id = cpu_to_le16(pf_q);
3953 ice_set_ctx((u8 *)&tlan_ctx, qg_buf->txqs[0].txq_ctx,
3956 /* init queue specific tail reg. It is referred as transmit
3957 * comm scheduler queue doorbell.
3959 vsi->tx_rings[i]->tail = pf->hw.hw_addr + QTX_COMM_DBELL(pf_q);
3960 status = ice_ena_vsi_txq(vsi->port_info, vsi->vsi_num, tc,
3961 num_q_grps, qg_buf, buf_len, NULL);
3963 dev_err(&vsi->back->pdev->dev,
3964 "Failed to set LAN Tx queue context, error: %d\n",
3970 /* Add Tx Queue TEID into the VSI tx ring from the response
3971 * This will complete configuring and enabling the queue.
3973 txq = &qg_buf->txqs[0];
3974 if (pf_q == le16_to_cpu(txq->txq_id))
3975 vsi->tx_rings[i]->txq_teid =
3976 le32_to_cpu(txq->q_teid);
3979 devm_kfree(&pf->pdev->dev, qg_buf);
3984 * ice_setup_rx_ctx - Configure a receive ring context
3985 * @ring: The Rx ring to configure
3987 * Configure the Rx descriptor ring in RLAN context.
3989 static int ice_setup_rx_ctx(struct ice_ring *ring)
3991 struct ice_vsi *vsi = ring->vsi;
3992 struct ice_hw *hw = &vsi->back->hw;
3993 u32 rxdid = ICE_RXDID_FLEX_NIC;
3994 struct ice_rlan_ctx rlan_ctx;
3999 /* what is RX queue number in global space of 2K rx queues */
4000 pf_q = vsi->rxq_map[ring->q_index];
4002 /* clear the context structure first */
4003 memset(&rlan_ctx, 0, sizeof(rlan_ctx));
4005 rlan_ctx.base = ring->dma >> ICE_RLAN_BASE_S;
4007 rlan_ctx.qlen = ring->count;
4009 /* Receive Packet Data Buffer Size.
4010 * The Packet Data Buffer Size is defined in 128 byte units.
4012 rlan_ctx.dbuf = vsi->rx_buf_len >> ICE_RLAN_CTX_DBUF_S;
4014 /* use 32 byte descriptors */
4017 /* Strip the Ethernet CRC bytes before the packet is posted to host
4020 rlan_ctx.crcstrip = 1;
4022 /* L2TSEL flag defines the reported L2 Tags in the receive descriptor */
4023 rlan_ctx.l2tsel = 1;
4025 rlan_ctx.dtype = ICE_RX_DTYPE_NO_SPLIT;
4026 rlan_ctx.hsplit_0 = ICE_RLAN_RX_HSPLIT_0_NO_SPLIT;
4027 rlan_ctx.hsplit_1 = ICE_RLAN_RX_HSPLIT_1_NO_SPLIT;
4029 /* This controls whether VLAN is stripped from inner headers
4030 * The VLAN in the inner L2 header is stripped to the receive
4031 * descriptor if enabled by this flag.
4033 rlan_ctx.showiv = 0;
4035 /* Max packet size for this queue - must not be set to a larger value
4038 rlan_ctx.rxmax = min_t(u16, vsi->max_frame,
4039 ICE_MAX_CHAINED_RX_BUFS * vsi->rx_buf_len);
4041 /* Rx queue threshold in units of 64 */
4042 rlan_ctx.lrxqthresh = 1;
4044 /* Enable Flexible Descriptors in the queue context which
4045 * allows this driver to select a specific receive descriptor format
4047 regval = rd32(hw, QRXFLXP_CNTXT(pf_q));
4048 regval |= (rxdid << QRXFLXP_CNTXT_RXDID_IDX_S) &
4049 QRXFLXP_CNTXT_RXDID_IDX_M;
4051 /* increasing context priority to pick up profile id;
4052 * default is 0x01; setting to 0x03 to ensure profile
4053 * is programming if prev context is of same priority
4055 regval |= (0x03 << QRXFLXP_CNTXT_RXDID_PRIO_S) &
4056 QRXFLXP_CNTXT_RXDID_PRIO_M;
4058 wr32(hw, QRXFLXP_CNTXT(pf_q), regval);
4060 /* Absolute queue number out of 2K needs to be passed */
4061 err = ice_write_rxq_ctx(hw, &rlan_ctx, pf_q);
4063 dev_err(&vsi->back->pdev->dev,
4064 "Failed to set LAN Rx queue context for absolute Rx queue %d error: %d\n",
4069 /* init queue specific tail register */
4070 ring->tail = hw->hw_addr + QRX_TAIL(pf_q);
4071 writel(0, ring->tail);
4072 ice_alloc_rx_bufs(ring, ICE_DESC_UNUSED(ring));
4078 * ice_vsi_cfg_rxqs - Configure the VSI for Rx
4079 * @vsi: the VSI being configured
4081 * Return 0 on success and a negative value on error
4082 * Configure the Rx VSI for operation.
4084 static int ice_vsi_cfg_rxqs(struct ice_vsi *vsi)
4089 if (vsi->netdev && vsi->netdev->mtu > ETH_DATA_LEN)
4090 vsi->max_frame = vsi->netdev->mtu +
4091 ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
4093 vsi->max_frame = ICE_RXBUF_2048;
4095 vsi->rx_buf_len = ICE_RXBUF_2048;
4096 /* set up individual rings */
4097 for (i = 0; i < vsi->num_rxq && !err; i++)
4098 err = ice_setup_rx_ctx(vsi->rx_rings[i]);
4101 dev_err(&vsi->back->pdev->dev, "ice_setup_rx_ctx failed\n");
4108 * ice_vsi_cfg - Setup the VSI
4109 * @vsi: the VSI being configured
4111 * Return 0 on success and negative value on error
4113 static int ice_vsi_cfg(struct ice_vsi *vsi)
4118 ice_set_rx_mode(vsi->netdev);
4119 err = ice_restore_vlan(vsi);
4124 err = ice_vsi_cfg_txqs(vsi);
4126 err = ice_vsi_cfg_rxqs(vsi);
4132 * ice_vsi_stop_tx_rings - Disable Tx rings
4133 * @vsi: the VSI being configured
4135 static int ice_vsi_stop_tx_rings(struct ice_vsi *vsi)
4137 struct ice_pf *pf = vsi->back;
4138 struct ice_hw *hw = &pf->hw;
4139 enum ice_status status;
4144 if (vsi->num_txq > ICE_LAN_TXQ_MAX_QDIS)
4147 q_teids = devm_kcalloc(&pf->pdev->dev, vsi->num_txq, sizeof(*q_teids),
4152 q_ids = devm_kcalloc(&pf->pdev->dev, vsi->num_txq, sizeof(*q_ids),
4156 goto err_alloc_q_ids;
4159 /* set up the tx queue list to be disabled */
4160 ice_for_each_txq(vsi, i) {
4163 if (!vsi->tx_rings || !vsi->tx_rings[i]) {
4168 q_ids[i] = vsi->txq_map[i];
4169 q_teids[i] = vsi->tx_rings[i]->txq_teid;
4171 /* clear cause_ena bit for disabled queues */
4172 val = rd32(hw, QINT_TQCTL(vsi->tx_rings[i]->reg_idx));
4173 val &= ~QINT_TQCTL_CAUSE_ENA_M;
4174 wr32(hw, QINT_TQCTL(vsi->tx_rings[i]->reg_idx), val);
4176 /* software is expected to wait for 100 ns */
4179 /* trigger a software interrupt for the vector associated to
4180 * the queue to schedule napi handler
4182 v_idx = vsi->tx_rings[i]->q_vector->v_idx;
4183 wr32(hw, GLINT_DYN_CTL(vsi->base_vector + v_idx),
4184 GLINT_DYN_CTL_SWINT_TRIG_M | GLINT_DYN_CTL_INTENA_MSK_M);
4186 status = ice_dis_vsi_txq(vsi->port_info, vsi->num_txq, q_ids, q_teids,
4189 dev_err(&pf->pdev->dev,
4190 "Failed to disable LAN Tx queues, error: %d\n",
4196 devm_kfree(&pf->pdev->dev, q_ids);
4199 devm_kfree(&pf->pdev->dev, q_teids);
4205 * ice_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled
4206 * @pf: the PF being configured
4207 * @pf_q: the PF queue
4208 * @ena: enable or disable state of the queue
4210 * This routine will wait for the given Rx queue of the PF to reach the
4211 * enabled or disabled state.
4212 * Returns -ETIMEDOUT in case of failing to reach the requested state after
4213 * multiple retries; else will return 0 in case of success.
4215 static int ice_pf_rxq_wait(struct ice_pf *pf, int pf_q, bool ena)
4219 for (i = 0; i < ICE_Q_WAIT_RETRY_LIMIT; i++) {
4220 u32 rx_reg = rd32(&pf->hw, QRX_CTRL(pf_q));
4222 if (ena == !!(rx_reg & QRX_CTRL_QENA_STAT_M))
4225 usleep_range(10, 20);
4227 if (i >= ICE_Q_WAIT_RETRY_LIMIT)
4234 * ice_vsi_ctrl_rx_rings - Start or stop a VSI's rx rings
4235 * @vsi: the VSI being configured
4236 * @ena: start or stop the rx rings
4238 static int ice_vsi_ctrl_rx_rings(struct ice_vsi *vsi, bool ena)
4240 struct ice_pf *pf = vsi->back;
4241 struct ice_hw *hw = &pf->hw;
4244 for (i = 0; i < vsi->num_rxq; i++) {
4245 int pf_q = vsi->rxq_map[i];
4248 for (j = 0; j < ICE_Q_WAIT_MAX_RETRY; j++) {
4249 rx_reg = rd32(hw, QRX_CTRL(pf_q));
4250 if (((rx_reg >> QRX_CTRL_QENA_REQ_S) & 1) ==
4251 ((rx_reg >> QRX_CTRL_QENA_STAT_S) & 1))
4253 usleep_range(1000, 2000);
4256 /* Skip if the queue is already in the requested state */
4257 if (ena == !!(rx_reg & QRX_CTRL_QENA_STAT_M))
4260 /* turn on/off the queue */
4262 rx_reg |= QRX_CTRL_QENA_REQ_M;
4264 rx_reg &= ~QRX_CTRL_QENA_REQ_M;
4265 wr32(hw, QRX_CTRL(pf_q), rx_reg);
4267 /* wait for the change to finish */
4268 ret = ice_pf_rxq_wait(pf, pf_q, ena);
4270 dev_err(&pf->pdev->dev,
4271 "VSI idx %d Rx ring %d %sable timeout\n",
4272 vsi->idx, pf_q, (ena ? "en" : "dis"));
4281 * ice_vsi_start_rx_rings - start VSI's rx rings
4282 * @vsi: the VSI whose rings are to be started
4284 * Returns 0 on success and a negative value on error
4286 static int ice_vsi_start_rx_rings(struct ice_vsi *vsi)
4288 return ice_vsi_ctrl_rx_rings(vsi, true);
4292 * ice_vsi_stop_rx_rings - stop VSI's rx rings
4295 * Returns 0 on success and a negative value on error
4297 static int ice_vsi_stop_rx_rings(struct ice_vsi *vsi)
4299 return ice_vsi_ctrl_rx_rings(vsi, false);
4303 * ice_vsi_stop_tx_rx_rings - stop VSI's tx and rx rings
4305 * Returns 0 on success and a negative value on error
4307 static int ice_vsi_stop_tx_rx_rings(struct ice_vsi *vsi)
4311 err_tx = ice_vsi_stop_tx_rings(vsi);
4313 dev_dbg(&vsi->back->pdev->dev, "Failed to disable Tx rings\n");
4315 err_rx = ice_vsi_stop_rx_rings(vsi);
4317 dev_dbg(&vsi->back->pdev->dev, "Failed to disable Rx rings\n");
4319 if (err_tx || err_rx)
4326 * ice_napi_enable_all - Enable NAPI for all q_vectors in the VSI
4327 * @vsi: the VSI being configured
4329 static void ice_napi_enable_all(struct ice_vsi *vsi)
4336 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++)
4337 napi_enable(&vsi->q_vectors[q_idx]->napi);
4341 * ice_up_complete - Finish the last steps of bringing up a connection
4342 * @vsi: The VSI being configured
4344 * Return 0 on success and negative value on error
4346 static int ice_up_complete(struct ice_vsi *vsi)
4348 struct ice_pf *pf = vsi->back;
4351 if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags))
4352 ice_vsi_cfg_msix(vsi);
4356 /* Enable only Rx rings, Tx rings were enabled by the FW when the
4357 * Tx queue group list was configured and the context bits were
4358 * programmed using ice_vsi_cfg_txqs
4360 err = ice_vsi_start_rx_rings(vsi);
4364 clear_bit(__ICE_DOWN, vsi->state);
4365 ice_napi_enable_all(vsi);
4366 ice_vsi_ena_irq(vsi);
4368 if (vsi->port_info &&
4369 (vsi->port_info->phy.link_info.link_info & ICE_AQ_LINK_UP) &&
4371 ice_print_link_msg(vsi, true);
4372 netif_tx_start_all_queues(vsi->netdev);
4373 netif_carrier_on(vsi->netdev);
4376 ice_service_task_schedule(pf);
4382 * ice_up - Bring the connection back up after being down
4383 * @vsi: VSI being configured
4385 int ice_up(struct ice_vsi *vsi)
4389 err = ice_vsi_cfg(vsi);
4391 err = ice_up_complete(vsi);
4397 * ice_fetch_u64_stats_per_ring - get packets and bytes stats per ring
4398 * @ring: Tx or Rx ring to read stats from
4399 * @pkts: packets stats counter
4400 * @bytes: bytes stats counter
4402 * This function fetches stats from the ring considering the atomic operations
4403 * that needs to be performed to read u64 values in 32 bit machine.
4405 static void ice_fetch_u64_stats_per_ring(struct ice_ring *ring, u64 *pkts,
4415 start = u64_stats_fetch_begin_irq(&ring->syncp);
4416 *pkts = ring->stats.pkts;
4417 *bytes = ring->stats.bytes;
4418 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
4422 * ice_stat_update40 - read 40 bit stat from the chip and update stat values
4423 * @hw: ptr to the hardware info
4424 * @hireg: high 32 bit HW register to read from
4425 * @loreg: low 32 bit HW register to read from
4426 * @prev_stat_loaded: bool to specify if previous stats are loaded
4427 * @prev_stat: ptr to previous loaded stat value
4428 * @cur_stat: ptr to current stat value
4430 static void ice_stat_update40(struct ice_hw *hw, u32 hireg, u32 loreg,
4431 bool prev_stat_loaded, u64 *prev_stat,
4436 new_data = rd32(hw, loreg);
4437 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32;
4439 /* device stats are not reset at PFR, they likely will not be zeroed
4440 * when the driver starts. So save the first values read and use them as
4441 * offsets to be subtracted from the raw values in order to report stats
4442 * that count from zero.
4444 if (!prev_stat_loaded)
4445 *prev_stat = new_data;
4446 if (likely(new_data >= *prev_stat))
4447 *cur_stat = new_data - *prev_stat;
4449 /* to manage the potential roll-over */
4450 *cur_stat = (new_data + BIT_ULL(40)) - *prev_stat;
4451 *cur_stat &= 0xFFFFFFFFFFULL;
4455 * ice_stat_update32 - read 32 bit stat from the chip and update stat values
4456 * @hw: ptr to the hardware info
4457 * @reg: HW register to read from
4458 * @prev_stat_loaded: bool to specify if previous stats are loaded
4459 * @prev_stat: ptr to previous loaded stat value
4460 * @cur_stat: ptr to current stat value
4462 static void ice_stat_update32(struct ice_hw *hw, u32 reg, bool prev_stat_loaded,
4463 u64 *prev_stat, u64 *cur_stat)
4467 new_data = rd32(hw, reg);
4469 /* device stats are not reset at PFR, they likely will not be zeroed
4470 * when the driver starts. So save the first values read and use them as
4471 * offsets to be subtracted from the raw values in order to report stats
4472 * that count from zero.
4474 if (!prev_stat_loaded)
4475 *prev_stat = new_data;
4476 if (likely(new_data >= *prev_stat))
4477 *cur_stat = new_data - *prev_stat;
4479 /* to manage the potential roll-over */
4480 *cur_stat = (new_data + BIT_ULL(32)) - *prev_stat;
4484 * ice_update_eth_stats - Update VSI-specific ethernet statistics counters
4485 * @vsi: the VSI to be updated
4487 static void ice_update_eth_stats(struct ice_vsi *vsi)
4489 struct ice_eth_stats *prev_es, *cur_es;
4490 struct ice_hw *hw = &vsi->back->hw;
4491 u16 vsi_num = vsi->vsi_num; /* HW absolute index of a VSI */
4493 prev_es = &vsi->eth_stats_prev;
4494 cur_es = &vsi->eth_stats;
4496 ice_stat_update40(hw, GLV_GORCH(vsi_num), GLV_GORCL(vsi_num),
4497 vsi->stat_offsets_loaded, &prev_es->rx_bytes,
4500 ice_stat_update40(hw, GLV_UPRCH(vsi_num), GLV_UPRCL(vsi_num),
4501 vsi->stat_offsets_loaded, &prev_es->rx_unicast,
4502 &cur_es->rx_unicast);
4504 ice_stat_update40(hw, GLV_MPRCH(vsi_num), GLV_MPRCL(vsi_num),
4505 vsi->stat_offsets_loaded, &prev_es->rx_multicast,
4506 &cur_es->rx_multicast);
4508 ice_stat_update40(hw, GLV_BPRCH(vsi_num), GLV_BPRCL(vsi_num),
4509 vsi->stat_offsets_loaded, &prev_es->rx_broadcast,
4510 &cur_es->rx_broadcast);
4512 ice_stat_update32(hw, GLV_RDPC(vsi_num), vsi->stat_offsets_loaded,
4513 &prev_es->rx_discards, &cur_es->rx_discards);
4515 ice_stat_update40(hw, GLV_GOTCH(vsi_num), GLV_GOTCL(vsi_num),
4516 vsi->stat_offsets_loaded, &prev_es->tx_bytes,
4519 ice_stat_update40(hw, GLV_UPTCH(vsi_num), GLV_UPTCL(vsi_num),
4520 vsi->stat_offsets_loaded, &prev_es->tx_unicast,
4521 &cur_es->tx_unicast);
4523 ice_stat_update40(hw, GLV_MPTCH(vsi_num), GLV_MPTCL(vsi_num),
4524 vsi->stat_offsets_loaded, &prev_es->tx_multicast,
4525 &cur_es->tx_multicast);
4527 ice_stat_update40(hw, GLV_BPTCH(vsi_num), GLV_BPTCL(vsi_num),
4528 vsi->stat_offsets_loaded, &prev_es->tx_broadcast,
4529 &cur_es->tx_broadcast);
4531 ice_stat_update32(hw, GLV_TEPC(vsi_num), vsi->stat_offsets_loaded,
4532 &prev_es->tx_errors, &cur_es->tx_errors);
4534 vsi->stat_offsets_loaded = true;
4538 * ice_update_vsi_ring_stats - Update VSI stats counters
4539 * @vsi: the VSI to be updated
4541 static void ice_update_vsi_ring_stats(struct ice_vsi *vsi)
4543 struct rtnl_link_stats64 *vsi_stats = &vsi->net_stats;
4544 struct ice_ring *ring;
4548 /* reset netdev stats */
4549 vsi_stats->tx_packets = 0;
4550 vsi_stats->tx_bytes = 0;
4551 vsi_stats->rx_packets = 0;
4552 vsi_stats->rx_bytes = 0;
4554 /* reset non-netdev (extended) stats */
4555 vsi->tx_restart = 0;
4557 vsi->tx_linearize = 0;
4558 vsi->rx_buf_failed = 0;
4559 vsi->rx_page_failed = 0;
4563 /* update Tx rings counters */
4564 ice_for_each_txq(vsi, i) {
4565 ring = READ_ONCE(vsi->tx_rings[i]);
4566 ice_fetch_u64_stats_per_ring(ring, &pkts, &bytes);
4567 vsi_stats->tx_packets += pkts;
4568 vsi_stats->tx_bytes += bytes;
4569 vsi->tx_restart += ring->tx_stats.restart_q;
4570 vsi->tx_busy += ring->tx_stats.tx_busy;
4571 vsi->tx_linearize += ring->tx_stats.tx_linearize;
4574 /* update Rx rings counters */
4575 ice_for_each_rxq(vsi, i) {
4576 ring = READ_ONCE(vsi->rx_rings[i]);
4577 ice_fetch_u64_stats_per_ring(ring, &pkts, &bytes);
4578 vsi_stats->rx_packets += pkts;
4579 vsi_stats->rx_bytes += bytes;
4580 vsi->rx_buf_failed += ring->rx_stats.alloc_buf_failed;
4581 vsi->rx_page_failed += ring->rx_stats.alloc_page_failed;
4588 * ice_update_vsi_stats - Update VSI stats counters
4589 * @vsi: the VSI to be updated
4591 static void ice_update_vsi_stats(struct ice_vsi *vsi)
4593 struct rtnl_link_stats64 *cur_ns = &vsi->net_stats;
4594 struct ice_eth_stats *cur_es = &vsi->eth_stats;
4595 struct ice_pf *pf = vsi->back;
4597 if (test_bit(__ICE_DOWN, vsi->state) ||
4598 test_bit(__ICE_CFG_BUSY, pf->state))
4601 /* get stats as recorded by Tx/Rx rings */
4602 ice_update_vsi_ring_stats(vsi);
4604 /* get VSI stats as recorded by the hardware */
4605 ice_update_eth_stats(vsi);
4607 cur_ns->tx_errors = cur_es->tx_errors;
4608 cur_ns->rx_dropped = cur_es->rx_discards;
4609 cur_ns->tx_dropped = cur_es->tx_discards;
4610 cur_ns->multicast = cur_es->rx_multicast;
4612 /* update some more netdev stats if this is main VSI */
4613 if (vsi->type == ICE_VSI_PF) {
4614 cur_ns->rx_crc_errors = pf->stats.crc_errors;
4615 cur_ns->rx_errors = pf->stats.crc_errors +
4616 pf->stats.illegal_bytes;
4617 cur_ns->rx_length_errors = pf->stats.rx_len_errors;
4622 * ice_update_pf_stats - Update PF port stats counters
4623 * @pf: PF whose stats needs to be updated
4625 static void ice_update_pf_stats(struct ice_pf *pf)
4627 struct ice_hw_port_stats *prev_ps, *cur_ps;
4628 struct ice_hw *hw = &pf->hw;
4631 prev_ps = &pf->stats_prev;
4632 cur_ps = &pf->stats;
4635 ice_stat_update40(hw, GLPRT_GORCH(pf_id), GLPRT_GORCL(pf_id),
4636 pf->stat_prev_loaded, &prev_ps->eth.rx_bytes,
4637 &cur_ps->eth.rx_bytes);
4639 ice_stat_update40(hw, GLPRT_UPRCH(pf_id), GLPRT_UPRCL(pf_id),
4640 pf->stat_prev_loaded, &prev_ps->eth.rx_unicast,
4641 &cur_ps->eth.rx_unicast);
4643 ice_stat_update40(hw, GLPRT_MPRCH(pf_id), GLPRT_MPRCL(pf_id),
4644 pf->stat_prev_loaded, &prev_ps->eth.rx_multicast,
4645 &cur_ps->eth.rx_multicast);
4647 ice_stat_update40(hw, GLPRT_BPRCH(pf_id), GLPRT_BPRCL(pf_id),
4648 pf->stat_prev_loaded, &prev_ps->eth.rx_broadcast,
4649 &cur_ps->eth.rx_broadcast);
4651 ice_stat_update40(hw, GLPRT_GOTCH(pf_id), GLPRT_GOTCL(pf_id),
4652 pf->stat_prev_loaded, &prev_ps->eth.tx_bytes,
4653 &cur_ps->eth.tx_bytes);
4655 ice_stat_update40(hw, GLPRT_UPTCH(pf_id), GLPRT_UPTCL(pf_id),
4656 pf->stat_prev_loaded, &prev_ps->eth.tx_unicast,
4657 &cur_ps->eth.tx_unicast);
4659 ice_stat_update40(hw, GLPRT_MPTCH(pf_id), GLPRT_MPTCL(pf_id),
4660 pf->stat_prev_loaded, &prev_ps->eth.tx_multicast,
4661 &cur_ps->eth.tx_multicast);
4663 ice_stat_update40(hw, GLPRT_BPTCH(pf_id), GLPRT_BPTCL(pf_id),
4664 pf->stat_prev_loaded, &prev_ps->eth.tx_broadcast,
4665 &cur_ps->eth.tx_broadcast);
4667 ice_stat_update32(hw, GLPRT_TDOLD(pf_id), pf->stat_prev_loaded,
4668 &prev_ps->tx_dropped_link_down,
4669 &cur_ps->tx_dropped_link_down);
4671 ice_stat_update40(hw, GLPRT_PRC64H(pf_id), GLPRT_PRC64L(pf_id),
4672 pf->stat_prev_loaded, &prev_ps->rx_size_64,
4673 &cur_ps->rx_size_64);
4675 ice_stat_update40(hw, GLPRT_PRC127H(pf_id), GLPRT_PRC127L(pf_id),
4676 pf->stat_prev_loaded, &prev_ps->rx_size_127,
4677 &cur_ps->rx_size_127);
4679 ice_stat_update40(hw, GLPRT_PRC255H(pf_id), GLPRT_PRC255L(pf_id),
4680 pf->stat_prev_loaded, &prev_ps->rx_size_255,
4681 &cur_ps->rx_size_255);
4683 ice_stat_update40(hw, GLPRT_PRC511H(pf_id), GLPRT_PRC511L(pf_id),
4684 pf->stat_prev_loaded, &prev_ps->rx_size_511,
4685 &cur_ps->rx_size_511);
4687 ice_stat_update40(hw, GLPRT_PRC1023H(pf_id),
4688 GLPRT_PRC1023L(pf_id), pf->stat_prev_loaded,
4689 &prev_ps->rx_size_1023, &cur_ps->rx_size_1023);
4691 ice_stat_update40(hw, GLPRT_PRC1522H(pf_id),
4692 GLPRT_PRC1522L(pf_id), pf->stat_prev_loaded,
4693 &prev_ps->rx_size_1522, &cur_ps->rx_size_1522);
4695 ice_stat_update40(hw, GLPRT_PRC9522H(pf_id),
4696 GLPRT_PRC9522L(pf_id), pf->stat_prev_loaded,
4697 &prev_ps->rx_size_big, &cur_ps->rx_size_big);
4699 ice_stat_update40(hw, GLPRT_PTC64H(pf_id), GLPRT_PTC64L(pf_id),
4700 pf->stat_prev_loaded, &prev_ps->tx_size_64,
4701 &cur_ps->tx_size_64);
4703 ice_stat_update40(hw, GLPRT_PTC127H(pf_id), GLPRT_PTC127L(pf_id),
4704 pf->stat_prev_loaded, &prev_ps->tx_size_127,
4705 &cur_ps->tx_size_127);
4707 ice_stat_update40(hw, GLPRT_PTC255H(pf_id), GLPRT_PTC255L(pf_id),
4708 pf->stat_prev_loaded, &prev_ps->tx_size_255,
4709 &cur_ps->tx_size_255);
4711 ice_stat_update40(hw, GLPRT_PTC511H(pf_id), GLPRT_PTC511L(pf_id),
4712 pf->stat_prev_loaded, &prev_ps->tx_size_511,
4713 &cur_ps->tx_size_511);
4715 ice_stat_update40(hw, GLPRT_PTC1023H(pf_id),
4716 GLPRT_PTC1023L(pf_id), pf->stat_prev_loaded,
4717 &prev_ps->tx_size_1023, &cur_ps->tx_size_1023);
4719 ice_stat_update40(hw, GLPRT_PTC1522H(pf_id),
4720 GLPRT_PTC1522L(pf_id), pf->stat_prev_loaded,
4721 &prev_ps->tx_size_1522, &cur_ps->tx_size_1522);
4723 ice_stat_update40(hw, GLPRT_PTC9522H(pf_id),
4724 GLPRT_PTC9522L(pf_id), pf->stat_prev_loaded,
4725 &prev_ps->tx_size_big, &cur_ps->tx_size_big);
4727 ice_stat_update32(hw, GLPRT_LXONRXC(pf_id), pf->stat_prev_loaded,
4728 &prev_ps->link_xon_rx, &cur_ps->link_xon_rx);
4730 ice_stat_update32(hw, GLPRT_LXOFFRXC(pf_id), pf->stat_prev_loaded,
4731 &prev_ps->link_xoff_rx, &cur_ps->link_xoff_rx);
4733 ice_stat_update32(hw, GLPRT_LXONTXC(pf_id), pf->stat_prev_loaded,
4734 &prev_ps->link_xon_tx, &cur_ps->link_xon_tx);
4736 ice_stat_update32(hw, GLPRT_LXOFFTXC(pf_id), pf->stat_prev_loaded,
4737 &prev_ps->link_xoff_tx, &cur_ps->link_xoff_tx);
4739 ice_stat_update32(hw, GLPRT_CRCERRS(pf_id), pf->stat_prev_loaded,
4740 &prev_ps->crc_errors, &cur_ps->crc_errors);
4742 ice_stat_update32(hw, GLPRT_ILLERRC(pf_id), pf->stat_prev_loaded,
4743 &prev_ps->illegal_bytes, &cur_ps->illegal_bytes);
4745 ice_stat_update32(hw, GLPRT_MLFC(pf_id), pf->stat_prev_loaded,
4746 &prev_ps->mac_local_faults,
4747 &cur_ps->mac_local_faults);
4749 ice_stat_update32(hw, GLPRT_MRFC(pf_id), pf->stat_prev_loaded,
4750 &prev_ps->mac_remote_faults,
4751 &cur_ps->mac_remote_faults);
4753 ice_stat_update32(hw, GLPRT_RLEC(pf_id), pf->stat_prev_loaded,
4754 &prev_ps->rx_len_errors, &cur_ps->rx_len_errors);
4756 ice_stat_update32(hw, GLPRT_RUC(pf_id), pf->stat_prev_loaded,
4757 &prev_ps->rx_undersize, &cur_ps->rx_undersize);
4759 ice_stat_update32(hw, GLPRT_RFC(pf_id), pf->stat_prev_loaded,
4760 &prev_ps->rx_fragments, &cur_ps->rx_fragments);
4762 ice_stat_update32(hw, GLPRT_ROC(pf_id), pf->stat_prev_loaded,
4763 &prev_ps->rx_oversize, &cur_ps->rx_oversize);
4765 ice_stat_update32(hw, GLPRT_RJC(pf_id), pf->stat_prev_loaded,
4766 &prev_ps->rx_jabber, &cur_ps->rx_jabber);
4768 pf->stat_prev_loaded = true;
4772 * ice_get_stats64 - get statistics for network device structure
4773 * @netdev: network interface device structure
4774 * @stats: main device statistics structure
4777 void ice_get_stats64(struct net_device *netdev, struct rtnl_link_stats64 *stats)
4779 struct ice_netdev_priv *np = netdev_priv(netdev);
4780 struct rtnl_link_stats64 *vsi_stats;
4781 struct ice_vsi *vsi = np->vsi;
4783 vsi_stats = &vsi->net_stats;
4785 if (test_bit(__ICE_DOWN, vsi->state) || !vsi->num_txq || !vsi->num_rxq)
4787 /* netdev packet/byte stats come from ring counter. These are obtained
4788 * by summing up ring counters (done by ice_update_vsi_ring_stats).
4790 ice_update_vsi_ring_stats(vsi);
4791 stats->tx_packets = vsi_stats->tx_packets;
4792 stats->tx_bytes = vsi_stats->tx_bytes;
4793 stats->rx_packets = vsi_stats->rx_packets;
4794 stats->rx_bytes = vsi_stats->rx_bytes;
4796 /* The rest of the stats can be read from the hardware but instead we
4797 * just return values that the watchdog task has already obtained from
4800 stats->multicast = vsi_stats->multicast;
4801 stats->tx_errors = vsi_stats->tx_errors;
4802 stats->tx_dropped = vsi_stats->tx_dropped;
4803 stats->rx_errors = vsi_stats->rx_errors;
4804 stats->rx_dropped = vsi_stats->rx_dropped;
4805 stats->rx_crc_errors = vsi_stats->rx_crc_errors;
4806 stats->rx_length_errors = vsi_stats->rx_length_errors;
4810 * ice_napi_disable_all - Disable NAPI for all q_vectors in the VSI
4811 * @vsi: VSI having NAPI disabled
4813 static void ice_napi_disable_all(struct ice_vsi *vsi)
4820 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++)
4821 napi_disable(&vsi->q_vectors[q_idx]->napi);
4825 * ice_down - Shutdown the connection
4826 * @vsi: The VSI being stopped
4828 int ice_down(struct ice_vsi *vsi)
4832 /* Caller of this function is expected to set the
4833 * vsi->state __ICE_DOWN bit
4836 netif_carrier_off(vsi->netdev);
4837 netif_tx_disable(vsi->netdev);
4840 ice_vsi_dis_irq(vsi);
4841 err = ice_vsi_stop_tx_rx_rings(vsi);
4842 ice_napi_disable_all(vsi);
4844 ice_for_each_txq(vsi, i)
4845 ice_clean_tx_ring(vsi->tx_rings[i]);
4847 ice_for_each_rxq(vsi, i)
4848 ice_clean_rx_ring(vsi->rx_rings[i]);
4851 netdev_err(vsi->netdev, "Failed to close VSI 0x%04X on switch 0x%04X\n",
4852 vsi->vsi_num, vsi->vsw->sw_id);
4857 * ice_vsi_setup_tx_rings - Allocate VSI Tx queue resources
4858 * @vsi: VSI having resources allocated
4860 * Return 0 on success, negative on failure
4862 static int ice_vsi_setup_tx_rings(struct ice_vsi *vsi)
4866 if (!vsi->num_txq) {
4867 dev_err(&vsi->back->pdev->dev, "VSI %d has 0 Tx queues\n",
4872 ice_for_each_txq(vsi, i) {
4873 err = ice_setup_tx_ring(vsi->tx_rings[i]);
4882 * ice_vsi_setup_rx_rings - Allocate VSI Rx queue resources
4883 * @vsi: VSI having resources allocated
4885 * Return 0 on success, negative on failure
4887 static int ice_vsi_setup_rx_rings(struct ice_vsi *vsi)
4891 if (!vsi->num_rxq) {
4892 dev_err(&vsi->back->pdev->dev, "VSI %d has 0 Rx queues\n",
4897 ice_for_each_rxq(vsi, i) {
4898 err = ice_setup_rx_ring(vsi->rx_rings[i]);
4907 * ice_vsi_req_irq - Request IRQ from the OS
4908 * @vsi: The VSI IRQ is being requested for
4909 * @basename: name for the vector
4911 * Return 0 on success and a negative value on error
4913 static int ice_vsi_req_irq(struct ice_vsi *vsi, char *basename)
4915 struct ice_pf *pf = vsi->back;
4918 if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags))
4919 err = ice_vsi_req_irq_msix(vsi, basename);
4925 * ice_vsi_free_tx_rings - Free Tx resources for VSI queues
4926 * @vsi: the VSI having resources freed
4928 static void ice_vsi_free_tx_rings(struct ice_vsi *vsi)
4935 ice_for_each_txq(vsi, i)
4936 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc)
4937 ice_free_tx_ring(vsi->tx_rings[i]);
4941 * ice_vsi_free_rx_rings - Free Rx resources for VSI queues
4942 * @vsi: the VSI having resources freed
4944 static void ice_vsi_free_rx_rings(struct ice_vsi *vsi)
4951 ice_for_each_rxq(vsi, i)
4952 if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc)
4953 ice_free_rx_ring(vsi->rx_rings[i]);
4957 * ice_vsi_open - Called when a network interface is made active
4958 * @vsi: the VSI to open
4960 * Initialization of the VSI
4962 * Returns 0 on success, negative value on error
4964 static int ice_vsi_open(struct ice_vsi *vsi)
4966 char int_name[ICE_INT_NAME_STR_LEN];
4967 struct ice_pf *pf = vsi->back;
4970 /* allocate descriptors */
4971 err = ice_vsi_setup_tx_rings(vsi);
4975 err = ice_vsi_setup_rx_rings(vsi);
4979 err = ice_vsi_cfg(vsi);
4983 snprintf(int_name, sizeof(int_name) - 1, "%s-%s",
4984 dev_driver_string(&pf->pdev->dev), vsi->netdev->name);
4985 err = ice_vsi_req_irq(vsi, int_name);
4989 /* Notify the stack of the actual queue counts. */
4990 err = netif_set_real_num_tx_queues(vsi->netdev, vsi->num_txq);
4994 err = netif_set_real_num_rx_queues(vsi->netdev, vsi->num_rxq);
4998 err = ice_up_complete(vsi);
5000 goto err_up_complete;
5007 ice_vsi_free_irq(vsi);
5009 ice_vsi_free_rx_rings(vsi);
5011 ice_vsi_free_tx_rings(vsi);
5017 * ice_vsi_close - Shut down a VSI
5018 * @vsi: the VSI being shut down
5020 static void ice_vsi_close(struct ice_vsi *vsi)
5022 if (!test_and_set_bit(__ICE_DOWN, vsi->state))
5025 ice_vsi_free_irq(vsi);
5026 ice_vsi_free_tx_rings(vsi);
5027 ice_vsi_free_rx_rings(vsi);
5031 * ice_rss_clean - Delete RSS related VSI structures that hold user inputs
5032 * @vsi: the VSI being removed
5034 static void ice_rss_clean(struct ice_vsi *vsi)
5040 if (vsi->rss_hkey_user)
5041 devm_kfree(&pf->pdev->dev, vsi->rss_hkey_user);
5042 if (vsi->rss_lut_user)
5043 devm_kfree(&pf->pdev->dev, vsi->rss_lut_user);
5047 * ice_vsi_release - Delete a VSI and free its resources
5048 * @vsi: the VSI being removed
5050 * Returns 0 on success or < 0 on error
5052 static int ice_vsi_release(struct ice_vsi *vsi)
5061 unregister_netdev(vsi->netdev);
5062 free_netdev(vsi->netdev);
5066 if (test_bit(ICE_FLAG_RSS_ENA, pf->flags))
5069 /* Disable VSI and free resources */
5070 ice_vsi_dis_irq(vsi);
5073 /* reclaim interrupt vectors back to PF */
5074 ice_free_res(vsi->back->irq_tracker, vsi->base_vector, vsi->idx);
5075 pf->num_avail_msix += vsi->num_q_vectors;
5077 ice_remove_vsi_fltr(&pf->hw, vsi->vsi_num);
5078 ice_vsi_delete(vsi);
5079 ice_vsi_free_q_vectors(vsi);
5080 ice_vsi_clear_rings(vsi);
5082 ice_vsi_put_qs(vsi);
5083 pf->q_left_tx += vsi->alloc_txq;
5084 pf->q_left_rx += vsi->alloc_rxq;
5092 * ice_dis_vsi - pause a VSI
5093 * @vsi: the VSI being paused
5095 static void ice_dis_vsi(struct ice_vsi *vsi)
5097 if (test_bit(__ICE_DOWN, vsi->state))
5100 set_bit(__ICE_NEEDS_RESTART, vsi->state);
5102 if (vsi->netdev && netif_running(vsi->netdev) &&
5103 vsi->type == ICE_VSI_PF)
5104 vsi->netdev->netdev_ops->ndo_stop(vsi->netdev);
5110 * ice_ena_vsi - resume a VSI
5111 * @vsi: the VSI being resume
5113 static void ice_ena_vsi(struct ice_vsi *vsi)
5115 if (!test_and_clear_bit(__ICE_NEEDS_RESTART, vsi->state))
5118 if (vsi->netdev && netif_running(vsi->netdev))
5119 vsi->netdev->netdev_ops->ndo_open(vsi->netdev);
5120 else if (ice_vsi_open(vsi))
5121 /* this clears the DOWN bit */
5122 dev_dbg(&vsi->back->pdev->dev, "Failed open VSI 0x%04X on switch 0x%04X\n",
5123 vsi->vsi_num, vsi->vsw->sw_id);
5127 * ice_pf_dis_all_vsi - Pause all VSIs on a PF
5130 static void ice_pf_dis_all_vsi(struct ice_pf *pf)
5134 ice_for_each_vsi(pf, v)
5136 ice_dis_vsi(pf->vsi[v]);
5140 * ice_pf_ena_all_vsi - Resume all VSIs on a PF
5143 static void ice_pf_ena_all_vsi(struct ice_pf *pf)
5147 ice_for_each_vsi(pf, v)
5149 ice_ena_vsi(pf->vsi[v]);
5153 * ice_rebuild - rebuild after reset
5154 * @pf: pf to rebuild
5156 static void ice_rebuild(struct ice_pf *pf)
5158 struct device *dev = &pf->pdev->dev;
5159 struct ice_hw *hw = &pf->hw;
5160 enum ice_status ret;
5163 if (test_bit(__ICE_DOWN, pf->state))
5164 goto clear_recovery;
5166 dev_dbg(dev, "rebuilding pf\n");
5168 ret = ice_init_all_ctrlq(hw);
5170 dev_err(dev, "control queues init failed %d\n", ret);
5174 ret = ice_clear_pf_cfg(hw);
5176 dev_err(dev, "clear PF configuration failed %d\n", ret);
5180 ice_clear_pxe_mode(hw);
5182 ret = ice_get_caps(hw);
5184 dev_err(dev, "ice_get_caps failed %d\n", ret);
5188 /* basic nic switch setup */
5189 err = ice_setup_pf_sw(pf);
5191 dev_err(dev, "ice_setup_pf_sw failed\n");
5195 /* start misc vector */
5196 if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags)) {
5197 err = ice_req_irq_msix_misc(pf);
5199 dev_err(dev, "misc vector setup failed: %d\n", err);
5204 /* restart the VSIs that were rebuilt and running before the reset */
5205 ice_pf_ena_all_vsi(pf);
5210 ice_shutdown_all_ctrlq(hw);
5211 set_bit(__ICE_RESET_FAILED, pf->state);
5213 set_bit(__ICE_RESET_RECOVERY_PENDING, pf->state);
5217 * ice_change_mtu - NDO callback to change the MTU
5218 * @netdev: network interface device structure
5219 * @new_mtu: new value for maximum frame size
5221 * Returns 0 on success, negative on failure
5223 static int ice_change_mtu(struct net_device *netdev, int new_mtu)
5225 struct ice_netdev_priv *np = netdev_priv(netdev);
5226 struct ice_vsi *vsi = np->vsi;
5227 struct ice_pf *pf = vsi->back;
5230 if (new_mtu == netdev->mtu) {
5231 netdev_warn(netdev, "mtu is already %u\n", netdev->mtu);
5235 if (new_mtu < netdev->min_mtu) {
5236 netdev_err(netdev, "new mtu invalid. min_mtu is %d\n",
5239 } else if (new_mtu > netdev->max_mtu) {
5240 netdev_err(netdev, "new mtu invalid. max_mtu is %d\n",
5244 /* if a reset is in progress, wait for some time for it to complete */
5246 if (ice_is_reset_recovery_pending(pf->state)) {
5248 usleep_range(1000, 2000);
5253 } while (count < 100);
5256 netdev_err(netdev, "can't change mtu. Device is busy\n");
5260 netdev->mtu = new_mtu;
5262 /* if VSI is up, bring it down and then back up */
5263 if (!test_and_set_bit(__ICE_DOWN, vsi->state)) {
5266 err = ice_down(vsi);
5268 netdev_err(netdev, "change mtu if_up err %d\n", err);
5274 netdev_err(netdev, "change mtu if_up err %d\n", err);
5279 netdev_dbg(netdev, "changed mtu to %d\n", new_mtu);
5284 * ice_set_rss - Set RSS keys and lut
5285 * @vsi: Pointer to VSI structure
5286 * @seed: RSS hash seed
5287 * @lut: Lookup table
5288 * @lut_size: Lookup table size
5290 * Returns 0 on success, negative on failure
5292 int ice_set_rss(struct ice_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
5294 struct ice_pf *pf = vsi->back;
5295 struct ice_hw *hw = &pf->hw;
5296 enum ice_status status;
5299 struct ice_aqc_get_set_rss_keys *buf =
5300 (struct ice_aqc_get_set_rss_keys *)seed;
5302 status = ice_aq_set_rss_key(hw, vsi->vsi_num, buf);
5305 dev_err(&pf->pdev->dev,
5306 "Cannot set RSS key, err %d aq_err %d\n",
5307 status, hw->adminq.rq_last_status);
5313 status = ice_aq_set_rss_lut(hw, vsi->vsi_num,
5314 vsi->rss_lut_type, lut, lut_size);
5316 dev_err(&pf->pdev->dev,
5317 "Cannot set RSS lut, err %d aq_err %d\n",
5318 status, hw->adminq.rq_last_status);
5327 * ice_get_rss - Get RSS keys and lut
5328 * @vsi: Pointer to VSI structure
5329 * @seed: Buffer to store the keys
5330 * @lut: Buffer to store the lookup table entries
5331 * @lut_size: Size of buffer to store the lookup table entries
5333 * Returns 0 on success, negative on failure
5335 int ice_get_rss(struct ice_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
5337 struct ice_pf *pf = vsi->back;
5338 struct ice_hw *hw = &pf->hw;
5339 enum ice_status status;
5342 struct ice_aqc_get_set_rss_keys *buf =
5343 (struct ice_aqc_get_set_rss_keys *)seed;
5345 status = ice_aq_get_rss_key(hw, vsi->vsi_num, buf);
5347 dev_err(&pf->pdev->dev,
5348 "Cannot get RSS key, err %d aq_err %d\n",
5349 status, hw->adminq.rq_last_status);
5355 status = ice_aq_get_rss_lut(hw, vsi->vsi_num,
5356 vsi->rss_lut_type, lut, lut_size);
5358 dev_err(&pf->pdev->dev,
5359 "Cannot get RSS lut, err %d aq_err %d\n",
5360 status, hw->adminq.rq_last_status);
5369 * ice_open - Called when a network interface becomes active
5370 * @netdev: network interface device structure
5372 * The open entry point is called when a network interface is made
5373 * active by the system (IFF_UP). At this point all resources needed
5374 * for transmit and receive operations are allocated, the interrupt
5375 * handler is registered with the OS, the netdev watchdog is enabled,
5376 * and the stack is notified that the interface is ready.
5378 * Returns 0 on success, negative value on failure
5380 static int ice_open(struct net_device *netdev)
5382 struct ice_netdev_priv *np = netdev_priv(netdev);
5383 struct ice_vsi *vsi = np->vsi;
5386 netif_carrier_off(netdev);
5388 err = ice_vsi_open(vsi);
5391 netdev_err(netdev, "Failed to open VSI 0x%04X on switch 0x%04X\n",
5392 vsi->vsi_num, vsi->vsw->sw_id);
5397 * ice_stop - Disables a network interface
5398 * @netdev: network interface device structure
5400 * The stop entry point is called when an interface is de-activated by the OS,
5401 * and the netdevice enters the DOWN state. The hardware is still under the
5402 * driver's control, but the netdev interface is disabled.
5404 * Returns success only - not allowed to fail
5406 static int ice_stop(struct net_device *netdev)
5408 struct ice_netdev_priv *np = netdev_priv(netdev);
5409 struct ice_vsi *vsi = np->vsi;
5417 * ice_features_check - Validate encapsulated packet conforms to limits
5419 * @netdev: This port's netdev
5420 * @features: Offload features that the stack believes apply
5422 static netdev_features_t
5423 ice_features_check(struct sk_buff *skb,
5424 struct net_device __always_unused *netdev,
5425 netdev_features_t features)
5429 /* No point in doing any of this if neither checksum nor GSO are
5430 * being requested for this frame. We can rule out both by just
5431 * checking for CHECKSUM_PARTIAL
5433 if (skb->ip_summed != CHECKSUM_PARTIAL)
5436 /* We cannot support GSO if the MSS is going to be less than
5437 * 64 bytes. If it is then we need to drop support for GSO.
5439 if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64))
5440 features &= ~NETIF_F_GSO_MASK;
5442 len = skb_network_header(skb) - skb->data;
5443 if (len & ~(ICE_TXD_MACLEN_MAX))
5444 goto out_rm_features;
5446 len = skb_transport_header(skb) - skb_network_header(skb);
5447 if (len & ~(ICE_TXD_IPLEN_MAX))
5448 goto out_rm_features;
5450 if (skb->encapsulation) {
5451 len = skb_inner_network_header(skb) - skb_transport_header(skb);
5452 if (len & ~(ICE_TXD_L4LEN_MAX))
5453 goto out_rm_features;
5455 len = skb_inner_transport_header(skb) -
5456 skb_inner_network_header(skb);
5457 if (len & ~(ICE_TXD_IPLEN_MAX))
5458 goto out_rm_features;
5463 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
5466 static const struct net_device_ops ice_netdev_ops = {
5467 .ndo_open = ice_open,
5468 .ndo_stop = ice_stop,
5469 .ndo_start_xmit = ice_start_xmit,
5470 .ndo_features_check = ice_features_check,
5471 .ndo_set_rx_mode = ice_set_rx_mode,
5472 .ndo_set_mac_address = ice_set_mac_address,
5473 .ndo_validate_addr = eth_validate_addr,
5474 .ndo_change_mtu = ice_change_mtu,
5475 .ndo_get_stats64 = ice_get_stats64,
5476 .ndo_vlan_rx_add_vid = ice_vlan_rx_add_vid,
5477 .ndo_vlan_rx_kill_vid = ice_vlan_rx_kill_vid,
5478 .ndo_set_features = ice_set_features,
5479 .ndo_fdb_add = ice_fdb_add,
5480 .ndo_fdb_del = ice_fdb_del,