drivers/infiniband/hw/nes/nes_hw.c

   1 /*
   2  * Copyright (c) 2006 - 2009 Intel Corporation.  All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  *
  32  */
  33
  34 #include <linux/module.h>
  35 #include <linux/moduleparam.h>
  36 #include <linux/netdevice.h>
  37 #include <linux/etherdevice.h>
  38 #include <linux/ip.h>
  39 #include <linux/tcp.h>
  40 #include <linux/if_vlan.h>
  41 #include <linux/inet_lro.h>
  42
  43 #include "nes.h"
  44
  45 static unsigned int nes_lro_max_aggr = NES_LRO_MAX_AGGR;
  46 module_param(nes_lro_max_aggr, uint, 0444);
  47 MODULE_PARM_DESC(nes_lro_max_aggr, "NIC LRO max packet aggregation");
  48
  49 static int wide_ppm_offset;
  50 module_param(wide_ppm_offset, int, 0644);
  51 MODULE_PARM_DESC(wide_ppm_offset, "Increase CX4 interface clock ppm offset, 0=100ppm (default), 1=300ppm");
  52
  53 static u32 crit_err_count;
  54 u32 int_mod_timer_init;
  55 u32 int_mod_cq_depth_256;
  56 u32 int_mod_cq_depth_128;
  57 u32 int_mod_cq_depth_32;
  58 u32 int_mod_cq_depth_24;
  59 u32 int_mod_cq_depth_16;
  60 u32 int_mod_cq_depth_4;
  61 u32 int_mod_cq_depth_1;
  62 static const u8 nes_max_critical_error_count = 100;
  63 #include "nes_cm.h"
  64
  65 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq);
  66 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count);
  67 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
  68                                 struct nes_adapter *nesadapter, u8  OneG_Mode);
  69 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq);
  70 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq);
  71 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq);
  72 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
  73                                    struct nes_hw_aeqe *aeqe);
  74 static void process_critical_error(struct nes_device *nesdev);
  75 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number);
  76 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode);
  77 static void nes_terminate_timeout(unsigned long context);
  78 static void nes_terminate_start_timer(struct nes_qp *nesqp);
  79
  80 #ifdef CONFIG_INFINIBAND_NES_DEBUG
  81 static unsigned char *nes_iwarp_state_str[] = {
  82         "Non-Existant",
  83         "Idle",
  84         "RTS",
  85         "Closing",
  86         "RSVD1",
  87         "Terminate",
  88         "Error",
  89         "RSVD2",
  90 };
  91
  92 static unsigned char *nes_tcp_state_str[] = {
  93         "Non-Existant",
  94         "Closed",
  95         "Listen",
  96         "SYN Sent",
  97         "SYN Rcvd",
  98         "Established",
  99         "Close Wait",
 100         "FIN Wait 1",
 101         "Closing",
 102         "Last Ack",
 103         "FIN Wait 2",
 104         "Time Wait",
 105         "RSVD1",
 106         "RSVD2",
 107         "RSVD3",
 108         "RSVD4",
 109 };
 110 #endif
 111
 112
 113 /**
 114  * nes_nic_init_timer_defaults
 115  */
 116 void  nes_nic_init_timer_defaults(struct nes_device *nesdev, u8 jumbomode)
 117 {
 118         unsigned long flags;
 119         struct nes_adapter *nesadapter = nesdev->nesadapter;
 120         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 121
 122         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 123
 124         shared_timer->timer_in_use_min = NES_NIC_FAST_TIMER_LOW;
 125         shared_timer->timer_in_use_max = NES_NIC_FAST_TIMER_HIGH;
 126         if (jumbomode) {
 127                 shared_timer->threshold_low    = DEFAULT_JUMBO_NES_QL_LOW;
 128                 shared_timer->threshold_target = DEFAULT_JUMBO_NES_QL_TARGET;
 129                 shared_timer->threshold_high   = DEFAULT_JUMBO_NES_QL_HIGH;
 130         } else {
 131                 shared_timer->threshold_low    = DEFAULT_NES_QL_LOW;
 132                 shared_timer->threshold_target = DEFAULT_NES_QL_TARGET;
 133                 shared_timer->threshold_high   = DEFAULT_NES_QL_HIGH;
 134         }
 135
 136         /* todo use netdev->mtu to set thresholds */
 137         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 138 }
 139
 140
 141 /**
 142  * nes_nic_init_timer
 143  */
 144 static void  nes_nic_init_timer(struct nes_device *nesdev)
 145 {
 146         unsigned long flags;
 147         struct nes_adapter *nesadapter = nesdev->nesadapter;
 148         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 149
 150         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 151
 152         if (shared_timer->timer_in_use_old == 0) {
 153                 nesdev->deepcq_count = 0;
 154                 shared_timer->timer_direction_upward = 0;
 155                 shared_timer->timer_direction_downward = 0;
 156                 shared_timer->timer_in_use = NES_NIC_FAST_TIMER;
 157                 shared_timer->timer_in_use_old = 0;
 158
 159         }
 160         if (shared_timer->timer_in_use != shared_timer->timer_in_use_old) {
 161                 shared_timer->timer_in_use_old = shared_timer->timer_in_use;
 162                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL,
 163                         0x80000000 | ((u32)(shared_timer->timer_in_use*8)));
 164         }
 165         /* todo use netdev->mtu to set thresholds */
 166         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 167 }
 168
 169
 170 /**
 171  * nes_nic_tune_timer
 172  */
 173 static void nes_nic_tune_timer(struct nes_device *nesdev)
 174 {
 175         unsigned long flags;
 176         struct nes_adapter *nesadapter = nesdev->nesadapter;
 177         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 178         u16 cq_count = nesdev->currcq_count;
 179
 180         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 181
 182         if (shared_timer->cq_count_old <= cq_count)
 183                 shared_timer->cq_direction_downward = 0;
 184         else
 185                 shared_timer->cq_direction_downward++;
 186         shared_timer->cq_count_old = cq_count;
 187         if (shared_timer->cq_direction_downward > NES_NIC_CQ_DOWNWARD_TREND) {
 188                 if (cq_count <= shared_timer->threshold_low &&
 189                     shared_timer->threshold_low > 4) {
 190                         shared_timer->threshold_low = shared_timer->threshold_low/2;
 191                         shared_timer->cq_direction_downward=0;
 192                         nesdev->currcq_count = 0;
 193                         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 194                         return;
 195                 }
 196         }
 197
 198         if (cq_count > 1) {
 199                 nesdev->deepcq_count += cq_count;
 200                 if (cq_count <= shared_timer->threshold_low) {       /* increase timer gently */
 201                         shared_timer->timer_direction_upward++;
 202                         shared_timer->timer_direction_downward = 0;
 203                 } else if (cq_count <= shared_timer->threshold_target) { /* balanced */
 204                         shared_timer->timer_direction_upward = 0;
 205                         shared_timer->timer_direction_downward = 0;
 206                 } else if (cq_count <= shared_timer->threshold_high) {  /* decrease timer gently */
 207                         shared_timer->timer_direction_downward++;
 208                         shared_timer->timer_direction_upward = 0;
 209                 } else if (cq_count <= (shared_timer->threshold_high) * 2) {
 210                         shared_timer->timer_in_use -= 2;
 211                         shared_timer->timer_direction_upward = 0;
 212                         shared_timer->timer_direction_downward++;
 213                 } else {
 214                         shared_timer->timer_in_use -= 4;
 215                         shared_timer->timer_direction_upward = 0;
 216                         shared_timer->timer_direction_downward++;
 217                 }
 218
 219                 if (shared_timer->timer_direction_upward > 3 ) {  /* using history */
 220                         shared_timer->timer_in_use += 3;
 221                         shared_timer->timer_direction_upward = 0;
 222                         shared_timer->timer_direction_downward = 0;
 223                 }
 224                 if (shared_timer->timer_direction_downward > 5) { /* using history */
 225                         shared_timer->timer_in_use -= 4 ;
 226                         shared_timer->timer_direction_downward = 0;
 227                         shared_timer->timer_direction_upward = 0;
 228                 }
 229         }
 230
 231         /* boundary checking */
 232         if (shared_timer->timer_in_use > shared_timer->threshold_high)
 233                 shared_timer->timer_in_use = shared_timer->threshold_high;
 234         else if (shared_timer->timer_in_use < shared_timer->threshold_low)
 235                 shared_timer->timer_in_use = shared_timer->threshold_low;
 236
 237         nesdev->currcq_count = 0;
 238
 239         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 240 }
 241
 242
 243 /**
 244  * nes_init_adapter - initialize adapter
 245  */
 246 struct nes_adapter *nes_init_adapter(struct nes_device *nesdev, u8 hw_rev) {
 247         struct nes_adapter *nesadapter = NULL;
 248         unsigned long num_pds;
 249         u32 u32temp;
 250         u32 port_count;
 251         u16 max_rq_wrs;
 252         u16 max_sq_wrs;
 253         u32 max_mr;
 254         u32 max_256pbl;
 255         u32 max_4kpbl;
 256         u32 max_qp;
 257         u32 max_irrq;
 258         u32 max_cq;
 259         u32 hte_index_mask;
 260         u32 adapter_size;
 261         u32 arp_table_size;
 262         u16 vendor_id;
 263         u16 device_id;
 264         u8  OneG_Mode;
 265         u8  func_index;
 266
 267         /* search the list of existing adapters */
 268         list_for_each_entry(nesadapter, &nes_adapter_list, list) {
 269                 nes_debug(NES_DBG_INIT, "Searching Adapter list for PCI devfn = 0x%X,"
 270                                 " adapter PCI slot/bus = %u/%u, pci devices PCI slot/bus = %u/%u, .\n",
 271                                 nesdev->pcidev->devfn,
 272                                 PCI_SLOT(nesadapter->devfn),
 273                                 nesadapter->bus_number,
 274                                 PCI_SLOT(nesdev->pcidev->devfn),
 275                                 nesdev->pcidev->bus->number );
 276                 if ((PCI_SLOT(nesadapter->devfn) == PCI_SLOT(nesdev->pcidev->devfn)) &&
 277                                 (nesadapter->bus_number == nesdev->pcidev->bus->number)) {
 278                         nesadapter->ref_count++;
 279                         return nesadapter;
 280                 }
 281         }
 282
 283         /* no adapter found */
 284         num_pds = pci_resource_len(nesdev->pcidev, BAR_1) >> PAGE_SHIFT;
 285         if ((hw_rev != NE020_REV) && (hw_rev != NE020_REV1)) {
 286                 nes_debug(NES_DBG_INIT, "NE020 driver detected unknown hardware revision 0x%x\n",
 287                                 hw_rev);
 288                 return NULL;
 289         }
 290
 291         nes_debug(NES_DBG_INIT, "Determine Soft Reset, QP_control=0x%x, CPU0=0x%x, CPU1=0x%x, CPU2=0x%x\n",
 292                         nes_read_indexed(nesdev, NES_IDX_QP_CONTROL + PCI_FUNC(nesdev->pcidev->devfn) * 8),
 293                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS),
 294                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 4),
 295                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 8));
 296
 297         nes_debug(NES_DBG_INIT, "Reset and init NE020\n");
 298
 299
 300         if ((port_count = nes_reset_adapter_ne020(nesdev, &OneG_Mode)) == 0)
 301                 return NULL;
 302
 303         max_qp = nes_read_indexed(nesdev, NES_IDX_QP_CTX_SIZE);
 304         nes_debug(NES_DBG_INIT, "QP_CTX_SIZE=%u\n", max_qp);
 305
 306         u32temp = nes_read_indexed(nesdev, NES_IDX_QUAD_HASH_TABLE_SIZE);
 307         if (max_qp > ((u32)1 << (u32temp & 0x001f))) {
 308                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to hash table size = 0x%08X\n",
 309                                 max_qp, u32temp);
 310                 max_qp = (u32)1 << (u32temp & 0x001f);
 311         }
 312
 313         hte_index_mask = ((u32)1 << ((u32temp & 0x001f)+1))-1;
 314         nes_debug(NES_DBG_INIT, "Max QP = %u, hte_index_mask = 0x%08X.\n",
 315                         max_qp, hte_index_mask);
 316
 317         u32temp = nes_read_indexed(nesdev, NES_IDX_IRRQ_COUNT);
 318
 319         max_irrq = 1 << (u32temp & 0x001f);
 320
 321         if (max_qp > max_irrq) {
 322                 max_qp = max_irrq;
 323                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to Available Q1s.\n",
 324                                 max_qp);
 325         }
 326
 327         /* there should be no reason to allocate more pds than qps */
 328         if (num_pds > max_qp)
 329                 num_pds = max_qp;
 330
 331         u32temp = nes_read_indexed(nesdev, NES_IDX_MRT_SIZE);
 332         max_mr = (u32)8192 << (u32temp & 0x7);
 333
 334         u32temp = nes_read_indexed(nesdev, NES_IDX_PBL_REGION_SIZE);
 335         max_256pbl = (u32)1 << (u32temp & 0x0000001f);
 336         max_4kpbl = (u32)1 << ((u32temp >> 16) & 0x0000001f);
 337         max_cq = nes_read_indexed(nesdev, NES_IDX_CQ_CTX_SIZE);
 338
 339         u32temp = nes_read_indexed(nesdev, NES_IDX_ARP_CACHE_SIZE);
 340         arp_table_size = 1 << u32temp;
 341
 342         adapter_size = (sizeof(struct nes_adapter) +
 343                         (sizeof(unsigned long)-1)) & (~(sizeof(unsigned long)-1));
 344         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_qp);
 345         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_mr);
 346         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_cq);
 347         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(num_pds);
 348         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(arp_table_size);
 349         adapter_size += sizeof(struct nes_qp **) * max_qp;
 350
 351         /* allocate a new adapter struct */
 352         nesadapter = kzalloc(adapter_size, GFP_KERNEL);
 353         if (nesadapter == NULL) {
 354                 return NULL;
 355         }
 356
 357         nes_debug(NES_DBG_INIT, "Allocating new nesadapter @ %p, size = %u (actual size = %u).\n",
 358                         nesadapter, (u32)sizeof(struct nes_adapter), adapter_size);
 359
 360         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 361                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 362                 kfree(nesadapter);
 363                 return NULL;
 364         }
 365
 366         nesadapter->vendor_id = (((u32) nesadapter->mac_addr_high) << 8) |
 367                                 (nesadapter->mac_addr_low >> 24);
 368
 369         pci_bus_read_config_word(nesdev->pcidev->bus, nesdev->pcidev->devfn,
 370                                  PCI_DEVICE_ID, &device_id);
 371         nesadapter->vendor_part_id = device_id;
 372
 373         if (nes_init_serdes(nesdev, hw_rev, port_count, nesadapter,
 374                                                         OneG_Mode)) {
 375                 kfree(nesadapter);
 376                 return NULL;
 377         }
 378         nes_init_csr_ne020(nesdev, hw_rev, port_count);
 379
 380         memset(nesadapter->pft_mcast_map, 255,
 381                sizeof nesadapter->pft_mcast_map);
 382
 383         /* populate the new nesadapter */
 384         nesadapter->devfn = nesdev->pcidev->devfn;
 385         nesadapter->bus_number = nesdev->pcidev->bus->number;
 386         nesadapter->ref_count = 1;
 387         nesadapter->timer_int_req = 0xffff0000;
 388         nesadapter->OneG_Mode = OneG_Mode;
 389         nesadapter->doorbell_start = nesdev->doorbell_region;
 390
 391         /* nesadapter->tick_delta = clk_divisor; */
 392         nesadapter->hw_rev = hw_rev;
 393         nesadapter->port_count = port_count;
 394
 395         nesadapter->max_qp = max_qp;
 396         nesadapter->hte_index_mask = hte_index_mask;
 397         nesadapter->max_irrq = max_irrq;
 398         nesadapter->max_mr = max_mr;
 399         nesadapter->max_256pbl = max_256pbl - 1;
 400         nesadapter->max_4kpbl = max_4kpbl - 1;
 401         nesadapter->max_cq = max_cq;
 402         nesadapter->free_256pbl = max_256pbl - 1;
 403         nesadapter->free_4kpbl = max_4kpbl - 1;
 404         nesadapter->max_pd = num_pds;
 405         nesadapter->arp_table_size = arp_table_size;
 406
 407         nesadapter->et_pkt_rate_low = NES_TIMER_ENABLE_LIMIT;
 408         if (nes_drv_opt & NES_DRV_OPT_DISABLE_INT_MOD) {
 409                 nesadapter->et_use_adaptive_rx_coalesce = 0;
 410                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT;
 411                 nesadapter->et_rx_coalesce_usecs_irq = interrupt_mod_interval;
 412         } else {
 413                 nesadapter->et_use_adaptive_rx_coalesce = 1;
 414                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT_DYNAMIC;
 415                 nesadapter->et_rx_coalesce_usecs_irq = 0;
 416                 printk(PFX "%s: Using Adaptive Interrupt Moderation\n", __func__);
 417         }
 418         /* Setup and enable the periodic timer */
 419         if (nesadapter->et_rx_coalesce_usecs_irq)
 420                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x80000000 |
 421                                 ((u32)(nesadapter->et_rx_coalesce_usecs_irq * 8)));
 422         else
 423                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x00000000);
 424
 425         nesadapter->base_pd = 1;
 426
 427         nesadapter->device_cap_flags = IB_DEVICE_LOCAL_DMA_LKEY |
 428                                        IB_DEVICE_MEM_WINDOW |
 429                                        IB_DEVICE_MEM_MGT_EXTENSIONS;
 430
 431         nesadapter->allocated_qps = (unsigned long *)&(((unsigned char *)nesadapter)
 432                         [(sizeof(struct nes_adapter)+(sizeof(unsigned long)-1))&(~(sizeof(unsigned long)-1))]);
 433         nesadapter->allocated_cqs = &nesadapter->allocated_qps[BITS_TO_LONGS(max_qp)];
 434         nesadapter->allocated_mrs = &nesadapter->allocated_cqs[BITS_TO_LONGS(max_cq)];
 435         nesadapter->allocated_pds = &nesadapter->allocated_mrs[BITS_TO_LONGS(max_mr)];
 436         nesadapter->allocated_arps = &nesadapter->allocated_pds[BITS_TO_LONGS(num_pds)];
 437         nesadapter->qp_table = (struct nes_qp **)(&nesadapter->allocated_arps[BITS_TO_LONGS(arp_table_size)]);
 438
 439
 440         /* mark the usual suspect QPs, MR and CQs as in use */
 441         for (u32temp = 0; u32temp < NES_FIRST_QPN; u32temp++) {
 442                 set_bit(u32temp, nesadapter->allocated_qps);
 443                 set_bit(u32temp, nesadapter->allocated_cqs);
 444         }
 445         set_bit(0, nesadapter->allocated_mrs);
 446
 447         for (u32temp = 0; u32temp < 20; u32temp++)
 448                 set_bit(u32temp, nesadapter->allocated_pds);
 449         u32temp = nes_read_indexed(nesdev, NES_IDX_QP_MAX_CFG_SIZES);
 450
 451         max_rq_wrs = ((u32temp >> 8) & 3);
 452         switch (max_rq_wrs) {
 453                 case 0:
 454                         max_rq_wrs = 4;
 455                         break;
 456                 case 1:
 457                         max_rq_wrs = 16;
 458                         break;
 459                 case 2:
 460                         max_rq_wrs = 32;
 461                         break;
 462                 case 3:
 463                         max_rq_wrs = 512;
 464                         break;
 465         }
 466
 467         max_sq_wrs = (u32temp & 3);
 468         switch (max_sq_wrs) {
 469                 case 0:
 470                         max_sq_wrs = 4;
 471                         break;
 472                 case 1:
 473                         max_sq_wrs = 16;
 474                         break;
 475                 case 2:
 476                         max_sq_wrs = 32;
 477                         break;
 478                 case 3:
 479                         max_sq_wrs = 512;
 480                         break;
 481         }
 482         nesadapter->max_qp_wr = min(max_rq_wrs, max_sq_wrs);
 483         nesadapter->max_irrq_wr = (u32temp >> 16) & 3;
 484
 485         nesadapter->max_sge = 4;
 486         nesadapter->max_cqe = 32766;
 487
 488         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 489                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 490                 kfree(nesadapter);
 491                 return NULL;
 492         }
 493
 494         u32temp = nes_read_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG);
 495         nes_write_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG,
 496                         (u32temp & 0xff000000) | (nesadapter->tcp_timer_core_clk_divisor & 0x00ffffff));
 497
 498         /* setup port configuration */
 499         if (nesadapter->port_count == 1) {
 500                 nesadapter->log_port = 0x00000000;
 501                 if (nes_drv_opt & NES_DRV_OPT_DUAL_LOGICAL_PORT)
 502                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000002);
 503                 else
 504                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 505         } else {
 506                 if (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) {
 507                         nesadapter->log_port = 0x000000D8;
 508                 } else {
 509                         if (nesadapter->port_count == 2)
 510                                 nesadapter->log_port = 0x00000044;
 511                         else
 512                                 nesadapter->log_port = 0x000000e4;
 513                 }
 514                 nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 515         }
 516
 517         nes_write_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT,
 518                                                 nesadapter->log_port);
 519         nes_debug(NES_DBG_INIT, "Probe time, LOG2PHY=%u\n",
 520                         nes_read_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT));
 521
 522         spin_lock_init(&nesadapter->resource_lock);
 523         spin_lock_init(&nesadapter->phy_lock);
 524         spin_lock_init(&nesadapter->pbl_lock);
 525         spin_lock_init(&nesadapter->periodic_timer_lock);
 526
 527         INIT_LIST_HEAD(&nesadapter->nesvnic_list[0]);
 528         INIT_LIST_HEAD(&nesadapter->nesvnic_list[1]);
 529         INIT_LIST_HEAD(&nesadapter->nesvnic_list[2]);
 530         INIT_LIST_HEAD(&nesadapter->nesvnic_list[3]);
 531
 532         if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
 533                 u32 pcs_control_status0, pcs_control_status1;
 534                 u32 reset_value;
 535                 u32 i = 0;
 536                 u32 int_cnt = 0;
 537                 u32 ext_cnt = 0;
 538                 unsigned long flags;
 539                 u32 j = 0;
 540
 541                 pcs_control_status0 = nes_read_indexed(nesdev,
 542                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 543                 pcs_control_status1 = nes_read_indexed(nesdev,
 544                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 545
 546                 for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 547                         pcs_control_status0 = nes_read_indexed(nesdev,
 548                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 549                         pcs_control_status1 = nes_read_indexed(nesdev,
 550                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 551                         if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 552                             || (0x0F000100 == (pcs_control_status1 & 0x0F000100)))
 553                                 int_cnt++;
 554                         msleep(1);
 555                 }
 556                 if (int_cnt > 1) {
 557                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
 558                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
 559                         mh_detected++;
 560                         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 561                         reset_value |= 0x0000003d;
 562                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 563
 564                         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 565                                 & 0x00000040) != 0x00000040) && (j++ < 5000));
 566                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 567
 568                         pcs_control_status0 = nes_read_indexed(nesdev,
 569                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 570                         pcs_control_status1 = nes_read_indexed(nesdev,
 571                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 572
 573                         for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 574                                 pcs_control_status0 = nes_read_indexed(nesdev,
 575                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 576                                 pcs_control_status1 = nes_read_indexed(nesdev,
 577                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 578                                 if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 579                                         || (0x0F000100 == (pcs_control_status1 & 0x0F000100))) {
 580                                         if (++ext_cnt > int_cnt) {
 581                                                 spin_lock_irqsave(&nesadapter->phy_lock, flags);
 582                                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1,
 583                                                                 0x0000F088);
 584                                                 mh_detected++;
 585                                                 reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 586                                                 reset_value |= 0x0000003d;
 587                                                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 588
 589                                                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 590                                                         & 0x00000040) != 0x00000040) && (j++ < 5000));
 591                                                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 592                                                 break;
 593                                         }
 594                                 }
 595                                 msleep(1);
 596                         }
 597                 }
 598         }
 599
 600         if (nesadapter->hw_rev == NE020_REV) {
 601                 init_timer(&nesadapter->mh_timer);
 602                 nesadapter->mh_timer.function = nes_mh_fix;
 603                 nesadapter->mh_timer.expires = jiffies + (HZ/5);  /* 1 second */
 604                 nesadapter->mh_timer.data = (unsigned long)nesdev;
 605                 add_timer(&nesadapter->mh_timer);
 606         } else {
 607                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, 0x0f000000);
 608         }
 609
 610         init_timer(&nesadapter->lc_timer);
 611         nesadapter->lc_timer.function = nes_clc;
 612         nesadapter->lc_timer.expires = jiffies + 3600 * HZ;  /* 1 hour */
 613         nesadapter->lc_timer.data = (unsigned long)nesdev;
 614         add_timer(&nesadapter->lc_timer);
 615
 616         list_add_tail(&nesadapter->list, &nes_adapter_list);
 617
 618         for (func_index = 0; func_index < 8; func_index++) {
 619                 pci_bus_read_config_word(nesdev->pcidev->bus,
 620                                         PCI_DEVFN(PCI_SLOT(nesdev->pcidev->devfn),
 621                                         func_index), 0, &vendor_id);
 622                 if (vendor_id == 0xffff)
 623                         break;
 624         }
 625         nes_debug(NES_DBG_INIT, "%s %d functions found for %s.\n", __func__,
 626                 func_index, pci_name(nesdev->pcidev));
 627         nesadapter->adapter_fcn_count = func_index;
 628
 629         return nesadapter;
 630 }
 631
 632
 633 /**
 634  * nes_reset_adapter_ne020
 635  */
 636 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode)
 637 {
 638         u32 port_count;
 639         u32 u32temp;
 640         u32 i;
 641
 642         u32temp = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 643         port_count = ((u32temp & 0x00000300) >> 8) + 1;
 644         /* TODO: assuming that both SERDES are set the same for now */
 645         *OneG_Mode = (u32temp & 0x00003c00) ? 0 : 1;
 646         nes_debug(NES_DBG_INIT, "Initial Software Reset = 0x%08X, port_count=%u\n",
 647                         u32temp, port_count);
 648         if (*OneG_Mode)
 649                 nes_debug(NES_DBG_INIT, "Running in 1G mode.\n");
 650         u32temp &= 0xff00ffc0;
 651         switch (port_count) {
 652                 case 1:
 653                         u32temp |= 0x00ee0000;
 654                         break;
 655                 case 2:
 656                         u32temp |= 0x00cc0000;
 657                         break;
 658                 case 4:
 659                         u32temp |= 0x00000000;
 660                         break;
 661                 default:
 662                         return 0;
 663                         break;
 664         }
 665
 666         /* check and do full reset if needed */
 667         if (nes_read_indexed(nesdev, NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8))) {
 668                 nes_debug(NES_DBG_INIT, "Issuing Full Soft reset = 0x%08X\n", u32temp | 0xd);
 669                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 670
 671                 i = 0;
 672                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 673                         mdelay(1);
 674                 if (i > 10000) {
 675                         nes_debug(NES_DBG_INIT, "Did not see full soft reset done.\n");
 676                         return 0;
 677                 }
 678
 679                 i = 0;
 680                 while ((nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS) != 0x80) && i++ < 10000)
 681                         mdelay(1);
 682                 if (i > 10000) {
 683                         printk(KERN_ERR PFX "Internal CPU not ready, status = %02X\n",
 684                                nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS));
 685                         return 0;
 686                 }
 687         }
 688
 689         /* port reset */
 690         switch (port_count) {
 691                 case 1:
 692                         u32temp |= 0x00ee0010;
 693                         break;
 694                 case 2:
 695                         u32temp |= 0x00cc0030;
 696                         break;
 697                 case 4:
 698                         u32temp |= 0x00000030;
 699                         break;
 700         }
 701
 702         nes_debug(NES_DBG_INIT, "Issuing Port Soft reset = 0x%08X\n", u32temp | 0xd);
 703         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 704
 705         i = 0;
 706         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 707                 mdelay(1);
 708         if (i > 10000) {
 709                 nes_debug(NES_DBG_INIT, "Did not see port soft reset done.\n");
 710                 return 0;
 711         }
 712
 713         /* serdes 0 */
 714         i = 0;
 715         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 716                         & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 717                 mdelay(1);
 718         if (i > 5000) {
 719                 nes_debug(NES_DBG_INIT, "Serdes 0 not ready, status=%x\n", u32temp);
 720                 return 0;
 721         }
 722
 723         /* serdes 1 */
 724         if (port_count > 1) {
 725                 i = 0;
 726                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 727                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 728                         mdelay(1);
 729                 if (i > 5000) {
 730                         nes_debug(NES_DBG_INIT, "Serdes 1 not ready, status=%x\n", u32temp);
 731                         return 0;
 732                 }
 733         }
 734
 735         return port_count;
 736 }
 737
 738
 739 /**
 740  * nes_init_serdes
 741  */
 742 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
 743                                 struct nes_adapter *nesadapter, u8  OneG_Mode)
 744 {
 745         int i;
 746         u32 u32temp;
 747         u32 sds;
 748
 749         if (hw_rev != NE020_REV) {
 750                 /* init serdes 0 */
 751                 switch (nesadapter->phy_type[0]) {
 752                 case NES_PHY_TYPE_CX4:
 753                         if (wide_ppm_offset)
 754                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000FFFAA);
 755                         else
 756                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 757                         break;
 758                 case NES_PHY_TYPE_KR:
 759                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 760                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x00000000);
 761                         break;
 762                 case NES_PHY_TYPE_PUMA_1G:
 763                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 764                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0);
 765                         sds |= 0x00000100;
 766                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, sds);
 767                         break;
 768                 default:
 769                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 770                         break;
 771                 }
 772
 773                 if (!OneG_Mode)
 774                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE0, 0x11110000);
 775
 776                 if (port_count < 2)
 777                         return 0;
 778
 779                 /* init serdes 1 */
 780                 if (!(OneG_Mode && (nesadapter->phy_type[1] != NES_PHY_TYPE_PUMA_1G)))
 781                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000FF);
 782
 783                 switch (nesadapter->phy_type[1]) {
 784                 case NES_PHY_TYPE_ARGUS:
 785                 case NES_PHY_TYPE_SFP_D:
 786                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x00000000);
 787                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x00000000);
 788                         break;
 789                 case NES_PHY_TYPE_CX4:
 790                         if (wide_ppm_offset)
 791                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000FFFAA);
 792                         break;
 793                 case NES_PHY_TYPE_KR:
 794                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x00000000);
 795                         break;
 796                 case NES_PHY_TYPE_PUMA_1G:
 797                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
 798                         sds |= 0x000000100;
 799                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
 800                 }
 801                 if (!OneG_Mode) {
 802                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE1, 0x11110000);
 803                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
 804                         sds &= 0xFFFFFFBF;
 805                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
 806                 }
 807         } else {
 808                 /* init serdes 0 */
 809                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, 0x00000008);
 810                 i = 0;
 811                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 812                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 813                         mdelay(1);
 814                 if (i > 5000) {
 815                         nes_debug(NES_DBG_PHY, "Init: serdes 0 not ready, status=%x\n", u32temp);
 816                         return 1;
 817                 }
 818                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x000bdef7);
 819                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE0, 0x9ce73000);
 820                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE0, 0x0ff00000);
 821                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET0, 0x00000000);
 822                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS0, 0x00000000);
 823                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL0, 0x00000000);
 824                 if (OneG_Mode)
 825                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0182222);
 826                 else
 827                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0042222);
 828
 829                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000ff);
 830                 if (port_count > 1) {
 831                         /* init serdes 1 */
 832                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x00000048);
 833                         i = 0;
 834                         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 835                                 & 0x0000000f)) != 0x0000000f) && (i++ < 5000))
 836                                 mdelay(1);
 837                         if (i > 5000) {
 838                                 printk("%s: Init: serdes 1 not ready, status=%x\n", __func__, u32temp);
 839                                 /* return 1; */
 840                         }
 841                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x000bdef7);
 842                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE1, 0x9ce73000);
 843                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE1, 0x0ff00000);
 844                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET1, 0x00000000);
 845                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS1, 0x00000000);
 846                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL1, 0x00000000);
 847                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL1, 0xf0002222);
 848                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000ff);
 849                 }
 850         }
 851         return 0;
 852 }
 853
 854
 855 /**
 856  * nes_init_csr_ne020
 857  * Initialize registers for ne020 hardware
 858  */
 859 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count)
 860 {
 861         u32 u32temp;
 862
 863         nes_debug(NES_DBG_INIT, "port_count=%d\n", port_count);
 864
 865         nes_write_indexed(nesdev, 0x000001E4, 0x00000007);
 866         /* nes_write_indexed(nesdev, 0x000001E8, 0x000208C4); */
 867         nes_write_indexed(nesdev, 0x000001E8, 0x00020874);
 868         nes_write_indexed(nesdev, 0x000001D8, 0x00048002);
 869         /* nes_write_indexed(nesdev, 0x000001D8, 0x0004B002); */
 870         nes_write_indexed(nesdev, 0x000001FC, 0x00050005);
 871         nes_write_indexed(nesdev, 0x00000600, 0x55555555);
 872         nes_write_indexed(nesdev, 0x00000604, 0x55555555);
 873
 874         /* TODO: move these MAC register settings to NIC bringup */
 875         nes_write_indexed(nesdev, 0x00002000, 0x00000001);
 876         nes_write_indexed(nesdev, 0x00002004, 0x00000001);
 877         nes_write_indexed(nesdev, 0x00002008, 0x0000FFFF);
 878         nes_write_indexed(nesdev, 0x0000200C, 0x00000001);
 879         nes_write_indexed(nesdev, 0x00002010, 0x000003c1);
 880         nes_write_indexed(nesdev, 0x0000201C, 0x75345678);
 881         if (port_count > 1) {
 882                 nes_write_indexed(nesdev, 0x00002200, 0x00000001);
 883                 nes_write_indexed(nesdev, 0x00002204, 0x00000001);
 884                 nes_write_indexed(nesdev, 0x00002208, 0x0000FFFF);
 885                 nes_write_indexed(nesdev, 0x0000220C, 0x00000001);
 886                 nes_write_indexed(nesdev, 0x00002210, 0x000003c1);
 887                 nes_write_indexed(nesdev, 0x0000221C, 0x75345678);
 888                 nes_write_indexed(nesdev, 0x00000908, 0x20000001);
 889         }
 890         if (port_count > 2) {
 891                 nes_write_indexed(nesdev, 0x00002400, 0x00000001);
 892                 nes_write_indexed(nesdev, 0x00002404, 0x00000001);
 893                 nes_write_indexed(nesdev, 0x00002408, 0x0000FFFF);
 894                 nes_write_indexed(nesdev, 0x0000240C, 0x00000001);
 895                 nes_write_indexed(nesdev, 0x00002410, 0x000003c1);
 896                 nes_write_indexed(nesdev, 0x0000241C, 0x75345678);
 897                 nes_write_indexed(nesdev, 0x00000910, 0x20000001);
 898
 899                 nes_write_indexed(nesdev, 0x00002600, 0x00000001);
 900                 nes_write_indexed(nesdev, 0x00002604, 0x00000001);
 901                 nes_write_indexed(nesdev, 0x00002608, 0x0000FFFF);
 902                 nes_write_indexed(nesdev, 0x0000260C, 0x00000001);
 903                 nes_write_indexed(nesdev, 0x00002610, 0x000003c1);
 904                 nes_write_indexed(nesdev, 0x0000261C, 0x75345678);
 905                 nes_write_indexed(nesdev, 0x00000918, 0x20000001);
 906         }
 907
 908         nes_write_indexed(nesdev, 0x00005000, 0x00018000);
 909         /* nes_write_indexed(nesdev, 0x00005000, 0x00010000); */
 910         nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG1, (wqm_quanta << 1) |
 911                                                          0x00000001);
 912         nes_write_indexed(nesdev, 0x00005008, 0x1F1F1F1F);
 913         nes_write_indexed(nesdev, 0x00005010, 0x1F1F1F1F);
 914         nes_write_indexed(nesdev, 0x00005018, 0x1F1F1F1F);
 915         nes_write_indexed(nesdev, 0x00005020, 0x1F1F1F1F);
 916         nes_write_indexed(nesdev, 0x00006090, 0xFFFFFFFF);
 917
 918         /* TODO: move this to code, get from EEPROM */
 919         nes_write_indexed(nesdev, 0x00000900, 0x20000001);
 920         nes_write_indexed(nesdev, 0x000060C0, 0x0000028e);
 921         nes_write_indexed(nesdev, 0x000060C8, 0x00000020);
 922
 923         nes_write_indexed(nesdev, 0x000001EC, 0x7b2625a0);
 924         /* nes_write_indexed(nesdev, 0x000001EC, 0x5f2625a0); */
 925
 926         if (hw_rev != NE020_REV) {
 927                 u32temp = nes_read_indexed(nesdev, 0x000008e8);
 928                 u32temp |= 0x80000000;
 929                 nes_write_indexed(nesdev, 0x000008e8, u32temp);
 930                 u32temp = nes_read_indexed(nesdev, 0x000021f8);
 931                 u32temp &= 0x7fffffff;
 932                 u32temp |= 0x7fff0010;
 933                 nes_write_indexed(nesdev, 0x000021f8, u32temp);
 934                 if (port_count > 1) {
 935                         u32temp = nes_read_indexed(nesdev, 0x000023f8);
 936                         u32temp &= 0x7fffffff;
 937                         u32temp |= 0x7fff0010;
 938                         nes_write_indexed(nesdev, 0x000023f8, u32temp);
 939                 }
 940         }
 941 }
 942
 943
 944 /**
 945  * nes_destroy_adapter - destroy the adapter structure
 946  */
 947 void nes_destroy_adapter(struct nes_adapter *nesadapter)
 948 {
 949         struct nes_adapter *tmp_adapter;
 950
 951         list_for_each_entry(tmp_adapter, &nes_adapter_list, list) {
 952                 nes_debug(NES_DBG_SHUTDOWN, "Nes Adapter list entry = 0x%p.\n",
 953                                 tmp_adapter);
 954         }
 955
 956         nesadapter->ref_count--;
 957         if (!nesadapter->ref_count) {
 958                 if (nesadapter->hw_rev == NE020_REV) {
 959                         del_timer(&nesadapter->mh_timer);
 960                 }
 961                 del_timer(&nesadapter->lc_timer);
 962
 963                 list_del(&nesadapter->list);
 964                 kfree(nesadapter);
 965         }
 966 }
 967
 968
 969 /**
 970  * nes_init_cqp
 971  */
 972 int nes_init_cqp(struct nes_device *nesdev)
 973 {
 974         struct nes_adapter *nesadapter = nesdev->nesadapter;
 975         struct nes_hw_cqp_qp_context *cqp_qp_context;
 976         struct nes_hw_cqp_wqe *cqp_wqe;
 977         struct nes_hw_ceq *ceq;
 978         struct nes_hw_ceq *nic_ceq;
 979         struct nes_hw_aeq *aeq;
 980         void *vmem;
 981         dma_addr_t pmem;
 982         u32 count=0;
 983         u32 cqp_head;
 984         u64 u64temp;
 985         u32 u32temp;
 986
 987         /* allocate CQP memory */
 988         /* Need to add max_cq to the aeq size once cq overflow checking is added back */
 989         /* SQ is 512 byte aligned, others are 256 byte aligned */
 990         nesdev->cqp_mem_size = 512 +
 991                         (sizeof(struct nes_hw_cqp_wqe) * NES_CQP_SQ_SIZE) +
 992                         (sizeof(struct nes_hw_cqe) * NES_CCQ_SIZE) +
 993                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_CCEQ_SIZE), (u32)256) +
 994                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_NIC_CEQ_SIZE), (u32)256) +
 995                         (sizeof(struct nes_hw_aeqe) * nesadapter->max_qp) +
 996                         sizeof(struct nes_hw_cqp_qp_context);
 997
 998         nesdev->cqp_vbase = pci_alloc_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
 999                         &nesdev->cqp_pbase);
1000         if (!nesdev->cqp_vbase) {
1001                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for host descriptor rings\n");
1002                 return -ENOMEM;
1003         }
1004         memset(nesdev->cqp_vbase, 0, nesdev->cqp_mem_size);
1005
1006         /* Allocate a twice the number of CQP requests as the SQ size */
1007         nesdev->nes_cqp_requests = kzalloc(sizeof(struct nes_cqp_request) *
1008                         2 * NES_CQP_SQ_SIZE, GFP_KERNEL);
1009         if (nesdev->nes_cqp_requests == NULL) {
1010                 nes_debug(NES_DBG_INIT, "Unable to allocate memory CQP request entries.\n");
1011                 pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1012                                 nesdev->cqp.sq_pbase);
1013                 return -ENOMEM;
1014         }
1015
1016         nes_debug(NES_DBG_INIT, "Allocated CQP structures at %p (phys = %016lX), size = %u.\n",
1017                         nesdev->cqp_vbase, (unsigned long)nesdev->cqp_pbase, nesdev->cqp_mem_size);
1018
1019         spin_lock_init(&nesdev->cqp.lock);
1020         init_waitqueue_head(&nesdev->cqp.waitq);
1021
1022         /* Setup Various Structures */
1023         vmem = (void *)(((unsigned long)nesdev->cqp_vbase + (512 - 1)) &
1024                         ~(unsigned long)(512 - 1));
1025         pmem = (dma_addr_t)(((unsigned long long)nesdev->cqp_pbase + (512 - 1)) &
1026                         ~(unsigned long long)(512 - 1));
1027
1028         nesdev->cqp.sq_vbase = vmem;
1029         nesdev->cqp.sq_pbase = pmem;
1030         nesdev->cqp.sq_size = NES_CQP_SQ_SIZE;
1031         nesdev->cqp.sq_head = 0;
1032         nesdev->cqp.sq_tail = 0;
1033         nesdev->cqp.qp_id = PCI_FUNC(nesdev->pcidev->devfn);
1034
1035         vmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
1036         pmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
1037
1038         nesdev->ccq.cq_vbase = vmem;
1039         nesdev->ccq.cq_pbase = pmem;
1040         nesdev->ccq.cq_size = NES_CCQ_SIZE;
1041         nesdev->ccq.cq_head = 0;
1042         nesdev->ccq.ce_handler = nes_cqp_ce_handler;
1043         nesdev->ccq.cq_number = PCI_FUNC(nesdev->pcidev->devfn);
1044
1045         vmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1046         pmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1047
1048         nesdev->ceq_index = PCI_FUNC(nesdev->pcidev->devfn);
1049         ceq = &nesadapter->ceq[nesdev->ceq_index];
1050         ceq->ceq_vbase = vmem;
1051         ceq->ceq_pbase = pmem;
1052         ceq->ceq_size = NES_CCEQ_SIZE;
1053         ceq->ceq_head = 0;
1054
1055         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1056         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1057
1058         nesdev->nic_ceq_index = PCI_FUNC(nesdev->pcidev->devfn) + 8;
1059         nic_ceq = &nesadapter->ceq[nesdev->nic_ceq_index];
1060         nic_ceq->ceq_vbase = vmem;
1061         nic_ceq->ceq_pbase = pmem;
1062         nic_ceq->ceq_size = NES_NIC_CEQ_SIZE;
1063         nic_ceq->ceq_head = 0;
1064
1065         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1066         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1067
1068         aeq = &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)];
1069         aeq->aeq_vbase = vmem;
1070         aeq->aeq_pbase = pmem;
1071         aeq->aeq_size = nesadapter->max_qp;
1072         aeq->aeq_head = 0;
1073
1074         /* Setup QP Context */
1075         vmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1076         pmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1077
1078         cqp_qp_context = vmem;
1079         cqp_qp_context->context_words[0] =
1080                         cpu_to_le32((PCI_FUNC(nesdev->pcidev->devfn) << 12) + (2 << 10));
1081         cqp_qp_context->context_words[1] = 0;
1082         cqp_qp_context->context_words[2] = cpu_to_le32((u32)nesdev->cqp.sq_pbase);
1083         cqp_qp_context->context_words[3] = cpu_to_le32(((u64)nesdev->cqp.sq_pbase) >> 32);
1084
1085
1086         /* Write the address to Create CQP */
1087         if ((sizeof(dma_addr_t) > 4)) {
1088                 nes_write_indexed(nesdev,
1089                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1090                                 ((u64)pmem) >> 32);
1091         } else {
1092                 nes_write_indexed(nesdev,
1093                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8), 0);
1094         }
1095         nes_write_indexed(nesdev,
1096                         NES_IDX_CREATE_CQP_LOW + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1097                         (u32)pmem);
1098
1099         INIT_LIST_HEAD(&nesdev->cqp_avail_reqs);
1100         INIT_LIST_HEAD(&nesdev->cqp_pending_reqs);
1101
1102         for (count = 0; count < 2*NES_CQP_SQ_SIZE; count++) {
1103                 init_waitqueue_head(&nesdev->nes_cqp_requests[count].waitq);
1104                 list_add_tail(&nesdev->nes_cqp_requests[count].list, &nesdev->cqp_avail_reqs);
1105         }
1106
1107         /* Write Create CCQ WQE */
1108         cqp_head = nesdev->cqp.sq_head++;
1109         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1110         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1111         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1112                         (NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1113                         NES_CQP_CQ_CHK_OVERFLOW | ((u32)nesdev->ccq.cq_size << 16)));
1114         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1115                             (nesdev->ccq.cq_number |
1116                              ((u32)nesdev->ceq_index << 16)));
1117         u64temp = (u64)nesdev->ccq.cq_pbase;
1118         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1119         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0;
1120         u64temp = (unsigned long)&nesdev->ccq;
1121         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =
1122                         cpu_to_le32((u32)(u64temp >> 1));
1123         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1124                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1125         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1126
1127         /* Write Create CEQ WQE */
1128         cqp_head = nesdev->cqp.sq_head++;
1129         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1130         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1131         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1132                             (NES_CQP_CREATE_CEQ + ((u32)nesdev->ceq_index << 8)));
1133         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, ceq->ceq_size);
1134         u64temp = (u64)ceq->ceq_pbase;
1135         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1136
1137         /* Write Create AEQ WQE */
1138         cqp_head = nesdev->cqp.sq_head++;
1139         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1140         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1141         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1142                         (NES_CQP_CREATE_AEQ + ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8)));
1143         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_AEQ_WQE_ELEMENT_COUNT_IDX, aeq->aeq_size);
1144         u64temp = (u64)aeq->aeq_pbase;
1145         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1146
1147         /* Write Create NIC CEQ WQE */
1148         cqp_head = nesdev->cqp.sq_head++;
1149         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1150         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1151         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1152                         (NES_CQP_CREATE_CEQ + ((u32)nesdev->nic_ceq_index << 8)));
1153         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, nic_ceq->ceq_size);
1154         u64temp = (u64)nic_ceq->ceq_pbase;
1155         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1156
1157         /* Poll until CCQP done */
1158         count = 0;
1159         do {
1160                 if (count++ > 1000) {
1161                         printk(KERN_ERR PFX "Error creating CQP\n");
1162                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1163                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1164                         return -1;
1165                 }
1166                 udelay(10);
1167         } while (!(nes_read_indexed(nesdev,
1168                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn) * 8)) & (1 << 8)));
1169
1170         nes_debug(NES_DBG_INIT, "CQP Status = 0x%08X\n", nes_read_indexed(nesdev,
1171                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1172
1173         u32temp = 0x04800000;
1174         nes_write32(nesdev->regs+NES_WQE_ALLOC, u32temp | nesdev->cqp.qp_id);
1175
1176         /* wait for the CCQ, CEQ, and AEQ to get created */
1177         count = 0;
1178         do {
1179                 if (count++ > 1000) {
1180                         printk(KERN_ERR PFX "Error creating CCQ, CEQ, and AEQ\n");
1181                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1182                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1183                         return -1;
1184                 }
1185                 udelay(10);
1186         } while (((nes_read_indexed(nesdev,
1187                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15<<8)) != (15<<8)));
1188
1189         /* dump the QP status value */
1190         nes_debug(NES_DBG_INIT, "QP Status = 0x%08X\n", nes_read_indexed(nesdev,
1191                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1192
1193         nesdev->cqp.sq_tail++;
1194
1195         return 0;
1196 }
1197
1198
1199 /**
1200  * nes_destroy_cqp
1201  */
1202 int nes_destroy_cqp(struct nes_device *nesdev)
1203 {
1204         struct nes_hw_cqp_wqe *cqp_wqe;
1205         u32 count = 0;
1206         u32 cqp_head;
1207         unsigned long flags;
1208
1209         do {
1210                 if (count++ > 1000)
1211                         break;
1212                 udelay(10);
1213         } while (!(nesdev->cqp.sq_head == nesdev->cqp.sq_tail));
1214
1215         /* Reset CCQ */
1216         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_RESET |
1217                         nesdev->ccq.cq_number);
1218
1219         /* Disable device interrupts */
1220         nes_write32(nesdev->regs+NES_INT_MASK, 0x7fffffff);
1221
1222         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1223
1224         /* Destroy the AEQ */
1225         cqp_head = nesdev->cqp.sq_head++;
1226         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1227         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1228         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_AEQ |
1229                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8));
1230         cqp_wqe->wqe_words[NES_CQP_WQE_COMP_CTX_HIGH_IDX] = 0;
1231
1232         /* Destroy the NIC CEQ */
1233         cqp_head = nesdev->cqp.sq_head++;
1234         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1235         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1236         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1237                         ((u32)nesdev->nic_ceq_index << 8));
1238
1239         /* Destroy the CEQ */
1240         cqp_head = nesdev->cqp.sq_head++;
1241         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1242         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1243         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1244                         (nesdev->ceq_index << 8));
1245
1246         /* Destroy the CCQ */
1247         cqp_head = nesdev->cqp.sq_head++;
1248         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1249         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1250         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CQ);
1251         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->ccq.cq_number |
1252                         ((u32)nesdev->ceq_index << 16));
1253
1254         /* Destroy CQP */
1255         cqp_head = nesdev->cqp.sq_head++;
1256         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1257         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1258         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_QP |
1259                         NES_CQP_QP_TYPE_CQP);
1260         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->cqp.qp_id);
1261
1262         barrier();
1263         /* Ring doorbell (5 WQEs) */
1264         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x05800000 | nesdev->cqp.qp_id);
1265
1266         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1267
1268         /* wait for the CCQ, CEQ, and AEQ to get destroyed */
1269         count = 0;
1270         do {
1271                 if (count++ > 1000) {
1272                         printk(KERN_ERR PFX "Function%d: Error destroying CCQ, CEQ, and AEQ\n",
1273                                         PCI_FUNC(nesdev->pcidev->devfn));
1274                         break;
1275                 }
1276                 udelay(10);
1277         } while (((nes_read_indexed(nesdev,
1278                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15 << 8)) != 0));
1279
1280         /* dump the QP status value */
1281         nes_debug(NES_DBG_SHUTDOWN, "Function%d: QP Status = 0x%08X\n",
1282                         PCI_FUNC(nesdev->pcidev->devfn),
1283                         nes_read_indexed(nesdev,
1284                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1285
1286         kfree(nesdev->nes_cqp_requests);
1287
1288         /* Free the control structures */
1289         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1290                         nesdev->cqp.sq_pbase);
1291
1292         return 0;
1293 }
1294
1295
1296 /**
1297  * nes_init_1g_phy
1298  */
1299 int nes_init_1g_phy(struct nes_device *nesdev, u8 phy_type, u8 phy_index)
1300 {
1301         u32 counter = 0;
1302         u16 phy_data;
1303         int ret = 0;
1304
1305         nes_read_1G_phy_reg(nesdev, 1, phy_index, &phy_data);
1306         nes_write_1G_phy_reg(nesdev, 23, phy_index, 0xb000);
1307
1308         /* Reset the PHY */
1309         nes_write_1G_phy_reg(nesdev, 0, phy_index, 0x8000);
1310         udelay(100);
1311         counter = 0;
1312         do {
1313                 nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1314                 if (counter++ > 100) {
1315                         ret = -1;
1316                         break;
1317                 }
1318         } while (phy_data & 0x8000);
1319
1320         /* Setting no phy loopback */
1321         phy_data &= 0xbfff;
1322         phy_data |= 0x1140;
1323         nes_write_1G_phy_reg(nesdev, 0, phy_index,  phy_data);
1324         nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1325         nes_read_1G_phy_reg(nesdev, 0x17, phy_index, &phy_data);
1326         nes_read_1G_phy_reg(nesdev, 0x1e, phy_index, &phy_data);
1327
1328         /* Setting the interrupt mask */
1329         nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data);
1330         nes_write_1G_phy_reg(nesdev, 0x19, phy_index, 0xffee);
1331         nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data);
1332
1333         /* turning on flow control */
1334         nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data);
1335         nes_write_1G_phy_reg(nesdev, 4, phy_index, (phy_data & ~(0x03E0)) | 0xc00);
1336         nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data);
1337
1338         /* Clear Half duplex */
1339         nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data);
1340         nes_write_1G_phy_reg(nesdev, 9, phy_index, phy_data & ~(0x0100));
1341         nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data);
1342
1343         nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1344         nes_write_1G_phy_reg(nesdev, 0, phy_index, phy_data | 0x0300);
1345
1346         return ret;
1347 }
1348
1349
1350 /**
1351  * nes_init_2025_phy
1352  */
1353 int nes_init_2025_phy(struct nes_device *nesdev, u8 phy_type, u8 phy_index)
1354 {
1355         u32 temp_phy_data = 0;
1356         u32 temp_phy_data2 = 0;
1357         u32 counter = 0;
1358         u32 sds;
1359         u32 mac_index = nesdev->mac_index;
1360         int ret = 0;
1361         unsigned int first_attempt = 1;
1362
1363         /* Check firmware heartbeat */
1364         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1365         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1366         udelay(1500);
1367         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1368         temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1369
1370         if (temp_phy_data != temp_phy_data2) {
1371                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7fd);
1372                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1373                 if ((temp_phy_data & 0xff) > 0x20)
1374                         return 0;
1375                 printk(PFX "Reinitialize external PHY\n");
1376         }
1377
1378         /* no heartbeat, configure the PHY */
1379         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0x0000, 0x8000);
1380         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0000);
1381         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1382         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1383
1384         switch (phy_type) {
1385         case NES_PHY_TYPE_ARGUS:
1386                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1387                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1388                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x000C);
1389                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0008);
1390                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0001);
1391                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0098);
1392                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1393
1394                 /* setup LEDs */
1395                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x0007);
1396                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x000A);
1397                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0009);
1398                 break;
1399
1400         case NES_PHY_TYPE_SFP_D:
1401                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1402                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1403                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x0004);
1404                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0038);
1405                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0013);
1406                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0098);
1407                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1408
1409                 /* setup LEDs */
1410                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x0007);
1411                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x000A);
1412                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0009);
1413                 break;
1414
1415         case NES_PHY_TYPE_KR:
1416                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1417                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1418                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x000C);
1419                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0010);
1420                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0013);
1421                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0080);
1422                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1423
1424                 /* setup LEDs */
1425                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x000B);
1426                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x0003);
1427                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0004);
1428
1429                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0022, 0x406D);
1430                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0023, 0x0020);
1431                 break;
1432         }
1433
1434         nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0028, 0xA528);
1435
1436         /* Bring PHY out of reset */
1437         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0002);
1438
1439         /* Check for heartbeat */
1440         counter = 0;
1441         mdelay(690);
1442         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1443         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1444         do {
1445                 if (counter++ > 150) {
1446                         printk(PFX "No PHY heartbeat\n");
1447                         break;
1448                 }
1449                 mdelay(1);
1450                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1451                 temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1452         } while ((temp_phy_data2 == temp_phy_data));
1453
1454         /* wait for tracking */
1455         counter = 0;
1456         do {
1457                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7fd);
1458                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1459                 if (counter++ > 300) {
1460                         if (((temp_phy_data & 0xff) == 0x0) && first_attempt) {
1461                                 first_attempt = 0;
1462                                 counter = 0;
1463                                 /* reset AMCC PHY and try again */
1464                                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0xe854, 0x00c0);
1465                                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0xe854, 0x0040);
1466                                 continue;
1467                         } else {
1468                                 ret = 1;
1469                                 break;
1470                         }
1471                 }
1472                 mdelay(10);
1473         } while ((temp_phy_data & 0xff) < 0x30);
1474
1475         /* setup signal integrity */
1476         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd003, 0x0000);
1477         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00D, 0x00FE);
1478         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00E, 0x0032);
1479         if (phy_type == NES_PHY_TYPE_KR) {
1480                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00F, 0x000C);
1481         } else {
1482                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00F, 0x0002);
1483                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc314, 0x0063);
1484         }
1485
1486         /* reset serdes */
1487         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200);
1488         sds |= 0x1;
1489         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200, sds);
1490         sds &= 0xfffffffe;
1491         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200, sds);
1492
1493         counter = 0;
1494         while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET) & 0x00000040) != 0x00000040)
1495                         && (counter++ < 5000))
1496                 ;
1497
1498         return ret;
1499 }
1500
1501
1502 /**
1503  * nes_init_phy
1504  */
1505 int nes_init_phy(struct nes_device *nesdev)
1506 {
1507         struct nes_adapter *nesadapter = nesdev->nesadapter;
1508         u32 mac_index = nesdev->mac_index;
1509         u32 tx_config = 0;
1510         unsigned long flags;
1511         u8  phy_type = nesadapter->phy_type[mac_index];
1512         u8  phy_index = nesadapter->phy_index[mac_index];
1513         int ret = 0;
1514
1515         tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1516         if (phy_type == NES_PHY_TYPE_1G) {
1517                 /* setup 1G MDIO operation */
1518                 tx_config &= 0xFFFFFFE3;
1519                 tx_config |= 0x04;
1520         } else {
1521                 /* setup 10G MDIO operation */
1522                 tx_config &= 0xFFFFFFE3;
1523                 tx_config |= 0x15;
1524         }
1525         nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1526
1527         spin_lock_irqsave(&nesdev->nesadapter->phy_lock, flags);
1528
1529         switch (phy_type) {
1530         case NES_PHY_TYPE_1G:
1531                 ret = nes_init_1g_phy(nesdev, phy_type, phy_index);
1532                 break;
1533         case NES_PHY_TYPE_ARGUS:
1534         case NES_PHY_TYPE_SFP_D:
1535         case NES_PHY_TYPE_KR:
1536                 ret = nes_init_2025_phy(nesdev, phy_type, phy_index);
1537                 break;
1538         }
1539
1540         spin_unlock_irqrestore(&nesdev->nesadapter->phy_lock, flags);
1541
1542         return ret;
1543 }
1544
1545
1546 /**
1547  * nes_replenish_nic_rq
1548  */
1549 static void nes_replenish_nic_rq(struct nes_vnic *nesvnic)
1550 {
1551         unsigned long flags;
1552         dma_addr_t bus_address;
1553         struct sk_buff *skb;
1554         struct nes_hw_nic_rq_wqe *nic_rqe;
1555         struct nes_hw_nic *nesnic;
1556         struct nes_device *nesdev;
1557         u32 rx_wqes_posted = 0;
1558
1559         nesnic = &nesvnic->nic;
1560         nesdev = nesvnic->nesdev;
1561         spin_lock_irqsave(&nesnic->rq_lock, flags);
1562         if (nesnic->replenishing_rq !=0) {
1563                 if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1564                                 (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1565                         atomic_set(&nesvnic->rx_skb_timer_running, 1);
1566                         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1567                         nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1568                         add_timer(&nesvnic->rq_wqes_timer);
1569                 } else
1570                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1571                 return;
1572         }
1573         nesnic->replenishing_rq = 1;
1574         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1575         do {
1576                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1577                 if (skb) {
1578                         skb->dev = nesvnic->netdev;
1579
1580                         bus_address = pci_map_single(nesdev->pcidev,
1581                                         skb->data, nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1582
1583                         nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_head];
1584                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] =
1585                                         cpu_to_le32(nesvnic->max_frame_size);
1586                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1587                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] =
1588                                         cpu_to_le32((u32)bus_address);
1589                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] =
1590                                         cpu_to_le32((u32)((u64)bus_address >> 32));
1591                         nesnic->rx_skb[nesnic->rq_head] = skb;
1592                         nesnic->rq_head++;
1593                         nesnic->rq_head &= nesnic->rq_size - 1;
1594                         atomic_dec(&nesvnic->rx_skbs_needed);
1595                         barrier();
1596                         if (++rx_wqes_posted == 255) {
1597                                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1598                                 rx_wqes_posted = 0;
1599                         }
1600                 } else {
1601                         spin_lock_irqsave(&nesnic->rq_lock, flags);
1602                         if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1603                                         (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1604                                 atomic_set(&nesvnic->rx_skb_timer_running, 1);
1605                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1606                                 nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1607                                 add_timer(&nesvnic->rq_wqes_timer);
1608                         } else
1609                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1610                         break;
1611                 }
1612         } while (atomic_read(&nesvnic->rx_skbs_needed));
1613         barrier();
1614         if (rx_wqes_posted)
1615                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1616         nesnic->replenishing_rq = 0;
1617 }
1618
1619
1620 /**
1621  * nes_rq_wqes_timeout
1622  */
1623 static void nes_rq_wqes_timeout(unsigned long parm)
1624 {
1625         struct nes_vnic *nesvnic = (struct nes_vnic *)parm;
1626         printk("%s: Timer fired.\n", __func__);
1627         atomic_set(&nesvnic->rx_skb_timer_running, 0);
1628         if (atomic_read(&nesvnic->rx_skbs_needed))
1629                 nes_replenish_nic_rq(nesvnic);
1630 }
1631
1632
1633 static int nes_lro_get_skb_hdr(struct sk_buff *skb, void **iphdr,
1634                                void **tcph, u64 *hdr_flags, void *priv)
1635 {
1636         unsigned int ip_len;
1637         struct iphdr *iph;
1638         skb_reset_network_header(skb);
1639         iph = ip_hdr(skb);
1640         if (iph->protocol != IPPROTO_TCP)
1641                 return -1;
1642         ip_len = ip_hdrlen(skb);
1643         skb_set_transport_header(skb, ip_len);
1644         *tcph = tcp_hdr(skb);
1645
1646         *hdr_flags = LRO_IPV4 | LRO_TCP;
1647         *iphdr = iph;
1648         return 0;
1649 }
1650
1651
1652 /**
1653  * nes_init_nic_qp
1654  */
1655 int nes_init_nic_qp(struct nes_device *nesdev, struct net_device *netdev)
1656 {
1657         struct nes_hw_cqp_wqe *cqp_wqe;
1658         struct nes_hw_nic_sq_wqe *nic_sqe;
1659         struct nes_hw_nic_qp_context *nic_context;
1660         struct sk_buff *skb;
1661         struct nes_hw_nic_rq_wqe *nic_rqe;
1662         struct nes_vnic *nesvnic = netdev_priv(netdev);
1663         unsigned long flags;
1664         void *vmem;
1665         dma_addr_t pmem;
1666         u64 u64temp;
1667         int ret;
1668         u32 cqp_head;
1669         u32 counter;
1670         u32 wqe_count;
1671         u8 jumbomode=0;
1672
1673         /* Allocate fragment, SQ, RQ, and CQ; Reuse CEQ based on the PCI function */
1674         nesvnic->nic_mem_size = 256 +
1675                         (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag)) +
1676                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe)) +
1677                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe)) +
1678                         (NES_NIC_WQ_SIZE * 2 * sizeof(struct nes_hw_nic_cqe)) +
1679                         sizeof(struct nes_hw_nic_qp_context);
1680
1681         nesvnic->nic_vbase = pci_alloc_consistent(nesdev->pcidev, nesvnic->nic_mem_size,
1682                         &nesvnic->nic_pbase);
1683         if (!nesvnic->nic_vbase) {
1684                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for NIC host descriptor rings\n");
1685                 return -ENOMEM;
1686         }
1687         memset(nesvnic->nic_vbase, 0, nesvnic->nic_mem_size);
1688         nes_debug(NES_DBG_INIT, "Allocated NIC QP structures at %p (phys = %016lX), size = %u.\n",
1689                         nesvnic->nic_vbase, (unsigned long)nesvnic->nic_pbase, nesvnic->nic_mem_size);
1690
1691         vmem = (void *)(((unsigned long)nesvnic->nic_vbase + (256 - 1)) &
1692                         ~(unsigned long)(256 - 1));
1693         pmem = (dma_addr_t)(((unsigned long long)nesvnic->nic_pbase + (256 - 1)) &
1694                         ~(unsigned long long)(256 - 1));
1695
1696         /* Setup the first Fragment buffers */
1697         nesvnic->nic.first_frag_vbase = vmem;
1698
1699         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1700                 nesvnic->nic.frag_paddr[counter] = pmem;
1701                 pmem += sizeof(struct nes_first_frag);
1702         }
1703
1704         /* setup the SQ */
1705         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag));
1706
1707         nesvnic->nic.sq_vbase = (void *)vmem;
1708         nesvnic->nic.sq_pbase = pmem;
1709         nesvnic->nic.sq_head = 0;
1710         nesvnic->nic.sq_tail = 0;
1711         nesvnic->nic.sq_size = NES_NIC_WQ_SIZE;
1712         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1713                 nic_sqe = &nesvnic->nic.sq_vbase[counter];
1714                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_MISC_IDX] =
1715                                 cpu_to_le32(NES_NIC_SQ_WQE_DISABLE_CHKSUM |
1716                                 NES_NIC_SQ_WQE_COMPLETION);
1717                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX] =
1718                                 cpu_to_le32((u32)NES_FIRST_FRAG_SIZE << 16);
1719                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX] =
1720                                 cpu_to_le32((u32)nesvnic->nic.frag_paddr[counter]);
1721                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX] =
1722                                 cpu_to_le32((u32)((u64)nesvnic->nic.frag_paddr[counter] >> 32));
1723         }
1724
1725         nesvnic->get_cqp_request = nes_get_cqp_request;
1726         nesvnic->post_cqp_request = nes_post_cqp_request;
1727         nesvnic->mcrq_mcast_filter = NULL;
1728
1729         spin_lock_init(&nesvnic->nic.rq_lock);
1730
1731         /* setup the RQ */
1732         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1733         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1734
1735
1736         nesvnic->nic.rq_vbase = vmem;
1737         nesvnic->nic.rq_pbase = pmem;
1738         nesvnic->nic.rq_head = 0;
1739         nesvnic->nic.rq_tail = 0;
1740         nesvnic->nic.rq_size = NES_NIC_WQ_SIZE;
1741
1742         /* setup the CQ */
1743         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1744         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1745
1746         if (nesdev->nesadapter->netdev_count > 2)
1747                 nesvnic->mcrq_qp_id = nesvnic->nic_index + 32;
1748         else
1749                 nesvnic->mcrq_qp_id = nesvnic->nic.qp_id + 4;
1750
1751         nesvnic->nic_cq.cq_vbase = vmem;
1752         nesvnic->nic_cq.cq_pbase = pmem;
1753         nesvnic->nic_cq.cq_head = 0;
1754         nesvnic->nic_cq.cq_size = NES_NIC_WQ_SIZE * 2;
1755
1756         nesvnic->nic_cq.ce_handler = nes_nic_napi_ce_handler;
1757
1758         /* Send CreateCQ request to CQP */
1759         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1760         cqp_head = nesdev->cqp.sq_head;
1761
1762         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1763         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1764
1765         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
1766                         NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1767                         ((u32)nesvnic->nic_cq.cq_size << 16));
1768         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(
1769                         nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16));
1770         u64temp = (u64)nesvnic->nic_cq.cq_pbase;
1771         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1772         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =  0;
1773         u64temp = (unsigned long)&nesvnic->nic_cq;
1774         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =  cpu_to_le32((u32)(u64temp >> 1));
1775         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1776                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1777         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1778         if (++cqp_head >= nesdev->cqp.sq_size)
1779                 cqp_head = 0;
1780         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1781         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1782
1783         /* Send CreateQP request to CQP */
1784         nic_context = (void *)(&nesvnic->nic_cq.cq_vbase[nesvnic->nic_cq.cq_size]);
1785         nic_context->context_words[NES_NIC_CTX_MISC_IDX] =
1786                         cpu_to_le32((u32)NES_NIC_CTX_SIZE |
1787                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 12));
1788         nes_debug(NES_DBG_INIT, "RX_WINDOW_BUFFER_PAGE_TABLE_SIZE = 0x%08X, RX_WINDOW_BUFFER_SIZE = 0x%08X\n",
1789                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_PAGE_TABLE_SIZE),
1790                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE));
1791         if (nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE) != 0) {
1792                 nic_context->context_words[NES_NIC_CTX_MISC_IDX] |= cpu_to_le32(NES_NIC_BACK_STORE);
1793         }
1794
1795         u64temp = (u64)nesvnic->nic.sq_pbase;
1796         nic_context->context_words[NES_NIC_CTX_SQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1797         nic_context->context_words[NES_NIC_CTX_SQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1798         u64temp = (u64)nesvnic->nic.rq_pbase;
1799         nic_context->context_words[NES_NIC_CTX_RQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1800         nic_context->context_words[NES_NIC_CTX_RQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1801
1802         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_CREATE_QP |
1803                         NES_CQP_QP_TYPE_NIC);
1804         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesvnic->nic.qp_id);
1805         u64temp = (u64)nesvnic->nic_cq.cq_pbase +
1806                         (nesvnic->nic_cq.cq_size * sizeof(struct nes_hw_nic_cqe));
1807         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp);
1808
1809         if (++cqp_head >= nesdev->cqp.sq_size)
1810                 cqp_head = 0;
1811         nesdev->cqp.sq_head = cqp_head;
1812
1813         barrier();
1814
1815         /* Ring doorbell (2 WQEs) */
1816         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1817
1818         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1819         nes_debug(NES_DBG_INIT, "Waiting for create NIC QP%u to complete.\n",
1820                         nesvnic->nic.qp_id);
1821
1822         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1823                         NES_EVENT_TIMEOUT);
1824         nes_debug(NES_DBG_INIT, "Create NIC QP%u completed, wait_event_timeout ret = %u.\n",
1825                         nesvnic->nic.qp_id, ret);
1826         if (!ret) {
1827                 nes_debug(NES_DBG_INIT, "NIC QP%u create timeout expired\n", nesvnic->nic.qp_id);
1828                 pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1829                                 nesvnic->nic_pbase);
1830                 return -EIO;
1831         }
1832
1833         /* Populate the RQ */
1834         for (counter = 0; counter < (NES_NIC_WQ_SIZE - 1); counter++) {
1835                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1836                 if (!skb) {
1837                         nes_debug(NES_DBG_INIT, "%s: out of memory for receive skb\n", netdev->name);
1838
1839                         nes_destroy_nic_qp(nesvnic);
1840                         return -ENOMEM;
1841                 }
1842
1843                 skb->dev = netdev;
1844
1845                 pmem = pci_map_single(nesdev->pcidev, skb->data,
1846                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1847
1848                 nic_rqe = &nesvnic->nic.rq_vbase[counter];
1849                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] = cpu_to_le32(nesvnic->max_frame_size);
1850                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1851                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]  = cpu_to_le32((u32)pmem);
1852                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] = cpu_to_le32((u32)((u64)pmem >> 32));
1853                 nesvnic->nic.rx_skb[counter] = skb;
1854         }
1855
1856         wqe_count = NES_NIC_WQ_SIZE - 1;
1857         nesvnic->nic.rq_head = wqe_count;
1858         barrier();
1859         do {
1860                 counter = min(wqe_count, ((u32)255));
1861                 wqe_count -= counter;
1862                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (counter << 24) | nesvnic->nic.qp_id);
1863         } while (wqe_count);
1864         init_timer(&nesvnic->rq_wqes_timer);
1865         nesvnic->rq_wqes_timer.function = nes_rq_wqes_timeout;
1866         nesvnic->rq_wqes_timer.data = (unsigned long)nesvnic;
1867         nes_debug(NES_DBG_INIT, "NAPI support Enabled\n");
1868         if (nesdev->nesadapter->et_use_adaptive_rx_coalesce)
1869         {
1870                 nes_nic_init_timer(nesdev);
1871                 if (netdev->mtu > 1500)
1872                         jumbomode = 1;
1873                 nes_nic_init_timer_defaults(nesdev, jumbomode);
1874         }
1875         nesvnic->lro_mgr.max_aggr       = nes_lro_max_aggr;
1876         nesvnic->lro_mgr.max_desc       = NES_MAX_LRO_DESCRIPTORS;
1877         nesvnic->lro_mgr.lro_arr        = nesvnic->lro_desc;
1878         nesvnic->lro_mgr.get_skb_header = nes_lro_get_skb_hdr;
1879         nesvnic->lro_mgr.features       = LRO_F_NAPI | LRO_F_EXTRACT_VLAN_ID;
1880         nesvnic->lro_mgr.dev            = netdev;
1881         nesvnic->lro_mgr.ip_summed      = CHECKSUM_UNNECESSARY;
1882         nesvnic->lro_mgr.ip_summed_aggr = CHECKSUM_UNNECESSARY;
1883         return 0;
1884 }
1885
1886
1887 /**
1888  * nes_destroy_nic_qp
1889  */
1890 void nes_destroy_nic_qp(struct nes_vnic *nesvnic)
1891 {
1892         u64 u64temp;
1893         dma_addr_t bus_address;
1894         struct nes_device *nesdev = nesvnic->nesdev;
1895         struct nes_hw_cqp_wqe *cqp_wqe;
1896         struct nes_hw_nic_sq_wqe *nic_sqe;
1897         struct nes_hw_nic_rq_wqe *nic_rqe;
1898         __le16 *wqe_fragment_length;
1899         u16  wqe_fragment_index;
1900         u64 wqe_frag;
1901         u32 cqp_head;
1902         u32 wqm_cfg0;
1903         unsigned long flags;
1904         int ret;
1905
1906         /* clear wqe stall before destroying NIC QP */
1907         wqm_cfg0 = nes_read_indexed(nesdev, NES_IDX_WQM_CONFIG0);
1908         nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG0, wqm_cfg0 & 0xFFFF7FFF);
1909
1910         /* Free remaining NIC receive buffers */
1911         while (nesvnic->nic.rq_head != nesvnic->nic.rq_tail) {
1912                 nic_rqe   = &nesvnic->nic.rq_vbase[nesvnic->nic.rq_tail];
1913                 wqe_frag  = (u64)le32_to_cpu(
1914                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
1915                 wqe_frag |= ((u64)le32_to_cpu(
1916                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX]))<<32;
1917                 pci_unmap_single(nesdev->pcidev, (dma_addr_t)wqe_frag,
1918                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1919                 dev_kfree_skb(nesvnic->nic.rx_skb[nesvnic->nic.rq_tail++]);
1920                 nesvnic->nic.rq_tail &= (nesvnic->nic.rq_size - 1);
1921         }
1922
1923         /* Free remaining NIC transmit buffers */
1924         while (nesvnic->nic.sq_head != nesvnic->nic.sq_tail) {
1925                 nic_sqe = &nesvnic->nic.sq_vbase[nesvnic->nic.sq_tail];
1926                 wqe_fragment_index = 1;
1927                 wqe_fragment_length = (__le16 *)
1928                         &nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
1929                 /* bump past the vlan tag */
1930                 wqe_fragment_length++;
1931                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
1932                         u64temp = (u64)le32_to_cpu(
1933                                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1934                                 wqe_fragment_index*2]);
1935                         u64temp += ((u64)le32_to_cpu(
1936                                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
1937                                 + wqe_fragment_index*2]))<<32;
1938                         bus_address = (dma_addr_t)u64temp;
1939                         if (test_and_clear_bit(nesvnic->nic.sq_tail,
1940                                         nesvnic->nic.first_frag_overflow)) {
1941                                 pci_unmap_single(nesdev->pcidev,
1942                                                 bus_address,
1943                                                 le16_to_cpu(wqe_fragment_length[
1944                                                         wqe_fragment_index++]),
1945                                                 PCI_DMA_TODEVICE);
1946                         }
1947                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
1948                                 if (wqe_fragment_length[wqe_fragment_index]) {
1949                                         u64temp = le32_to_cpu(
1950                                                 nic_sqe->wqe_words[
1951                                                 NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1952                                                 wqe_fragment_index*2]);
1953                                         u64temp += ((u64)le32_to_cpu(
1954                                                 nic_sqe->wqe_words[
1955                                                 NES_NIC_SQ_WQE_FRAG0_HIGH_IDX+
1956                                                 wqe_fragment_index*2]))<<32;
1957                                         bus_address = (dma_addr_t)u64temp;
1958                                         pci_unmap_page(nesdev->pcidev,
1959                                                         bus_address,
1960                                                         le16_to_cpu(
1961                                                         wqe_fragment_length[
1962                                                         wqe_fragment_index]),
1963                                                         PCI_DMA_TODEVICE);
1964                                 } else
1965                                         break;
1966                         }
1967                 }
1968                 if (nesvnic->nic.tx_skb[nesvnic->nic.sq_tail])
1969                         dev_kfree_skb(
1970                                 nesvnic->nic.tx_skb[nesvnic->nic.sq_tail]);
1971
1972                 nesvnic->nic.sq_tail = (++nesvnic->nic.sq_tail)
1973                                         & (nesvnic->nic.sq_size - 1);
1974         }
1975
1976         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1977
1978         /* Destroy NIC QP */
1979         cqp_head = nesdev->cqp.sq_head;
1980         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1981         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1982
1983         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1984                 (NES_CQP_DESTROY_QP | NES_CQP_QP_TYPE_NIC));
1985         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1986                 nesvnic->nic.qp_id);
1987
1988         if (++cqp_head >= nesdev->cqp.sq_size)
1989                 cqp_head = 0;
1990
1991         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1992
1993         /* Destroy NIC CQ */
1994         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1995         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1996                 (NES_CQP_DESTROY_CQ | ((u32)nesvnic->nic_cq.cq_size << 16)));
1997         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1998                 (nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16)));
1999
2000         if (++cqp_head >= nesdev->cqp.sq_size)
2001                 cqp_head = 0;
2002
2003         nesdev->cqp.sq_head = cqp_head;
2004         barrier();
2005
2006         /* Ring doorbell (2 WQEs) */
2007         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
2008
2009         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2010         nes_debug(NES_DBG_SHUTDOWN, "Waiting for CQP, cqp_head=%u, cqp.sq_head=%u,"
2011                         " cqp.sq_tail=%u, cqp.sq_size=%u\n",
2012                         cqp_head, nesdev->cqp.sq_head,
2013                         nesdev->cqp.sq_tail, nesdev->cqp.sq_size);
2014
2015         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
2016                         NES_EVENT_TIMEOUT);
2017
2018         nes_debug(NES_DBG_SHUTDOWN, "Destroy NIC QP returned, wait_event_timeout ret = %u, cqp_head=%u,"
2019                         " cqp.sq_head=%u, cqp.sq_tail=%u\n",
2020                         ret, cqp_head, nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
2021         if (!ret) {
2022                 nes_debug(NES_DBG_SHUTDOWN, "NIC QP%u destroy timeout expired\n",
2023                                 nesvnic->nic.qp_id);
2024         }
2025
2026         pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
2027                         nesvnic->nic_pbase);
2028
2029         /* restore old wqm_cfg0 value */
2030         nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG0, wqm_cfg0);
2031 }
2032
2033 /**
2034  * nes_napi_isr
2035  */
2036 int nes_napi_isr(struct nes_device *nesdev)
2037 {
2038         struct nes_adapter *nesadapter = nesdev->nesadapter;
2039         u32 int_stat;
2040
2041         if (nesdev->napi_isr_ran) {
2042                 /* interrupt status has already been read in ISR */
2043                 int_stat = nesdev->int_stat;
2044         } else {
2045                 int_stat = nes_read32(nesdev->regs + NES_INT_STAT);
2046                 nesdev->int_stat = int_stat;
2047                 nesdev->napi_isr_ran = 1;
2048         }
2049
2050         int_stat &= nesdev->int_req;
2051         /* iff NIC, process here, else wait for DPC */
2052         if ((int_stat) && ((int_stat & 0x0000ff00) == int_stat)) {
2053                 nesdev->napi_isr_ran = 0;
2054                 nes_write32(nesdev->regs + NES_INT_STAT,
2055                         (int_stat &
2056                         ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 | NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
2057
2058                 /* Process the CEQs */
2059                 nes_process_ceq(nesdev, &nesdev->nesadapter->ceq[nesdev->nic_ceq_index]);
2060
2061                 if (unlikely((((nesadapter->et_rx_coalesce_usecs_irq) &&
2062                                         (!nesadapter->et_use_adaptive_rx_coalesce)) ||
2063                                         ((nesadapter->et_use_adaptive_rx_coalesce) &&
2064                                          (nesdev->deepcq_count > nesadapter->et_pkt_rate_low))))) {
2065                         if ((nesdev->int_req & NES_INT_TIMER) == 0) {
2066                                 /* Enable Periodic timer interrupts */
2067                                 nesdev->int_req |= NES_INT_TIMER;
2068                                 /* ack any pending periodic timer interrupts so we don't get an immediate interrupt */
2069                                 /* TODO: need to also ack other unused periodic timer values, get from nesadapter */
2070                                 nes_write32(nesdev->regs+NES_TIMER_STAT,
2071                                                 nesdev->timer_int_req  | ~(nesdev->nesadapter->timer_int_req));
2072                                 nes_write32(nesdev->regs+NES_INTF_INT_MASK,
2073                                                 ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
2074                         }
2075
2076                         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2077                         {
2078                                 nes_nic_init_timer(nesdev);
2079                         }
2080                         /* Enable interrupts, except CEQs */
2081                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2082                 } else {
2083                         /* Enable interrupts, make sure timer is off */
2084                         nesdev->int_req &= ~NES_INT_TIMER;
2085                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2086                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2087                 }
2088                 nesdev->deepcq_count = 0;
2089                 return 1;
2090         } else {
2091                 return 0;
2092         }
2093 }
2094
2095 static void process_critical_error(struct nes_device *nesdev)
2096 {
2097         u32 debug_error;
2098         u32 nes_idx_debug_error_masks0 = 0;
2099         u16 error_module = 0;
2100
2101         debug_error = nes_read_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS);
2102         printk(KERN_ERR PFX "Critical Error reported by device!!! 0x%02X\n",
2103                         (u16)debug_error);
2104         nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS,
2105                         0x01010000 | (debug_error & 0x0000ffff));
2106         if (crit_err_count++ > 10)
2107                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS1, 1 << 0x17);
2108         error_module = (u16) (debug_error & 0x1F00) >> 8;
2109         if (++nesdev->nesadapter->crit_error_count[error_module-1] >=
2110                         nes_max_critical_error_count) {
2111                 printk(KERN_ERR PFX "Masking off critical error for module "
2112                         "0x%02X\n", (u16)error_module);
2113                 nes_idx_debug_error_masks0 = nes_read_indexed(nesdev,
2114                         NES_IDX_DEBUG_ERROR_MASKS0);
2115                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS0,
2116                         nes_idx_debug_error_masks0 | (1 << error_module));
2117         }
2118 }
2119 /**
2120  * nes_dpc
2121  */
2122 void nes_dpc(unsigned long param)
2123 {
2124         struct nes_device *nesdev = (struct nes_device *)param;
2125         struct nes_adapter *nesadapter = nesdev->nesadapter;
2126         u32 counter;
2127         u32 loop_counter = 0;
2128         u32 int_status_bit;
2129         u32 int_stat;
2130         u32 timer_stat;
2131         u32 temp_int_stat;
2132         u32 intf_int_stat;
2133         u32 processed_intf_int = 0;
2134         u16 processed_timer_int = 0;
2135         u16 completion_ints = 0;
2136         u16 timer_ints = 0;
2137
2138         /* nes_debug(NES_DBG_ISR, "\n"); */
2139
2140         do {
2141                 timer_stat = 0;
2142                 if (nesdev->napi_isr_ran) {
2143                         nesdev->napi_isr_ran = 0;
2144                         int_stat = nesdev->int_stat;
2145                 } else
2146                         int_stat = nes_read32(nesdev->regs+NES_INT_STAT);
2147                 if (processed_intf_int != 0)
2148                         int_stat &= nesdev->int_req & ~NES_INT_INTF;
2149                 else
2150                         int_stat &= nesdev->int_req;
2151                 if (processed_timer_int == 0) {
2152                         processed_timer_int = 1;
2153                         if (int_stat & NES_INT_TIMER) {
2154                                 timer_stat = nes_read32(nesdev->regs + NES_TIMER_STAT);
2155                                 if ((timer_stat & nesdev->timer_int_req) == 0) {
2156                                         int_stat &= ~NES_INT_TIMER;
2157                                 }
2158                         }
2159                 } else {
2160                         int_stat &= ~NES_INT_TIMER;
2161                 }
2162
2163                 if (int_stat) {
2164                         if (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2165                                         NES_INT_MAC1|NES_INT_MAC2 | NES_INT_MAC3)) {
2166                                 /* Ack the interrupts */
2167                                 nes_write32(nesdev->regs+NES_INT_STAT,
2168                                         (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2169                                         NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
2170                         }
2171
2172                         temp_int_stat = int_stat;
2173                         for (counter = 0, int_status_bit = 1; counter < 16; counter++) {
2174                                 if (int_stat & int_status_bit) {
2175                                         nes_process_ceq(nesdev, &nesadapter->ceq[counter]);
2176                                         temp_int_stat &= ~int_status_bit;
2177                                         completion_ints = 1;
2178                                 }
2179                                 if (!(temp_int_stat & 0x0000ffff))
2180                                         break;
2181                                 int_status_bit <<= 1;
2182                         }
2183
2184                         /* Process the AEQ for this pci function */
2185                         int_status_bit = 1 << (16 + PCI_FUNC(nesdev->pcidev->devfn));
2186                         if (int_stat & int_status_bit) {
2187                                 nes_process_aeq(nesdev, &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)]);
2188                         }
2189
2190                         /* Process the MAC interrupt for this pci function */
2191                         int_status_bit = 1 << (24 + nesdev->mac_index);
2192                         if (int_stat & int_status_bit) {
2193                                 nes_process_mac_intr(nesdev, nesdev->mac_index);
2194                         }
2195
2196                         if (int_stat & NES_INT_TIMER) {
2197                                 if (timer_stat & nesdev->timer_int_req) {
2198                                         nes_write32(nesdev->regs + NES_TIMER_STAT,
2199                                                         (timer_stat & nesdev->timer_int_req) |
2200                                                         ~(nesdev->nesadapter->timer_int_req));
2201                                         timer_ints = 1;
2202                                 }
2203                         }
2204
2205                         if (int_stat & NES_INT_INTF) {
2206                                 processed_intf_int = 1;
2207                                 intf_int_stat = nes_read32(nesdev->regs+NES_INTF_INT_STAT);
2208                                 intf_int_stat &= nesdev->intf_int_req;
2209                                 if (NES_INTF_INT_CRITERR & intf_int_stat) {
2210                                         process_critical_error(nesdev);
2211                                 }
2212                                 if (NES_INTF_INT_PCIERR & intf_int_stat) {
2213                                         printk(KERN_ERR PFX "PCI Error reported by device!!!\n");
2214                                         BUG();
2215                                 }
2216                                 if (NES_INTF_INT_AEQ_OFLOW & intf_int_stat) {
2217                                         printk(KERN_ERR PFX "AEQ Overflow reported by device!!!\n");
2218                                         BUG();
2219                                 }
2220                                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, intf_int_stat);
2221                         }
2222
2223                         if (int_stat & NES_INT_TSW) {
2224                         }
2225                 }
2226                 /* Don't use the interface interrupt bit stay in loop */
2227                 int_stat &= ~NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 |
2228                                 NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3;
2229         } while ((int_stat != 0) && (loop_counter++ < MAX_DPC_ITERATIONS));
2230
2231         if (timer_ints == 1) {
2232                 if ((nesadapter->et_rx_coalesce_usecs_irq) || (nesadapter->et_use_adaptive_rx_coalesce)) {
2233                         if (completion_ints == 0) {
2234                                 nesdev->timer_only_int_count++;
2235                                 if (nesdev->timer_only_int_count>=nesadapter->timer_int_limit) {
2236                                         nesdev->timer_only_int_count = 0;
2237                                         nesdev->int_req &= ~NES_INT_TIMER;
2238                                         nes_write32(nesdev->regs + NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2239                                         nes_write32(nesdev->regs + NES_INT_MASK, ~nesdev->int_req);
2240                                 } else {
2241                                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2242                                 }
2243                         } else {
2244                                 if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2245                                 {
2246                                         nes_nic_init_timer(nesdev);
2247                                 }
2248                                 nesdev->timer_only_int_count = 0;
2249                                 nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2250                         }
2251                 } else {
2252                         nesdev->timer_only_int_count = 0;
2253                         nesdev->int_req &= ~NES_INT_TIMER;
2254                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2255                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2256                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2257                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2258                 }
2259         } else {
2260                 if ( (completion_ints == 1) &&
2261                          (((nesadapter->et_rx_coalesce_usecs_irq) &&
2262                            (!nesadapter->et_use_adaptive_rx_coalesce)) ||
2263                           ((nesdev->deepcq_count > nesadapter->et_pkt_rate_low) &&
2264                            (nesadapter->et_use_adaptive_rx_coalesce) )) ) {
2265                         /* nes_debug(NES_DBG_ISR, "Enabling periodic timer interrupt.\n" ); */
2266                         nesdev->timer_only_int_count = 0;
2267                         nesdev->int_req |= NES_INT_TIMER;
2268                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2269                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2270                         nes_write32(nesdev->regs+NES_INTF_INT_MASK,
2271                                         ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
2272                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2273                 } else {
2274                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2275                 }
2276         }
2277         nesdev->deepcq_count = 0;
2278 }
2279
2280
2281 /**
2282  * nes_process_ceq
2283  */
2284 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq)
2285 {
2286         u64 u64temp;
2287         struct nes_hw_cq *cq;
2288         u32 head;
2289         u32 ceq_size;
2290
2291         /* nes_debug(NES_DBG_CQ, "\n"); */
2292         head = ceq->ceq_head;
2293         ceq_size = ceq->ceq_size;
2294
2295         do {
2296                 if (le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]) &
2297                                 NES_CEQE_VALID) {
2298                         u64temp = (((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]))) << 32) |
2299                                                 ((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_LOW_IDX])));
2300                         u64temp <<= 1;
2301                         cq = *((struct nes_hw_cq **)&u64temp);
2302                         /* nes_debug(NES_DBG_CQ, "pCQ = %p\n", cq); */
2303                         barrier();
2304                         ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX] = 0;
2305
2306                         /* call the event handler */
2307                         cq->ce_handler(nesdev, cq);
2308
2309                         if (++head >= ceq_size)
2310                                 head = 0;
2311                 } else {
2312                         break;
2313                 }
2314
2315         } while (1);
2316
2317         ceq->ceq_head = head;
2318 }
2319
2320
2321 /**
2322  * nes_process_aeq
2323  */
2324 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq)
2325 {
2326         /* u64 u64temp; */
2327         u32 head;
2328         u32 aeq_size;
2329         u32 aeqe_misc;
2330         u32 aeqe_cq_id;
2331         struct nes_hw_aeqe volatile *aeqe;
2332
2333         head = aeq->aeq_head;
2334         aeq_size = aeq->aeq_size;
2335
2336         do {
2337                 aeqe = &aeq->aeq_vbase[head];
2338                 if ((le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]) & NES_AEQE_VALID) == 0)
2339                         break;
2340                 aeqe_misc  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2341                 aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
2342                 if (aeqe_misc & (NES_AEQE_QP|NES_AEQE_CQ)) {
2343                         if (aeqe_cq_id >= NES_FIRST_QPN) {
2344                                 /* dealing with an accelerated QP related AE */
2345                                 /*
2346                                  * u64temp = (((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX]))) << 32) |
2347                                  *           ((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX])));
2348                                  */
2349                                 nes_process_iwarp_aeqe(nesdev, (struct nes_hw_aeqe *)aeqe);
2350                         } else {
2351                                 /* TODO: dealing with a CQP related AE */
2352                                 nes_debug(NES_DBG_AEQ, "Processing CQP related AE, misc = 0x%04X\n",
2353                                                 (u16)(aeqe_misc >> 16));
2354                         }
2355                 }
2356
2357                 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = 0;
2358
2359                 if (++head >= aeq_size)
2360                         head = 0;
2361
2362                 nes_write32(nesdev->regs + NES_AEQ_ALLOC, 1 << 16);
2363         }
2364         while (1);
2365         aeq->aeq_head = head;
2366 }
2367
2368 static void nes_reset_link(struct nes_device *nesdev, u32 mac_index)
2369 {
2370         struct nes_adapter *nesadapter = nesdev->nesadapter;
2371         u32 reset_value;
2372         u32 i=0;
2373         u32 u32temp;
2374
2375         if (nesadapter->hw_rev == NE020_REV) {
2376                 return;
2377         }
2378         mh_detected++;
2379
2380         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
2381
2382         if ((mac_index == 0) || ((mac_index == 1) && (nesadapter->OneG_Mode)))
2383                 reset_value |= 0x0000001d;
2384         else
2385                 reset_value |= 0x0000002d;
2386
2387         if (4 <= (nesadapter->link_interrupt_count[mac_index] / ((u16)NES_MAX_LINK_INTERRUPTS))) {
2388                 if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
2389                         nesadapter->link_interrupt_count[0] = 0;
2390                         nesadapter->link_interrupt_count[1] = 0;
2391                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2392                         if (0x00000040 & u32temp)
2393                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2394                         else
2395                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2396
2397                         reset_value |= 0x0000003d;
2398                 }
2399                 nesadapter->link_interrupt_count[mac_index] = 0;
2400         }
2401
2402         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2403
2404         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
2405                         & 0x00000040) != 0x00000040) && (i++ < 5000));
2406
2407         if (0x0000003d == (reset_value & 0x0000003d)) {
2408                 u32 pcs_control_status0, pcs_control_status1;
2409
2410                 for (i = 0; i < 10; i++) {
2411                         pcs_control_status0 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0);
2412                         pcs_control_status1 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2413                         if (((0x0F000000 == (pcs_control_status0 & 0x0F000000))
2414                              && (pcs_control_status0 & 0x00100000))
2415                             || ((0x0F000000 == (pcs_control_status1 & 0x0F000000))
2416                                 && (pcs_control_status1 & 0x00100000)))
2417                                 continue;
2418                         else
2419                                 break;
2420                 }
2421                 if (10 == i) {
2422                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2423                         if (0x00000040 & u32temp)
2424                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2425                         else
2426                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2427
2428                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2429
2430                         while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET)
2431                                  & 0x00000040) != 0x00000040) && (i++ < 5000));
2432                 }
2433         }
2434 }
2435
2436 /**
2437  * nes_process_mac_intr
2438  */
2439 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number)
2440 {
2441         unsigned long flags;
2442         u32 pcs_control_status;
2443         struct nes_adapter *nesadapter = nesdev->nesadapter;
2444         struct nes_vnic *nesvnic;
2445         u32 mac_status;
2446         u32 mac_index = nesdev->mac_index;
2447         u32 u32temp;
2448         u16 phy_data;
2449         u16 temp_phy_data;
2450         u32 pcs_val  = 0x0f0f0000;
2451         u32 pcs_mask = 0x0f1f0000;
2452         u32 cdr_ctrl;
2453
2454         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2455         if (nesadapter->mac_sw_state[mac_number] != NES_MAC_SW_IDLE) {
2456                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2457                 return;
2458         }
2459         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_INTERRUPT;
2460         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2461
2462         /* ack the MAC interrupt */
2463         mac_status = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200));
2464         /* Clear the interrupt */
2465         nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200), mac_status);
2466
2467         nes_debug(NES_DBG_PHY, "MAC%u interrupt status = 0x%X.\n", mac_number, mac_status);
2468
2469         if (mac_status & (NES_MAC_INT_LINK_STAT_CHG | NES_MAC_INT_XGMII_EXT)) {
2470                 nesdev->link_status_interrupts++;
2471                 if (0 == (++nesadapter->link_interrupt_count[mac_index] % ((u16)NES_MAX_LINK_INTERRUPTS))) {
2472                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2473                         nes_reset_link(nesdev, mac_index);
2474                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2475                 }
2476                 /* read the PHY interrupt status register */
2477                 if ((nesadapter->OneG_Mode) &&
2478                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2479                         do {
2480                                 nes_read_1G_phy_reg(nesdev, 0x1a,
2481                                                 nesadapter->phy_index[mac_index], &phy_data);
2482                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1a = 0x%X.\n",
2483                                                 nesadapter->phy_index[mac_index], phy_data);
2484                         } while (phy_data&0x8000);
2485
2486                         temp_phy_data = 0;
2487                         do {
2488                                 nes_read_1G_phy_reg(nesdev, 0x11,
2489                                                 nesadapter->phy_index[mac_index], &phy_data);
2490                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x11 = 0x%X.\n",
2491                                                 nesadapter->phy_index[mac_index], phy_data);
2492                                 if (temp_phy_data == phy_data)
2493                                         break;
2494                                 temp_phy_data = phy_data;
2495                         } while (1);
2496
2497                         nes_read_1G_phy_reg(nesdev, 0x1e,
2498                                         nesadapter->phy_index[mac_index], &phy_data);
2499                         nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1e = 0x%X.\n",
2500                                         nesadapter->phy_index[mac_index], phy_data);
2501
2502                         nes_read_1G_phy_reg(nesdev, 1,
2503                                         nesadapter->phy_index[mac_index], &phy_data);
2504                         nes_debug(NES_DBG_PHY, "1G phy%u data from register 1 = 0x%X\n",
2505                                         nesadapter->phy_index[mac_index], phy_data);
2506
2507                         if (temp_phy_data & 0x1000) {
2508                                 nes_debug(NES_DBG_PHY, "The Link is up according to the PHY\n");
2509                                 phy_data = 4;
2510                         } else {
2511                                 nes_debug(NES_DBG_PHY, "The Link is down according to the PHY\n");
2512                         }
2513                 }
2514                 nes_debug(NES_DBG_PHY, "Eth SERDES Common Status: 0=0x%08X, 1=0x%08X\n",
2515                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0),
2516                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0+0x200));
2517
2518                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_PUMA_1G) {
2519                         switch (mac_index) {
2520                         case 1:
2521                         case 3:
2522                                 pcs_control_status = nes_read_indexed(nesdev,
2523                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2524                                 break;
2525                         default:
2526                                 pcs_control_status = nes_read_indexed(nesdev,
2527                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0);
2528                                 break;
2529                         }
2530                 } else {
2531                         pcs_control_status = nes_read_indexed(nesdev,
2532                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2533                         pcs_control_status = nes_read_indexed(nesdev,
2534                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2535                 }
2536
2537                 nes_debug(NES_DBG_PHY, "PCS PHY Control/Status%u: 0x%08X\n",
2538                                 mac_index, pcs_control_status);
2539                 if ((nesadapter->OneG_Mode) &&
2540                                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2541                         u32temp = 0x01010000;
2542                         if (nesadapter->port_count > 2) {
2543                                 u32temp |= 0x02020000;
2544                         }
2545                         if ((pcs_control_status & u32temp)!= u32temp) {
2546                                 phy_data = 0;
2547                                 nes_debug(NES_DBG_PHY, "PCS says the link is down\n");
2548                         }
2549                 } else {
2550                         switch (nesadapter->phy_type[mac_index]) {
2551                         case NES_PHY_TYPE_ARGUS:
2552                         case NES_PHY_TYPE_SFP_D:
2553                         case NES_PHY_TYPE_KR:
2554                                 /* clear the alarms */
2555                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0x0008);
2556                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc001);
2557                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc002);
2558                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc005);
2559                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc006);
2560                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2561                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9004);
2562                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9005);
2563                                 /* check link status */
2564                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2565                                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2566
2567                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2568                                 nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2569                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2570                                 phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2571
2572                                 phy_data = (!temp_phy_data && (phy_data == 0x8000)) ? 0x4 : 0x0;
2573
2574                                 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2575                                         __func__, phy_data, nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2576                                 break;
2577
2578                         case NES_PHY_TYPE_PUMA_1G:
2579                                 if (mac_index < 2)
2580                                         pcs_val = pcs_mask = 0x01010000;
2581                                 else
2582                                         pcs_val = pcs_mask = 0x02020000;
2583                                 /* fall through */
2584                         default:
2585                                 phy_data = (pcs_val == (pcs_control_status & pcs_mask)) ? 0x4 : 0x0;
2586                                 break;
2587                         }
2588                 }
2589
2590                 if (phy_data & 0x0004) {
2591                         if (wide_ppm_offset &&
2592                             (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) &&
2593                             (nesadapter->hw_rev != NE020_REV)) {
2594                                 cdr_ctrl = nes_read_indexed(nesdev,
2595                                                             NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2596                                                             mac_index * 0x200);
2597                                 nes_write_indexed(nesdev,
2598                                                   NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2599                                                   mac_index * 0x200,
2600                                                   cdr_ctrl | 0x000F0000);
2601                         }
2602                         nesadapter->mac_link_down[mac_index] = 0;
2603                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2604                                 nes_debug(NES_DBG_PHY, "The Link is UP!!.  linkup was %d\n",
2605                                                 nesvnic->linkup);
2606                                 if (nesvnic->linkup == 0) {
2607                                         printk(PFX "The Link is now up for port %s, netdev %p.\n",
2608                                                         nesvnic->netdev->name, nesvnic->netdev);
2609                                         if (netif_queue_stopped(nesvnic->netdev))
2610                                                 netif_start_queue(nesvnic->netdev);
2611                                         nesvnic->linkup = 1;
2612                                         netif_carrier_on(nesvnic->netdev);
2613                                 }
2614                         }
2615                 } else {
2616                         if (wide_ppm_offset &&
2617                             (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) &&
2618                             (nesadapter->hw_rev != NE020_REV)) {
2619                                 cdr_ctrl = nes_read_indexed(nesdev,
2620                                                             NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2621                                                             mac_index * 0x200);
2622                                 nes_write_indexed(nesdev,
2623                                                   NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2624                                                   mac_index * 0x200,
2625                                                   cdr_ctrl & 0xFFF0FFFF);
2626                         }
2627                         nesadapter->mac_link_down[mac_index] = 1;
2628                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2629                                 nes_debug(NES_DBG_PHY, "The Link is Down!!. linkup was %d\n",
2630                                                 nesvnic->linkup);
2631                                 if (nesvnic->linkup == 1) {
2632                                         printk(PFX "The Link is now down for port %s, netdev %p.\n",
2633                                                         nesvnic->netdev->name, nesvnic->netdev);
2634                                         if (!(netif_queue_stopped(nesvnic->netdev)))
2635                                                 netif_stop_queue(nesvnic->netdev);
2636                                         nesvnic->linkup = 0;
2637                                         netif_carrier_off(nesvnic->netdev);
2638                                 }
2639                         }
2640                 }
2641         }
2642
2643         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_IDLE;
2644 }
2645
2646
2647
2648 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2649 {
2650         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2651
2652         napi_schedule(&nesvnic->napi);
2653 }
2654
2655
2656 /* The MAX_RQES_TO_PROCESS defines how many max read requests to complete before
2657 * getting out of nic_ce_handler
2658 */
2659 #define MAX_RQES_TO_PROCESS     384
2660
2661 /**
2662  * nes_nic_ce_handler
2663  */
2664 void nes_nic_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2665 {
2666         u64 u64temp;
2667         dma_addr_t bus_address;
2668         struct nes_hw_nic *nesnic;
2669         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2670         struct nes_adapter *nesadapter = nesdev->nesadapter;
2671         struct nes_hw_nic_rq_wqe *nic_rqe;
2672         struct nes_hw_nic_sq_wqe *nic_sqe;
2673         struct sk_buff *skb;
2674         struct sk_buff *rx_skb;
2675         __le16 *wqe_fragment_length;
2676         u32 head;
2677         u32 cq_size;
2678         u32 rx_pkt_size;
2679         u32 cqe_count=0;
2680         u32 cqe_errv;
2681         u32 cqe_misc;
2682         u16 wqe_fragment_index = 1;     /* first fragment (0) is used by copy buffer */
2683         u16 vlan_tag;
2684         u16 pkt_type;
2685         u16 rqes_processed = 0;
2686         u8 sq_cqes = 0;
2687         u8 nes_use_lro = 0;
2688
2689         head = cq->cq_head;
2690         cq_size = cq->cq_size;
2691         cq->cqes_pending = 1;
2692         if (nesvnic->netdev->features & NETIF_F_LRO)
2693                 nes_use_lro = 1;
2694         do {
2695                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]) &
2696                                 NES_NIC_CQE_VALID) {
2697                         nesnic = &nesvnic->nic;
2698                         cqe_misc = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]);
2699                         if (cqe_misc & NES_NIC_CQE_SQ) {
2700                                 sq_cqes++;
2701                                 wqe_fragment_index = 1;
2702                                 nic_sqe = &nesnic->sq_vbase[nesnic->sq_tail];
2703                                 skb = nesnic->tx_skb[nesnic->sq_tail];
2704                                 wqe_fragment_length = (__le16 *)&nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
2705                                 /* bump past the vlan tag */
2706                                 wqe_fragment_length++;
2707                                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
2708                                         u64temp = (u64) le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2709                                                         wqe_fragment_index * 2]);
2710                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX +
2711                                                         wqe_fragment_index * 2])) << 32;
2712                                         bus_address = (dma_addr_t)u64temp;
2713                                         if (test_and_clear_bit(nesnic->sq_tail, nesnic->first_frag_overflow)) {
2714                                                 pci_unmap_single(nesdev->pcidev,
2715                                                                 bus_address,
2716                                                                 le16_to_cpu(wqe_fragment_length[wqe_fragment_index++]),
2717                                                                 PCI_DMA_TODEVICE);
2718                                         }
2719                                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
2720                                                 if (wqe_fragment_length[wqe_fragment_index]) {
2721                                                         u64temp = le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2722                                                                                 wqe_fragment_index * 2]);
2723                                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
2724                                                                                 + wqe_fragment_index * 2])) <<32;
2725                                                         bus_address = (dma_addr_t)u64temp;
2726                                                         pci_unmap_page(nesdev->pcidev,
2727                                                                         bus_address,
2728                                                                         le16_to_cpu(wqe_fragment_length[wqe_fragment_index]),
2729                                                                         PCI_DMA_TODEVICE);
2730                                                 } else
2731                                                         break;
2732                                         }
2733                                 }
2734                                 if (skb)
2735                                         dev_kfree_skb_any(skb);
2736                                 nesnic->sq_tail++;
2737                                 nesnic->sq_tail &= nesnic->sq_size-1;
2738                                 if (sq_cqes > 128) {
2739                                         barrier();
2740                                 /* restart the queue if it had been stopped */
2741                                 if (netif_queue_stopped(nesvnic->netdev))
2742                                         netif_wake_queue(nesvnic->netdev);
2743                                         sq_cqes = 0;
2744                                 }
2745                         } else {
2746                                 rqes_processed ++;
2747
2748                                 cq->rx_cqes_completed++;
2749                                 cq->rx_pkts_indicated++;
2750                                 rx_pkt_size = cqe_misc & 0x0000ffff;
2751                                 nic_rqe = &nesnic->rq_vbase[nesnic->rq_tail];
2752                                 /* Get the skb */
2753                                 rx_skb = nesnic->rx_skb[nesnic->rq_tail];
2754                                 nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_tail];
2755                                 bus_address = (dma_addr_t)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
2756                                 bus_address += ((u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX])) << 32;
2757                                 pci_unmap_single(nesdev->pcidev, bus_address,
2758                                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
2759                                 /* rx_skb->tail = rx_skb->data + rx_pkt_size; */
2760                                 /* rx_skb->len = rx_pkt_size; */
2761                                 rx_skb->len = 0;  /* TODO: see if this is necessary */
2762                                 skb_put(rx_skb, rx_pkt_size);
2763                                 rx_skb->protocol = eth_type_trans(rx_skb, nesvnic->netdev);
2764                                 nesnic->rq_tail++;
2765                                 nesnic->rq_tail &= nesnic->rq_size - 1;
2766
2767                                 atomic_inc(&nesvnic->rx_skbs_needed);
2768                                 if (atomic_read(&nesvnic->rx_skbs_needed) > (nesvnic->nic.rq_size>>1)) {
2769                                         nes_write32(nesdev->regs+NES_CQE_ALLOC,
2770                                                         cq->cq_number | (cqe_count << 16));
2771                                         /* nesadapter->tune_timer.cq_count += cqe_count; */
2772                                         nesdev->currcq_count += cqe_count;
2773                                         cqe_count = 0;
2774                                         nes_replenish_nic_rq(nesvnic);
2775                                 }
2776                                 pkt_type = (u16)(le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX]));
2777                                 cqe_errv = (cqe_misc & NES_NIC_CQE_ERRV_MASK) >> NES_NIC_CQE_ERRV_SHIFT;
2778                                 rx_skb->ip_summed = CHECKSUM_NONE;
2779
2780                                 if ((NES_PKT_TYPE_TCPV4_BITS == (pkt_type & NES_PKT_TYPE_TCPV4_MASK)) ||
2781                                                 (NES_PKT_TYPE_UDPV4_BITS == (pkt_type & NES_PKT_TYPE_UDPV4_MASK))) {
2782                                         if ((cqe_errv &
2783                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_TCPUDP_CSUM_ERR |
2784                                                         NES_NIC_ERRV_BITS_IPH_ERR | NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2785                                                 if (nesvnic->rx_checksum_disabled == 0) {
2786                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2787                                                 }
2788                                         } else
2789                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2790                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2791                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2792
2793                                 } else if ((pkt_type & NES_PKT_TYPE_IPV4_MASK) == NES_PKT_TYPE_IPV4_BITS) {
2794                                         if ((cqe_errv &
2795                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_IPH_ERR |
2796                                                         NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2797                                                 if (nesvnic->rx_checksum_disabled == 0) {
2798                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2799                                                         /* nes_debug(NES_DBG_CQ, "%s: Reporting successfully checksummed IPv4 packet.\n",
2800                                                                   nesvnic->netdev->name); */
2801                                                 }
2802                                         } else
2803                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2804                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2805                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2806                                         }
2807                                 /* nes_debug(NES_DBG_CQ, "pkt_type=%x, APBVT_MASK=%x\n",
2808                                                         pkt_type, (pkt_type & NES_PKT_TYPE_APBVT_MASK)); */
2809
2810                                 if ((pkt_type & NES_PKT_TYPE_APBVT_MASK) == NES_PKT_TYPE_APBVT_BITS) {
2811                                         if (nes_cm_recv(rx_skb, nesvnic->netdev))
2812                                                 rx_skb = NULL;
2813                                 }
2814                                 if (rx_skb == NULL)
2815                                         goto skip_rx_indicate0;
2816
2817
2818                                 if ((cqe_misc & NES_NIC_CQE_TAG_VALID) &&
2819                                     (nesvnic->vlan_grp != NULL)) {
2820                                         vlan_tag = (u16)(le32_to_cpu(
2821                                                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX])
2822                                                         >> 16);
2823                                         nes_debug(NES_DBG_CQ, "%s: Reporting stripped VLAN packet. Tag = 0x%04X\n",
2824                                                         nesvnic->netdev->name, vlan_tag);
2825                                         if (nes_use_lro)
2826                                                 lro_vlan_hwaccel_receive_skb(&nesvnic->lro_mgr, rx_skb,
2827                                                                 nesvnic->vlan_grp, vlan_tag, NULL);
2828                                         else
2829                                                 nes_vlan_rx(rx_skb, nesvnic->vlan_grp, vlan_tag);
2830                                 } else {
2831                                         if (nes_use_lro)
2832                                                 lro_receive_skb(&nesvnic->lro_mgr, rx_skb, NULL);
2833                                         else
2834                                                 nes_netif_rx(rx_skb);
2835                                 }
2836
2837 skip_rx_indicate0:
2838                                 ;
2839                                 /* nesvnic->netstats.rx_packets++; */
2840                                 /* nesvnic->netstats.rx_bytes += rx_pkt_size; */
2841                         }
2842
2843                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX] = 0;
2844                         /* Accounting... */
2845                         cqe_count++;
2846                         if (++head >= cq_size)
2847                                 head = 0;
2848                         if (cqe_count == 255) {
2849                                 /* Replenish Nic CQ */
2850                                 nes_write32(nesdev->regs+NES_CQE_ALLOC,
2851                                                 cq->cq_number | (cqe_count << 16));
2852                                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2853                                 nesdev->currcq_count += cqe_count;
2854                                 cqe_count = 0;
2855                         }
2856
2857                         if (cq->rx_cqes_completed >= nesvnic->budget)
2858                                 break;
2859                 } else {
2860                         cq->cqes_pending = 0;
2861                         break;
2862                 }
2863
2864         } while (1);
2865
2866         if (nes_use_lro)
2867                 lro_flush_all(&nesvnic->lro_mgr);
2868         if (sq_cqes) {
2869                 barrier();
2870                 /* restart the queue if it had been stopped */
2871                 if (netif_queue_stopped(nesvnic->netdev))
2872                         netif_wake_queue(nesvnic->netdev);
2873         }
2874         cq->cq_head = head;
2875         /* nes_debug(NES_DBG_CQ, "CQ%u Processed = %u cqes, new head = %u.\n",
2876                         cq->cq_number, cqe_count, cq->cq_head); */
2877         cq->cqe_allocs_pending = cqe_count;
2878         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2879         {
2880                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2881                 nesdev->currcq_count += cqe_count;
2882                 nes_nic_tune_timer(nesdev);
2883         }
2884         if (atomic_read(&nesvnic->rx_skbs_needed))
2885                 nes_replenish_nic_rq(nesvnic);
2886 }
2887
2888
2889 /**
2890  * nes_cqp_ce_handler
2891  */
2892 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq)
2893 {
2894         u64 u64temp;
2895         unsigned long flags;
2896         struct nes_hw_cqp *cqp = NULL;
2897         struct nes_cqp_request *cqp_request;
2898         struct nes_hw_cqp_wqe *cqp_wqe;
2899         u32 head;
2900         u32 cq_size;
2901         u32 cqe_count=0;
2902         u32 error_code;
2903         /* u32 counter; */
2904
2905         head = cq->cq_head;
2906         cq_size = cq->cq_size;
2907
2908         do {
2909                 /* process the CQE */
2910                 /* nes_debug(NES_DBG_CQP, "head=%u cqe_words=%08X\n", head,
2911                           le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])); */
2912
2913                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_VALID) {
2914                         u64temp = (((u64)(le32_to_cpu(cq->cq_vbase[head].
2915                                         cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX]))) << 32) |
2916                                         ((u64)(le32_to_cpu(cq->cq_vbase[head].
2917                                         cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX])));
2918                         cqp = *((struct nes_hw_cqp **)&u64temp);
2919
2920                         error_code = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_ERROR_CODE_IDX]);
2921                         if (error_code) {
2922                                 nes_debug(NES_DBG_CQP, "Bad Completion code for opcode 0x%02X from CQP,"
2923                                                 " Major/Minor codes = 0x%04X:%04X.\n",
2924                                                 le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])&0x3f,
2925                                                 (u16)(error_code >> 16),
2926                                                 (u16)error_code);
2927                                 nes_debug(NES_DBG_CQP, "cqp: qp_id=%u, sq_head=%u, sq_tail=%u\n",
2928                                                 cqp->qp_id, cqp->sq_head, cqp->sq_tail);
2929                         }
2930
2931                         u64temp = (((u64)(le32_to_cpu(nesdev->cqp.sq_vbase[cqp->sq_tail].
2932                                         wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX]))) << 32) |
2933                                         ((u64)(le32_to_cpu(nesdev->cqp.sq_vbase[cqp->sq_tail].
2934                                         wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX])));
2935                         cqp_request = *((struct nes_cqp_request **)&u64temp);
2936                         if (cqp_request) {
2937                                 if (cqp_request->waiting) {
2938                                         /* nes_debug(NES_DBG_CQP, "%s: Waking up requestor\n"); */
2939                                         cqp_request->major_code = (u16)(error_code >> 16);
2940                                         cqp_request->minor_code = (u16)error_code;
2941                                         barrier();
2942                                         cqp_request->request_done = 1;
2943                                         wake_up(&cqp_request->waitq);
2944                                         nes_put_cqp_request(nesdev, cqp_request);
2945                                 } else {
2946                                         if (cqp_request->callback)
2947                                                 cqp_request->cqp_callback(nesdev, cqp_request);
2948                                         nes_free_cqp_request(nesdev, cqp_request);
2949                                 }
2950                         } else {
2951                                 wake_up(&nesdev->cqp.waitq);
2952                         }
2953
2954                         cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0;
2955                         nes_write32(nesdev->regs + NES_CQE_ALLOC, cq->cq_number | (1 << 16));
2956                         if (++cqp->sq_tail >= cqp->sq_size)
2957                                 cqp->sq_tail = 0;
2958
2959                         /* Accounting... */
2960                         cqe_count++;
2961                         if (++head >= cq_size)
2962                                 head = 0;
2963                 } else {
2964                         break;
2965                 }
2966         } while (1);
2967         cq->cq_head = head;
2968
2969         spin_lock_irqsave(&nesdev->cqp.lock, flags);
2970         while ((!list_empty(&nesdev->cqp_pending_reqs)) &&
2971                         ((((nesdev->cqp.sq_tail+nesdev->cqp.sq_size)-nesdev->cqp.sq_head) &
2972                         (nesdev->cqp.sq_size - 1)) != 1)) {
2973                 cqp_request = list_entry(nesdev->cqp_pending_reqs.next,
2974                                 struct nes_cqp_request, list);
2975                 list_del_init(&cqp_request->list);
2976                 head = nesdev->cqp.sq_head++;
2977                 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
2978                 cqp_wqe = &nesdev->cqp.sq_vbase[head];
2979                 memcpy(cqp_wqe, &cqp_request->cqp_wqe, sizeof(*cqp_wqe));
2980                 barrier();
2981                 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX] =
2982                         cpu_to_le32((u32)((unsigned long)cqp_request));
2983                 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX] =
2984                         cpu_to_le32((u32)(upper_32_bits((unsigned long)cqp_request)));
2985                 nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) put on CQPs SQ wqe%u.\n",
2986                                 cqp_request, le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f, head);
2987                 /* Ring doorbell (1 WQEs) */
2988                 barrier();
2989                 nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x01800000 | nesdev->cqp.qp_id);
2990         }
2991         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2992
2993         /* Arm the CCQ */
2994         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT |
2995                         cq->cq_number);
2996         nes_read32(nesdev->regs+NES_CQE_ALLOC);
2997 }
2998
2999
3000 static u8 *locate_mpa(u8 *pkt, u32 aeq_info)
3001 {
3002         u16 pkt_len;
3003
3004         if (aeq_info & NES_AEQE_Q2_DATA_ETHERNET) {
3005                 /* skip over ethernet header */
3006                 pkt_len = be16_to_cpu(*(u16 *)(pkt + ETH_HLEN - 2));
3007                 pkt += ETH_HLEN;
3008
3009                 /* Skip over IP and TCP headers */
3010                 pkt += 4 * (pkt[0] & 0x0f);
3011                 pkt += 4 * ((pkt[12] >> 4) & 0x0f);
3012         }
3013         return pkt;
3014 }
3015
3016 /* Determine if incoming error pkt is rdma layer */
3017 static u32 iwarp_opcode(struct nes_qp *nesqp, u32 aeq_info)
3018 {
3019         u8 *pkt;
3020         u16 *mpa;
3021         u32 opcode = 0xffffffff;
3022
3023         if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
3024                 pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
3025                 mpa = (u16 *)locate_mpa(pkt, aeq_info);
3026                 opcode = be16_to_cpu(mpa[1]) & 0xf;
3027         }
3028
3029         return opcode;
3030 }
3031
3032 /* Build iWARP terminate header */
3033 static int nes_bld_terminate_hdr(struct nes_qp *nesqp, u16 async_event_id, u32 aeq_info)
3034 {
3035         u8 *pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
3036         u16 ddp_seg_len;
3037         int copy_len = 0;
3038         u8 is_tagged = 0;
3039         u8 flush_code = 0;
3040         struct nes_terminate_hdr *termhdr;
3041
3042         termhdr = (struct nes_terminate_hdr *)nesqp->hwqp.q2_vbase;
3043         memset(termhdr, 0, 64);
3044
3045         if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
3046
3047                 /* Use data from offending packet to fill in ddp & rdma hdrs */
3048                 pkt = locate_mpa(pkt, aeq_info);
3049                 ddp_seg_len = be16_to_cpu(*(u16 *)pkt);
3050                 if (ddp_seg_len) {
3051                         copy_len = 2;
3052                         termhdr->hdrct = DDP_LEN_FLAG;
3053                         if (pkt[2] & 0x80) {
3054                                 is_tagged = 1;
3055                                 if (ddp_seg_len >= TERM_DDP_LEN_TAGGED) {
3056                                         copy_len += TERM_DDP_LEN_TAGGED;
3057                                         termhdr->hdrct |= DDP_HDR_FLAG;
3058                                 }
3059                         } else {
3060                                 if (ddp_seg_len >= TERM_DDP_LEN_UNTAGGED) {
3061                                         copy_len += TERM_DDP_LEN_UNTAGGED;
3062                                         termhdr->hdrct |= DDP_HDR_FLAG;
3063                                 }
3064
3065                                 if (ddp_seg_len >= (TERM_DDP_LEN_UNTAGGED + TERM_RDMA_LEN)) {
3066                                         if ((pkt[3] & RDMA_OPCODE_MASK) == RDMA_READ_REQ_OPCODE) {
3067                                                 copy_len += TERM_RDMA_LEN;
3068                                                 termhdr->hdrct |= RDMA_HDR_FLAG;
3069                                         }
3070                                 }
3071                         }
3072                 }
3073         }
3074
3075         switch (async_event_id) {
3076         case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
3077                 switch (iwarp_opcode(nesqp, aeq_info)) {
3078                 case IWARP_OPCODE_WRITE:
3079                         flush_code = IB_WC_LOC_PROT_ERR;
3080                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3081                         termhdr->error_code = DDP_TAGGED_INV_STAG;
3082                         break;
3083                 default:
3084                         flush_code = IB_WC_REM_ACCESS_ERR;
3085                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3086                         termhdr->error_code = RDMAP_INV_STAG;
3087                 }
3088                 break;
3089         case NES_AEQE_AEID_AMP_INVALID_STAG:
3090                 flush_code = IB_WC_REM_ACCESS_ERR;
3091                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3092                 termhdr->error_code = RDMAP_INV_STAG;
3093                 break;
3094         case NES_AEQE_AEID_AMP_BAD_QP:
3095                 flush_code = IB_WC_LOC_QP_OP_ERR;
3096                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3097                 termhdr->error_code = DDP_UNTAGGED_INV_QN;
3098                 break;
3099         case NES_AEQE_AEID_AMP_BAD_STAG_KEY:
3100         case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
3101                 switch (iwarp_opcode(nesqp, aeq_info)) {
3102                 case IWARP_OPCODE_SEND_INV:
3103                 case IWARP_OPCODE_SEND_SE_INV:
3104                         flush_code = IB_WC_REM_OP_ERR;
3105                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3106                         termhdr->error_code = RDMAP_CANT_INV_STAG;
3107                         break;
3108                 default:
3109                         flush_code = IB_WC_REM_ACCESS_ERR;
3110                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3111                         termhdr->error_code = RDMAP_INV_STAG;
3112                 }
3113                 break;
3114         case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION:
3115                 if (aeq_info & (NES_AEQE_Q2_DATA_ETHERNET | NES_AEQE_Q2_DATA_MPA)) {
3116                         flush_code = IB_WC_LOC_PROT_ERR;
3117                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3118                         termhdr->error_code = DDP_TAGGED_BOUNDS;
3119                 } else {
3120                         flush_code = IB_WC_REM_ACCESS_ERR;
3121                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3122                         termhdr->error_code = RDMAP_INV_BOUNDS;
3123                 }
3124                 break;
3125         case NES_AEQE_AEID_AMP_RIGHTS_VIOLATION:
3126         case NES_AEQE_AEID_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
3127         case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
3128                 flush_code = IB_WC_REM_ACCESS_ERR;
3129                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3130                 termhdr->error_code = RDMAP_ACCESS;
3131                 break;
3132         case NES_AEQE_AEID_AMP_TO_WRAP:
3133                 flush_code = IB_WC_REM_ACCESS_ERR;
3134                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3135                 termhdr->error_code = RDMAP_TO_WRAP;
3136                 break;
3137         case NES_AEQE_AEID_AMP_BAD_PD:
3138                 switch (iwarp_opcode(nesqp, aeq_info)) {
3139                 case IWARP_OPCODE_WRITE:
3140                         flush_code = IB_WC_LOC_PROT_ERR;
3141                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3142                         termhdr->error_code = DDP_TAGGED_UNASSOC_STAG;
3143                         break;
3144                 case IWARP_OPCODE_SEND_INV:
3145                 case IWARP_OPCODE_SEND_SE_INV:
3146                         flush_code = IB_WC_REM_ACCESS_ERR;
3147                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3148                         termhdr->error_code = RDMAP_CANT_INV_STAG;
3149                         break;
3150                 default:
3151                         flush_code = IB_WC_REM_ACCESS_ERR;
3152                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3153                         termhdr->error_code = RDMAP_UNASSOC_STAG;
3154                 }
3155                 break;
3156         case NES_AEQE_AEID_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH:
3157                 flush_code = IB_WC_LOC_LEN_ERR;
3158                 termhdr->layer_etype = (LAYER_MPA << 4) | DDP_LLP;
3159                 termhdr->error_code = MPA_MARKER;
3160                 break;
3161         case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
3162                 flush_code = IB_WC_GENERAL_ERR;
3163                 termhdr->layer_etype = (LAYER_MPA << 4) | DDP_LLP;
3164                 termhdr->error_code = MPA_CRC;
3165                 break;
3166         case NES_AEQE_AEID_LLP_SEGMENT_TOO_LARGE:
3167         case NES_AEQE_AEID_LLP_SEGMENT_TOO_SMALL:
3168                 flush_code = IB_WC_LOC_LEN_ERR;
3169                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_CATASTROPHIC;
3170                 termhdr->error_code = DDP_CATASTROPHIC_LOCAL;
3171                 break;
3172         case NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC:
3173         case NES_AEQE_AEID_DDP_NO_L_BIT:
3174                 flush_code = IB_WC_FATAL_ERR;
3175                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_CATASTROPHIC;
3176                 termhdr->error_code = DDP_CATASTROPHIC_LOCAL;
3177                 break;
3178         case NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN:
3179         case NES_AEQE_AEID_DDP_INVALID_MSN_RANGE_IS_NOT_VALID:
3180                 flush_code = IB_WC_GENERAL_ERR;
3181                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3182                 termhdr->error_code = DDP_UNTAGGED_INV_MSN_RANGE;
3183                 break;
3184         case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
3185                 flush_code = IB_WC_LOC_LEN_ERR;
3186                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3187                 termhdr->error_code = DDP_UNTAGGED_INV_TOO_LONG;
3188                 break;
3189         case NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION:
3190                 flush_code = IB_WC_GENERAL_ERR;
3191                 if (is_tagged) {
3192                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3193                         termhdr->error_code = DDP_TAGGED_INV_DDP_VER;
3194                 } else {
3195                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3196                         termhdr->error_code = DDP_UNTAGGED_INV_DDP_VER;
3197                 }
3198                 break;
3199         case NES_AEQE_AEID_DDP_UBE_INVALID_MO:
3200                 flush_code = IB_WC_GENERAL_ERR;
3201                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3202                 termhdr->error_code = DDP_UNTAGGED_INV_MO;
3203                 break;
3204         case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
3205                 flush_code = IB_WC_REM_OP_ERR;
3206                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3207                 termhdr->error_code = DDP_UNTAGGED_INV_MSN_NO_BUF;
3208                 break;
3209         case NES_AEQE_AEID_DDP_UBE_INVALID_QN:
3210                 flush_code = IB_WC_GENERAL_ERR;
3211                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3212                 termhdr->error_code = DDP_UNTAGGED_INV_QN;
3213                 break;
3214         case NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION:
3215                 flush_code = IB_WC_GENERAL_ERR;
3216                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3217                 termhdr->error_code = RDMAP_INV_RDMAP_VER;
3218                 break;
3219         case NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE:
3220                 flush_code = IB_WC_LOC_QP_OP_ERR;
3221                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3222                 termhdr->error_code = RDMAP_UNEXPECTED_OP;
3223                 break;
3224         default:
3225                 flush_code = IB_WC_FATAL_ERR;
3226                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3227                 termhdr->error_code = RDMAP_UNSPECIFIED;
3228                 break;
3229         }
3230
3231         if (copy_len)
3232                 memcpy(termhdr + 1, pkt, copy_len);
3233
3234         if ((flush_code) && ((NES_AEQE_INBOUND_RDMA & aeq_info) == 0)) {
3235                 if (aeq_info & NES_AEQE_SQ)
3236                         nesqp->term_sq_flush_code = flush_code;
3237                 else
3238                         nesqp->term_rq_flush_code = flush_code;
3239         }
3240
3241         return sizeof(struct nes_terminate_hdr) + copy_len;
3242 }
3243
3244 static void nes_terminate_connection(struct nes_device *nesdev, struct nes_qp *nesqp,
3245                  struct nes_hw_aeqe *aeqe, enum ib_event_type eventtype)
3246 {
3247         u64 context;
3248         unsigned long flags;
3249         u32 aeq_info;
3250         u16 async_event_id;
3251         u8 tcp_state;
3252         u8 iwarp_state;
3253         u32 termlen = 0;
3254         u32 mod_qp_flags = NES_CQP_QP_IWARP_STATE_TERMINATE |
3255                            NES_CQP_QP_TERM_DONT_SEND_FIN;
3256         struct nes_adapter *nesadapter = nesdev->nesadapter;
3257
3258         if (nesqp->term_flags & NES_TERM_SENT)
3259                 return; /* Sanity check */
3260
3261         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3262         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3263         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3264         async_event_id = (u16)aeq_info;
3265
3266         context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
3267                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN];
3268         if (!context) {
3269                 WARN_ON(!context);
3270                 return;
3271         }
3272
3273         nesqp = (struct nes_qp *)(unsigned long)context;
3274         spin_lock_irqsave(&nesqp->lock, flags);
3275         nesqp->hw_iwarp_state = iwarp_state;
3276         nesqp->hw_tcp_state = tcp_state;
3277         nesqp->last_aeq = async_event_id;
3278         nesqp->terminate_eventtype = eventtype;
3279         spin_unlock_irqrestore(&nesqp->lock, flags);
3280
3281         if (nesadapter->send_term_ok)
3282                 termlen = nes_bld_terminate_hdr(nesqp, async_event_id, aeq_info);
3283         else
3284                 mod_qp_flags |= NES_CQP_QP_TERM_DONT_SEND_TERM_MSG;
3285
3286         nes_terminate_start_timer(nesqp);
3287         nesqp->term_flags |= NES_TERM_SENT;
3288         nes_hw_modify_qp(nesdev, nesqp, mod_qp_flags, termlen, 0);
3289 }
3290
3291 static void nes_terminate_send_fin(struct nes_device *nesdev,
3292                           struct nes_qp *nesqp, struct nes_hw_aeqe *aeqe)
3293 {
3294         u32 aeq_info;
3295         u16 async_event_id;
3296         u8 tcp_state;
3297         u8 iwarp_state;
3298         unsigned long flags;
3299
3300         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3301         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3302         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3303         async_event_id = (u16)aeq_info;
3304
3305         spin_lock_irqsave(&nesqp->lock, flags);
3306         nesqp->hw_iwarp_state = iwarp_state;
3307         nesqp->hw_tcp_state = tcp_state;
3308         nesqp->last_aeq = async_event_id;
3309         spin_unlock_irqrestore(&nesqp->lock, flags);
3310
3311         /* Send the fin only */
3312         nes_hw_modify_qp(nesdev, nesqp, NES_CQP_QP_IWARP_STATE_TERMINATE |
3313                 NES_CQP_QP_TERM_DONT_SEND_TERM_MSG, 0, 0);
3314 }
3315
3316 /* Cleanup after a terminate sent or received */
3317 static void nes_terminate_done(struct nes_qp *nesqp, int timeout_occurred)
3318 {
3319         u32 next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR;
3320         unsigned long flags;
3321         struct nes_vnic *nesvnic = to_nesvnic(nesqp->ibqp.device);
3322         struct nes_device *nesdev = nesvnic->nesdev;
3323         u8 first_time = 0;
3324
3325         spin_lock_irqsave(&nesqp->lock, flags);
3326         if (nesqp->hte_added) {
3327                 nesqp->hte_added = 0;
3328                 next_iwarp_state |= NES_CQP_QP_DEL_HTE;
3329         }
3330
3331         first_time = (nesqp->term_flags & NES_TERM_DONE) == 0;
3332         nesqp->term_flags |= NES_TERM_DONE;
3333         spin_unlock_irqrestore(&nesqp->lock, flags);
3334
3335         /* Make sure we go through this only once */
3336         if (first_time) {
3337                 if (timeout_occurred == 0)
3338                         del_timer(&nesqp->terminate_timer);
3339                 else
3340                         next_iwarp_state |= NES_CQP_QP_RESET;
3341
3342                 nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
3343                 nes_cm_disconn(nesqp);
3344         }
3345 }
3346
3347 static void nes_terminate_received(struct nes_device *nesdev,
3348                                 struct nes_qp *nesqp, struct nes_hw_aeqe *aeqe)
3349 {
3350         u32 aeq_info;
3351         u8 *pkt;
3352         u32 *mpa;
3353         u8 ddp_ctl;
3354         u8 rdma_ctl;
3355         u16 aeq_id = 0;
3356
3357         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3358         if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
3359                 /* Terminate is not a performance path so the silicon */
3360                 /* did not validate the frame - do it now */
3361                 pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
3362                 mpa = (u32 *)locate_mpa(pkt, aeq_info);
3363                 ddp_ctl = (be32_to_cpu(mpa[0]) >> 8) & 0xff;
3364                 rdma_ctl = be32_to_cpu(mpa[0]) & 0xff;
3365                 if ((ddp_ctl & 0xc0) != 0x40)
3366                         aeq_id = NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC;
3367                 else if ((ddp_ctl & 0x03) != 1)
3368                         aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION;
3369                 else if (be32_to_cpu(mpa[2]) != 2)
3370                         aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_QN;
3371                 else if (be32_to_cpu(mpa[3]) != 1)
3372                         aeq_id = NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN;
3373                 else if (be32_to_cpu(mpa[4]) != 0)
3374                         aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_MO;
3375                 else if ((rdma_ctl & 0xc0) != 0x40)
3376                         aeq_id = NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION;
3377
3378                 if (aeq_id) {
3379                         /* Bad terminate recvd - send back a terminate */
3380                         aeq_info = (aeq_info & 0xffff0000) | aeq_id;
3381                         aeqe->aeqe_words[NES_AEQE_MISC_IDX] = cpu_to_le32(aeq_info);
3382                         nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_FATAL);
3383                         return;
3384                 }
3385         }
3386
3387         nesqp->term_flags |= NES_TERM_RCVD;
3388         nesqp->terminate_eventtype = IB_EVENT_QP_FATAL;
3389         nes_terminate_start_timer(nesqp);
3390         nes_terminate_send_fin(nesdev, nesqp, aeqe);
3391 }
3392
3393 /* Timeout routine in case terminate fails to complete */
3394 static void nes_terminate_timeout(unsigned long context)
3395 {
3396         struct nes_qp *nesqp = (struct nes_qp *)(unsigned long)context;
3397
3398         nes_terminate_done(nesqp, 1);
3399 }
3400
3401 /* Set a timer in case hw cannot complete the terminate sequence */
3402 static void nes_terminate_start_timer(struct nes_qp *nesqp)
3403 {
3404         init_timer(&nesqp->terminate_timer);
3405         nesqp->terminate_timer.function = nes_terminate_timeout;
3406         nesqp->terminate_timer.expires = jiffies + HZ;
3407         nesqp->terminate_timer.data = (unsigned long)nesqp;
3408         add_timer(&nesqp->terminate_timer);
3409 }
3410
3411 /**
3412  * nes_process_iwarp_aeqe
3413  */
3414 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
3415                                    struct nes_hw_aeqe *aeqe)
3416 {
3417         u64 context;
3418         unsigned long flags;
3419         struct nes_qp *nesqp;
3420         struct nes_hw_cq *hw_cq;
3421         struct nes_cq *nescq;
3422         int resource_allocated;
3423         struct nes_adapter *nesadapter = nesdev->nesadapter;
3424         u32 aeq_info;
3425         u32 next_iwarp_state = 0;
3426         u16 async_event_id;
3427         u8 tcp_state;
3428         u8 iwarp_state;
3429         struct ib_event ibevent;
3430
3431         nes_debug(NES_DBG_AEQ, "\n");
3432         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3433         if ((NES_AEQE_INBOUND_RDMA & aeq_info) || (!(NES_AEQE_QP & aeq_info))) {
3434                 context  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
3435                 context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
3436         } else {
3437                 context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
3438                                                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN];
3439                 BUG_ON(!context);
3440         }
3441
3442         /* context is nesqp unless async_event_id == CQ ERROR */
3443         nesqp = (struct nes_qp *)(unsigned long)context;
3444         async_event_id = (u16)aeq_info;
3445         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3446         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3447         nes_debug(NES_DBG_AEQ, "aeid = 0x%04X, qp-cq id = %d, aeqe = %p,"
3448                         " Tcp state = %s, iWARP state = %s\n",
3449                         async_event_id,
3450                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), aeqe,
3451                         nes_tcp_state_str[tcp_state], nes_iwarp_state_str[iwarp_state]);
3452
3453         switch (async_event_id) {
3454                 case NES_AEQE_AEID_LLP_FIN_RECEIVED:
3455                         if (nesqp->term_flags)
3456                                 return; /* Ignore it, wait for close complete */
3457
3458                         if (atomic_inc_return(&nesqp->close_timer_started) == 1) {
3459                                 nesqp->cm_id->add_ref(nesqp->cm_id);
3460                                 schedule_nes_timer(nesqp->cm_node, (struct sk_buff *)nesqp,
3461                                                 NES_TIMER_TYPE_CLOSE, 1, 0);
3462                                 nes_debug(NES_DBG_AEQ, "QP%u Not decrementing QP refcount (%d),"
3463                                                 " need ae to finish up, original_last_aeq = 0x%04X."
3464                                                 " last_aeq = 0x%04X, scheduling timer. TCP state = %d\n",
3465                                                 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),
3466                                                 async_event_id, nesqp->last_aeq, tcp_state);
3467                         }
3468
3469                         break;
3470                 case NES_AEQE_AEID_LLP_CLOSE_COMPLETE:
3471                         if (nesqp->term_flags) {
3472                                 nes_terminate_done(nesqp, 0);
3473                                 return;
3474                         }
3475                         spin_lock_irqsave(&nesqp->lock, flags);
3476                         nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING;
3477                         spin_unlock_irqrestore(&nesqp->lock, flags);
3478                         nes_hw_modify_qp(nesdev, nesqp, NES_CQP_QP_IWARP_STATE_CLOSING, 0, 0);
3479                         nes_cm_disconn(nesqp);
3480                         break;
3481
3482                 case NES_AEQE_AEID_RESET_SENT:
3483                         tcp_state = NES_AEQE_TCP_STATE_CLOSED;
3484                         spin_lock_irqsave(&nesqp->lock, flags);
3485                         nesqp->hw_iwarp_state = iwarp_state;
3486                         nesqp->hw_tcp_state = tcp_state;
3487                         nesqp->last_aeq = async_event_id;
3488                         nesqp->hte_added = 0;
3489                         spin_unlock_irqrestore(&nesqp->lock, flags);
3490                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_DEL_HTE;
3491                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
3492                         nes_cm_disconn(nesqp);
3493                         break;
3494
3495                 case NES_AEQE_AEID_LLP_CONNECTION_RESET:
3496                         if (atomic_read(&nesqp->close_timer_started))
3497                                 return;
3498                         spin_lock_irqsave(&nesqp->lock, flags);
3499                         nesqp->hw_iwarp_state = iwarp_state;
3500                         nesqp->hw_tcp_state = tcp_state;
3501                         nesqp->last_aeq = async_event_id;
3502                         spin_unlock_irqrestore(&nesqp->lock, flags);
3503                         nes_cm_disconn(nesqp);
3504                         break;
3505
3506                 case NES_AEQE_AEID_TERMINATE_SENT:
3507                         nes_terminate_send_fin(nesdev, nesqp, aeqe);
3508                         break;
3509
3510                 case NES_AEQE_AEID_LLP_TERMINATE_RECEIVED:
3511                         nes_terminate_received(nesdev, nesqp, aeqe);
3512                         break;
3513
3514                 case NES_AEQE_AEID_AMP_BAD_STAG_KEY:
3515                 case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
3516                 case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
3517                 case NES_AEQE_AEID_AMP_INVALID_STAG:
3518                 case NES_AEQE_AEID_AMP_RIGHTS_VIOLATION:
3519                 case NES_AEQE_AEID_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
3520                 case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
3521                 case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
3522                 case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION:
3523                 case NES_AEQE_AEID_AMP_TO_WRAP:
3524                         printk(KERN_ERR PFX "QP[%u] async_event_id=0x%04X IB_EVENT_QP_ACCESS_ERR\n",
3525                                         nesqp->hwqp.qp_id, async_event_id);
3526                         nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_ACCESS_ERR);
3527                         break;
3528
3529                 case NES_AEQE_AEID_LLP_SEGMENT_TOO_LARGE:
3530                 case NES_AEQE_AEID_LLP_SEGMENT_TOO_SMALL:
3531                 case NES_AEQE_AEID_DDP_UBE_INVALID_MO:
3532                 case NES_AEQE_AEID_DDP_UBE_INVALID_QN:
3533                         if (iwarp_opcode(nesqp, aeq_info) > IWARP_OPCODE_TERM) {
3534                                 aeq_info &= 0xffff0000;
3535                                 aeq_info |= NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE;
3536                                 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = cpu_to_le32(aeq_info);
3537                         }
3538
3539                 case NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE:
3540                 case NES_AEQE_AEID_LLP_TOO_MANY_RETRIES:
3541                 case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
3542                 case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
3543                 case NES_AEQE_AEID_AMP_BAD_QP:
3544                 case NES_AEQE_AEID_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH:
3545                 case NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC:
3546                 case NES_AEQE_AEID_DDP_NO_L_BIT:
3547                 case NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN:
3548                 case NES_AEQE_AEID_DDP_INVALID_MSN_RANGE_IS_NOT_VALID:
3549                 case NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION:
3550                 case NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION:
3551                 case NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE:
3552                 case NES_AEQE_AEID_AMP_BAD_PD:
3553                 case NES_AEQE_AEID_AMP_FASTREG_SHARED:
3554                 case NES_AEQE_AEID_AMP_FASTREG_VALID_STAG:
3555                 case NES_AEQE_AEID_AMP_FASTREG_MW_STAG:
3556                 case NES_AEQE_AEID_AMP_FASTREG_INVALID_RIGHTS:
3557                 case NES_AEQE_AEID_AMP_FASTREG_PBL_TABLE_OVERFLOW:
3558                 case NES_AEQE_AEID_AMP_FASTREG_INVALID_LENGTH:
3559                 case NES_AEQE_AEID_AMP_INVALIDATE_SHARED:
3560                 case NES_AEQE_AEID_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS:
3561                 case NES_AEQE_AEID_AMP_MWBIND_VALID_STAG:
3562                 case NES_AEQE_AEID_AMP_MWBIND_OF_MR_STAG:
3563                 case NES_AEQE_AEID_AMP_MWBIND_TO_ZERO_BASED_STAG:
3564                 case NES_AEQE_AEID_AMP_MWBIND_TO_MW_STAG:
3565                 case NES_AEQE_AEID_AMP_MWBIND_INVALID_RIGHTS:
3566                 case NES_AEQE_AEID_AMP_MWBIND_INVALID_BOUNDS:
3567                 case NES_AEQE_AEID_AMP_MWBIND_TO_INVALID_PARENT:
3568                 case NES_AEQE_AEID_AMP_MWBIND_BIND_DISABLED:
3569                 case NES_AEQE_AEID_BAD_CLOSE:
3570                 case NES_AEQE_AEID_RDMA_READ_WHILE_ORD_ZERO:
3571                 case NES_AEQE_AEID_STAG_ZERO_INVALID:
3572                 case NES_AEQE_AEID_ROE_INVALID_RDMA_READ_REQUEST:
3573                 case NES_AEQE_AEID_ROE_INVALID_RDMA_WRITE_OR_READ_RESP:
3574                         printk(KERN_ERR PFX "QP[%u] async_event_id=0x%04X IB_EVENT_QP_FATAL\n",
3575                                         nesqp->hwqp.qp_id, async_event_id);
3576                         nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_FATAL);
3577                         break;
3578
3579                 case NES_AEQE_AEID_CQ_OPERATION_ERROR:
3580                         context <<= 1;
3581                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u, %p\n",
3582                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), (void *)(unsigned long)context);
3583                         resource_allocated = nes_is_resource_allocated(nesadapter, nesadapter->allocated_cqs,
3584                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3585                         if (resource_allocated) {
3586                                 printk(KERN_ERR PFX "%s: Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u\n",
3587                                                 __func__, le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3588                                 hw_cq = (struct nes_hw_cq *)(unsigned long)context;
3589                                 if (hw_cq) {
3590                                         nescq = container_of(hw_cq, struct nes_cq, hw_cq);
3591                                         if (nescq->ibcq.event_handler) {
3592                                                 ibevent.device = nescq->ibcq.device;
3593                                                 ibevent.event = IB_EVENT_CQ_ERR;
3594                                                 ibevent.element.cq = &nescq->ibcq;
3595                                                 nescq->ibcq.event_handler(&ibevent, nescq->ibcq.cq_context);
3596                                         }
3597                                 }
3598                         }
3599                         break;
3600
3601                 default:
3602                         nes_debug(NES_DBG_AEQ, "Processing an iWARP related AE for QP, misc = 0x%04X\n",
3603                                         async_event_id);
3604                         break;
3605         }
3606
3607 }
3608
3609 /**
3610  * nes_iwarp_ce_handler
3611  */
3612 void nes_iwarp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *hw_cq)
3613 {
3614         struct nes_cq *nescq = container_of(hw_cq, struct nes_cq, hw_cq);
3615
3616         /* nes_debug(NES_DBG_CQ, "Processing completion event for iWARP CQ%u.\n",
3617                         nescq->hw_cq.cq_number); */
3618         nes_write32(nesdev->regs+NES_CQ_ACK, nescq->hw_cq.cq_number);
3619
3620         if (nescq->ibcq.comp_handler)
3621                 nescq->ibcq.comp_handler(&nescq->ibcq, nescq->ibcq.cq_context);
3622
3623         return;
3624 }
3625
3626
3627 /**
3628  * nes_manage_apbvt()
3629  */
3630 int nes_manage_apbvt(struct nes_vnic *nesvnic, u32 accel_local_port,
3631                 u32 nic_index, u32 add_port)
3632 {
3633         struct nes_device *nesdev = nesvnic->nesdev;
3634         struct nes_hw_cqp_wqe *cqp_wqe;
3635         struct nes_cqp_request *cqp_request;
3636         int ret = 0;
3637         u16 major_code;
3638
3639         /* Send manage APBVT request to CQP */
3640         cqp_request = nes_get_cqp_request(nesdev);
3641         if (cqp_request == NULL) {
3642                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3643                 return -ENOMEM;
3644         }
3645         cqp_request->waiting = 1;
3646         cqp_wqe = &cqp_request->cqp_wqe;
3647
3648         nes_debug(NES_DBG_QP, "%s APBV for local port=%u(0x%04x), nic_index=%u\n",
3649                         (add_port == NES_MANAGE_APBVT_ADD) ? "ADD" : "DEL",
3650                         accel_local_port, accel_local_port, nic_index);
3651
3652         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3653         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, (NES_CQP_MANAGE_APBVT |
3654                         ((add_port == NES_MANAGE_APBVT_ADD) ? NES_CQP_APBVT_ADD : 0)));
3655         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
3656                         ((nic_index << NES_CQP_APBVT_NIC_SHIFT) | accel_local_port));
3657
3658         nes_debug(NES_DBG_QP, "Waiting for CQP completion for APBVT.\n");
3659
3660         atomic_set(&cqp_request->refcount, 2);
3661         nes_post_cqp_request(nesdev, cqp_request);
3662
3663         if (add_port == NES_MANAGE_APBVT_ADD)
3664                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3665                                 NES_EVENT_TIMEOUT);
3666         nes_debug(NES_DBG_QP, "Completed, ret=%u,  CQP Major:Minor codes = 0x%04X:0x%04X\n",
3667                         ret, cqp_request->major_code, cqp_request->minor_code);
3668         major_code = cqp_request->major_code;
3669
3670         nes_put_cqp_request(nesdev, cqp_request);
3671
3672         if (!ret)
3673                 return -ETIME;
3674         else if (major_code)
3675                 return -EIO;
3676         else
3677                 return 0;
3678 }
3679
3680
3681 /**
3682  * nes_manage_arp_cache
3683  */
3684 void nes_manage_arp_cache(struct net_device *netdev, unsigned char *mac_addr,
3685                 u32 ip_addr, u32 action)
3686 {
3687         struct nes_hw_cqp_wqe *cqp_wqe;
3688         struct nes_vnic *nesvnic = netdev_priv(netdev);
3689         struct nes_device *nesdev;
3690         struct nes_cqp_request *cqp_request;
3691         int arp_index;
3692
3693         nesdev = nesvnic->nesdev;
3694         arp_index = nes_arp_table(nesdev, ip_addr, mac_addr, action);
3695         if (arp_index == -1) {
3696                 return;
3697         }
3698
3699         /* update the ARP entry */
3700         cqp_request = nes_get_cqp_request(nesdev);
3701         if (cqp_request == NULL) {
3702                 nes_debug(NES_DBG_NETDEV, "Failed to get a cqp_request.\n");
3703                 return;
3704         }
3705         cqp_request->waiting = 0;
3706         cqp_wqe = &cqp_request->cqp_wqe;
3707         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3708
3709         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
3710                         NES_CQP_MANAGE_ARP_CACHE | NES_CQP_ARP_PERM);
3711         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(
3712                         (u32)PCI_FUNC(nesdev->pcidev->devfn) << NES_CQP_ARP_AEQ_INDEX_SHIFT);
3713         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(arp_index);
3714
3715         if (action == NES_ARP_ADD) {
3716                 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_ARP_VALID);
3717                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = cpu_to_le32(
3718                                 (((u32)mac_addr[2]) << 24) | (((u32)mac_addr[3]) << 16) |
3719                                 (((u32)mac_addr[4]) << 8)  | (u32)mac_addr[5]);
3720                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = cpu_to_le32(
3721                                 (((u32)mac_addr[0]) << 16) | (u32)mac_addr[1]);
3722         } else {
3723                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = 0;
3724                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = 0;
3725         }
3726
3727         nes_debug(NES_DBG_NETDEV, "Not waiting for CQP, cqp.sq_head=%u, cqp.sq_tail=%u\n",
3728                         nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
3729
3730         atomic_set(&cqp_request->refcount, 1);
3731         nes_post_cqp_request(nesdev, cqp_request);
3732 }
3733
3734
3735 /**
3736  * flush_wqes
3737  */
3738 void flush_wqes(struct nes_device *nesdev, struct nes_qp *nesqp,
3739                 u32 which_wq, u32 wait_completion)
3740 {
3741         struct nes_cqp_request *cqp_request;
3742         struct nes_hw_cqp_wqe *cqp_wqe;
3743         u32 sq_code = (NES_IWARP_CQE_MAJOR_FLUSH << 16) | NES_IWARP_CQE_MINOR_FLUSH;
3744         u32 rq_code = (NES_IWARP_CQE_MAJOR_FLUSH << 16) | NES_IWARP_CQE_MINOR_FLUSH;
3745         int ret;
3746
3747         cqp_request = nes_get_cqp_request(nesdev);
3748         if (cqp_request == NULL) {
3749                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3750                 return;
3751         }
3752         if (wait_completion) {
3753                 cqp_request->waiting = 1;
3754                 atomic_set(&cqp_request->refcount, 2);
3755         } else {
3756                 cqp_request->waiting = 0;
3757         }
3758         cqp_wqe = &cqp_request->cqp_wqe;
3759         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3760
3761         /* If wqe in error was identified, set code to be put into cqe */
3762         if ((nesqp->term_sq_flush_code) && (which_wq & NES_CQP_FLUSH_SQ)) {
3763                 which_wq |= NES_CQP_FLUSH_MAJ_MIN;
3764                 sq_code = (CQE_MAJOR_DRV << 16) | nesqp->term_sq_flush_code;
3765                 nesqp->term_sq_flush_code = 0;
3766         }
3767
3768         if ((nesqp->term_rq_flush_code) && (which_wq & NES_CQP_FLUSH_RQ)) {
3769                 which_wq |= NES_CQP_FLUSH_MAJ_MIN;
3770                 rq_code = (CQE_MAJOR_DRV << 16) | nesqp->term_rq_flush_code;
3771                 nesqp->term_rq_flush_code = 0;
3772         }
3773
3774         if (which_wq & NES_CQP_FLUSH_MAJ_MIN) {
3775                 cqp_wqe->wqe_words[NES_CQP_QP_WQE_FLUSH_SQ_CODE] = cpu_to_le32(sq_code);
3776                 cqp_wqe->wqe_words[NES_CQP_QP_WQE_FLUSH_RQ_CODE] = cpu_to_le32(rq_code);
3777         }
3778
3779         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] =
3780                         cpu_to_le32(NES_CQP_FLUSH_WQES | which_wq);
3781         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesqp->hwqp.qp_id);
3782
3783         nes_post_cqp_request(nesdev, cqp_request);
3784
3785         if (wait_completion) {
3786                 /* Wait for CQP */
3787                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3788                                 NES_EVENT_TIMEOUT);
3789                 nes_debug(NES_DBG_QP, "Flush SQ QP WQEs completed, ret=%u,"
3790                                 " CQP Major:Minor codes = 0x%04X:0x%04X\n",
3791                                 ret, cqp_request->major_code, cqp_request->minor_code);
3792                 nes_put_cqp_request(nesdev, cqp_request);
3793         }
3794 }