drivers/net/vxge/vxge-config.c

   1 /******************************************************************************
   2  * This software may be used and distributed according to the terms of
   3  * the GNU General Public License (GPL), incorporated herein by reference.
   4  * Drivers based on or derived from this code fall under the GPL and must
   5  * retain the authorship, copyright and license notice.  This file is not
   6  * a complete program and may only be used when the entire operating
   7  * system is licensed under the GPL.
   8  * See the file COPYING in this distribution for more information.
   9  *
  10  * vxge-config.c: Driver for Neterion Inc's X3100 Series 10GbE PCIe I/O
  11  *                Virtualized Server Adapter.
  12  * Copyright(c) 2002-2009 Neterion Inc.
  13  ******************************************************************************/
  14 #include <linux/vmalloc.h>
  15 #include <linux/etherdevice.h>
  16 #include <linux/pci.h>
  17 #include <linux/pci_hotplug.h>
  18
  19 #include "vxge-traffic.h"
  20 #include "vxge-config.h"
  21
  22 /*
  23  * __vxge_hw_channel_allocate - Allocate memory for channel
  24  * This function allocates required memory for the channel and various arrays
  25  * in the channel
  26  */
  27 struct __vxge_hw_channel*
  28 __vxge_hw_channel_allocate(struct __vxge_hw_vpath_handle *vph,
  29                            enum __vxge_hw_channel_type type,
  30         u32 length, u32 per_dtr_space, void *userdata)
  31 {
  32         struct __vxge_hw_channel *channel;
  33         struct __vxge_hw_device *hldev;
  34         int size = 0;
  35         u32 vp_id;
  36
  37         hldev = vph->vpath->hldev;
  38         vp_id = vph->vpath->vp_id;
  39
  40         switch (type) {
  41         case VXGE_HW_CHANNEL_TYPE_FIFO:
  42                 size = sizeof(struct __vxge_hw_fifo);
  43                 break;
  44         case VXGE_HW_CHANNEL_TYPE_RING:
  45                 size = sizeof(struct __vxge_hw_ring);
  46                 break;
  47         default:
  48                 break;
  49         }
  50
  51         channel = kzalloc(size, GFP_KERNEL);
  52         if (channel == NULL)
  53                 goto exit0;
  54         INIT_LIST_HEAD(&channel->item);
  55
  56         channel->common_reg = hldev->common_reg;
  57         channel->first_vp_id = hldev->first_vp_id;
  58         channel->type = type;
  59         channel->devh = hldev;
  60         channel->vph = vph;
  61         channel->userdata = userdata;
  62         channel->per_dtr_space = per_dtr_space;
  63         channel->length = length;
  64         channel->vp_id = vp_id;
  65
  66         channel->work_arr = kzalloc(sizeof(void *)*length, GFP_KERNEL);
  67         if (channel->work_arr == NULL)
  68                 goto exit1;
  69
  70         channel->free_arr = kzalloc(sizeof(void *)*length, GFP_KERNEL);
  71         if (channel->free_arr == NULL)
  72                 goto exit1;
  73         channel->free_ptr = length;
  74
  75         channel->reserve_arr = kzalloc(sizeof(void *)*length, GFP_KERNEL);
  76         if (channel->reserve_arr == NULL)
  77                 goto exit1;
  78         channel->reserve_ptr = length;
  79         channel->reserve_top = 0;
  80
  81         channel->orig_arr = kzalloc(sizeof(void *)*length, GFP_KERNEL);
  82         if (channel->orig_arr == NULL)
  83                 goto exit1;
  84
  85         return channel;
  86 exit1:
  87         __vxge_hw_channel_free(channel);
  88
  89 exit0:
  90         return NULL;
  91 }
  92
  93 /*
  94  * __vxge_hw_channel_free - Free memory allocated for channel
  95  * This function deallocates memory from the channel and various arrays
  96  * in the channel
  97  */
  98 void __vxge_hw_channel_free(struct __vxge_hw_channel *channel)
  99 {
 100         kfree(channel->work_arr);
 101         kfree(channel->free_arr);
 102         kfree(channel->reserve_arr);
 103         kfree(channel->orig_arr);
 104         kfree(channel);
 105 }
 106
 107 /*
 108  * __vxge_hw_channel_initialize - Initialize a channel
 109  * This function initializes a channel by properly setting the
 110  * various references
 111  */
 112 enum vxge_hw_status
 113 __vxge_hw_channel_initialize(struct __vxge_hw_channel *channel)
 114 {
 115         u32 i;
 116         struct __vxge_hw_virtualpath *vpath;
 117
 118         vpath = channel->vph->vpath;
 119
 120         if ((channel->reserve_arr != NULL) && (channel->orig_arr != NULL)) {
 121                 for (i = 0; i < channel->length; i++)
 122                         channel->orig_arr[i] = channel->reserve_arr[i];
 123         }
 124
 125         switch (channel->type) {
 126         case VXGE_HW_CHANNEL_TYPE_FIFO:
 127                 vpath->fifoh = (struct __vxge_hw_fifo *)channel;
 128                 channel->stats = &((struct __vxge_hw_fifo *)
 129                                 channel)->stats->common_stats;
 130                 break;
 131         case VXGE_HW_CHANNEL_TYPE_RING:
 132                 vpath->ringh = (struct __vxge_hw_ring *)channel;
 133                 channel->stats = &((struct __vxge_hw_ring *)
 134                                 channel)->stats->common_stats;
 135                 break;
 136         default:
 137                 break;
 138         }
 139
 140         return VXGE_HW_OK;
 141 }
 142
 143 /*
 144  * __vxge_hw_channel_reset - Resets a channel
 145  * This function resets a channel by properly setting the various references
 146  */
 147 enum vxge_hw_status
 148 __vxge_hw_channel_reset(struct __vxge_hw_channel *channel)
 149 {
 150         u32 i;
 151
 152         for (i = 0; i < channel->length; i++) {
 153                 if (channel->reserve_arr != NULL)
 154                         channel->reserve_arr[i] = channel->orig_arr[i];
 155                 if (channel->free_arr != NULL)
 156                         channel->free_arr[i] = NULL;
 157                 if (channel->work_arr != NULL)
 158                         channel->work_arr[i] = NULL;
 159         }
 160         channel->free_ptr = channel->length;
 161         channel->reserve_ptr = channel->length;
 162         channel->reserve_top = 0;
 163         channel->post_index = 0;
 164         channel->compl_index = 0;
 165
 166         return VXGE_HW_OK;
 167 }
 168
 169 /*
 170  * __vxge_hw_device_pci_e_init
 171  * Initialize certain PCI/PCI-X configuration registers
 172  * with recommended values. Save config space for future hw resets.
 173  */
 174 void
 175 __vxge_hw_device_pci_e_init(struct __vxge_hw_device *hldev)
 176 {
 177         u16 cmd = 0;
 178
 179         /* Set the PErr Repconse bit and SERR in PCI command register. */
 180         pci_read_config_word(hldev->pdev, PCI_COMMAND, &cmd);
 181         cmd |= 0x140;
 182         pci_write_config_word(hldev->pdev, PCI_COMMAND, cmd);
 183
 184         pci_save_state(hldev->pdev);
 185
 186         return;
 187 }
 188
 189 /*
 190  * __vxge_hw_device_register_poll
 191  * Will poll certain register for specified amount of time.
 192  * Will poll until masked bit is not cleared.
 193  */
 194 enum vxge_hw_status
 195 __vxge_hw_device_register_poll(void __iomem *reg, u64 mask, u32 max_millis)
 196 {
 197         u64 val64;
 198         u32 i = 0;
 199         enum vxge_hw_status ret = VXGE_HW_FAIL;
 200
 201         udelay(10);
 202
 203         do {
 204                 val64 = readq(reg);
 205                 if (!(val64 & mask))
 206                         return VXGE_HW_OK;
 207                 udelay(100);
 208         } while (++i <= 9);
 209
 210         i = 0;
 211         do {
 212                 val64 = readq(reg);
 213                 if (!(val64 & mask))
 214                         return VXGE_HW_OK;
 215                 mdelay(1);
 216         } while (++i <= max_millis);
 217
 218         return ret;
 219 }
 220
 221  /* __vxge_hw_device_vpath_reset_in_prog_check - Check if vpath reset
 222  * in progress
 223  * This routine checks the vpath reset in progress register is turned zero
 224  */
 225 enum vxge_hw_status
 226 __vxge_hw_device_vpath_reset_in_prog_check(u64 __iomem *vpath_rst_in_prog)
 227 {
 228         enum vxge_hw_status status;
 229         status = __vxge_hw_device_register_poll(vpath_rst_in_prog,
 230                         VXGE_HW_VPATH_RST_IN_PROG_VPATH_RST_IN_PROG(0x1ffff),
 231                         VXGE_HW_DEF_DEVICE_POLL_MILLIS);
 232         return status;
 233 }
 234
 235 /*
 236  * __vxge_hw_device_toc_get
 237  * This routine sets the swapper and reads the toc pointer and returns the
 238  * memory mapped address of the toc
 239  */
 240 struct vxge_hw_toc_reg __iomem *
 241 __vxge_hw_device_toc_get(void __iomem *bar0)
 242 {
 243         u64 val64;
 244         struct vxge_hw_toc_reg __iomem *toc = NULL;
 245         enum vxge_hw_status status;
 246
 247         struct vxge_hw_legacy_reg __iomem *legacy_reg =
 248                 (struct vxge_hw_legacy_reg __iomem *)bar0;
 249
 250         status = __vxge_hw_legacy_swapper_set(legacy_reg);
 251         if (status != VXGE_HW_OK)
 252                 goto exit;
 253
 254         val64 = readq(&legacy_reg->toc_first_pointer);
 255         toc = (struct vxge_hw_toc_reg __iomem *)(bar0+val64);
 256 exit:
 257         return toc;
 258 }
 259
 260 /*
 261  * __vxge_hw_device_reg_addr_get
 262  * This routine sets the swapper and reads the toc pointer and initializes the
 263  * register location pointers in the device object. It waits until the ric is
 264  * completed initializing registers.
 265  */
 266 enum vxge_hw_status
 267 __vxge_hw_device_reg_addr_get(struct __vxge_hw_device *hldev)
 268 {
 269         u64 val64;
 270         u32 i;
 271         enum vxge_hw_status status = VXGE_HW_OK;
 272
 273         hldev->legacy_reg = (struct vxge_hw_legacy_reg __iomem *)hldev->bar0;
 274
 275         hldev->toc_reg = __vxge_hw_device_toc_get(hldev->bar0);
 276         if (hldev->toc_reg  == NULL) {
 277                 status = VXGE_HW_FAIL;
 278                 goto exit;
 279         }
 280
 281         val64 = readq(&hldev->toc_reg->toc_common_pointer);
 282         hldev->common_reg =
 283         (struct vxge_hw_common_reg __iomem *)(hldev->bar0 + val64);
 284
 285         val64 = readq(&hldev->toc_reg->toc_mrpcim_pointer);
 286         hldev->mrpcim_reg =
 287                 (struct vxge_hw_mrpcim_reg __iomem *)(hldev->bar0 + val64);
 288
 289         for (i = 0; i < VXGE_HW_TITAN_SRPCIM_REG_SPACES; i++) {
 290                 val64 = readq(&hldev->toc_reg->toc_srpcim_pointer[i]);
 291                 hldev->srpcim_reg[i] =
 292                         (struct vxge_hw_srpcim_reg __iomem *)
 293                                 (hldev->bar0 + val64);
 294         }
 295
 296         for (i = 0; i < VXGE_HW_TITAN_VPMGMT_REG_SPACES; i++) {
 297                 val64 = readq(&hldev->toc_reg->toc_vpmgmt_pointer[i]);
 298                 hldev->vpmgmt_reg[i] =
 299                 (struct vxge_hw_vpmgmt_reg __iomem *)(hldev->bar0 + val64);
 300         }
 301
 302         for (i = 0; i < VXGE_HW_TITAN_VPATH_REG_SPACES; i++) {
 303                 val64 = readq(&hldev->toc_reg->toc_vpath_pointer[i]);
 304                 hldev->vpath_reg[i] =
 305                         (struct vxge_hw_vpath_reg __iomem *)
 306                                 (hldev->bar0 + val64);
 307         }
 308
 309         val64 = readq(&hldev->toc_reg->toc_kdfc);
 310
 311         switch (VXGE_HW_TOC_GET_KDFC_INITIAL_BIR(val64)) {
 312         case 0:
 313                 hldev->kdfc = (u8 __iomem *)(hldev->bar0 +
 314                         VXGE_HW_TOC_GET_KDFC_INITIAL_OFFSET(val64));
 315                 break;
 316         case 2:
 317                 hldev->kdfc = (u8 __iomem *)(hldev->bar1 +
 318                         VXGE_HW_TOC_GET_KDFC_INITIAL_OFFSET(val64));
 319                 break;
 320         case 4:
 321                 hldev->kdfc = (u8 __iomem *)(hldev->bar2 +
 322                         VXGE_HW_TOC_GET_KDFC_INITIAL_OFFSET(val64));
 323                 break;
 324         default:
 325                 break;
 326         }
 327
 328         status = __vxge_hw_device_vpath_reset_in_prog_check(
 329                         (u64 __iomem *)&hldev->common_reg->vpath_rst_in_prog);
 330 exit:
 331         return status;
 332 }
 333
 334 /*
 335  * __vxge_hw_device_id_get
 336  * This routine returns sets the device id and revision numbers into the device
 337  * structure
 338  */
 339 void __vxge_hw_device_id_get(struct __vxge_hw_device *hldev)
 340 {
 341         u64 val64;
 342
 343         val64 = readq(&hldev->common_reg->titan_asic_id);
 344         hldev->device_id =
 345                 (u16)VXGE_HW_TITAN_ASIC_ID_GET_INITIAL_DEVICE_ID(val64);
 346
 347         hldev->major_revision =
 348                 (u8)VXGE_HW_TITAN_ASIC_ID_GET_INITIAL_MAJOR_REVISION(val64);
 349
 350         hldev->minor_revision =
 351                 (u8)VXGE_HW_TITAN_ASIC_ID_GET_INITIAL_MINOR_REVISION(val64);
 352
 353         return;
 354 }
 355
 356 /*
 357  * __vxge_hw_device_access_rights_get: Get Access Rights of the driver
 358  * This routine returns the Access Rights of the driver
 359  */
 360 static u32
 361 __vxge_hw_device_access_rights_get(u32 host_type, u32 func_id)
 362 {
 363         u32 access_rights = VXGE_HW_DEVICE_ACCESS_RIGHT_VPATH;
 364
 365         switch (host_type) {
 366         case VXGE_HW_NO_MR_NO_SR_NORMAL_FUNCTION:
 367                 if (func_id == 0) {
 368                         access_rights |= VXGE_HW_DEVICE_ACCESS_RIGHT_MRPCIM |
 369                                         VXGE_HW_DEVICE_ACCESS_RIGHT_SRPCIM;
 370                 }
 371                 break;
 372         case VXGE_HW_MR_NO_SR_VH0_BASE_FUNCTION:
 373                 access_rights |= VXGE_HW_DEVICE_ACCESS_RIGHT_MRPCIM |
 374                                 VXGE_HW_DEVICE_ACCESS_RIGHT_SRPCIM;
 375                 break;
 376         case VXGE_HW_NO_MR_SR_VH0_FUNCTION0:
 377                 access_rights |= VXGE_HW_DEVICE_ACCESS_RIGHT_MRPCIM |
 378                                 VXGE_HW_DEVICE_ACCESS_RIGHT_SRPCIM;
 379                 break;
 380         case VXGE_HW_NO_MR_SR_VH0_VIRTUAL_FUNCTION:
 381         case VXGE_HW_SR_VH_VIRTUAL_FUNCTION:
 382         case VXGE_HW_MR_SR_VH0_INVALID_CONFIG:
 383                 break;
 384         case VXGE_HW_SR_VH_FUNCTION0:
 385         case VXGE_HW_VH_NORMAL_FUNCTION:
 386                 access_rights |= VXGE_HW_DEVICE_ACCESS_RIGHT_SRPCIM;
 387                 break;
 388         }
 389
 390         return access_rights;
 391 }
 392 /*
 393  * __vxge_hw_device_host_info_get
 394  * This routine returns the host type assignments
 395  */
 396 void __vxge_hw_device_host_info_get(struct __vxge_hw_device *hldev)
 397 {
 398         u64 val64;
 399         u32 i;
 400
 401         val64 = readq(&hldev->common_reg->host_type_assignments);
 402
 403         hldev->host_type =
 404            (u32)VXGE_HW_HOST_TYPE_ASSIGNMENTS_GET_HOST_TYPE_ASSIGNMENTS(val64);
 405
 406         hldev->vpath_assignments = readq(&hldev->common_reg->vpath_assignments);
 407
 408         for (i = 0; i < VXGE_HW_MAX_VIRTUAL_PATHS; i++) {
 409
 410                 if (!(hldev->vpath_assignments & vxge_mBIT(i)))
 411                         continue;
 412
 413                 hldev->func_id =
 414                         __vxge_hw_vpath_func_id_get(i, hldev->vpmgmt_reg[i]);
 415
 416                 hldev->access_rights = __vxge_hw_device_access_rights_get(
 417                         hldev->host_type, hldev->func_id);
 418
 419                 hldev->first_vp_id = i;
 420                 break;
 421         }
 422
 423         return;
 424 }
 425
 426 /*
 427  * __vxge_hw_verify_pci_e_info - Validate the pci-e link parameters such as
 428  * link width and signalling rate.
 429  */
 430 static enum vxge_hw_status
 431 __vxge_hw_verify_pci_e_info(struct __vxge_hw_device *hldev)
 432 {
 433         int exp_cap;
 434         u16 lnk;
 435
 436         /* Get the negotiated link width and speed from PCI config space */
 437         exp_cap = pci_find_capability(hldev->pdev, PCI_CAP_ID_EXP);
 438         pci_read_config_word(hldev->pdev, exp_cap + PCI_EXP_LNKSTA, &lnk);
 439
 440         if ((lnk & PCI_EXP_LNKSTA_CLS) != 1)
 441                 return VXGE_HW_ERR_INVALID_PCI_INFO;
 442
 443         switch ((lnk & PCI_EXP_LNKSTA_NLW) >> 4) {
 444         case PCIE_LNK_WIDTH_RESRV:
 445         case PCIE_LNK_X1:
 446         case PCIE_LNK_X2:
 447         case PCIE_LNK_X4:
 448         case PCIE_LNK_X8:
 449                 break;
 450         default:
 451                 return VXGE_HW_ERR_INVALID_PCI_INFO;
 452         }
 453
 454         return VXGE_HW_OK;
 455 }
 456
 457 enum vxge_hw_status
 458 __vxge_hw_device_is_privilaged(struct __vxge_hw_device *hldev)
 459 {
 460         if ((hldev->host_type == VXGE_HW_NO_MR_NO_SR_NORMAL_FUNCTION ||
 461         hldev->host_type == VXGE_HW_MR_NO_SR_VH0_BASE_FUNCTION ||
 462         hldev->host_type == VXGE_HW_NO_MR_SR_VH0_FUNCTION0) &&
 463         (hldev->func_id == 0))
 464                 return VXGE_HW_OK;
 465         else
 466                 return VXGE_HW_ERR_PRIVILAGED_OPEARATION;
 467 }
 468
 469 /*
 470  * vxge_hw_wrr_rebalance - Rebalance the RX_WRR and KDFC_WRR calandars.
 471  * Rebalance the RX_WRR and KDFC_WRR calandars.
 472  */
 473 static enum
 474 vxge_hw_status vxge_hw_wrr_rebalance(struct __vxge_hw_device *hldev)
 475 {
 476         u64 val64;
 477         u32 wrr_states[VXGE_HW_WEIGHTED_RR_SERVICE_STATES];
 478         u32 i, j, how_often = 1;
 479         enum vxge_hw_status status = VXGE_HW_OK;
 480
 481         status = __vxge_hw_device_is_privilaged(hldev);
 482         if (status != VXGE_HW_OK)
 483                 goto exit;
 484
 485         /* Reset the priorities assigned to the WRR arbitration
 486         phases for the receive traffic */
 487         for (i = 0; i < VXGE_HW_WRR_RING_COUNT; i++)
 488                 writeq(0, ((&hldev->mrpcim_reg->rx_w_round_robin_0) + i));
 489
 490         /* Reset the transmit FIFO servicing calendar for FIFOs */
 491         for (i = 0; i < VXGE_HW_WRR_FIFO_COUNT; i++) {
 492                 writeq(0, ((&hldev->mrpcim_reg->kdfc_w_round_robin_0) + i));
 493                 writeq(0, ((&hldev->mrpcim_reg->kdfc_w_round_robin_20) + i));
 494         }
 495
 496         /* Assign WRR priority  0 for all FIFOs */
 497         for (i = 1; i < VXGE_HW_MAX_VIRTUAL_PATHS; i++) {
 498                 writeq(VXGE_HW_KDFC_FIFO_0_CTRL_WRR_NUMBER(0),
 499                                 ((&hldev->mrpcim_reg->kdfc_fifo_0_ctrl)  + i));
 500
 501                 writeq(VXGE_HW_KDFC_FIFO_17_CTRL_WRR_NUMBER(0),
 502                         ((&hldev->mrpcim_reg->kdfc_fifo_17_ctrl) + i));
 503         }
 504
 505         /* Reset to service non-offload doorbells */
 506         writeq(0, &hldev->mrpcim_reg->kdfc_entry_type_sel_0);
 507         writeq(0, &hldev->mrpcim_reg->kdfc_entry_type_sel_1);
 508
 509         /* Set priority 0 to all receive queues */
 510         writeq(0, &hldev->mrpcim_reg->rx_queue_priority_0);
 511         writeq(0, &hldev->mrpcim_reg->rx_queue_priority_1);
 512         writeq(0, &hldev->mrpcim_reg->rx_queue_priority_2);
 513
 514         /* Initialize all the slots as unused */
 515         for (i = 0; i < VXGE_HW_WEIGHTED_RR_SERVICE_STATES; i++)
 516                 wrr_states[i] = -1;
 517
 518         /* Prepare the Fifo service states */
 519         for (i = 0; i < VXGE_HW_MAX_VIRTUAL_PATHS; i++) {
 520
 521                 if (!hldev->config.vp_config[i].min_bandwidth)
 522                         continue;
 523
 524                 how_often = VXGE_HW_VPATH_BANDWIDTH_MAX /
 525                                 hldev->config.vp_config[i].min_bandwidth;
 526                 if (how_often) {
 527
 528                         for (j = 0; j < VXGE_HW_WRR_FIFO_SERVICE_STATES;) {
 529                                 if (wrr_states[j] == -1) {
 530                                         wrr_states[j] = i;
 531                                         /* Make sure each fifo is serviced
 532                                          * atleast once */
 533                                         if (i == j)
 534                                                 j += VXGE_HW_MAX_VIRTUAL_PATHS;
 535                                         else
 536                                                 j += how_often;
 537                                 } else
 538                                         j++;
 539                         }
 540                 }
 541         }
 542
 543         /* Fill the unused slots with 0 */
 544         for (j = 0; j < VXGE_HW_WEIGHTED_RR_SERVICE_STATES; j++) {
 545                 if (wrr_states[j] == -1)
 546                         wrr_states[j] = 0;
 547         }
 548
 549         /* Assign WRR priority number for FIFOs */
 550         for (i = 0; i < VXGE_HW_MAX_VIRTUAL_PATHS; i++) {
 551                 writeq(VXGE_HW_KDFC_FIFO_0_CTRL_WRR_NUMBER(i),
 552                                 ((&hldev->mrpcim_reg->kdfc_fifo_0_ctrl) + i));
 553
 554                 writeq(VXGE_HW_KDFC_FIFO_17_CTRL_WRR_NUMBER(i),
 555                         ((&hldev->mrpcim_reg->kdfc_fifo_17_ctrl) + i));
 556         }
 557
 558         /* Modify the servicing algorithm applied to the 3 types of doorbells.
 559         i.e, none-offload, message and offload */
 560         writeq(VXGE_HW_KDFC_ENTRY_TYPE_SEL_0_NUMBER_0(0) |
 561                                 VXGE_HW_KDFC_ENTRY_TYPE_SEL_0_NUMBER_1(0) |
 562                                 VXGE_HW_KDFC_ENTRY_TYPE_SEL_0_NUMBER_2(0) |
 563                                 VXGE_HW_KDFC_ENTRY_TYPE_SEL_0_NUMBER_3(0) |
 564                                 VXGE_HW_KDFC_ENTRY_TYPE_SEL_0_NUMBER_4(1) |
 565                                 VXGE_HW_KDFC_ENTRY_TYPE_SEL_0_NUMBER_5(0) |
 566                                 VXGE_HW_KDFC_ENTRY_TYPE_SEL_0_NUMBER_6(0) |
 567                                 VXGE_HW_KDFC_ENTRY_TYPE_SEL_0_NUMBER_7(0),
 568                                 &hldev->mrpcim_reg->kdfc_entry_type_sel_0);
 569
 570         writeq(VXGE_HW_KDFC_ENTRY_TYPE_SEL_1_NUMBER_8(1),
 571                                 &hldev->mrpcim_reg->kdfc_entry_type_sel_1);
 572
 573         for (i = 0, j = 0; i < VXGE_HW_WRR_FIFO_COUNT; i++) {
 574
 575                 val64 = VXGE_HW_KDFC_W_ROUND_ROBIN_0_NUMBER_0(wrr_states[j++]);
 576                 val64 |= VXGE_HW_KDFC_W_ROUND_ROBIN_0_NUMBER_1(wrr_states[j++]);
 577                 val64 |= VXGE_HW_KDFC_W_ROUND_ROBIN_0_NUMBER_2(wrr_states[j++]);
 578                 val64 |= VXGE_HW_KDFC_W_ROUND_ROBIN_0_NUMBER_3(wrr_states[j++]);
 579                 val64 |= VXGE_HW_KDFC_W_ROUND_ROBIN_0_NUMBER_4(wrr_states[j++]);
 580                 val64 |= VXGE_HW_KDFC_W_ROUND_ROBIN_0_NUMBER_5(wrr_states[j++]);
 581                 val64 |= VXGE_HW_KDFC_W_ROUND_ROBIN_0_NUMBER_6(wrr_states[j++]);
 582                 val64 |= VXGE_HW_KDFC_W_ROUND_ROBIN_0_NUMBER_7(wrr_states[j++]);
 583
 584                 writeq(val64, (&hldev->mrpcim_reg->kdfc_w_round_robin_0 + i));
 585                 writeq(val64, (&hldev->mrpcim_reg->kdfc_w_round_robin_20 + i));
 586         }
 587
 588         /* Set up the priorities assigned to receive queues */
 589         writeq(VXGE_HW_RX_QUEUE_PRIORITY_0_RX_Q_NUMBER_0(0) |
 590                         VXGE_HW_RX_QUEUE_PRIORITY_0_RX_Q_NUMBER_1(1) |
 591                         VXGE_HW_RX_QUEUE_PRIORITY_0_RX_Q_NUMBER_2(2) |
 592                         VXGE_HW_RX_QUEUE_PRIORITY_0_RX_Q_NUMBER_3(3) |
 593                         VXGE_HW_RX_QUEUE_PRIORITY_0_RX_Q_NUMBER_4(4) |
 594                         VXGE_HW_RX_QUEUE_PRIORITY_0_RX_Q_NUMBER_5(5) |
 595                         VXGE_HW_RX_QUEUE_PRIORITY_0_RX_Q_NUMBER_6(6) |
 596                         VXGE_HW_RX_QUEUE_PRIORITY_0_RX_Q_NUMBER_7(7),
 597                         &hldev->mrpcim_reg->rx_queue_priority_0);
 598
 599         writeq(VXGE_HW_RX_QUEUE_PRIORITY_1_RX_Q_NUMBER_8(8) |
 600                         VXGE_HW_RX_QUEUE_PRIORITY_1_RX_Q_NUMBER_9(9) |
 601                         VXGE_HW_RX_QUEUE_PRIORITY_1_RX_Q_NUMBER_10(10) |
 602                         VXGE_HW_RX_QUEUE_PRIORITY_1_RX_Q_NUMBER_11(11) |
 603                         VXGE_HW_RX_QUEUE_PRIORITY_1_RX_Q_NUMBER_12(12) |
 604                         VXGE_HW_RX_QUEUE_PRIORITY_1_RX_Q_NUMBER_13(13) |
 605                         VXGE_HW_RX_QUEUE_PRIORITY_1_RX_Q_NUMBER_14(14) |
 606                         VXGE_HW_RX_QUEUE_PRIORITY_1_RX_Q_NUMBER_15(15),
 607                         &hldev->mrpcim_reg->rx_queue_priority_1);
 608
 609         writeq(VXGE_HW_RX_QUEUE_PRIORITY_2_RX_Q_NUMBER_16(16),
 610                                 &hldev->mrpcim_reg->rx_queue_priority_2);
 611
 612         /* Initialize all the slots as unused */
 613         for (i = 0; i < VXGE_HW_WEIGHTED_RR_SERVICE_STATES; i++)
 614                 wrr_states[i] = -1;
 615
 616         /* Prepare the Ring service states */
 617         for (i = 0; i < VXGE_HW_MAX_VIRTUAL_PATHS; i++) {
 618
 619                 if (!hldev->config.vp_config[i].min_bandwidth)
 620                         continue;
 621
 622                 how_often = VXGE_HW_VPATH_BANDWIDTH_MAX /
 623                                 hldev->config.vp_config[i].min_bandwidth;
 624
 625                 if (how_often) {
 626                         for (j = 0; j < VXGE_HW_WRR_RING_SERVICE_STATES;) {
 627                                 if (wrr_states[j] == -1) {
 628                                         wrr_states[j] = i;
 629                                         /* Make sure each ring is
 630                                          * serviced atleast once */
 631                                         if (i == j)
 632                                                 j += VXGE_HW_MAX_VIRTUAL_PATHS;
 633                                         else
 634                                                 j += how_often;
 635                                 } else
 636                                         j++;
 637                         }
 638                 }
 639         }
 640
 641         /* Fill the unused slots with 0 */
 642         for (j = 0; j < VXGE_HW_WEIGHTED_RR_SERVICE_STATES; j++) {
 643                 if (wrr_states[j] == -1)
 644                         wrr_states[j] = 0;
 645         }
 646
 647         for (i = 0, j = 0; i < VXGE_HW_WRR_RING_COUNT; i++) {
 648                 val64 =  VXGE_HW_RX_W_ROUND_ROBIN_0_RX_W_PRIORITY_SS_0(
 649                                 wrr_states[j++]);
 650                 val64 |=  VXGE_HW_RX_W_ROUND_ROBIN_0_RX_W_PRIORITY_SS_1(
 651                                 wrr_states[j++]);
 652                 val64 |=  VXGE_HW_RX_W_ROUND_ROBIN_0_RX_W_PRIORITY_SS_2(
 653                                 wrr_states[j++]);
 654                 val64 |=  VXGE_HW_RX_W_ROUND_ROBIN_0_RX_W_PRIORITY_SS_3(
 655                                 wrr_states[j++]);
 656                 val64 |=  VXGE_HW_RX_W_ROUND_ROBIN_0_RX_W_PRIORITY_SS_4(
 657                                 wrr_states[j++]);
 658                 val64 |=  VXGE_HW_RX_W_ROUND_ROBIN_0_RX_W_PRIORITY_SS_5(
 659                                 wrr_states[j++]);
 660                 val64 |=  VXGE_HW_RX_W_ROUND_ROBIN_0_RX_W_PRIORITY_SS_6(
 661                                 wrr_states[j++]);
 662                 val64 |=  VXGE_HW_RX_W_ROUND_ROBIN_0_RX_W_PRIORITY_SS_7(
 663                                 wrr_states[j++]);
 664
 665                 writeq(val64, ((&hldev->mrpcim_reg->rx_w_round_robin_0) + i));
 666         }
 667 exit:
 668         return status;
 669 }
 670
 671 /*
 672  * __vxge_hw_device_initialize
 673  * Initialize Titan-V hardware.
 674  */
 675 enum vxge_hw_status __vxge_hw_device_initialize(struct __vxge_hw_device *hldev)
 676 {
 677         enum vxge_hw_status status = VXGE_HW_OK;
 678
 679         if (VXGE_HW_OK == __vxge_hw_device_is_privilaged(hldev)) {
 680                 /* Validate the pci-e link width and speed */
 681                 status = __vxge_hw_verify_pci_e_info(hldev);
 682                 if (status != VXGE_HW_OK)
 683                         goto exit;
 684         }
 685
 686         vxge_hw_wrr_rebalance(hldev);
 687 exit:
 688         return status;
 689 }
 690
 691 /**
 692  * vxge_hw_device_hw_info_get - Get the hw information
 693  * Returns the vpath mask that has the bits set for each vpath allocated
 694  * for the driver, FW version information and the first mac addresse for
 695  * each vpath
 696  */
 697 enum vxge_hw_status __devinit
 698 vxge_hw_device_hw_info_get(void __iomem *bar0,
 699                            struct vxge_hw_device_hw_info *hw_info)
 700 {
 701         u32 i;
 702         u64 val64;
 703         struct vxge_hw_toc_reg __iomem *toc;
 704         struct vxge_hw_mrpcim_reg __iomem *mrpcim_reg;
 705         struct vxge_hw_common_reg __iomem *common_reg;
 706         struct vxge_hw_vpath_reg __iomem *vpath_reg;
 707         struct vxge_hw_vpmgmt_reg __iomem *vpmgmt_reg;
 708         enum vxge_hw_status status;
 709
 710         memset(hw_info, 0, sizeof(struct vxge_hw_device_hw_info));
 711
 712         toc = __vxge_hw_device_toc_get(bar0);
 713         if (toc == NULL) {
 714                 status = VXGE_HW_ERR_CRITICAL;
 715                 goto exit;
 716         }
 717
 718         val64 = readq(&toc->toc_common_pointer);
 719         common_reg = (struct vxge_hw_common_reg __iomem *)(bar0 + val64);
 720
 721         status = __vxge_hw_device_vpath_reset_in_prog_check(
 722                 (u64 __iomem *)&common_reg->vpath_rst_in_prog);
 723         if (status != VXGE_HW_OK)
 724                 goto exit;
 725
 726         hw_info->vpath_mask = readq(&common_reg->vpath_assignments);
 727
 728         val64 = readq(&common_reg->host_type_assignments);
 729
 730         hw_info->host_type =
 731            (u32)VXGE_HW_HOST_TYPE_ASSIGNMENTS_GET_HOST_TYPE_ASSIGNMENTS(val64);
 732
 733         for (i = 0; i < VXGE_HW_MAX_VIRTUAL_PATHS; i++) {
 734
 735                 if (!((hw_info->vpath_mask) & vxge_mBIT(i)))
 736                         continue;
 737
 738                 val64 = readq(&toc->toc_vpmgmt_pointer[i]);
 739
 740                 vpmgmt_reg = (struct vxge_hw_vpmgmt_reg __iomem *)
 741                                 (bar0 + val64);
 742
 743                 hw_info->func_id = __vxge_hw_vpath_func_id_get(i, vpmgmt_reg);
 744                 if (__vxge_hw_device_access_rights_get(hw_info->host_type,
 745                         hw_info->func_id) &
 746                         VXGE_HW_DEVICE_ACCESS_RIGHT_MRPCIM) {
 747
 748                         val64 = readq(&toc->toc_mrpcim_pointer);
 749
 750                         mrpcim_reg = (struct vxge_hw_mrpcim_reg __iomem *)
 751                                         (bar0 + val64);
 752
 753                         writeq(0, &mrpcim_reg->xgmac_gen_fw_memo_mask);
 754                         wmb();
 755                 }
 756
 757                 val64 = readq(&toc->toc_vpath_pointer[i]);
 758
 759                 vpath_reg = (struct vxge_hw_vpath_reg __iomem *)(bar0 + val64);
 760
 761                 hw_info->function_mode =
 762                         __vxge_hw_vpath_pci_func_mode_get(i, vpath_reg);
 763
 764                 status = __vxge_hw_vpath_fw_ver_get(i, vpath_reg, hw_info);
 765                 if (status != VXGE_HW_OK)
 766                         goto exit;
 767
 768                 status = __vxge_hw_vpath_card_info_get(i, vpath_reg, hw_info);
 769                 if (status != VXGE_HW_OK)
 770                         goto exit;
 771
 772                 break;
 773         }
 774
 775         for (i = 0; i < VXGE_HW_MAX_VIRTUAL_PATHS; i++) {
 776
 777                 if (!((hw_info->vpath_mask) & vxge_mBIT(i)))
 778                         continue;
 779
 780                 val64 = readq(&toc->toc_vpath_pointer[i]);
 781                 vpath_reg = (struct vxge_hw_vpath_reg __iomem *)(bar0 + val64);
 782
 783                 status =  __vxge_hw_vpath_addr_get(i, vpath_reg,
 784                                 hw_info->mac_addrs[i],
 785                                 hw_info->mac_addr_masks[i]);
 786                 if (status != VXGE_HW_OK)
 787                         goto exit;
 788         }
 789 exit:
 790         return status;
 791 }
 792
 793 /*
 794  * vxge_hw_device_initialize - Initialize Titan device.
 795  * Initialize Titan device. Note that all the arguments of this public API
 796  * are 'IN', including @hldev. Driver cooperates with
 797  * OS to find new Titan device, locate its PCI and memory spaces.
 798  *
 799  * When done, the driver allocates sizeof(struct __vxge_hw_device) bytes for HW
 800  * to enable the latter to perform Titan hardware initialization.
 801  */
 802 enum vxge_hw_status __devinit
 803 vxge_hw_device_initialize(
 804         struct __vxge_hw_device **devh,
 805         struct vxge_hw_device_attr *attr,
 806         struct vxge_hw_device_config *device_config)
 807 {
 808         u32 i;
 809         u32 nblocks = 0;
 810         struct __vxge_hw_device *hldev = NULL;
 811         enum vxge_hw_status status = VXGE_HW_OK;
 812
 813         status = __vxge_hw_device_config_check(device_config);
 814         if (status != VXGE_HW_OK)
 815                 goto exit;
 816
 817         hldev = (struct __vxge_hw_device *)
 818                         vmalloc(sizeof(struct __vxge_hw_device));
 819         if (hldev == NULL) {
 820                 status = VXGE_HW_ERR_OUT_OF_MEMORY;
 821                 goto exit;
 822         }
 823
 824         memset(hldev, 0, sizeof(struct __vxge_hw_device));
 825         hldev->magic = VXGE_HW_DEVICE_MAGIC;
 826
 827         vxge_hw_device_debug_set(hldev, VXGE_ERR, VXGE_COMPONENT_ALL);
 828
 829         /* apply config */
 830         memcpy(&hldev->config, device_config,
 831                 sizeof(struct vxge_hw_device_config));
 832
 833         hldev->bar0 = attr->bar0;
 834         hldev->bar1 = attr->bar1;
 835         hldev->bar2 = attr->bar2;
 836         hldev->pdev = attr->pdev;
 837
 838         hldev->uld_callbacks.link_up = attr->uld_callbacks.link_up;
 839         hldev->uld_callbacks.link_down = attr->uld_callbacks.link_down;
 840         hldev->uld_callbacks.crit_err = attr->uld_callbacks.crit_err;
 841
 842         __vxge_hw_device_pci_e_init(hldev);
 843
 844         status = __vxge_hw_device_reg_addr_get(hldev);
 845         if (status != VXGE_HW_OK)
 846                 goto exit;
 847         __vxge_hw_device_id_get(hldev);
 848
 849         __vxge_hw_device_host_info_get(hldev);
 850
 851         /* Incrementing for stats blocks */
 852         nblocks++;
 853
 854         for (i = 0; i < VXGE_HW_MAX_VIRTUAL_PATHS; i++) {
 855
 856                 if (!(hldev->vpath_assignments & vxge_mBIT(i)))
 857                         continue;
 858
 859                 if (device_config->vp_config[i].ring.enable ==
 860                         VXGE_HW_RING_ENABLE)
 861                         nblocks += device_config->vp_config[i].ring.ring_blocks;
 862
 863                 if (device_config->vp_config[i].fifo.enable ==
 864                         VXGE_HW_FIFO_ENABLE)
 865                         nblocks += device_config->vp_config[i].fifo.fifo_blocks;
 866                 nblocks++;
 867         }
 868
 869         if (__vxge_hw_blockpool_create(hldev,
 870                 &hldev->block_pool,
 871                 device_config->dma_blockpool_initial + nblocks,
 872                 device_config->dma_blockpool_max + nblocks) != VXGE_HW_OK) {
 873
 874                 vxge_hw_device_terminate(hldev);
 875                 status = VXGE_HW_ERR_OUT_OF_MEMORY;
 876                 goto exit;
 877         }
 878
 879         status = __vxge_hw_device_initialize(hldev);
 880
 881         if (status != VXGE_HW_OK) {
 882                 vxge_hw_device_terminate(hldev);
 883                 goto exit;
 884         }
 885
 886         *devh = hldev;
 887 exit:
 888         return status;
 889 }
 890
 891 /*
 892  * vxge_hw_device_terminate - Terminate Titan device.
 893  * Terminate HW device.
 894  */
 895 void
 896 vxge_hw_device_terminate(struct __vxge_hw_device *hldev)
 897 {
 898         vxge_assert(hldev->magic == VXGE_HW_DEVICE_MAGIC);
 899
 900         hldev->magic = VXGE_HW_DEVICE_DEAD;
 901         __vxge_hw_blockpool_destroy(&hldev->block_pool);
 902         vfree(hldev);
 903 }
 904
 905 /*
 906  * vxge_hw_device_stats_get - Get the device hw statistics.
 907  * Returns the vpath h/w stats for the device.
 908  */
 909 enum vxge_hw_status
 910 vxge_hw_device_stats_get(struct __vxge_hw_device *hldev,
 911                         struct vxge_hw_device_stats_hw_info *hw_stats)
 912 {
 913         u32 i;
 914         enum vxge_hw_status status = VXGE_HW_OK;
 915
 916         for (i = 0; i < VXGE_HW_MAX_VIRTUAL_PATHS; i++) {
 917
 918                 if (!(hldev->vpaths_deployed & vxge_mBIT(i)) ||
 919                         (hldev->virtual_paths[i].vp_open ==
 920                                 VXGE_HW_VP_NOT_OPEN))
 921                         continue;
 922
 923                 memcpy(hldev->virtual_paths[i].hw_stats_sav,
 924                                 hldev->virtual_paths[i].hw_stats,
 925                                 sizeof(struct vxge_hw_vpath_stats_hw_info));
 926
 927                 status = __vxge_hw_vpath_stats_get(
 928                         &hldev->virtual_paths[i],
 929                         hldev->virtual_paths[i].hw_stats);
 930         }
 931
 932         memcpy(hw_stats, &hldev->stats.hw_dev_info_stats,
 933                         sizeof(struct vxge_hw_device_stats_hw_info));
 934
 935         return status;
 936 }
 937
 938 /*
 939  * vxge_hw_driver_stats_get - Get the device sw statistics.
 940  * Returns the vpath s/w stats for the device.
 941  */
 942 enum vxge_hw_status vxge_hw_driver_stats_get(
 943                         struct __vxge_hw_device *hldev,
 944                         struct vxge_hw_device_stats_sw_info *sw_stats)
 945 {
 946         enum vxge_hw_status status = VXGE_HW_OK;
 947
 948         memcpy(sw_stats, &hldev->stats.sw_dev_info_stats,
 949                 sizeof(struct vxge_hw_device_stats_sw_info));
 950
 951         return status;
 952 }
 953
 954 /*
 955  * vxge_hw_mrpcim_stats_access - Access the statistics from the given location
 956  *                           and offset and perform an operation
 957  * Get the statistics from the given location and offset.
 958  */
 959 enum vxge_hw_status
 960 vxge_hw_mrpcim_stats_access(struct __vxge_hw_device *hldev,
 961                             u32 operation, u32 location, u32 offset, u64 *stat)
 962 {
 963         u64 val64;
 964         enum vxge_hw_status status = VXGE_HW_OK;
 965
 966         status = __vxge_hw_device_is_privilaged(hldev);
 967         if (status != VXGE_HW_OK)
 968                 goto exit;
 969
 970         val64 = VXGE_HW_XMAC_STATS_SYS_CMD_OP(operation) |
 971                 VXGE_HW_XMAC_STATS_SYS_CMD_STROBE |
 972                 VXGE_HW_XMAC_STATS_SYS_CMD_LOC_SEL(location) |
 973                 VXGE_HW_XMAC_STATS_SYS_CMD_OFFSET_SEL(offset);
 974
 975         status = __vxge_hw_pio_mem_write64(val64,
 976                                 &hldev->mrpcim_reg->xmac_stats_sys_cmd,
 977                                 VXGE_HW_XMAC_STATS_SYS_CMD_STROBE,
 978                                 hldev->config.device_poll_millis);
 979
 980         if ((status == VXGE_HW_OK) && (operation == VXGE_HW_STATS_OP_READ))
 981                 *stat = readq(&hldev->mrpcim_reg->xmac_stats_sys_data);
 982         else
 983                 *stat = 0;
 984 exit:
 985         return status;
 986 }
 987
 988 /*
 989  * vxge_hw_device_xmac_aggr_stats_get - Get the Statistics on aggregate port
 990  * Get the Statistics on aggregate port
 991  */
 992 enum vxge_hw_status
 993 vxge_hw_device_xmac_aggr_stats_get(struct __vxge_hw_device *hldev, u32 port,
 994                                    struct vxge_hw_xmac_aggr_stats *aggr_stats)
 995 {
 996         u64 *val64;
 997         int i;
 998         u32 offset = VXGE_HW_STATS_AGGRn_OFFSET;
 999         enum vxge_hw_status status = VXGE_HW_OK;
1000
1001         val64 = (u64 *)aggr_stats;
1002
1003         status = __vxge_hw_device_is_privilaged(hldev);
1004         if (status != VXGE_HW_OK)
1005                 goto exit;
1006
1007         for (i = 0; i < sizeof(struct vxge_hw_xmac_aggr_stats) / 8; i++) {
1008                 status = vxge_hw_mrpcim_stats_access(hldev,
1009                                         VXGE_HW_STATS_OP_READ,
1010                                         VXGE_HW_STATS_LOC_AGGR,
1011                                         ((offset + (104 * port)) >> 3), val64);
1012                 if (status != VXGE_HW_OK)
1013                         goto exit;
1014
1015                 offset += 8;
1016                 val64++;
1017         }
1018 exit:
1019         return status;
1020 }
1021
1022 /*
1023  * vxge_hw_device_xmac_port_stats_get - Get the Statistics on a port
1024  * Get the Statistics on port
1025  */
1026 enum vxge_hw_status
1027 vxge_hw_device_xmac_port_stats_get(struct __vxge_hw_device *hldev, u32 port,
1028                                    struct vxge_hw_xmac_port_stats *port_stats)
1029 {
1030         u64 *val64;
1031         enum vxge_hw_status status = VXGE_HW_OK;
1032         int i;
1033         u32 offset = 0x0;
1034         val64 = (u64 *) port_stats;
1035
1036         status = __vxge_hw_device_is_privilaged(hldev);
1037         if (status != VXGE_HW_OK)
1038                 goto exit;
1039
1040         for (i = 0; i < sizeof(struct vxge_hw_xmac_port_stats) / 8; i++) {
1041                 status = vxge_hw_mrpcim_stats_access(hldev,
1042                                         VXGE_HW_STATS_OP_READ,
1043                                         VXGE_HW_STATS_LOC_AGGR,
1044                                         ((offset + (608 * port)) >> 3), val64);
1045                 if (status != VXGE_HW_OK)
1046                         goto exit;
1047
1048                 offset += 8;
1049                 val64++;
1050         }
1051
1052 exit:
1053         return status;
1054 }
1055
1056 /*
1057  * vxge_hw_device_xmac_stats_get - Get the XMAC Statistics
1058  * Get the XMAC Statistics
1059  */
1060 enum vxge_hw_status
1061 vxge_hw_device_xmac_stats_get(struct __vxge_hw_device *hldev,
1062                               struct vxge_hw_xmac_stats *xmac_stats)
1063 {
1064         enum vxge_hw_status status = VXGE_HW_OK;
1065         u32 i;
1066
1067         status = vxge_hw_device_xmac_aggr_stats_get(hldev,
1068                                         0, &xmac_stats->aggr_stats[0]);
1069
1070         if (status != VXGE_HW_OK)
1071                 goto exit;
1072
1073         status = vxge_hw_device_xmac_aggr_stats_get(hldev,
1074                                 1, &xmac_stats->aggr_stats[1]);
1075         if (status != VXGE_HW_OK)
1076                 goto exit;
1077
1078         for (i = 0; i <= VXGE_HW_MAC_MAX_MAC_PORT_ID; i++) {
1079
1080                 status = vxge_hw_device_xmac_port_stats_get(hldev,
1081                                         i, &xmac_stats->port_stats[i]);
1082                 if (status != VXGE_HW_OK)
1083                         goto exit;
1084         }
1085
1086         for (i = 0; i < VXGE_HW_MAX_VIRTUAL_PATHS; i++) {
1087
1088                 if (!(hldev->vpaths_deployed & vxge_mBIT(i)))
1089                         continue;
1090
1091                 status = __vxge_hw_vpath_xmac_tx_stats_get(
1092                                         &hldev->virtual_paths[i],
1093                                         &xmac_stats->vpath_tx_stats[i]);
1094                 if (status != VXGE_HW_OK)
1095                         goto exit;
1096
1097                 status = __vxge_hw_vpath_xmac_rx_stats_get(
1098                                         &hldev->virtual_paths[i],
1099                                         &xmac_stats->vpath_rx_stats[i]);
1100                 if (status != VXGE_HW_OK)
1101                         goto exit;
1102         }
1103 exit:
1104         return status;
1105 }
1106
1107 /*
1108  * vxge_hw_device_debug_set - Set the debug module, level and timestamp
1109  * This routine is used to dynamically change the debug output
1110  */
1111 void vxge_hw_device_debug_set(struct __vxge_hw_device *hldev,
1112                               enum vxge_debug_level level, u32 mask)
1113 {
1114         if (hldev == NULL)
1115                 return;
1116
1117 #if defined(VXGE_DEBUG_TRACE_MASK) || \
1118         defined(VXGE_DEBUG_ERR_MASK)
1119         hldev->debug_module_mask = mask;
1120         hldev->debug_level = level;
1121 #endif
1122
1123 #if defined(VXGE_DEBUG_ERR_MASK)
1124         hldev->level_err = level & VXGE_ERR;
1125 #endif
1126
1127 #if defined(VXGE_DEBUG_TRACE_MASK)
1128         hldev->level_trace = level & VXGE_TRACE;
1129 #endif
1130 }
1131
1132 /*
1133  * vxge_hw_device_error_level_get - Get the error level
1134  * This routine returns the current error level set
1135  */
1136 u32 vxge_hw_device_error_level_get(struct __vxge_hw_device *hldev)
1137 {
1138 #if defined(VXGE_DEBUG_ERR_MASK)
1139         if (hldev == NULL)
1140                 return VXGE_ERR;
1141         else
1142                 return hldev->level_err;
1143 #else
1144         return 0;
1145 #endif
1146 }
1147
1148 /*
1149  * vxge_hw_device_trace_level_get - Get the trace level
1150  * This routine returns the current trace level set
1151  */
1152 u32 vxge_hw_device_trace_level_get(struct __vxge_hw_device *hldev)
1153 {
1154 #if defined(VXGE_DEBUG_TRACE_MASK)
1155         if (hldev == NULL)
1156                 return VXGE_TRACE;
1157         else
1158                 return hldev->level_trace;
1159 #else
1160         return 0;
1161 #endif
1162 }
1163 /*
1164  * vxge_hw_device_debug_mask_get - Get the debug mask
1165  * This routine returns the current debug mask set
1166  */
1167 u32 vxge_hw_device_debug_mask_get(struct __vxge_hw_device *hldev)
1168 {
1169 #if defined(VXGE_DEBUG_TRACE_MASK) || defined(VXGE_DEBUG_ERR_MASK)
1170         if (hldev == NULL)
1171                 return 0;
1172         return hldev->debug_module_mask;
1173 #else
1174         return 0;
1175 #endif
1176 }
1177
1178 /*
1179  * vxge_hw_getpause_data -Pause frame frame generation and reception.
1180  * Returns the Pause frame generation and reception capability of the NIC.
1181  */
1182 enum vxge_hw_status vxge_hw_device_getpause_data(struct __vxge_hw_device *hldev,
1183                                                  u32 port, u32 *tx, u32 *rx)
1184 {
1185         u64 val64;
1186         enum vxge_hw_status status = VXGE_HW_OK;
1187
1188         if ((hldev == NULL) || (hldev->magic != VXGE_HW_DEVICE_MAGIC)) {
1189                 status = VXGE_HW_ERR_INVALID_DEVICE;
1190                 goto exit;
1191         }
1192
1193         if (port > VXGE_HW_MAC_MAX_MAC_PORT_ID) {
1194                 status = VXGE_HW_ERR_INVALID_PORT;
1195                 goto exit;
1196         }
1197
1198         if (!(hldev->access_rights & VXGE_HW_DEVICE_ACCESS_RIGHT_MRPCIM)) {
1199                 status = VXGE_HW_ERR_PRIVILAGED_OPEARATION;
1200                 goto exit;
1201         }
1202
1203         val64 = readq(&hldev->mrpcim_reg->rxmac_pause_cfg_port[port]);
1204         if (val64 & VXGE_HW_RXMAC_PAUSE_CFG_PORT_GEN_EN)
1205                 *tx = 1;
1206         if (val64 & VXGE_HW_RXMAC_PAUSE_CFG_PORT_RCV_EN)
1207                 *rx = 1;
1208 exit:
1209         return status;
1210 }
1211
1212 /*
1213  * vxge_hw_device_setpause_data -  set/reset pause frame generation.
1214  * It can be used to set or reset Pause frame generation or reception
1215  * support of the NIC.
1216  */
1217
1218 enum vxge_hw_status vxge_hw_device_setpause_data(struct __vxge_hw_device *hldev,
1219                                                  u32 port, u32 tx, u32 rx)
1220 {
1221         u64 val64;
1222         enum vxge_hw_status status = VXGE_HW_OK;
1223
1224         if ((hldev == NULL) || (hldev->magic != VXGE_HW_DEVICE_MAGIC)) {
1225                 status = VXGE_HW_ERR_INVALID_DEVICE;
1226                 goto exit;
1227         }
1228
1229         if (port > VXGE_HW_MAC_MAX_MAC_PORT_ID) {
1230                 status = VXGE_HW_ERR_INVALID_PORT;
1231                 goto exit;
1232         }
1233
1234         status = __vxge_hw_device_is_privilaged(hldev);
1235         if (status != VXGE_HW_OK)
1236                 goto exit;
1237
1238         val64 = readq(&hldev->mrpcim_reg->rxmac_pause_cfg_port[port]);
1239         if (tx)
1240                 val64 |= VXGE_HW_RXMAC_PAUSE_CFG_PORT_GEN_EN;
1241         else
1242                 val64 &= ~VXGE_HW_RXMAC_PAUSE_CFG_PORT_GEN_EN;
1243         if (rx)
1244                 val64 |= VXGE_HW_RXMAC_PAUSE_CFG_PORT_RCV_EN;
1245         else
1246                 val64 &= ~VXGE_HW_RXMAC_PAUSE_CFG_PORT_RCV_EN;
1247
1248         writeq(val64, &hldev->mrpcim_reg->rxmac_pause_cfg_port[port]);
1249 exit:
1250         return status;
1251 }
1252
1253 u16 vxge_hw_device_link_width_get(struct __vxge_hw_device *hldev)
1254 {
1255         int link_width, exp_cap;
1256         u16 lnk;
1257
1258         exp_cap = pci_find_capability(hldev->pdev, PCI_CAP_ID_EXP);
1259         pci_read_config_word(hldev->pdev, exp_cap + PCI_EXP_LNKSTA, &lnk);
1260         link_width = (lnk & VXGE_HW_PCI_EXP_LNKCAP_LNK_WIDTH) >> 4;
1261         return link_width;
1262 }
1263
1264 /*
1265  * __vxge_hw_ring_block_memblock_idx - Return the memblock index
1266  * This function returns the index of memory block
1267  */
1268 static inline u32
1269 __vxge_hw_ring_block_memblock_idx(u8 *block)
1270 {
1271         return (u32)*((u64 *)(block + VXGE_HW_RING_MEMBLOCK_IDX_OFFSET));
1272 }
1273
1274 /*
1275  * __vxge_hw_ring_block_memblock_idx_set - Sets the memblock index
1276  * This function sets index to a memory block
1277  */
1278 static inline void
1279 __vxge_hw_ring_block_memblock_idx_set(u8 *block, u32 memblock_idx)
1280 {
1281         *((u64 *)(block + VXGE_HW_RING_MEMBLOCK_IDX_OFFSET)) = memblock_idx;
1282 }
1283
1284 /*
1285  * __vxge_hw_ring_block_next_pointer_set - Sets the next block pointer
1286  * in RxD block
1287  * Sets the next block pointer in RxD block
1288  */
1289 static inline void
1290 __vxge_hw_ring_block_next_pointer_set(u8 *block, dma_addr_t dma_next)
1291 {
1292         *((u64 *)(block + VXGE_HW_RING_NEXT_BLOCK_POINTER_OFFSET)) = dma_next;
1293 }
1294
1295 /*
1296  * __vxge_hw_ring_first_block_address_get - Returns the dma address of the
1297  *             first block
1298  * Returns the dma address of the first RxD block
1299  */
1300 u64 __vxge_hw_ring_first_block_address_get(struct __vxge_hw_ring *ring)
1301 {
1302         struct vxge_hw_mempool_dma *dma_object;
1303
1304         dma_object = ring->mempool->memblocks_dma_arr;
1305         vxge_assert(dma_object != NULL);
1306
1307         return dma_object->addr;
1308 }
1309
1310 /*
1311  * __vxge_hw_ring_item_dma_addr - Return the dma address of an item
1312  * This function returns the dma address of a given item
1313  */
1314 static dma_addr_t __vxge_hw_ring_item_dma_addr(struct vxge_hw_mempool *mempoolh,
1315                                                void *item)
1316 {
1317         u32 memblock_idx;
1318         void *memblock;
1319         struct vxge_hw_mempool_dma *memblock_dma_object;
1320         ptrdiff_t dma_item_offset;
1321
1322         /* get owner memblock index */
1323         memblock_idx = __vxge_hw_ring_block_memblock_idx(item);
1324
1325         /* get owner memblock by memblock index */
1326         memblock = mempoolh->memblocks_arr[memblock_idx];
1327
1328         /* get memblock DMA object by memblock index */
1329         memblock_dma_object = mempoolh->memblocks_dma_arr + memblock_idx;
1330
1331         /* calculate offset in the memblock of this item */
1332         dma_item_offset = (u8 *)item - (u8 *)memblock;
1333
1334         return memblock_dma_object->addr + dma_item_offset;
1335 }
1336
1337 /*
1338  * __vxge_hw_ring_rxdblock_link - Link the RxD blocks
1339  * This function returns the dma address of a given item
1340  */
1341 static void __vxge_hw_ring_rxdblock_link(struct vxge_hw_mempool *mempoolh,
1342                                          struct __vxge_hw_ring *ring, u32 from,
1343                                          u32 to)
1344 {
1345         u8 *to_item , *from_item;
1346         dma_addr_t to_dma;
1347
1348         /* get "from" RxD block */
1349         from_item = mempoolh->items_arr[from];
1350         vxge_assert(from_item);
1351
1352         /* get "to" RxD block */
1353         to_item = mempoolh->items_arr[to];
1354         vxge_assert(to_item);
1355
1356         /* return address of the beginning of previous RxD block */
1357         to_dma = __vxge_hw_ring_item_dma_addr(mempoolh, to_item);
1358
1359         /* set next pointer for this RxD block to point on
1360          * previous item's DMA start address */
1361         __vxge_hw_ring_block_next_pointer_set(from_item, to_dma);
1362 }
1363
1364 /*
1365  * __vxge_hw_ring_mempool_item_alloc - Allocate List blocks for RxD
1366  * block callback
1367  * This function is callback passed to __vxge_hw_mempool_create to create memory
1368  * pool for RxD block
1369  */
1370 static void
1371 __vxge_hw_ring_mempool_item_alloc(struct vxge_hw_mempool *mempoolh,
1372                                   u32 memblock_index,
1373                                   struct vxge_hw_mempool_dma *dma_object,
1374                                   u32 index, u32 is_last)
1375 {
1376         u32 i;
1377         void *item = mempoolh->items_arr[index];
1378         struct __vxge_hw_ring *ring =
1379                 (struct __vxge_hw_ring *)mempoolh->userdata;
1380
1381         /* format rxds array */
1382         for (i = 0; i < ring->rxds_per_block; i++) {
1383                 void *rxdblock_priv;
1384                 void *uld_priv;
1385                 struct vxge_hw_ring_rxd_1 *rxdp;
1386
1387                 u32 reserve_index = ring->channel.reserve_ptr -
1388                                 (index * ring->rxds_per_block + i + 1);
1389                 u32 memblock_item_idx;
1390
1391                 ring->channel.reserve_arr[reserve_index] = ((u8 *)item) +
1392                                                 i * ring->rxd_size;
1393
1394                 /* Note: memblock_item_idx is index of the item within
1395                  *       the memblock. For instance, in case of three RxD-blocks
1396                  *       per memblock this value can be 0, 1 or 2. */
1397                 rxdblock_priv = __vxge_hw_mempool_item_priv(mempoolh,
1398                                         memblock_index, item,
1399                                         &memblock_item_idx);
1400
1401                 rxdp = (struct vxge_hw_ring_rxd_1 *)
1402                                 ring->channel.reserve_arr[reserve_index];
1403
1404                 uld_priv = ((u8 *)rxdblock_priv + ring->rxd_priv_size * i);
1405
1406                 /* pre-format Host_Control */
1407                 rxdp->host_control = (u64)(size_t)uld_priv;
1408         }
1409
1410         __vxge_hw_ring_block_memblock_idx_set(item, memblock_index);
1411
1412         if (is_last) {
1413                 /* link last one with first one */
1414                 __vxge_hw_ring_rxdblock_link(mempoolh, ring, index, 0);
1415         }
1416
1417         if (index > 0) {
1418                 /* link this RxD block with previous one */
1419                 __vxge_hw_ring_rxdblock_link(mempoolh, ring, index - 1, index);
1420         }
1421
1422         return;
1423 }
1424
1425 /*
1426  * __vxge_hw_ring_initial_replenish - Initial replenish of RxDs
1427  * This function replenishes the RxDs from reserve array to work array
1428  */
1429 enum vxge_hw_status
1430 vxge_hw_ring_replenish(struct __vxge_hw_ring *ring, u16 min_flag)
1431 {
1432         void *rxd;
1433         int i = 0;
1434         struct __vxge_hw_channel *channel;
1435         enum vxge_hw_status status = VXGE_HW_OK;
1436
1437         channel = &ring->channel;
1438
1439         while (vxge_hw_channel_dtr_count(channel) > 0) {
1440
1441                 status = vxge_hw_ring_rxd_reserve(ring, &rxd);
1442
1443                 vxge_assert(status == VXGE_HW_OK);
1444
1445                 if (ring->rxd_init) {
1446                         status = ring->rxd_init(rxd, channel->userdata);
1447                         if (status != VXGE_HW_OK) {
1448                                 vxge_hw_ring_rxd_free(ring, rxd);
1449                                 goto exit;
1450                         }
1451                 }
1452
1453                 vxge_hw_ring_rxd_post(ring, rxd);
1454                 if (min_flag) {
1455                         i++;
1456                         if (i == VXGE_HW_RING_MIN_BUFF_ALLOCATION)
1457                                 break;
1458                 }
1459         }
1460         status = VXGE_HW_OK;
1461 exit:
1462         return status;
1463 }
1464
1465 /*
1466  * __vxge_hw_ring_create - Create a Ring
1467  * This function creates Ring and initializes it.
1468  *
1469  */
1470 enum vxge_hw_status
1471 __vxge_hw_ring_create(struct __vxge_hw_vpath_handle *vp,
1472                       struct vxge_hw_ring_attr *attr)
1473 {
1474         enum vxge_hw_status status = VXGE_HW_OK;
1475         struct __vxge_hw_ring *ring;
1476         u32 ring_length;
1477         struct vxge_hw_ring_config *config;
1478         struct __vxge_hw_device *hldev;
1479         u32 vp_id;
1480         struct vxge_hw_mempool_cbs ring_mp_callback;
1481
1482         if ((vp == NULL) || (attr == NULL)) {
1483                 status = VXGE_HW_FAIL;
1484                 goto exit;
1485         }
1486
1487         hldev = vp->vpath->hldev;
1488         vp_id = vp->vpath->vp_id;
1489
1490         config = &hldev->config.vp_config[vp_id].ring;
1491
1492         ring_length = config->ring_blocks *
1493                         vxge_hw_ring_rxds_per_block_get(config->buffer_mode);
1494
1495         ring = (struct __vxge_hw_ring *)__vxge_hw_channel_allocate(vp,
1496                                                 VXGE_HW_CHANNEL_TYPE_RING,
1497                                                 ring_length,
1498                                                 attr->per_rxd_space,
1499                                                 attr->userdata);
1500
1501         if (ring == NULL) {
1502                 status = VXGE_HW_ERR_OUT_OF_MEMORY;
1503                 goto exit;
1504         }
1505
1506         vp->vpath->ringh = ring;
1507         ring->vp_id = vp_id;
1508         ring->vp_reg = vp->vpath->vp_reg;
1509         ring->common_reg = hldev->common_reg;
1510         ring->stats = &vp->vpath->sw_stats->ring_stats;
1511         ring->config = config;
1512         ring->callback = attr->callback;
1513         ring->rxd_init = attr->rxd_init;
1514         ring->rxd_term = attr->rxd_term;
1515         ring->buffer_mode = config->buffer_mode;
1516         ring->rxds_limit = config->rxds_limit;
1517
1518         ring->rxd_size = vxge_hw_ring_rxd_size_get(config->buffer_mode);
1519         ring->rxd_priv_size =
1520                 sizeof(struct __vxge_hw_ring_rxd_priv) + attr->per_rxd_space;
1521         ring->per_rxd_space = attr->per_rxd_space;
1522
1523         ring->rxd_priv_size =
1524                 ((ring->rxd_priv_size + VXGE_CACHE_LINE_SIZE - 1) /
1525                 VXGE_CACHE_LINE_SIZE) * VXGE_CACHE_LINE_SIZE;
1526
1527         /* how many RxDs can fit into one block. Depends on configured
1528          * buffer_mode. */
1529         ring->rxds_per_block =
1530                 vxge_hw_ring_rxds_per_block_get(config->buffer_mode);
1531
1532         /* calculate actual RxD block private size */
1533         ring->rxdblock_priv_size = ring->rxd_priv_size * ring->rxds_per_block;
1534         ring_mp_callback.item_func_alloc = __vxge_hw_ring_mempool_item_alloc;
1535         ring->mempool = __vxge_hw_mempool_create(hldev,
1536                                 VXGE_HW_BLOCK_SIZE,
1537                                 VXGE_HW_BLOCK_SIZE,
1538                                 ring->rxdblock_priv_size,
1539                                 ring->config->ring_blocks,
1540                                 ring->config->ring_blocks,
1541                                 &ring_mp_callback,
1542                                 ring);
1543
1544         if (ring->mempool == NULL) {
1545                 __vxge_hw_ring_delete(vp);
1546                 return VXGE_HW_ERR_OUT_OF_MEMORY;
1547         }
1548
1549         status = __vxge_hw_channel_initialize(&ring->channel);
1550         if (status != VXGE_HW_OK) {
1551                 __vxge_hw_ring_delete(vp);
1552                 goto exit;
1553         }
1554
1555         /* Note:
1556          * Specifying rxd_init callback means two things:
1557          * 1) rxds need to be initialized by driver at channel-open time;
1558          * 2) rxds need to be posted at channel-open time
1559          *    (that's what the initial_replenish() below does)
1560          * Currently we don't have a case when the 1) is done without the 2).
1561          */
1562         if (ring->rxd_init) {
1563                 status = vxge_hw_ring_replenish(ring, 1);
1564                 if (status != VXGE_HW_OK) {
1565                         __vxge_hw_ring_delete(vp);
1566                         goto exit;
1567                 }
1568         }
1569
1570         /* initial replenish will increment the counter in its post() routine,
1571          * we have to reset it */
1572         ring->stats->common_stats.usage_cnt = 0;
1573 exit:
1574         return status;
1575 }
1576
1577 /*
1578  * __vxge_hw_ring_abort - Returns the RxD
1579  * This function terminates the RxDs of ring
1580  */
1581 enum vxge_hw_status __vxge_hw_ring_abort(struct __vxge_hw_ring *ring)
1582 {
1583         void *rxdh;
1584         struct __vxge_hw_channel *channel;
1585
1586         channel = &ring->channel;
1587
1588         for (;;) {
1589                 vxge_hw_channel_dtr_try_complete(channel, &rxdh);
1590
1591                 if (rxdh == NULL)
1592                         break;
1593
1594                 vxge_hw_channel_dtr_complete(channel);
1595
1596                 if (ring->rxd_term)
1597                         ring->rxd_term(rxdh, VXGE_HW_RXD_STATE_POSTED,
1598                                 channel->userdata);
1599
1600                 vxge_hw_channel_dtr_free(channel, rxdh);
1601         }
1602
1603         return VXGE_HW_OK;
1604 }
1605
1606 /*
1607  * __vxge_hw_ring_reset - Resets the ring
1608  * This function resets the ring during vpath reset operation
1609  */
1610 enum vxge_hw_status __vxge_hw_ring_reset(struct __vxge_hw_ring *ring)
1611 {
1612         enum vxge_hw_status status = VXGE_HW_OK;
1613         struct __vxge_hw_channel *channel;
1614
1615         channel = &ring->channel;
1616
1617         __vxge_hw_ring_abort(ring);
1618
1619         status = __vxge_hw_channel_reset(channel);
1620
1621         if (status != VXGE_HW_OK)
1622                 goto exit;
1623
1624         if (ring->rxd_init) {
1625                 status = vxge_hw_ring_replenish(ring, 1);
1626                 if (status != VXGE_HW_OK)
1627                         goto exit;
1628         }
1629 exit:
1630         return status;
1631 }
1632
1633 /*
1634  * __vxge_hw_ring_delete - Removes the ring
1635  * This function freeup the memory pool and removes the ring
1636  */
1637 enum vxge_hw_status __vxge_hw_ring_delete(struct __vxge_hw_vpath_handle *vp)
1638 {
1639         struct __vxge_hw_ring *ring = vp->vpath->ringh;
1640
1641         __vxge_hw_ring_abort(ring);
1642
1643         if (ring->mempool)
1644                 __vxge_hw_mempool_destroy(ring->mempool);
1645
1646         vp->vpath->ringh = NULL;
1647         __vxge_hw_channel_free(&ring->channel);
1648
1649         return VXGE_HW_OK;
1650 }
1651
1652 /*
1653  * __vxge_hw_mempool_grow
1654  * Will resize mempool up to %num_allocate value.
1655  */
1656 enum vxge_hw_status
1657 __vxge_hw_mempool_grow(struct vxge_hw_mempool *mempool, u32 num_allocate,
1658                        u32 *num_allocated)
1659 {
1660         u32 i, first_time = mempool->memblocks_allocated == 0 ? 1 : 0;
1661         u32 n_items = mempool->items_per_memblock;
1662         u32 start_block_idx = mempool->memblocks_allocated;
1663         u32 end_block_idx = mempool->memblocks_allocated + num_allocate;
1664         enum vxge_hw_status status = VXGE_HW_OK;
1665
1666         *num_allocated = 0;
1667
1668         if (end_block_idx > mempool->memblocks_max) {
1669                 status = VXGE_HW_ERR_OUT_OF_MEMORY;
1670                 goto exit;
1671         }
1672
1673         for (i = start_block_idx; i < end_block_idx; i++) {
1674                 u32 j;
1675                 u32 is_last = ((end_block_idx - 1) == i);
1676                 struct vxge_hw_mempool_dma *dma_object =
1677                         mempool->memblocks_dma_arr + i;
1678                 void *the_memblock;
1679
1680                 /* allocate memblock's private part. Each DMA memblock
1681                  * has a space allocated for item's private usage upon
1682                  * mempool's user request. Each time mempool grows, it will
1683                  * allocate new memblock and its private part at once.
1684                  * This helps to minimize memory usage a lot. */
1685                 mempool->memblocks_priv_arr[i] =
1686                                 vmalloc(mempool->items_priv_size * n_items);
1687                 if (mempool->memblocks_priv_arr[i] == NULL) {
1688                         status = VXGE_HW_ERR_OUT_OF_MEMORY;
1689                         goto exit;
1690                 }
1691
1692                 memset(mempool->memblocks_priv_arr[i], 0,
1693                              mempool->items_priv_size * n_items);
1694
1695                 /* allocate DMA-capable memblock */
1696                 mempool->memblocks_arr[i] =
1697                         __vxge_hw_blockpool_malloc(mempool->devh,
1698                                 mempool->memblock_size, dma_object);
1699                 if (mempool->memblocks_arr[i] == NULL) {
1700                         vfree(mempool->memblocks_priv_arr[i]);
1701                         status = VXGE_HW_ERR_OUT_OF_MEMORY;
1702                         goto exit;
1703                 }
1704
1705                 (*num_allocated)++;
1706                 mempool->memblocks_allocated++;
1707
1708                 memset(mempool->memblocks_arr[i], 0, mempool->memblock_size);
1709
1710                 the_memblock = mempool->memblocks_arr[i];
1711
1712                 /* fill the items hash array */
1713                 for (j = 0; j < n_items; j++) {
1714                         u32 index = i * n_items + j;
1715
1716                         if (first_time && index >= mempool->items_initial)
1717                                 break;
1718
1719                         mempool->items_arr[index] =
1720                                 ((char *)the_memblock + j*mempool->item_size);
1721
1722                         /* let caller to do more job on each item */
1723                         if (mempool->item_func_alloc != NULL)
1724                                 mempool->item_func_alloc(mempool, i,
1725                                         dma_object, index, is_last);
1726
1727                         mempool->items_current = index + 1;
1728                 }
1729
1730                 if (first_time && mempool->items_current ==
1731                                         mempool->items_initial)
1732                         break;
1733         }
1734 exit:
1735         return status;
1736 }
1737
1738 /*
1739  * vxge_hw_mempool_create
1740  * This function will create memory pool object. Pool may grow but will
1741  * never shrink. Pool consists of number of dynamically allocated blocks
1742  * with size enough to hold %items_initial number of items. Memory is
1743  * DMA-able but client must map/unmap before interoperating with the device.
1744  */
1745 struct vxge_hw_mempool*
1746 __vxge_hw_mempool_create(
1747         struct __vxge_hw_device *devh,
1748         u32 memblock_size,
1749         u32 item_size,
1750         u32 items_priv_size,
1751         u32 items_initial,
1752         u32 items_max,
1753         struct vxge_hw_mempool_cbs *mp_callback,
1754         void *userdata)
1755 {
1756         enum vxge_hw_status status = VXGE_HW_OK;
1757         u32 memblocks_to_allocate;
1758         struct vxge_hw_mempool *mempool = NULL;
1759         u32 allocated;
1760
1761         if (memblock_size < item_size) {
1762                 status = VXGE_HW_FAIL;
1763                 goto exit;
1764         }
1765
1766         mempool = (struct vxge_hw_mempool *)
1767                         vmalloc(sizeof(struct vxge_hw_mempool));
1768         if (mempool == NULL) {
1769                 status = VXGE_HW_ERR_OUT_OF_MEMORY;
1770                 goto exit;
1771         }
1772         memset(mempool, 0, sizeof(struct vxge_hw_mempool));
1773
1774         mempool->devh                   = devh;
1775         mempool->memblock_size          = memblock_size;
1776         mempool->items_max              = items_max;
1777         mempool->items_initial          = items_initial;
1778         mempool->item_size              = item_size;
1779         mempool->items_priv_size        = items_priv_size;
1780         mempool->item_func_alloc        = mp_callback->item_func_alloc;
1781         mempool->userdata               = userdata;
1782
1783         mempool->memblocks_allocated = 0;
1784
1785         mempool->items_per_memblock = memblock_size / item_size;
1786
1787         mempool->memblocks_max = (items_max + mempool->items_per_memblock - 1) /
1788                                         mempool->items_per_memblock;
1789
1790         /* allocate array of memblocks */
1791         mempool->memblocks_arr =
1792                 (void **) vmalloc(sizeof(void *) * mempool->memblocks_max);
1793         if (mempool->memblocks_arr == NULL) {
1794                 __vxge_hw_mempool_destroy(mempool);
1795                 status = VXGE_HW_ERR_OUT_OF_MEMORY;
1796                 mempool = NULL;
1797                 goto exit;
1798         }
1799         memset(mempool->memblocks_arr, 0,
1800                 sizeof(void *) * mempool->memblocks_max);
1801
1802         /* allocate array of private parts of items per memblocks */
1803         mempool->memblocks_priv_arr =
1804                 (void **) vmalloc(sizeof(void *) * mempool->memblocks_max);
1805         if (mempool->memblocks_priv_arr == NULL) {
1806                 __vxge_hw_mempool_destroy(mempool);
1807                 status = VXGE_HW_ERR_OUT_OF_MEMORY;
1808                 mempool = NULL;
1809                 goto exit;
1810         }
1811         memset(mempool->memblocks_priv_arr, 0,
1812                     sizeof(void *) * mempool->memblocks_max);
1813
1814         /* allocate array of memblocks DMA objects */
1815         mempool->memblocks_dma_arr = (struct vxge_hw_mempool_dma *)
1816                 vmalloc(sizeof(struct vxge_hw_mempool_dma) *
1817                         mempool->memblocks_max);
1818
1819         if (mempool->memblocks_dma_arr == NULL) {
1820                 __vxge_hw_mempool_destroy(mempool);
1821                 status = VXGE_HW_ERR_OUT_OF_MEMORY;
1822                 mempool = NULL;
1823                 goto exit;
1824         }
1825         memset(mempool->memblocks_dma_arr, 0,
1826                         sizeof(struct vxge_hw_mempool_dma) *
1827                         mempool->memblocks_max);
1828
1829         /* allocate hash array of items */
1830         mempool->items_arr =
1831                 (void **) vmalloc(sizeof(void *) * mempool->items_max);
1832         if (mempool->items_arr == NULL) {
1833                 __vxge_hw_mempool_destroy(mempool);
1834                 status = VXGE_HW_ERR_OUT_OF_MEMORY;
1835                 mempool = NULL;
1836                 goto exit;
1837         }
1838         memset(mempool->items_arr, 0, sizeof(void *) * mempool->items_max);
1839
1840         /* calculate initial number of memblocks */
1841         memblocks_to_allocate = (mempool->items_initial +
1842                                  mempool->items_per_memblock - 1) /
1843                                                 mempool->items_per_memblock;
1844
1845         /* pre-allocate the mempool */
1846         status = __vxge_hw_mempool_grow(mempool, memblocks_to_allocate,
1847                                         &allocated);
1848         if (status != VXGE_HW_OK) {
1849                 __vxge_hw_mempool_destroy(mempool);
1850                 status = VXGE_HW_ERR_OUT_OF_MEMORY;
1851                 mempool = NULL;
1852                 goto exit;
1853         }
1854
1855 exit:
1856         return mempool;
1857 }
1858
1859 /*
1860  * vxge_hw_mempool_destroy
1861  */
1862 void __vxge_hw_mempool_destroy(struct vxge_hw_mempool *mempool)
1863 {
1864         u32 i, j;
1865         struct __vxge_hw_device *devh = mempool->devh;
1866
1867         for (i = 0; i < mempool->memblocks_allocated; i++) {
1868                 struct vxge_hw_mempool_dma *dma_object;
1869
1870                 vxge_assert(mempool->memblocks_arr[i]);
1871                 vxge_assert(mempool->memblocks_dma_arr + i);
1872
1873                 dma_object = mempool->memblocks_dma_arr + i;
1874
1875                 for (j = 0; j < mempool->items_per_memblock; j++) {
1876                         u32 index = i * mempool->items_per_memblock + j;
1877
1878                         /* to skip last partially filled(if any) memblock */
1879                         if (index >= mempool->items_current)
1880                                 break;
1881                 }
1882
1883                 vfree(mempool->memblocks_priv_arr[i]);
1884
1885                 __vxge_hw_blockpool_free(devh, mempool->memblocks_arr[i],
1886                                 mempool->memblock_size, dma_object);
1887         }
1888
1889         vfree(mempool->items_arr);
1890
1891         vfree(mempool->memblocks_dma_arr);
1892
1893         vfree(mempool->memblocks_priv_arr);
1894
1895         vfree(mempool->memblocks_arr);
1896
1897         vfree(mempool);
1898 }
1899
1900 /*
1901  * __vxge_hw_device_fifo_config_check - Check fifo configuration.
1902  * Check the fifo configuration
1903  */
1904 enum vxge_hw_status
1905 __vxge_hw_device_fifo_config_check(struct vxge_hw_fifo_config *fifo_config)
1906 {
1907         if ((fifo_config->fifo_blocks < VXGE_HW_MIN_FIFO_BLOCKS) ||
1908              (fifo_config->fifo_blocks > VXGE_HW_MAX_FIFO_BLOCKS))
1909                 return VXGE_HW_BADCFG_FIFO_BLOCKS;
1910
1911         return VXGE_HW_OK;
1912 }
1913
1914 /*
1915  * __vxge_hw_device_vpath_config_check - Check vpath configuration.
1916  * Check the vpath configuration
1917  */
1918 enum vxge_hw_status
1919 __vxge_hw_device_vpath_config_check(struct vxge_hw_vp_config *vp_config)
1920 {
1921         enum vxge_hw_status status;
1922
1923         if ((vp_config->min_bandwidth < VXGE_HW_VPATH_BANDWIDTH_MIN) ||
1924                 (vp_config->min_bandwidth >
1925                                         VXGE_HW_VPATH_BANDWIDTH_MAX))
1926                 return VXGE_HW_BADCFG_VPATH_MIN_BANDWIDTH;
1927
1928         status = __vxge_hw_device_fifo_config_check(&vp_config->fifo);
1929         if (status != VXGE_HW_OK)
1930                 return status;
1931
1932         if ((vp_config->mtu != VXGE_HW_VPATH_USE_FLASH_DEFAULT_INITIAL_MTU) &&
1933                 ((vp_config->mtu < VXGE_HW_VPATH_MIN_INITIAL_MTU) ||
1934                 (vp_config->mtu > VXGE_HW_VPATH_MAX_INITIAL_MTU)))
1935                 return VXGE_HW_BADCFG_VPATH_MTU;
1936
1937         if ((vp_config->rpa_strip_vlan_tag !=
1938                 VXGE_HW_VPATH_RPA_STRIP_VLAN_TAG_USE_FLASH_DEFAULT) &&
1939                 (vp_config->rpa_strip_vlan_tag !=
1940                 VXGE_HW_VPATH_RPA_STRIP_VLAN_TAG_ENABLE) &&
1941                 (vp_config->rpa_strip_vlan_tag !=
1942                 VXGE_HW_VPATH_RPA_STRIP_VLAN_TAG_DISABLE))
1943                 return VXGE_HW_BADCFG_VPATH_RPA_STRIP_VLAN_TAG;
1944
1945         return VXGE_HW_OK;
1946 }
1947
1948 /*
1949  * __vxge_hw_device_config_check - Check device configuration.
1950  * Check the device configuration
1951  */
1952 enum vxge_hw_status
1953 __vxge_hw_device_config_check(struct vxge_hw_device_config *new_config)
1954 {
1955         u32 i;
1956         enum vxge_hw_status status;
1957
1958         if ((new_config->intr_mode != VXGE_HW_INTR_MODE_IRQLINE) &&
1959            (new_config->intr_mode != VXGE_HW_INTR_MODE_MSIX) &&
1960            (new_config->intr_mode != VXGE_HW_INTR_MODE_MSIX_ONE_SHOT) &&
1961            (new_config->intr_mode != VXGE_HW_INTR_MODE_DEF))
1962                 return VXGE_HW_BADCFG_INTR_MODE;
1963
1964         if ((new_config->rts_mac_en != VXGE_HW_RTS_MAC_DISABLE) &&
1965            (new_config->rts_mac_en != VXGE_HW_RTS_MAC_ENABLE))
1966                 return VXGE_HW_BADCFG_RTS_MAC_EN;
1967
1968         for (i = 0; i < VXGE_HW_MAX_VIRTUAL_PATHS; i++) {
1969                 status = __vxge_hw_device_vpath_config_check(
1970                                 &new_config->vp_config[i]);
1971                 if (status != VXGE_HW_OK)
1972                         return status;
1973         }
1974
1975         return VXGE_HW_OK;
1976 }
1977
1978 /*
1979  * vxge_hw_device_config_default_get - Initialize device config with defaults.
1980  * Initialize Titan device config with default values.
1981  */
1982 enum vxge_hw_status __devinit
1983 vxge_hw_device_config_default_get(struct vxge_hw_device_config *device_config)
1984 {
1985         u32 i;
1986
1987         device_config->dma_blockpool_initial =
1988                                         VXGE_HW_INITIAL_DMA_BLOCK_POOL_SIZE;
1989         device_config->dma_blockpool_max = VXGE_HW_MAX_DMA_BLOCK_POOL_SIZE;
1990         device_config->intr_mode = VXGE_HW_INTR_MODE_DEF;
1991         device_config->rth_en = VXGE_HW_RTH_DEFAULT;
1992         device_config->rth_it_type = VXGE_HW_RTH_IT_TYPE_DEFAULT;
1993         device_config->device_poll_millis =  VXGE_HW_DEF_DEVICE_POLL_MILLIS;
1994         device_config->rts_mac_en =  VXGE_HW_RTS_MAC_DEFAULT;
1995
1996         for (i = 0; i < VXGE_HW_MAX_VIRTUAL_PATHS; i++) {
1997
1998                 device_config->vp_config[i].vp_id = i;
1999
2000                 device_config->vp_config[i].min_bandwidth =
2001                                 VXGE_HW_VPATH_BANDWIDTH_DEFAULT;
2002
2003                 device_config->vp_config[i].ring.enable = VXGE_HW_RING_DEFAULT;
2004
2005                 device_config->vp_config[i].ring.ring_blocks =
2006                                 VXGE_HW_DEF_RING_BLOCKS;
2007
2008                 device_config->vp_config[i].ring.buffer_mode =
2009                                 VXGE_HW_RING_RXD_BUFFER_MODE_DEFAULT;
2010
2011                 device_config->vp_config[i].ring.scatter_mode =
2012                                 VXGE_HW_RING_SCATTER_MODE_USE_FLASH_DEFAULT;
2013
2014                 device_config->vp_config[i].ring.rxds_limit =
2015                                 VXGE_HW_DEF_RING_RXDS_LIMIT;
2016
2017                 device_config->vp_config[i].fifo.enable = VXGE_HW_FIFO_ENABLE;
2018
2019                 device_config->vp_config[i].fifo.fifo_blocks =
2020                                 VXGE_HW_MIN_FIFO_BLOCKS;
2021
2022                 device_config->vp_config[i].fifo.max_frags =
2023                                 VXGE_HW_MAX_FIFO_FRAGS;
2024
2025                 device_config->vp_config[i].fifo.memblock_size =
2026                                 VXGE_HW_DEF_FIFO_MEMBLOCK_SIZE;
2027
2028                 device_config->vp_config[i].fifo.alignment_size =
2029                                 VXGE_HW_DEF_FIFO_ALIGNMENT_SIZE;
2030
2031                 device_config->vp_config[i].fifo.intr =
2032                                 VXGE_HW_FIFO_QUEUE_INTR_DEFAULT;
2033
2034                 device_config->vp_config[i].fifo.no_snoop_bits =
2035                                 VXGE_HW_FIFO_NO_SNOOP_DEFAULT;
2036                 device_config->vp_config[i].tti.intr_enable =
2037                                 VXGE_HW_TIM_INTR_DEFAULT;
2038
2039                 device_config->vp_config[i].tti.btimer_val =
2040                                 VXGE_HW_USE_FLASH_DEFAULT;
2041
2042                 device_config->vp_config[i].tti.timer_ac_en =
2043                                 VXGE_HW_USE_FLASH_DEFAULT;
2044
2045                 device_config->vp_config[i].tti.timer_ci_en =
2046                                 VXGE_HW_USE_FLASH_DEFAULT;
2047
2048                 device_config->vp_config[i].tti.timer_ri_en =
2049                                 VXGE_HW_USE_FLASH_DEFAULT;
2050
2051                 device_config->vp_config[i].tti.rtimer_val =
2052                                 VXGE_HW_USE_FLASH_DEFAULT;
2053
2054                 device_config->vp_config[i].tti.util_sel =
2055                                 VXGE_HW_USE_FLASH_DEFAULT;
2056
2057                 device_config->vp_config[i].tti.ltimer_val =
2058                                 VXGE_HW_USE_FLASH_DEFAULT;
2059
2060                 device_config->vp_config[i].tti.urange_a =
2061                                 VXGE_HW_USE_FLASH_DEFAULT;
2062
2063                 device_config->vp_config[i].tti.uec_a =
2064                                 VXGE_HW_USE_FLASH_DEFAULT;
2065
2066                 device_config->vp_config[i].tti.urange_b =
2067                                 VXGE_HW_USE_FLASH_DEFAULT;
2068
2069                 device_config->vp_config[i].tti.uec_b =
2070                                 VXGE_HW_USE_FLASH_DEFAULT;
2071
2072                 device_config->vp_config[i].tti.urange_c =
2073                                 VXGE_HW_USE_FLASH_DEFAULT;
2074
2075                 device_config->vp_config[i].tti.uec_c =
2076                                 VXGE_HW_USE_FLASH_DEFAULT;
2077
2078                 device_config->vp_config[i].tti.uec_d =
2079                                 VXGE_HW_USE_FLASH_DEFAULT;
2080
2081                 device_config->vp_config[i].rti.intr_enable =
2082                                 VXGE_HW_TIM_INTR_DEFAULT;
2083
2084                 device_config->vp_config[i].rti.btimer_val =
2085                                 VXGE_HW_USE_FLASH_DEFAULT;
2086
2087                 device_config->vp_config[i].rti.timer_ac_en =
2088                                 VXGE_HW_USE_FLASH_DEFAULT;
2089
2090                 device_config->vp_config[i].rti.timer_ci_en =
2091                                 VXGE_HW_USE_FLASH_DEFAULT;
2092
2093                 device_config->vp_config[i].rti.timer_ri_en =
2094                                 VXGE_HW_USE_FLASH_DEFAULT;
2095
2096                 device_config->vp_config[i].rti.rtimer_val =
2097                                 VXGE_HW_USE_FLASH_DEFAULT;
2098
2099                 device_config->vp_config[i].rti.util_sel =
2100                                 VXGE_HW_USE_FLASH_DEFAULT;
2101
2102                 device_config->vp_config[i].rti.ltimer_val =
2103                                 VXGE_HW_USE_FLASH_DEFAULT;
2104
2105                 device_config->vp_config[i].rti.urange_a =
2106                                 VXGE_HW_USE_FLASH_DEFAULT;
2107
2108                 device_config->vp_config[i].rti.uec_a =
2109                                 VXGE_HW_USE_FLASH_DEFAULT;
2110
2111                 device_config->vp_config[i].rti.urange_b =
2112                                 VXGE_HW_USE_FLASH_DEFAULT;
2113
2114                 device_config->vp_config[i].rti.uec_b =
2115                                 VXGE_HW_USE_FLASH_DEFAULT;
2116
2117                 device_config->vp_config[i].rti.urange_c =
2118                                 VXGE_HW_USE_FLASH_DEFAULT;
2119
2120                 device_config->vp_config[i].rti.uec_c =
2121                                 VXGE_HW_USE_FLASH_DEFAULT;
2122
2123                 device_config->vp_config[i].rti.uec_d =
2124                                 VXGE_HW_USE_FLASH_DEFAULT;
2125
2126                 device_config->vp_config[i].mtu =
2127                                 VXGE_HW_VPATH_USE_FLASH_DEFAULT_INITIAL_MTU;
2128
2129                 device_config->vp_config[i].rpa_strip_vlan_tag =
2130                         VXGE_HW_VPATH_RPA_STRIP_VLAN_TAG_USE_FLASH_DEFAULT;
2131         }
2132
2133         return VXGE_HW_OK;
2134 }
2135
2136 /*
2137  * _hw_legacy_swapper_set - Set the swapper bits for the legacy secion.
2138  * Set the swapper bits appropriately for the lagacy section.
2139  */
2140 enum vxge_hw_status
2141 __vxge_hw_legacy_swapper_set(struct vxge_hw_legacy_reg __iomem *legacy_reg)
2142 {
2143         u64 val64;
2144         enum vxge_hw_status status = VXGE_HW_OK;
2145
2146         val64 = readq(&legacy_reg->toc_swapper_fb);
2147
2148         wmb();
2149
2150         switch (val64) {
2151
2152         case VXGE_HW_SWAPPER_INITIAL_VALUE:
2153                 return status;
2154
2155         case VXGE_HW_SWAPPER_BYTE_SWAPPED_BIT_FLIPPED:
2156                 writeq(VXGE_HW_SWAPPER_READ_BYTE_SWAP_ENABLE,
2157                         &legacy_reg->pifm_rd_swap_en);
2158                 writeq(VXGE_HW_SWAPPER_READ_BIT_FLAP_ENABLE,
2159                         &legacy_reg->pifm_rd_flip_en);
2160                 writeq(VXGE_HW_SWAPPER_WRITE_BYTE_SWAP_ENABLE,
2161                         &legacy_reg->pifm_wr_swap_en);
2162                 writeq(VXGE_HW_SWAPPER_WRITE_BIT_FLAP_ENABLE,
2163                         &legacy_reg->pifm_wr_flip_en);
2164                 break;
2165
2166         case VXGE_HW_SWAPPER_BYTE_SWAPPED:
2167                 writeq(VXGE_HW_SWAPPER_READ_BYTE_SWAP_ENABLE,
2168                         &legacy_reg->pifm_rd_swap_en);
2169                 writeq(VXGE_HW_SWAPPER_WRITE_BYTE_SWAP_ENABLE,
2170                         &legacy_reg->pifm_wr_swap_en);
2171                 break;
2172
2173         case VXGE_HW_SWAPPER_BIT_FLIPPED:
2174                 writeq(VXGE_HW_SWAPPER_READ_BIT_FLAP_ENABLE,
2175                         &legacy_reg->pifm_rd_flip_en);
2176                 writeq(VXGE_HW_SWAPPER_WRITE_BIT_FLAP_ENABLE,
2177                         &legacy_reg->pifm_wr_flip_en);
2178                 break;
2179         }
2180
2181         wmb();
2182
2183         val64 = readq(&legacy_reg->toc_swapper_fb);
2184
2185         if (val64 != VXGE_HW_SWAPPER_INITIAL_VALUE)
2186                 status = VXGE_HW_ERR_SWAPPER_CTRL;
2187
2188         return status;
2189 }
2190
2191 /*
2192  * __vxge_hw_vpath_swapper_set - Set the swapper bits for the vpath.
2193  * Set the swapper bits appropriately for the vpath.
2194  */
2195 enum vxge_hw_status
2196 __vxge_hw_vpath_swapper_set(struct vxge_hw_vpath_reg __iomem *vpath_reg)
2197 {
2198 #ifndef __BIG_ENDIAN
2199         u64 val64;
2200
2201         val64 = readq(&vpath_reg->vpath_general_cfg1);
2202         wmb();
2203         val64 |= VXGE_HW_VPATH_GENERAL_CFG1_CTL_BYTE_SWAPEN;
2204         writeq(val64, &vpath_reg->vpath_general_cfg1);
2205         wmb();
2206 #endif
2207         return VXGE_HW_OK;
2208 }
2209
2210 /*
2211  * __vxge_hw_kdfc_swapper_set - Set the swapper bits for the kdfc.
2212  * Set the swapper bits appropriately for the vpath.
2213  */
2214 enum vxge_hw_status
2215 __vxge_hw_kdfc_swapper_set(
2216         struct vxge_hw_legacy_reg __iomem *legacy_reg,
2217         struct vxge_hw_vpath_reg __iomem *vpath_reg)
2218 {
2219         u64 val64;
2220
2221         val64 = readq(&legacy_reg->pifm_wr_swap_en);
2222
2223         if (val64 == VXGE_HW_SWAPPER_WRITE_BYTE_SWAP_ENABLE) {
2224                 val64 = readq(&vpath_reg->kdfcctl_cfg0);
2225                 wmb();
2226
2227                 val64 |= VXGE_HW_KDFCCTL_CFG0_BYTE_SWAPEN_FIFO0 |
2228                         VXGE_HW_KDFCCTL_CFG0_BYTE_SWAPEN_FIFO1  |
2229                         VXGE_HW_KDFCCTL_CFG0_BYTE_SWAPEN_FIFO2;
2230
2231                 writeq(val64, &vpath_reg->kdfcctl_cfg0);
2232                 wmb();
2233         }
2234
2235         return VXGE_HW_OK;
2236 }
2237
2238 /*
2239  * vxge_hw_mgmt_device_config - Retrieve device configuration.
2240  * Get device configuration. Permits to retrieve at run-time configuration
2241  * values that were used to initialize and configure the device.
2242  */
2243 enum vxge_hw_status
2244 vxge_hw_mgmt_device_config(struct __vxge_hw_device *hldev,
2245                            struct vxge_hw_device_config *dev_config, int size)
2246 {
2247
2248         if ((hldev == NULL) || (hldev->magic != VXGE_HW_DEVICE_MAGIC))
2249                 return VXGE_HW_ERR_INVALID_DEVICE;
2250
2251         if (size != sizeof(struct vxge_hw_device_config))
2252                 return VXGE_HW_ERR_VERSION_CONFLICT;
2253
2254         memcpy(dev_config, &hldev->config,
2255                 sizeof(struct vxge_hw_device_config));
2256
2257         return VXGE_HW_OK;
2258 }
2259
2260 /*
2261  * vxge_hw_mgmt_reg_read - Read Titan register.
2262  */
2263 enum vxge_hw_status
2264 vxge_hw_mgmt_reg_read(struct __vxge_hw_device *hldev,
2265                       enum vxge_hw_mgmt_reg_type type,
2266                       u32 index, u32 offset, u64 *value)
2267 {
2268         enum vxge_hw_status status = VXGE_HW_OK;
2269
2270         if ((hldev == NULL) || (hldev->magic != VXGE_HW_DEVICE_MAGIC)) {
2271                 status = VXGE_HW_ERR_INVALID_DEVICE;
2272                 goto exit;
2273         }
2274
2275         switch (type) {
2276         case vxge_hw_mgmt_reg_type_legacy:
2277                 if (offset > sizeof(struct vxge_hw_legacy_reg) - 8) {
2278                         status = VXGE_HW_ERR_INVALID_OFFSET;
2279                         break;
2280                 }
2281                 *value = readq((void __iomem *)hldev->legacy_reg + offset);
2282                 break;
2283         case vxge_hw_mgmt_reg_type_toc:
2284                 if (offset > sizeof(struct vxge_hw_toc_reg) - 8) {
2285                         status = VXGE_HW_ERR_INVALID_OFFSET;
2286                         break;
2287                 }
2288                 *value = readq((void __iomem *)hldev->toc_reg + offset);
2289                 break;
2290         case vxge_hw_mgmt_reg_type_common:
2291                 if (offset > sizeof(struct vxge_hw_common_reg) - 8) {
2292                         status = VXGE_HW_ERR_INVALID_OFFSET;
2293                         break;
2294                 }
2295                 *value = readq((void __iomem *)hldev->common_reg + offset);
2296                 break;
2297         case vxge_hw_mgmt_reg_type_mrpcim:
2298                 if (!(hldev->access_rights &
2299                         VXGE_HW_DEVICE_ACCESS_RIGHT_MRPCIM)) {
2300                         status = VXGE_HW_ERR_PRIVILAGED_OPEARATION;
2301                         break;
2302                 }
2303                 if (offset > sizeof(struct vxge_hw_mrpcim_reg) - 8) {
2304                         status = VXGE_HW_ERR_INVALID_OFFSET;
2305                         break;
2306                 }
2307                 *value = readq((void __iomem *)hldev->mrpcim_reg + offset);
2308                 break;
2309         case vxge_hw_mgmt_reg_type_srpcim:
2310                 if (!(hldev->access_rights &
2311                         VXGE_HW_DEVICE_ACCESS_RIGHT_SRPCIM)) {
2312                         status = VXGE_HW_ERR_PRIVILAGED_OPEARATION;
2313                         break;
2314                 }
2315                 if (index > VXGE_HW_TITAN_SRPCIM_REG_SPACES - 1) {
2316                         status = VXGE_HW_ERR_INVALID_INDEX;
2317                         break;
2318                 }
2319                 if (offset > sizeof(struct vxge_hw_srpcim_reg) - 8) {
2320                         status = VXGE_HW_ERR_INVALID_OFFSET;
2321                         break;
2322                 }
2323                 *value = readq((void __iomem *)hldev->srpcim_reg[index] +
2324                                 offset);
2325                 break;
2326         case vxge_hw_mgmt_reg_type_vpmgmt:
2327                 if ((index > VXGE_HW_TITAN_VPMGMT_REG_SPACES - 1) ||
2328                         (!(hldev->vpath_assignments & vxge_mBIT(index)))) {
2329                         status = VXGE_HW_ERR_INVALID_INDEX;
2330                         break;
2331                 }
2332                 if (offset > sizeof(struct vxge_hw_vpmgmt_reg) - 8) {
2333                         status = VXGE_HW_ERR_INVALID_OFFSET;
2334                         break;
2335                 }
2336                 *value = readq((void __iomem *)hldev->vpmgmt_reg[index] +
2337                                 offset);
2338                 break;
2339         case vxge_hw_mgmt_reg_type_vpath:
2340                 if ((index > VXGE_HW_TITAN_VPATH_REG_SPACES - 1) ||
2341                         (!(hldev->vpath_assignments & vxge_mBIT(index)))) {
2342                         status = VXGE_HW_ERR_INVALID_INDEX;
2343                         break;
2344                 }
2345                 if (index > VXGE_HW_TITAN_VPATH_REG_SPACES - 1) {
2346                         status = VXGE_HW_ERR_INVALID_INDEX;
2347                         break;
2348                 }
2349                 if (offset > sizeof(struct vxge_hw_vpath_reg) - 8) {
2350                         status = VXGE_HW_ERR_INVALID_OFFSET;
2351                         break;
2352                 }
2353                 *value = readq((void __iomem *)hldev->vpath_reg[index] +
2354                                 offset);
2355                 break;
2356         default:
2357                 status = VXGE_HW_ERR_INVALID_TYPE;
2358                 break;
2359         }
2360
2361 exit:
2362         return status;
2363 }
2364
2365 /*
2366  * vxge_hw_mgmt_reg_Write - Write Titan register.
2367  */
2368 enum vxge_hw_status
2369 vxge_hw_mgmt_reg_write(struct __vxge_hw_device *hldev,
2370                       enum vxge_hw_mgmt_reg_type type,
2371                       u32 index, u32 offset, u64 value)
2372 {
2373         enum vxge_hw_status status = VXGE_HW_OK;
2374
2375         if ((hldev == NULL) || (hldev->magic != VXGE_HW_DEVICE_MAGIC)) {
2376                 status = VXGE_HW_ERR_INVALID_DEVICE;
2377                 goto exit;
2378         }
2379
2380         switch (type) {
2381         case vxge_hw_mgmt_reg_type_legacy:
2382                 if (offset > sizeof(struct vxge_hw_legacy_reg) - 8) {
2383                         status = VXGE_HW_ERR_INVALID_OFFSET;
2384                         break;
2385                 }
2386                 writeq(value, (void __iomem *)hldev->legacy_reg + offset);
2387                 break;
2388         case vxge_hw_mgmt_reg_type_toc:
2389                 if (offset > sizeof(struct vxge_hw_toc_reg) - 8) {
2390                         status = VXGE_HW_ERR_INVALID_OFFSET;
2391                         break;
2392                 }
2393                 writeq(value, (void __iomem *)hldev->toc_reg + offset);
2394                 break;
2395         case vxge_hw_mgmt_reg_type_common:
2396                 if (offset > sizeof(struct vxge_hw_common_reg) - 8) {
2397                         status = VXGE_HW_ERR_INVALID_OFFSET;
2398                         break;
2399                 }
2400                 writeq(value, (void __iomem *)hldev->common_reg + offset);
2401                 break;
2402         case vxge_hw_mgmt_reg_type_mrpcim:
2403                 if (!(hldev->access_rights &
2404                         VXGE_HW_DEVICE_ACCESS_RIGHT_MRPCIM)) {
2405                         status = VXGE_HW_ERR_PRIVILAGED_OPEARATION;
2406                         break;
2407                 }
2408                 if (offset > sizeof(struct vxge_hw_mrpcim_reg) - 8) {
2409                         status = VXGE_HW_ERR_INVALID_OFFSET;
2410                         break;
2411                 }
2412                 writeq(value, (void __iomem *)hldev->mrpcim_reg + offset);
2413                 break;
2414         case vxge_hw_mgmt_reg_type_srpcim:
2415                 if (!(hldev->access_rights &
2416                         VXGE_HW_DEVICE_ACCESS_RIGHT_SRPCIM)) {
2417                         status = VXGE_HW_ERR_PRIVILAGED_OPEARATION;
2418                         break;
2419                 }
2420                 if (index > VXGE_HW_TITAN_SRPCIM_REG_SPACES - 1) {
2421                         status = VXGE_HW_ERR_INVALID_INDEX;
2422                         break;
2423                 }
2424                 if (offset > sizeof(struct vxge_hw_srpcim_reg) - 8) {
2425                         status = VXGE_HW_ERR_INVALID_OFFSET;
2426                         break;
2427                 }
2428                 writeq(value, (void __iomem *)hldev->srpcim_reg[index] +
2429                         offset);
2430
2431                 break;
2432         case vxge_hw_mgmt_reg_type_vpmgmt:
2433                 if ((index > VXGE_HW_TITAN_VPMGMT_REG_SPACES - 1) ||
2434                         (!(hldev->vpath_assignments & vxge_mBIT(index)))) {
2435                         status = VXGE_HW_ERR_INVALID_INDEX;
2436                         break;
2437                 }
2438                 if (offset > sizeof(struct vxge_hw_vpmgmt_reg) - 8) {
2439                         status = VXGE_HW_ERR_INVALID_OFFSET;
2440                         break;
2441                 }
2442                 writeq(value, (void __iomem *)hldev->vpmgmt_reg[index] +
2443                         offset);
2444                 break;
2445         case vxge_hw_mgmt_reg_type_vpath:
2446                 if ((index > VXGE_HW_TITAN_VPATH_REG_SPACES-1) ||
2447                         (!(hldev->vpath_assignments & vxge_mBIT(index)))) {
2448                         status = VXGE_HW_ERR_INVALID_INDEX;
2449                         break;
2450                 }
2451                 if (offset > sizeof(struct vxge_hw_vpath_reg) - 8) {
2452                         status = VXGE_HW_ERR_INVALID_OFFSET;
2453                         break;
2454                 }
2455                 writeq(value, (void __iomem *)hldev->vpath_reg[index] +
2456                         offset);
2457                 break;
2458         default:
2459                 status = VXGE_HW_ERR_INVALID_TYPE;
2460                 break;
2461         }
2462 exit:
2463         return status;
2464 }
2465
2466 /*
2467  * __vxge_hw_fifo_mempool_item_alloc - Allocate List blocks for TxD
2468  * list callback
2469  * This function is callback passed to __vxge_hw_mempool_create to create memory
2470  * pool for TxD list
2471  */
2472 static void
2473 __vxge_hw_fifo_mempool_item_alloc(
2474         struct vxge_hw_mempool *mempoolh,
2475         u32 memblock_index, struct vxge_hw_mempool_dma *dma_object,
2476         u32 index, u32 is_last)
2477 {
2478         u32 memblock_item_idx;
2479         struct __vxge_hw_fifo_txdl_priv *txdl_priv;
2480         struct vxge_hw_fifo_txd *txdp =
2481                 (struct vxge_hw_fifo_txd *)mempoolh->items_arr[index];
2482         struct __vxge_hw_fifo *fifo =
2483                         (struct __vxge_hw_fifo *)mempoolh->userdata;
2484         void *memblock = mempoolh->memblocks_arr[memblock_index];
2485
2486         vxge_assert(txdp);
2487
2488         txdp->host_control = (u64) (size_t)
2489         __vxge_hw_mempool_item_priv(mempoolh, memblock_index, txdp,
2490                                         &memblock_item_idx);
2491
2492         txdl_priv = __vxge_hw_fifo_txdl_priv(fifo, txdp);
2493
2494         vxge_assert(txdl_priv);
2495
2496         fifo->channel.reserve_arr[fifo->channel.reserve_ptr - 1 - index] = txdp;
2497
2498         /* pre-format HW's TxDL's private */
2499         txdl_priv->dma_offset = (char *)txdp - (char *)memblock;
2500         txdl_priv->dma_addr = dma_object->addr + txdl_priv->dma_offset;
2501         txdl_priv->dma_handle = dma_object->handle;
2502         txdl_priv->memblock   = memblock;
2503         txdl_priv->first_txdp = txdp;
2504         txdl_priv->next_txdl_priv = NULL;
2505         txdl_priv->alloc_frags = 0;
2506
2507         return;
2508 }
2509
2510 /*
2511  * __vxge_hw_fifo_create - Create a FIFO
2512  * This function creates FIFO and initializes it.
2513  */
2514 enum vxge_hw_status
2515 __vxge_hw_fifo_create(struct __vxge_hw_vpath_handle *vp,
2516                       struct vxge_hw_fifo_attr *attr)
2517 {
2518         enum vxge_hw_status status = VXGE_HW_OK;
2519         struct __vxge_hw_fifo *fifo;
2520         struct vxge_hw_fifo_config *config;
2521         u32 txdl_size, txdl_per_memblock;
2522         struct vxge_hw_mempool_cbs fifo_mp_callback;
2523         struct __vxge_hw_virtualpath *vpath;
2524
2525         if ((vp == NULL) || (attr == NULL)) {
2526                 status = VXGE_HW_ERR_INVALID_HANDLE;
2527                 goto exit;
2528         }
2529         vpath = vp->vpath;
2530         config = &vpath->hldev->config.vp_config[vpath->vp_id].fifo;
2531
2532         txdl_size = config->max_frags * sizeof(struct vxge_hw_fifo_txd);
2533
2534         txdl_per_memblock = config->memblock_size / txdl_size;
2535
2536         fifo = (struct __vxge_hw_fifo *)__vxge_hw_channel_allocate(vp,
2537                                         VXGE_HW_CHANNEL_TYPE_FIFO,
2538                                         config->fifo_blocks * txdl_per_memblock,
2539                                         attr->per_txdl_space, attr->userdata);
2540
2541         if (fifo == NULL) {
2542                 status = VXGE_HW_ERR_OUT_OF_MEMORY;
2543                 goto exit;
2544         }
2545
2546         vpath->fifoh = fifo;
2547         fifo->nofl_db = vpath->nofl_db;
2548
2549         fifo->vp_id = vpath->vp_id;
2550         fifo->vp_reg = vpath->vp_reg;
2551         fifo->stats = &vpath->sw_stats->fifo_stats;
2552
2553         fifo->config = config;
2554
2555         /* apply "interrupts per txdl" attribute */
2556         fifo->interrupt_type = VXGE_HW_FIFO_TXD_INT_TYPE_UTILZ;
2557
2558         if (fifo->config->intr)
2559                 fifo->interrupt_type = VXGE_HW_FIFO_TXD_INT_TYPE_PER_LIST;
2560
2561         fifo->no_snoop_bits = config->no_snoop_bits;
2562
2563         /*
2564          * FIFO memory management strategy:
2565          *
2566          * TxDL split into three independent parts:
2567          *      - set of TxD's
2568          *      - TxD HW private part
2569          *      - driver private part
2570          *
2571          * Adaptative memory allocation used. i.e. Memory allocated on
2572          * demand with the size which will fit into one memory block.
2573          * One memory block may contain more than one TxDL.
2574          *
2575          * During "reserve" operations more memory can be allocated on demand
2576          * for example due to FIFO full condition.
2577          *
2578          * Pool of memory memblocks never shrinks except in __vxge_hw_fifo_close
2579          * routine which will essentially stop the channel and free resources.
2580          */
2581
2582         /* TxDL common private size == TxDL private  +  driver private */
2583         fifo->priv_size =
2584                 sizeof(struct __vxge_hw_fifo_txdl_priv) + attr->per_txdl_space;
2585         fifo->priv_size = ((fifo->priv_size  +  VXGE_CACHE_LINE_SIZE - 1) /
2586                         VXGE_CACHE_LINE_SIZE) * VXGE_CACHE_LINE_SIZE;
2587
2588         fifo->per_txdl_space = attr->per_txdl_space;
2589
2590         /* recompute txdl size to be cacheline aligned */
2591         fifo->txdl_size = txdl_size;
2592         fifo->txdl_per_memblock = txdl_per_memblock;
2593
2594         fifo->txdl_term = attr->txdl_term;
2595         fifo->callback = attr->callback;
2596
2597         if (fifo->txdl_per_memblock == 0) {
2598                 __vxge_hw_fifo_delete(vp);
2599                 status = VXGE_HW_ERR_INVALID_BLOCK_SIZE;
2600                 goto exit;
2601         }
2602
2603         fifo_mp_callback.item_func_alloc = __vxge_hw_fifo_mempool_item_alloc;
2604
2605         fifo->mempool =
2606                 __vxge_hw_mempool_create(vpath->hldev,
2607                         fifo->config->memblock_size,
2608                         fifo->txdl_size,
2609                         fifo->priv_size,
2610                         (fifo->config->fifo_blocks * fifo->txdl_per_memblock),
2611                         (fifo->config->fifo_blocks * fifo->txdl_per_memblock),
2612                         &fifo_mp_callback,
2613                         fifo);
2614
2615         if (fifo->mempool == NULL) {
2616                 __vxge_hw_fifo_delete(vp);
2617                 status = VXGE_HW_ERR_OUT_OF_MEMORY;
2618                 goto exit;
2619         }
2620
2621         status = __vxge_hw_channel_initialize(&fifo->channel);
2622         if (status != VXGE_HW_OK) {
2623                 __vxge_hw_fifo_delete(vp);
2624                 goto exit;
2625         }
2626
2627         vxge_assert(fifo->channel.reserve_ptr);
2628 exit:
2629         return status;
2630 }
2631
2632 /*
2633  * __vxge_hw_fifo_abort - Returns the TxD
2634  * This function terminates the TxDs of fifo
2635  */
2636 enum vxge_hw_status __vxge_hw_fifo_abort(struct __vxge_hw_fifo *fifo)
2637 {
2638         void *txdlh;
2639
2640         for (;;) {
2641                 vxge_hw_channel_dtr_try_complete(&fifo->channel, &txdlh);
2642
2643                 if (txdlh == NULL)
2644                         break;
2645
2646                 vxge_hw_channel_dtr_complete(&fifo->channel);
2647
2648                 if (fifo->txdl_term) {
2649                         fifo->txdl_term(txdlh,
2650                         VXGE_HW_TXDL_STATE_POSTED,
2651                         fifo->channel.userdata);
2652                 }
2653
2654                 vxge_hw_channel_dtr_free(&fifo->channel, txdlh);
2655         }
2656
2657         return VXGE_HW_OK;
2658 }
2659
2660 /*
2661  * __vxge_hw_fifo_reset - Resets the fifo
2662  * This function resets the fifo during vpath reset operation
2663  */
2664 enum vxge_hw_status __vxge_hw_fifo_reset(struct __vxge_hw_fifo *fifo)
2665 {
2666         enum vxge_hw_status status = VXGE_HW_OK;
2667
2668         __vxge_hw_fifo_abort(fifo);
2669         status = __vxge_hw_channel_reset(&fifo->channel);
2670
2671         return status;
2672 }
2673
2674 /*
2675  * __vxge_hw_fifo_delete - Removes the FIFO
2676  * This function freeup the memory pool and removes the FIFO
2677  */
2678 enum vxge_hw_status __vxge_hw_fifo_delete(struct __vxge_hw_vpath_handle *vp)
2679 {
2680         struct __vxge_hw_fifo *fifo = vp->vpath->fifoh;
2681
2682         __vxge_hw_fifo_abort(fifo);
2683
2684         if (fifo->mempool)
2685                 __vxge_hw_mempool_destroy(fifo->mempool);
2686
2687         vp->vpath->fifoh = NULL;
2688
2689         __vxge_hw_channel_free(&fifo->channel);
2690
2691         return VXGE_HW_OK;
2692 }
2693
2694 /*
2695  * __vxge_hw_vpath_pci_read - Read the content of given address
2696  *                          in pci config space.
2697  * Read from the vpath pci config space.
2698  */
2699 enum vxge_hw_status
2700 __vxge_hw_vpath_pci_read(struct __vxge_hw_virtualpath *vpath,
2701                          u32 phy_func_0, u32 offset, u32 *val)
2702 {
2703         u64 val64;
2704         enum vxge_hw_status status = VXGE_HW_OK;
2705         struct vxge_hw_vpath_reg __iomem *vp_reg = vpath->vp_reg;
2706
2707         val64 = VXGE_HW_PCI_CONFIG_ACCESS_CFG1_ADDRESS(offset);
2708
2709         if (phy_func_0)
2710                 val64 |= VXGE_HW_PCI_CONFIG_ACCESS_CFG1_SEL_FUNC0;
2711
2712         writeq(val64, &vp_reg->pci_config_access_cfg1);
2713         wmb();
2714         writeq(VXGE_HW_PCI_CONFIG_ACCESS_CFG2_REQ,
2715                         &vp_reg->pci_config_access_cfg2);
2716         wmb();
2717
2718         status = __vxge_hw_device_register_poll(
2719                         &vp_reg->pci_config_access_cfg2,
2720                         VXGE_HW_INTR_MASK_ALL, VXGE_HW_DEF_DEVICE_POLL_MILLIS);
2721
2722         if (status != VXGE_HW_OK)
2723                 goto exit;
2724
2725         val64 = readq(&vp_reg->pci_config_access_status);
2726
2727         if (val64 & VXGE_HW_PCI_CONFIG_ACCESS_STATUS_ACCESS_ERR) {
2728                 status = VXGE_HW_FAIL;
2729                 *val = 0;
2730         } else
2731                 *val = (u32)vxge_bVALn(val64, 32, 32);
2732 exit:
2733         return status;
2734 }
2735
2736 /*
2737  * __vxge_hw_vpath_func_id_get - Get the function id of the vpath.
2738  * Returns the function number of the vpath.
2739  */
2740 u32
2741 __vxge_hw_vpath_func_id_get(u32 vp_id,
2742         struct vxge_hw_vpmgmt_reg __iomem *vpmgmt_reg)
2743 {
2744         u64 val64;
2745
2746         val64 = readq(&vpmgmt_reg->vpath_to_func_map_cfg1);
2747
2748         return
2749          (u32)VXGE_HW_VPATH_TO_FUNC_MAP_CFG1_GET_VPATH_TO_FUNC_MAP_CFG1(val64);
2750 }
2751
2752 /*
2753  * __vxge_hw_read_rts_ds - Program RTS steering critieria
2754  */
2755 static inline void
2756 __vxge_hw_read_rts_ds(struct vxge_hw_vpath_reg __iomem *vpath_reg,
2757                       u64 dta_struct_sel)
2758 {
2759         writeq(0, &vpath_reg->rts_access_steer_ctrl);
2760         wmb();
2761         writeq(dta_struct_sel, &vpath_reg->rts_access_steer_data0);
2762         writeq(0, &vpath_reg->rts_access_steer_data1);
2763         wmb();
2764         return;
2765 }
2766
2767
2768 /*
2769  * __vxge_hw_vpath_card_info_get - Get the serial numbers,
2770  * part number and product description.
2771  */
2772 enum vxge_hw_status
2773 __vxge_hw_vpath_card_info_get(
2774         u32 vp_id,
2775         struct vxge_hw_vpath_reg __iomem *vpath_reg,
2776         struct vxge_hw_device_hw_info *hw_info)
2777 {
2778         u32 i, j;
2779         u64 val64;
2780         u64 data1 = 0ULL;
2781         u64 data2 = 0ULL;
2782         enum vxge_hw_status status = VXGE_HW_OK;
2783         u8 *serial_number = hw_info->serial_number;
2784         u8 *part_number = hw_info->part_number;
2785         u8 *product_desc = hw_info->product_desc;
2786
2787         __vxge_hw_read_rts_ds(vpath_reg,
2788                 VXGE_HW_RTS_ACCESS_STEER_DATA0_MEMO_ITEM_SERIAL_NUMBER);
2789
2790         val64 = VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION(
2791                         VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION_READ_MEMO_ENTRY) |
2792                 VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL(
2793                         VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL_FW_MEMO) |
2794                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE |
2795                 VXGE_HW_RTS_ACCESS_STEER_CTRL_OFFSET(0);
2796
2797         status = __vxge_hw_pio_mem_write64(val64,
2798                                 &vpath_reg->rts_access_steer_ctrl,
2799                                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE,
2800                                 VXGE_HW_DEF_DEVICE_POLL_MILLIS);
2801
2802         if (status != VXGE_HW_OK)
2803                 return status;
2804
2805         val64 = readq(&vpath_reg->rts_access_steer_ctrl);
2806
2807         if (val64 & VXGE_HW_RTS_ACCESS_STEER_CTRL_RMACJ_STATUS) {
2808                 data1 = readq(&vpath_reg->rts_access_steer_data0);
2809                 ((u64 *)serial_number)[0] = be64_to_cpu(data1);
2810
2811                 data2 = readq(&vpath_reg->rts_access_steer_data1);
2812                 ((u64 *)serial_number)[1] = be64_to_cpu(data2);
2813                 status = VXGE_HW_OK;
2814         } else
2815                 *serial_number = 0;
2816
2817         __vxge_hw_read_rts_ds(vpath_reg,
2818                         VXGE_HW_RTS_ACCESS_STEER_DATA0_MEMO_ITEM_PART_NUMBER);
2819
2820         val64 = VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION(
2821                         VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION_READ_MEMO_ENTRY) |
2822                 VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL(
2823                         VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL_FW_MEMO) |
2824                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE |
2825                 VXGE_HW_RTS_ACCESS_STEER_CTRL_OFFSET(0);
2826
2827         status = __vxge_hw_pio_mem_write64(val64,
2828                                 &vpath_reg->rts_access_steer_ctrl,
2829                                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE,
2830                                 VXGE_HW_DEF_DEVICE_POLL_MILLIS);
2831
2832         if (status != VXGE_HW_OK)
2833                 return status;
2834
2835         val64 = readq(&vpath_reg->rts_access_steer_ctrl);
2836
2837         if (val64 & VXGE_HW_RTS_ACCESS_STEER_CTRL_RMACJ_STATUS) {
2838
2839                 data1 = readq(&vpath_reg->rts_access_steer_data0);
2840                 ((u64 *)part_number)[0] = be64_to_cpu(data1);
2841
2842                 data2 = readq(&vpath_reg->rts_access_steer_data1);
2843                 ((u64 *)part_number)[1] = be64_to_cpu(data2);
2844
2845                 status = VXGE_HW_OK;
2846
2847         } else
2848                 *part_number = 0;
2849
2850         j = 0;
2851
2852         for (i = VXGE_HW_RTS_ACCESS_STEER_DATA0_MEMO_ITEM_DESC_0;
2853              i <= VXGE_HW_RTS_ACCESS_STEER_DATA0_MEMO_ITEM_DESC_3; i++) {
2854
2855                 __vxge_hw_read_rts_ds(vpath_reg, i);
2856
2857                 val64 = VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION(
2858                         VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION_READ_MEMO_ENTRY) |
2859                         VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL(
2860                         VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL_FW_MEMO) |
2861                         VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE |
2862                         VXGE_HW_RTS_ACCESS_STEER_CTRL_OFFSET(0);
2863
2864                 status = __vxge_hw_pio_mem_write64(val64,
2865                                 &vpath_reg->rts_access_steer_ctrl,
2866                                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE,
2867                                 VXGE_HW_DEF_DEVICE_POLL_MILLIS);
2868
2869                 if (status != VXGE_HW_OK)
2870                         return status;
2871
2872                 val64 = readq(&vpath_reg->rts_access_steer_ctrl);
2873
2874                 if (val64 & VXGE_HW_RTS_ACCESS_STEER_CTRL_RMACJ_STATUS) {
2875
2876                         data1 = readq(&vpath_reg->rts_access_steer_data0);
2877                         ((u64 *)product_desc)[j++] = be64_to_cpu(data1);
2878
2879                         data2 = readq(&vpath_reg->rts_access_steer_data1);
2880                         ((u64 *)product_desc)[j++] = be64_to_cpu(data2);
2881
2882                         status = VXGE_HW_OK;
2883                 } else
2884                         *product_desc = 0;
2885         }
2886
2887         return status;
2888 }
2889
2890 /*
2891  * __vxge_hw_vpath_fw_ver_get - Get the fw version
2892  * Returns FW Version
2893  */
2894 enum vxge_hw_status
2895 __vxge_hw_vpath_fw_ver_get(
2896         u32 vp_id,
2897         struct vxge_hw_vpath_reg __iomem *vpath_reg,
2898         struct vxge_hw_device_hw_info *hw_info)
2899 {
2900         u64 val64;
2901         u64 data1 = 0ULL;
2902         u64 data2 = 0ULL;
2903         struct vxge_hw_device_version *fw_version = &hw_info->fw_version;
2904         struct vxge_hw_device_date *fw_date = &hw_info->fw_date;
2905         struct vxge_hw_device_version *flash_version = &hw_info->flash_version;
2906         struct vxge_hw_device_date *flash_date = &hw_info->flash_date;
2907         enum vxge_hw_status status = VXGE_HW_OK;
2908
2909         val64 = VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION(
2910                 VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION_READ_ENTRY) |
2911                 VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL(
2912                 VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL_FW_MEMO) |
2913                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE |
2914                 VXGE_HW_RTS_ACCESS_STEER_CTRL_OFFSET(0);
2915
2916         status = __vxge_hw_pio_mem_write64(val64,
2917                                 &vpath_reg->rts_access_steer_ctrl,
2918                                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE,
2919                                 VXGE_HW_DEF_DEVICE_POLL_MILLIS);
2920
2921         if (status != VXGE_HW_OK)
2922                 goto exit;
2923
2924         val64 = readq(&vpath_reg->rts_access_steer_ctrl);
2925
2926         if (val64 & VXGE_HW_RTS_ACCESS_STEER_CTRL_RMACJ_STATUS) {
2927
2928                 data1 = readq(&vpath_reg->rts_access_steer_data0);
2929                 data2 = readq(&vpath_reg->rts_access_steer_data1);
2930
2931                 fw_date->day =
2932                         (u32)VXGE_HW_RTS_ACCESS_STEER_DATA0_GET_FW_VER_DAY(
2933                                                 data1);
2934                 fw_date->month =
2935                         (u32)VXGE_HW_RTS_ACCESS_STEER_DATA0_GET_FW_VER_MONTH(
2936                                                 data1);
2937                 fw_date->year =
2938                         (u32)VXGE_HW_RTS_ACCESS_STEER_DATA0_GET_FW_VER_YEAR(
2939                                                 data1);
2940
2941                 snprintf(fw_date->date, VXGE_HW_FW_STRLEN, "%2.2d/%2.2d/%4.4d",
2942                         fw_date->month, fw_date->day, fw_date->year);
2943
2944                 fw_version->major =
2945                     (u32)VXGE_HW_RTS_ACCESS_STEER_DATA0_GET_FW_VER_MAJOR(data1);
2946                 fw_version->minor =
2947                     (u32)VXGE_HW_RTS_ACCESS_STEER_DATA0_GET_FW_VER_MINOR(data1);
2948                 fw_version->build =
2949                     (u32)VXGE_HW_RTS_ACCESS_STEER_DATA0_GET_FW_VER_BUILD(data1);
2950
2951                 snprintf(fw_version->version, VXGE_HW_FW_STRLEN, "%d.%d.%d",
2952                     fw_version->major, fw_version->minor, fw_version->build);
2953
2954                 flash_date->day =
2955                   (u32)VXGE_HW_RTS_ACCESS_STEER_DATA1_GET_FLASH_VER_DAY(data2);
2956                 flash_date->month =
2957                  (u32)VXGE_HW_RTS_ACCESS_STEER_DATA1_GET_FLASH_VER_MONTH(data2);
2958                 flash_date->year =
2959                  (u32)VXGE_HW_RTS_ACCESS_STEER_DATA1_GET_FLASH_VER_YEAR(data2);
2960
2961                 snprintf(flash_date->date, VXGE_HW_FW_STRLEN,
2962                         "%2.2d/%2.2d/%4.4d",
2963                         flash_date->month, flash_date->day, flash_date->year);
2964
2965                 flash_version->major =
2966                  (u32)VXGE_HW_RTS_ACCESS_STEER_DATA1_GET_FLASH_VER_MAJOR(data2);
2967                 flash_version->minor =
2968                  (u32)VXGE_HW_RTS_ACCESS_STEER_DATA1_GET_FLASH_VER_MINOR(data2);
2969                 flash_version->build =
2970                  (u32)VXGE_HW_RTS_ACCESS_STEER_DATA1_GET_FLASH_VER_BUILD(data2);
2971
2972                 snprintf(flash_version->version, VXGE_HW_FW_STRLEN, "%d.%d.%d",
2973                         flash_version->major, flash_version->minor,
2974                         flash_version->build);
2975
2976                 status = VXGE_HW_OK;
2977
2978         } else
2979                 status = VXGE_HW_FAIL;
2980 exit:
2981         return status;
2982 }
2983
2984 /*
2985  * __vxge_hw_vpath_pci_func_mode_get - Get the pci mode
2986  * Returns pci function mode
2987  */
2988 u64
2989 __vxge_hw_vpath_pci_func_mode_get(
2990         u32  vp_id,
2991         struct vxge_hw_vpath_reg __iomem *vpath_reg)
2992 {
2993         u64 val64;
2994         u64 data1 = 0ULL;
2995         enum vxge_hw_status status = VXGE_HW_OK;
2996
2997         __vxge_hw_read_rts_ds(vpath_reg,
2998                 VXGE_HW_RTS_ACCESS_STEER_DATA0_MEMO_ITEM_PCI_MODE);
2999
3000         val64 = VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION(
3001                         VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION_READ_MEMO_ENTRY) |
3002                 VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL(
3003                         VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL_FW_MEMO) |
3004                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE |
3005                 VXGE_HW_RTS_ACCESS_STEER_CTRL_OFFSET(0);
3006
3007         status = __vxge_hw_pio_mem_write64(val64,
3008                                 &vpath_reg->rts_access_steer_ctrl,
3009                                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE,
3010                                 VXGE_HW_DEF_DEVICE_POLL_MILLIS);
3011
3012         if (status != VXGE_HW_OK)
3013                 goto exit;
3014
3015         val64 = readq(&vpath_reg->rts_access_steer_ctrl);
3016
3017         if (val64 & VXGE_HW_RTS_ACCESS_STEER_CTRL_RMACJ_STATUS) {
3018                 data1 = readq(&vpath_reg->rts_access_steer_data0);
3019                 status = VXGE_HW_OK;
3020         } else {
3021                 data1 = 0;
3022                 status = VXGE_HW_FAIL;
3023         }
3024 exit:
3025         return data1;
3026 }
3027
3028 /**
3029  * vxge_hw_device_flick_link_led - Flick (blink) link LED.
3030  * @hldev: HW device.
3031  * @on_off: TRUE if flickering to be on, FALSE to be off
3032  *
3033  * Flicker the link LED.
3034  */
3035 enum vxge_hw_status
3036 vxge_hw_device_flick_link_led(struct __vxge_hw_device *hldev,
3037                                u64 on_off)
3038 {
3039         u64 val64;
3040         enum vxge_hw_status status = VXGE_HW_OK;
3041         struct vxge_hw_vpath_reg __iomem *vp_reg;
3042
3043         if (hldev == NULL) {
3044                 status = VXGE_HW_ERR_INVALID_DEVICE;
3045                 goto exit;
3046         }
3047
3048         vp_reg = hldev->vpath_reg[hldev->first_vp_id];
3049
3050         writeq(0, &vp_reg->rts_access_steer_ctrl);
3051         wmb();
3052         writeq(on_off, &vp_reg->rts_access_steer_data0);
3053         writeq(0, &vp_reg->rts_access_steer_data1);
3054         wmb();
3055
3056         val64 = VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION(
3057                         VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION_LED_CONTROL) |
3058                 VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL(
3059                         VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL_FW_MEMO) |
3060                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE |
3061                 VXGE_HW_RTS_ACCESS_STEER_CTRL_OFFSET(0);
3062
3063         status = __vxge_hw_pio_mem_write64(val64,
3064                                 &vp_reg->rts_access_steer_ctrl,
3065                                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE,
3066                                 VXGE_HW_DEF_DEVICE_POLL_MILLIS);
3067 exit:
3068         return status;
3069 }
3070
3071 /*
3072  * __vxge_hw_vpath_rts_table_get - Get the entries from RTS access tables
3073  */
3074 enum vxge_hw_status
3075 __vxge_hw_vpath_rts_table_get(
3076         struct __vxge_hw_vpath_handle *vp,
3077         u32 action, u32 rts_table, u32 offset, u64 *data1, u64 *data2)
3078 {
3079         u64 val64;
3080         struct __vxge_hw_virtualpath *vpath;
3081         struct vxge_hw_vpath_reg __iomem *vp_reg;
3082
3083         enum vxge_hw_status status = VXGE_HW_OK;
3084
3085         if (vp == NULL) {
3086                 status = VXGE_HW_ERR_INVALID_HANDLE;
3087                 goto exit;
3088         }
3089
3090         vpath = vp->vpath;
3091         vp_reg = vpath->vp_reg;
3092
3093         val64 = VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION(action) |
3094                 VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL(rts_table) |
3095                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE |
3096                 VXGE_HW_RTS_ACCESS_STEER_CTRL_OFFSET(offset);
3097
3098         if ((rts_table ==
3099                 VXGE_HW_RTS_ACS_STEER_CTRL_DATA_STRUCT_SEL_RTH_SOLO_IT) ||
3100             (rts_table ==
3101                 VXGE_HW_RTS_ACS_STEER_CTRL_DATA_STRUCT_SEL_RTH_MULTI_IT) ||
3102             (rts_table ==
3103                 VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL_RTH_MASK) ||
3104             (rts_table ==
3105                 VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL_RTH_KEY)) {
3106                 val64 = val64 | VXGE_HW_RTS_ACCESS_STEER_CTRL_TABLE_SEL;
3107         }
3108
3109         status = __vxge_hw_pio_mem_write64(val64,
3110                                 &vp_reg->rts_access_steer_ctrl,
3111                                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE,
3112                                 vpath->hldev->config.device_poll_millis);
3113
3114         if (status != VXGE_HW_OK)
3115                 goto exit;
3116
3117         val64 = readq(&vp_reg->rts_access_steer_ctrl);
3118
3119         if (val64 & VXGE_HW_RTS_ACCESS_STEER_CTRL_RMACJ_STATUS) {
3120
3121                 *data1 = readq(&vp_reg->rts_access_steer_data0);
3122
3123                 if ((rts_table ==
3124                 VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL_DA) ||
3125                 (rts_table ==
3126                 VXGE_HW_RTS_ACS_STEER_CTRL_DATA_STRUCT_SEL_RTH_MULTI_IT)) {
3127                         *data2 = readq(&vp_reg->rts_access_steer_data1);
3128                 }
3129                 status = VXGE_HW_OK;
3130         } else
3131                 status = VXGE_HW_FAIL;
3132 exit:
3133         return status;
3134 }
3135
3136 /*
3137  * __vxge_hw_vpath_rts_table_set - Set the entries of RTS access tables
3138  */
3139 enum vxge_hw_status
3140 __vxge_hw_vpath_rts_table_set(
3141         struct __vxge_hw_vpath_handle *vp, u32 action, u32 rts_table,
3142         u32 offset, u64 data1, u64 data2)
3143 {
3144         u64 val64;
3145         struct __vxge_hw_virtualpath *vpath;
3146         enum vxge_hw_status status = VXGE_HW_OK;
3147         struct vxge_hw_vpath_reg __iomem *vp_reg;
3148
3149         if (vp == NULL) {
3150                 status = VXGE_HW_ERR_INVALID_HANDLE;
3151                 goto exit;
3152         }
3153
3154         vpath = vp->vpath;
3155         vp_reg = vpath->vp_reg;
3156
3157         writeq(data1, &vp_reg->rts_access_steer_data0);
3158         wmb();
3159
3160         if ((rts_table == VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL_DA) ||
3161             (rts_table ==
3162                 VXGE_HW_RTS_ACS_STEER_CTRL_DATA_STRUCT_SEL_RTH_MULTI_IT)) {
3163                 writeq(data2, &vp_reg->rts_access_steer_data1);
3164                 wmb();
3165         }
3166
3167         val64 = VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION(action) |
3168                 VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL(rts_table) |
3169                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE |
3170                 VXGE_HW_RTS_ACCESS_STEER_CTRL_OFFSET(offset);
3171
3172         status = __vxge_hw_pio_mem_write64(val64,
3173                                 &vp_reg->rts_access_steer_ctrl,
3174                                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE,
3175                                 vpath->hldev->config.device_poll_millis);
3176
3177         if (status != VXGE_HW_OK)
3178                 goto exit;
3179
3180         val64 = readq(&vp_reg->rts_access_steer_ctrl);
3181
3182         if (val64 & VXGE_HW_RTS_ACCESS_STEER_CTRL_RMACJ_STATUS)
3183                 status = VXGE_HW_OK;
3184         else
3185                 status = VXGE_HW_FAIL;
3186 exit:
3187         return status;
3188 }
3189
3190 /*
3191  * __vxge_hw_vpath_addr_get - Get the hw address entry for this vpath
3192  *               from MAC address table.
3193  */
3194 enum vxge_hw_status
3195 __vxge_hw_vpath_addr_get(
3196         u32 vp_id, struct vxge_hw_vpath_reg __iomem *vpath_reg,
3197         u8 (macaddr)[ETH_ALEN], u8 (macaddr_mask)[ETH_ALEN])
3198 {
3199         u32 i;
3200         u64 val64;
3201         u64 data1 = 0ULL;
3202         u64 data2 = 0ULL;
3203         enum vxge_hw_status status = VXGE_HW_OK;
3204
3205         val64 = VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION(
3206                 VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION_LIST_FIRST_ENTRY) |
3207                 VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL(
3208                 VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL_DA) |
3209                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE |
3210                 VXGE_HW_RTS_ACCESS_STEER_CTRL_OFFSET(0);
3211
3212         status = __vxge_hw_pio_mem_write64(val64,
3213                                 &vpath_reg->rts_access_steer_ctrl,
3214                                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE,
3215                                 VXGE_HW_DEF_DEVICE_POLL_MILLIS);
3216
3217         if (status != VXGE_HW_OK)
3218                 goto exit;
3219
3220         val64 = readq(&vpath_reg->rts_access_steer_ctrl);
3221
3222         if (val64 & VXGE_HW_RTS_ACCESS_STEER_CTRL_RMACJ_STATUS) {
3223
3224                 data1 = readq(&vpath_reg->rts_access_steer_data0);
3225                 data2 = readq(&vpath_reg->rts_access_steer_data1);
3226
3227                 data1 = VXGE_HW_RTS_ACCESS_STEER_DATA0_GET_DA_MAC_ADDR(data1);
3228                 data2 = VXGE_HW_RTS_ACCESS_STEER_DATA1_GET_DA_MAC_ADDR_MASK(
3229                                                         data2);
3230
3231                 for (i = ETH_ALEN; i > 0; i--) {
3232                         macaddr[i-1] = (u8)(data1 & 0xFF);
3233                         data1 >>= 8;
3234
3235                         macaddr_mask[i-1] = (u8)(data2 & 0xFF);
3236                         data2 >>= 8;
3237                 }
3238                 status = VXGE_HW_OK;
3239         } else
3240                 status = VXGE_HW_FAIL;
3241 exit:
3242         return status;
3243 }
3244
3245 /*
3246  * vxge_hw_vpath_rts_rth_set - Set/configure RTS hashing.
3247  */
3248 enum vxge_hw_status vxge_hw_vpath_rts_rth_set(
3249                         struct __vxge_hw_vpath_handle *vp,
3250                         enum vxge_hw_rth_algoritms algorithm,
3251                         struct vxge_hw_rth_hash_types *hash_type,
3252                         u16 bucket_size)
3253 {
3254         u64 data0, data1;
3255         enum vxge_hw_status status = VXGE_HW_OK;
3256
3257         if (vp == NULL) {
3258                 status = VXGE_HW_ERR_INVALID_HANDLE;
3259                 goto exit;
3260         }
3261
3262         status = __vxge_hw_vpath_rts_table_get(vp,
3263                      VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION_READ_ENTRY,
3264                      VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL_RTH_GEN_CFG,
3265                         0, &data0, &data1);
3266
3267         data0 &= ~(VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_GEN_BUCKET_SIZE(0xf) |
3268                         VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_GEN_ALG_SEL(0x3));
3269
3270         data0 |= VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_GEN_RTH_EN |
3271         VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_GEN_BUCKET_SIZE(bucket_size) |
3272         VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_GEN_ALG_SEL(algorithm);
3273
3274         if (hash_type->hash_type_tcpipv4_en)
3275                 data0 |= VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_GEN_RTH_TCP_IPV4_EN;
3276
3277         if (hash_type->hash_type_ipv4_en)
3278                 data0 |= VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_GEN_RTH_IPV4_EN;
3279
3280         if (hash_type->hash_type_tcpipv6_en)
3281                 data0 |= VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_GEN_RTH_TCP_IPV6_EN;
3282
3283         if (hash_type->hash_type_ipv6_en)
3284                 data0 |= VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_GEN_RTH_IPV6_EN;
3285
3286         if (hash_type->hash_type_tcpipv6ex_en)
3287                 data0 |=
3288                 VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_GEN_RTH_TCP_IPV6_EX_EN;
3289
3290         if (hash_type->hash_type_ipv6ex_en)
3291                 data0 |= VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_GEN_RTH_IPV6_EX_EN;
3292
3293         if (VXGE_HW_RTS_ACCESS_STEER_DATA0_GET_RTH_GEN_ACTIVE_TABLE(data0))
3294                 data0 &= ~VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_GEN_ACTIVE_TABLE;
3295         else
3296                 data0 |= VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_GEN_ACTIVE_TABLE;
3297
3298         status = __vxge_hw_vpath_rts_table_set(vp,
3299                 VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION_WRITE_ENTRY,
3300                 VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL_RTH_GEN_CFG,
3301                 0, data0, 0);
3302 exit:
3303         return status;
3304 }
3305
3306 static void
3307 vxge_hw_rts_rth_data0_data1_get(u32 j, u64 *data0, u64 *data1,
3308                                 u16 flag, u8 *itable)
3309 {
3310         switch (flag) {
3311         case 1:
3312                 *data0 = VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_ITEM0_BUCKET_NUM(j)|
3313                         VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_ITEM0_ENTRY_EN |
3314                         VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_ITEM0_BUCKET_DATA(
3315                         itable[j]);
3316         case 2:
3317                 *data0 |=
3318                         VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_ITEM1_BUCKET_NUM(j)|
3319                         VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_ITEM1_ENTRY_EN |
3320                         VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_ITEM1_BUCKET_DATA(
3321                         itable[j]);
3322         case 3:
3323                 *data1 = VXGE_HW_RTS_ACCESS_STEER_DATA1_RTH_ITEM0_BUCKET_NUM(j)|
3324                         VXGE_HW_RTS_ACCESS_STEER_DATA1_RTH_ITEM0_ENTRY_EN |
3325                         VXGE_HW_RTS_ACCESS_STEER_DATA1_RTH_ITEM0_BUCKET_DATA(
3326                         itable[j]);
3327         case 4:
3328                 *data1 |=
3329                         VXGE_HW_RTS_ACCESS_STEER_DATA1_RTH_ITEM1_BUCKET_NUM(j)|
3330                         VXGE_HW_RTS_ACCESS_STEER_DATA1_RTH_ITEM1_ENTRY_EN |
3331                         VXGE_HW_RTS_ACCESS_STEER_DATA1_RTH_ITEM1_BUCKET_DATA(
3332                         itable[j]);
3333         default:
3334                 return;
3335         }
3336 }
3337 /*
3338  * vxge_hw_vpath_rts_rth_itable_set - Set/configure indirection table (IT).
3339  */
3340 enum vxge_hw_status vxge_hw_vpath_rts_rth_itable_set(
3341                         struct __vxge_hw_vpath_handle **vpath_handles,
3342                         u32 vpath_count,
3343                         u8 *mtable,
3344                         u8 *itable,
3345                         u32 itable_size)
3346 {
3347         u32 i, j, action, rts_table;
3348         u64 data0;
3349         u64 data1;
3350         u32 max_entries;
3351         enum vxge_hw_status status = VXGE_HW_OK;
3352         struct __vxge_hw_vpath_handle *vp = vpath_handles[0];
3353
3354         if (vp == NULL) {
3355                 status = VXGE_HW_ERR_INVALID_HANDLE;
3356                 goto exit;
3357         }
3358
3359         max_entries = (((u32)1) << itable_size);
3360
3361         if (vp->vpath->hldev->config.rth_it_type
3362                                 == VXGE_HW_RTH_IT_TYPE_SOLO_IT) {
3363                 action = VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION_WRITE_ENTRY;
3364                 rts_table =
3365                         VXGE_HW_RTS_ACS_STEER_CTRL_DATA_STRUCT_SEL_RTH_SOLO_IT;
3366
3367                 for (j = 0; j < max_entries; j++) {
3368
3369                         data1 = 0;
3370
3371                         data0 =
3372                         VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_SOLO_IT_BUCKET_DATA(
3373                                 itable[j]);
3374
3375                         status = __vxge_hw_vpath_rts_table_set(vpath_handles[0],
3376                                 action, rts_table, j, data0, data1);
3377
3378                         if (status != VXGE_HW_OK)
3379                                 goto exit;
3380                 }
3381
3382                 for (j = 0; j < max_entries; j++) {
3383
3384                         data1 = 0;
3385
3386                         data0 =
3387                         VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_SOLO_IT_ENTRY_EN |
3388                         VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_SOLO_IT_BUCKET_DATA(
3389                                 itable[j]);
3390
3391                         status = __vxge_hw_vpath_rts_table_set(
3392                                 vpath_handles[mtable[itable[j]]], action,
3393                                 rts_table, j, data0, data1);
3394
3395                         if (status != VXGE_HW_OK)
3396                                 goto exit;
3397                 }
3398         } else {
3399                 action = VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION_WRITE_ENTRY;
3400                 rts_table =
3401                         VXGE_HW_RTS_ACS_STEER_CTRL_DATA_STRUCT_SEL_RTH_MULTI_IT;
3402                 for (i = 0; i < vpath_count; i++) {
3403
3404                         for (j = 0; j < max_entries;) {
3405
3406                                 data0 = 0;
3407                                 data1 = 0;
3408
3409                                 while (j < max_entries) {
3410                                         if (mtable[itable[j]] != i) {
3411                                                 j++;
3412                                                 continue;
3413                                         }
3414                                         vxge_hw_rts_rth_data0_data1_get(j,
3415                                                 &data0, &data1, 1, itable);
3416                                         j++;
3417                                         break;
3418                                 }
3419
3420                                 while (j < max_entries) {
3421                                         if (mtable[itable[j]] != i) {
3422                                                 j++;
3423                                                 continue;
3424                                         }
3425                                         vxge_hw_rts_rth_data0_data1_get(j,
3426                                                 &data0, &data1, 2, itable);
3427                                         j++;
3428                                         break;
3429                                 }
3430
3431                                 while (j < max_entries) {
3432                                         if (mtable[itable[j]] != i) {
3433                                                 j++;
3434                                                 continue;
3435                                         }
3436                                         vxge_hw_rts_rth_data0_data1_get(j,
3437                                                 &data0, &data1, 3, itable);
3438                                         j++;
3439                                         break;
3440                                 }
3441
3442                                 while (j < max_entries) {
3443                                         if (mtable[itable[j]] != i) {
3444                                                 j++;
3445                                                 continue;
3446                                         }
3447                                         vxge_hw_rts_rth_data0_data1_get(j,
3448                                                 &data0, &data1, 4, itable);
3449                                         j++;
3450                                         break;
3451                                 }
3452
3453                                 if (data0 != 0) {
3454                                         status = __vxge_hw_vpath_rts_table_set(
3455                                                         vpath_handles[i],
3456                                                         action, rts_table,
3457                                                         0, data0, data1);
3458
3459                                         if (status != VXGE_HW_OK)
3460                                                 goto exit;
3461                                 }
3462                         }
3463                 }
3464         }
3465 exit:
3466         return status;
3467 }
3468
3469 /**
3470  * vxge_hw_vpath_check_leak - Check for memory leak
3471  * @ringh: Handle to the ring object used for receive
3472  *
3473  * If PRC_RXD_DOORBELL_VPn.NEW_QW_CNT is larger or equal to
3474  * PRC_CFG6_VPn.RXD_SPAT then a leak has occurred.
3475  * Returns: VXGE_HW_FAIL, if leak has occurred.
3476  *
3477  */
3478 enum vxge_hw_status
3479 vxge_hw_vpath_check_leak(struct __vxge_hw_ring *ring)
3480 {
3481         enum vxge_hw_status status = VXGE_HW_OK;
3482         u64 rxd_new_count, rxd_spat;
3483
3484         if (ring == NULL)
3485                 return status;
3486
3487         rxd_new_count = readl(&ring->vp_reg->prc_rxd_doorbell);
3488         rxd_spat = readq(&ring->vp_reg->prc_cfg6);
3489         rxd_spat = VXGE_HW_PRC_CFG6_RXD_SPAT(rxd_spat);
3490
3491         if (rxd_new_count >= rxd_spat)
3492                 status = VXGE_HW_FAIL;
3493
3494         return status;
3495 }
3496
3497 /*
3498  * __vxge_hw_vpath_mgmt_read
3499  * This routine reads the vpath_mgmt registers
3500  */
3501 static enum vxge_hw_status
3502 __vxge_hw_vpath_mgmt_read(
3503         struct __vxge_hw_device *hldev,
3504         struct __vxge_hw_virtualpath *vpath)
3505 {
3506         u32 i, mtu = 0, max_pyld = 0;
3507         u64 val64;
3508         enum vxge_hw_status status = VXGE_HW_OK;
3509
3510         for (i = 0; i < VXGE_HW_MAC_MAX_MAC_PORT_ID; i++) {
3511
3512                 val64 = readq(&vpath->vpmgmt_reg->
3513                                 rxmac_cfg0_port_vpmgmt_clone[i]);
3514                 max_pyld =
3515                         (u32)
3516                         VXGE_HW_RXMAC_CFG0_PORT_VPMGMT_CLONE_GET_MAX_PYLD_LEN
3517                         (val64);
3518                 if (mtu < max_pyld)
3519                         mtu = max_pyld;
3520         }
3521
3522         vpath->max_mtu = mtu + VXGE_HW_MAC_HEADER_MAX_SIZE;
3523
3524         val64 = readq(&vpath->vpmgmt_reg->xmac_vsport_choices_vp);
3525
3526         for (i = 0; i < VXGE_HW_MAX_VIRTUAL_PATHS; i++) {
3527                 if (val64 & vxge_mBIT(i))
3528                         vpath->vsport_number = i;
3529         }
3530
3531         val64 = readq(&vpath->vpmgmt_reg->xgmac_gen_status_vpmgmt_clone);
3532
3533         if (val64 & VXGE_HW_XGMAC_GEN_STATUS_VPMGMT_CLONE_XMACJ_NTWK_OK)
3534                 VXGE_HW_DEVICE_LINK_STATE_SET(vpath->hldev, VXGE_HW_LINK_UP);
3535         else
3536                 VXGE_HW_DEVICE_LINK_STATE_SET(vpath->hldev, VXGE_HW_LINK_DOWN);
3537
3538         return status;
3539 }
3540
3541 /*
3542  * __vxge_hw_vpath_reset_check - Check if resetting the vpath completed
3543  * This routine checks the vpath_rst_in_prog register to see if
3544  * adapter completed the reset process for the vpath
3545  */
3546 enum vxge_hw_status
3547 __vxge_hw_vpath_reset_check(struct __vxge_hw_virtualpath *vpath)
3548 {
3549         enum vxge_hw_status status;
3550
3551         status = __vxge_hw_device_register_poll(
3552                         &vpath->hldev->common_reg->vpath_rst_in_prog,
3553                         VXGE_HW_VPATH_RST_IN_PROG_VPATH_RST_IN_PROG(
3554                                 1 << (16 - vpath->vp_id)),
3555                         vpath->hldev->config.device_poll_millis);
3556
3557         return status;
3558 }
3559
3560 /*
3561  * __vxge_hw_vpath_reset
3562  * This routine resets the vpath on the device
3563  */
3564 enum vxge_hw_status
3565 __vxge_hw_vpath_reset(struct __vxge_hw_device *hldev, u32 vp_id)
3566 {
3567         u64 val64;
3568         enum vxge_hw_status status = VXGE_HW_OK;
3569
3570         val64 = VXGE_HW_CMN_RSTHDLR_CFG0_SW_RESET_VPATH(1 << (16 - vp_id));
3571
3572         __vxge_hw_pio_mem_write32_upper((u32)vxge_bVALn(val64, 0, 32),
3573                                 &hldev->common_reg->cmn_rsthdlr_cfg0);
3574
3575         return status;
3576 }
3577
3578 /*
3579  * __vxge_hw_vpath_sw_reset
3580  * This routine resets the vpath structures
3581  */
3582 enum vxge_hw_status
3583 __vxge_hw_vpath_sw_reset(struct __vxge_hw_device *hldev, u32 vp_id)
3584 {
3585         enum vxge_hw_status status = VXGE_HW_OK;
3586         struct __vxge_hw_virtualpath *vpath;
3587
3588         vpath = (struct __vxge_hw_virtualpath *)&hldev->virtual_paths[vp_id];
3589
3590         if (vpath->ringh) {
3591                 status = __vxge_hw_ring_reset(vpath->ringh);
3592                 if (status != VXGE_HW_OK)
3593                         goto exit;
3594         }
3595
3596         if (vpath->fifoh)
3597                 status = __vxge_hw_fifo_reset(vpath->fifoh);
3598 exit:
3599         return status;
3600 }
3601
3602 /*
3603  * __vxge_hw_vpath_prc_configure
3604  * This routine configures the prc registers of virtual path using the config
3605  * passed
3606  */
3607 void
3608 __vxge_hw_vpath_prc_configure(struct __vxge_hw_device *hldev, u32 vp_id)
3609 {
3610         u64 val64;
3611         struct __vxge_hw_virtualpath *vpath;
3612         struct vxge_hw_vp_config *vp_config;
3613         struct vxge_hw_vpath_reg __iomem *vp_reg;
3614
3615         vpath = &hldev->virtual_paths[vp_id];
3616         vp_reg = vpath->vp_reg;
3617         vp_config = vpath->vp_config;
3618
3619         if (vp_config->ring.enable == VXGE_HW_RING_DISABLE)
3620                 return;
3621
3622         val64 = readq(&vp_reg->prc_cfg1);
3623         val64 |= VXGE_HW_PRC_CFG1_RTI_TINT_DISABLE;
3624         writeq(val64, &vp_reg->prc_cfg1);
3625
3626         val64 = readq(&vpath->vp_reg->prc_cfg6);
3627         val64 |= VXGE_HW_PRC_CFG6_DOORBELL_MODE_EN;
3628         writeq(val64, &vpath->vp_reg->prc_cfg6);
3629
3630         val64 = readq(&vp_reg->prc_cfg7);
3631
3632         if (vpath->vp_config->ring.scatter_mode !=
3633                 VXGE_HW_RING_SCATTER_MODE_USE_FLASH_DEFAULT) {
3634
3635                 val64 &= ~VXGE_HW_PRC_CFG7_SCATTER_MODE(0x3);
3636
3637                 switch (vpath->vp_config->ring.scatter_mode) {
3638                 case VXGE_HW_RING_SCATTER_MODE_A:
3639                         val64 |= VXGE_HW_PRC_CFG7_SCATTER_MODE(
3640                                         VXGE_HW_PRC_CFG7_SCATTER_MODE_A);
3641                         break;
3642                 case VXGE_HW_RING_SCATTER_MODE_B:
3643                         val64 |= VXGE_HW_PRC_CFG7_SCATTER_MODE(
3644                                         VXGE_HW_PRC_CFG7_SCATTER_MODE_B);
3645                         break;
3646                 case VXGE_HW_RING_SCATTER_MODE_C:
3647                         val64 |= VXGE_HW_PRC_CFG7_SCATTER_MODE(
3648                                         VXGE_HW_PRC_CFG7_SCATTER_MODE_C);
3649                         break;
3650                 }
3651         }
3652
3653         writeq(val64, &vp_reg->prc_cfg7);
3654
3655         writeq(VXGE_HW_PRC_CFG5_RXD0_ADD(
3656                                 __vxge_hw_ring_first_block_address_get(
3657                                         vpath->ringh) >> 3), &vp_reg->prc_cfg5);
3658
3659         val64 = readq(&vp_reg->prc_cfg4);
3660         val64 |= VXGE_HW_PRC_CFG4_IN_SVC;
3661         val64 &= ~VXGE_HW_PRC_CFG4_RING_MODE(0x3);
3662
3663         val64 |= VXGE_HW_PRC_CFG4_RING_MODE(
3664                         VXGE_HW_PRC_CFG4_RING_MODE_ONE_BUFFER);
3665
3666         if (hldev->config.rth_en == VXGE_HW_RTH_DISABLE)
3667                 val64 |= VXGE_HW_PRC_CFG4_RTH_DISABLE;
3668         else
3669                 val64 &= ~VXGE_HW_PRC_CFG4_RTH_DISABLE;
3670
3671         writeq(val64, &vp_reg->prc_cfg4);
3672         return;
3673 }
3674
3675 /*
3676  * __vxge_hw_vpath_kdfc_configure
3677  * This routine configures the kdfc registers of virtual path using the
3678  * config passed
3679  */
3680 enum vxge_hw_status
3681 __vxge_hw_vpath_kdfc_configure(struct __vxge_hw_device *hldev, u32 vp_id)
3682 {
3683         u64 val64;
3684         u64 vpath_stride;
3685         enum vxge_hw_status status = VXGE_HW_OK;
3686         struct __vxge_hw_virtualpath *vpath;
3687         struct vxge_hw_vpath_reg __iomem *vp_reg;
3688
3689         vpath = &hldev->virtual_paths[vp_id];
3690         vp_reg = vpath->vp_reg;
3691         status = __vxge_hw_kdfc_swapper_set(hldev->legacy_reg, vp_reg);
3692
3693         if (status != VXGE_HW_OK)
3694                 goto exit;
3695
3696         val64 = readq(&vp_reg->kdfc_drbl_triplet_total);
3697
3698         vpath->max_kdfc_db =
3699                 (u32)VXGE_HW_KDFC_DRBL_TRIPLET_TOTAL_GET_KDFC_MAX_SIZE(
3700                         val64+1)/2;
3701
3702         if (vpath->vp_config->fifo.enable == VXGE_HW_FIFO_ENABLE) {
3703
3704                 vpath->max_nofl_db = vpath->max_kdfc_db;
3705
3706                 if (vpath->max_nofl_db <
3707                         ((vpath->vp_config->fifo.memblock_size /
3708                         (vpath->vp_config->fifo.max_frags *
3709                         sizeof(struct vxge_hw_fifo_txd))) *
3710                         vpath->vp_config->fifo.fifo_blocks)) {
3711
3712                         return VXGE_HW_BADCFG_FIFO_BLOCKS;
3713                 }
3714                 val64 = VXGE_HW_KDFC_FIFO_TRPL_PARTITION_LENGTH_0(
3715                                 (vpath->max_nofl_db*2)-1);
3716         }
3717
3718         writeq(val64, &vp_reg->kdfc_fifo_trpl_partition);
3719
3720         writeq(VXGE_HW_KDFC_FIFO_TRPL_CTRL_TRIPLET_ENABLE,
3721                 &vp_reg->kdfc_fifo_trpl_ctrl);
3722
3723         val64 = readq(&vp_reg->kdfc_trpl_fifo_0_ctrl);
3724
3725         val64 &= ~(VXGE_HW_KDFC_TRPL_FIFO_0_CTRL_MODE(0x3) |
3726                    VXGE_HW_KDFC_TRPL_FIFO_0_CTRL_SELECT(0xFF));
3727
3728         val64 |= VXGE_HW_KDFC_TRPL_FIFO_0_CTRL_MODE(
3729                  VXGE_HW_KDFC_TRPL_FIFO_0_CTRL_MODE_NON_OFFLOAD_ONLY) |
3730 #ifndef __BIG_ENDIAN
3731                  VXGE_HW_KDFC_TRPL_FIFO_0_CTRL_SWAP_EN |
3732 #endif
3733                  VXGE_HW_KDFC_TRPL_FIFO_0_CTRL_SELECT(0);
3734
3735         writeq(val64, &vp_reg->kdfc_trpl_fifo_0_ctrl);
3736         writeq((u64)0, &vp_reg->kdfc_trpl_fifo_0_wb_address);
3737         wmb();
3738         vpath_stride = readq(&hldev->toc_reg->toc_kdfc_vpath_stride);
3739
3740         vpath->nofl_db =
3741                 (struct __vxge_hw_non_offload_db_wrapper __iomem *)
3742                 (hldev->kdfc + (vp_id *
3743                 VXGE_HW_TOC_KDFC_VPATH_STRIDE_GET_TOC_KDFC_VPATH_STRIDE(
3744                                         vpath_stride)));
3745 exit:
3746         return status;
3747 }
3748
3749 /*
3750  * __vxge_hw_vpath_mac_configure
3751  * This routine configures the mac of virtual path using the config passed
3752  */
3753 enum vxge_hw_status
3754 __vxge_hw_vpath_mac_configure(struct __vxge_hw_device *hldev, u32 vp_id)
3755 {
3756         u64 val64;
3757         enum vxge_hw_status status = VXGE_HW_OK;
3758         struct __vxge_hw_virtualpath *vpath;
3759         struct vxge_hw_vp_config *vp_config;
3760         struct vxge_hw_vpath_reg __iomem *vp_reg;
3761
3762         vpath = &hldev->virtual_paths[vp_id];
3763         vp_reg = vpath->vp_reg;
3764         vp_config = vpath->vp_config;
3765
3766         writeq(VXGE_HW_XMAC_VSPORT_CHOICE_VSPORT_NUMBER(
3767                         vpath->vsport_number), &vp_reg->xmac_vsport_choice);
3768
3769         if (vp_config->ring.enable == VXGE_HW_RING_ENABLE) {
3770
3771                 val64 = readq(&vp_reg->xmac_rpa_vcfg);
3772
3773                 if (vp_config->rpa_strip_vlan_tag !=
3774                         VXGE_HW_VPATH_RPA_STRIP_VLAN_TAG_USE_FLASH_DEFAULT) {
3775                         if (vp_config->rpa_strip_vlan_tag)
3776                                 val64 |= VXGE_HW_XMAC_RPA_VCFG_STRIP_VLAN_TAG;
3777                         else
3778                                 val64 &= ~VXGE_HW_XMAC_RPA_VCFG_STRIP_VLAN_TAG;
3779                 }
3780
3781                 writeq(val64, &vp_reg->xmac_rpa_vcfg);
3782                 val64 = readq(&vp_reg->rxmac_vcfg0);
3783
3784                 if (vp_config->mtu !=
3785                                 VXGE_HW_VPATH_USE_FLASH_DEFAULT_INITIAL_MTU) {
3786                         val64 &= ~VXGE_HW_RXMAC_VCFG0_RTS_MAX_FRM_LEN(0x3fff);
3787                         if ((vp_config->mtu  +
3788                                 VXGE_HW_MAC_HEADER_MAX_SIZE) < vpath->max_mtu)
3789                                 val64 |= VXGE_HW_RXMAC_VCFG0_RTS_MAX_FRM_LEN(
3790                                         vp_config->mtu  +
3791                                         VXGE_HW_MAC_HEADER_MAX_SIZE);
3792                         else
3793                                 val64 |= VXGE_HW_RXMAC_VCFG0_RTS_MAX_FRM_LEN(
3794                                         vpath->max_mtu);
3795                 }
3796
3797                 writeq(val64, &vp_reg->rxmac_vcfg0);
3798
3799                 val64 = readq(&vp_reg->rxmac_vcfg1);
3800
3801                 val64 &= ~(VXGE_HW_RXMAC_VCFG1_RTS_RTH_MULTI_IT_BD_MODE(0x3) |
3802                         VXGE_HW_RXMAC_VCFG1_RTS_RTH_MULTI_IT_EN_MODE);
3803
3804                 if (hldev->config.rth_it_type ==
3805                                 VXGE_HW_RTH_IT_TYPE_MULTI_IT) {
3806                         val64 |= VXGE_HW_RXMAC_VCFG1_RTS_RTH_MULTI_IT_BD_MODE(
3807                                 0x2) |
3808                                 VXGE_HW_RXMAC_VCFG1_RTS_RTH_MULTI_IT_EN_MODE;
3809                 }
3810
3811                 writeq(val64, &vp_reg->rxmac_vcfg1);
3812         }
3813         return status;
3814 }
3815
3816 /*
3817  * __vxge_hw_vpath_tim_configure
3818  * This routine configures the tim registers of virtual path using the config
3819  * passed
3820  */
3821 enum vxge_hw_status
3822 __vxge_hw_vpath_tim_configure(struct __vxge_hw_device *hldev, u32 vp_id)
3823 {
3824         u64 val64;
3825         enum vxge_hw_status status = VXGE_HW_OK;
3826         struct __vxge_hw_virtualpath *vpath;
3827         struct vxge_hw_vpath_reg __iomem *vp_reg;
3828         struct vxge_hw_vp_config *config;
3829
3830         vpath = &hldev->virtual_paths[vp_id];
3831         vp_reg = vpath->vp_reg;
3832         config = vpath->vp_config;
3833
3834         writeq((u64)0, &vp_reg->tim_dest_addr);
3835         writeq((u64)0, &vp_reg->tim_vpath_map);
3836         writeq((u64)0, &vp_reg->tim_bitmap);
3837         writeq((u64)0, &vp_reg->tim_remap);
3838
3839         if (config->ring.enable == VXGE_HW_RING_ENABLE)
3840                 writeq(VXGE_HW_TIM_RING_ASSN_INT_NUM(
3841                         (vp_id * VXGE_HW_MAX_INTR_PER_VP) +
3842                         VXGE_HW_VPATH_INTR_RX), &vp_reg->tim_ring_assn);
3843
3844         val64 = readq(&vp_reg->tim_pci_cfg);
3845         val64 |= VXGE_HW_TIM_PCI_CFG_ADD_PAD;
3846         writeq(val64, &vp_reg->tim_pci_cfg);
3847
3848         if (config->fifo.enable == VXGE_HW_FIFO_ENABLE) {
3849
3850                 val64 = readq(&vp_reg->tim_cfg1_int_num[VXGE_HW_VPATH_INTR_TX]);
3851
3852                 if (config->tti.btimer_val != VXGE_HW_USE_FLASH_DEFAULT) {
3853                         val64 &= ~VXGE_HW_TIM_CFG1_INT_NUM_BTIMER_VAL(
3854                                 0x3ffffff);
3855                         val64 |= VXGE_HW_TIM_CFG1_INT_NUM_BTIMER_VAL(
3856                                         config->tti.btimer_val);
3857                 }
3858
3859                 val64 &= ~VXGE_HW_TIM_CFG1_INT_NUM_BITMP_EN;
3860
3861                 if (config->tti.timer_ac_en != VXGE_HW_USE_FLASH_DEFAULT) {
3862                         if (config->tti.timer_ac_en)
3863                                 val64 |= VXGE_HW_TIM_CFG1_INT_NUM_TIMER_AC;
3864                         else
3865                                 val64 &= ~VXGE_HW_TIM_CFG1_INT_NUM_TIMER_AC;
3866                 }
3867
3868                 if (config->tti.timer_ci_en != VXGE_HW_USE_FLASH_DEFAULT) {
3869                         if (config->tti.timer_ci_en)
3870                                 val64 |= VXGE_HW_TIM_CFG1_INT_NUM_TIMER_CI;
3871                         else
3872                                 val64 &= ~VXGE_HW_TIM_CFG1_INT_NUM_TIMER_CI;
3873                 }
3874
3875                 if (config->tti.urange_a != VXGE_HW_USE_FLASH_DEFAULT) {
3876                         val64 &= ~VXGE_HW_TIM_CFG1_INT_NUM_URNG_A(0x3f);
3877                         val64 |= VXGE_HW_TIM_CFG1_INT_NUM_URNG_A(
3878                                         config->tti.urange_a);
3879                 }
3880
3881                 if (config->tti.urange_b != VXGE_HW_USE_FLASH_DEFAULT) {
3882                         val64 &= ~VXGE_HW_TIM_CFG1_INT_NUM_URNG_B(0x3f);
3883                         val64 |= VXGE_HW_TIM_CFG1_INT_NUM_URNG_B(
3884                                         config->tti.urange_b);
3885                 }
3886
3887                 if (config->tti.urange_c != VXGE_HW_USE_FLASH_DEFAULT) {
3888                         val64 &= ~VXGE_HW_TIM_CFG1_INT_NUM_URNG_C(0x3f);
3889                         val64 |= VXGE_HW_TIM_CFG1_INT_NUM_URNG_C(
3890                                         config->tti.urange_c);
3891                 }
3892
3893                 writeq(val64, &vp_reg->tim_cfg1_int_num[VXGE_HW_VPATH_INTR_TX]);
3894                 val64 = readq(&vp_reg->tim_cfg2_int_num[VXGE_HW_VPATH_INTR_TX]);
3895
3896                 if (config->tti.uec_a != VXGE_HW_USE_FLASH_DEFAULT) {
3897                         val64 &= ~VXGE_HW_TIM_CFG2_INT_NUM_UEC_A(0xffff);
3898                         val64 |= VXGE_HW_TIM_CFG2_INT_NUM_UEC_A(
3899                                                 config->tti.uec_a);
3900                 }
3901
3902                 if (config->tti.uec_b != VXGE_HW_USE_FLASH_DEFAULT) {
3903                         val64 &= ~VXGE_HW_TIM_CFG2_INT_NUM_UEC_B(0xffff);
3904                         val64 |= VXGE_HW_TIM_CFG2_INT_NUM_UEC_B(
3905                                                 config->tti.uec_b);
3906                 }
3907
3908                 if (config->tti.uec_c != VXGE_HW_USE_FLASH_DEFAULT) {
3909                         val64 &= ~VXGE_HW_TIM_CFG2_INT_NUM_UEC_C(0xffff);
3910                         val64 |= VXGE_HW_TIM_CFG2_INT_NUM_UEC_C(
3911                                                 config->tti.uec_c);
3912                 }
3913
3914                 if (config->tti.uec_d != VXGE_HW_USE_FLASH_DEFAULT) {
3915                         val64 &= ~VXGE_HW_TIM_CFG2_INT_NUM_UEC_D(0xffff);
3916                         val64 |= VXGE_HW_TIM_CFG2_INT_NUM_UEC_D(
3917                                                 config->tti.uec_d);
3918                 }
3919
3920                 writeq(val64, &vp_reg->tim_cfg2_int_num[VXGE_HW_VPATH_INTR_TX]);
3921                 val64 = readq(&vp_reg->tim_cfg3_int_num[VXGE_HW_VPATH_INTR_TX]);
3922
3923                 if (config->tti.timer_ri_en != VXGE_HW_USE_FLASH_DEFAULT) {
3924                         if (config->tti.timer_ri_en)
3925                                 val64 |= VXGE_HW_TIM_CFG3_INT_NUM_TIMER_RI;
3926                         else
3927                                 val64 &= ~VXGE_HW_TIM_CFG3_INT_NUM_TIMER_RI;
3928                 }
3929
3930                 if (config->tti.rtimer_val != VXGE_HW_USE_FLASH_DEFAULT) {
3931                         val64 &= ~VXGE_HW_TIM_CFG3_INT_NUM_RTIMER_VAL(
3932                                         0x3ffffff);
3933                         val64 |= VXGE_HW_TIM_CFG3_INT_NUM_RTIMER_VAL(
3934                                         config->tti.rtimer_val);
3935                 }
3936
3937                 if (config->tti.util_sel != VXGE_HW_USE_FLASH_DEFAULT) {
3938                         val64 &= ~VXGE_HW_TIM_CFG3_INT_NUM_UTIL_SEL(0x3f);
3939                         val64 |= VXGE_HW_TIM_CFG3_INT_NUM_UTIL_SEL(
3940                                         config->tti.util_sel);
3941                 }
3942
3943                 if (config->tti.ltimer_val != VXGE_HW_USE_FLASH_DEFAULT) {
3944                         val64 &= ~VXGE_HW_TIM_CFG3_INT_NUM_LTIMER_VAL(
3945                                         0x3ffffff);
3946                         val64 |= VXGE_HW_TIM_CFG3_INT_NUM_LTIMER_VAL(
3947                                         config->tti.ltimer_val);
3948                 }
3949
3950                 writeq(val64, &vp_reg->tim_cfg3_int_num[VXGE_HW_VPATH_INTR_TX]);
3951         }
3952
3953         if (config->ring.enable == VXGE_HW_RING_ENABLE) {
3954
3955                 val64 = readq(&vp_reg->tim_cfg1_int_num[VXGE_HW_VPATH_INTR_RX]);
3956
3957                 if (config->rti.btimer_val != VXGE_HW_USE_FLASH_DEFAULT) {
3958                         val64 &= ~VXGE_HW_TIM_CFG1_INT_NUM_BTIMER_VAL(
3959                                         0x3ffffff);
3960                         val64 |= VXGE_HW_TIM_CFG1_INT_NUM_BTIMER_VAL(
3961                                         config->rti.btimer_val);
3962                 }
3963
3964                 val64 &= ~VXGE_HW_TIM_CFG1_INT_NUM_BITMP_EN;
3965
3966                 if (config->rti.timer_ac_en != VXGE_HW_USE_FLASH_DEFAULT) {
3967                         if (config->rti.timer_ac_en)
3968                                 val64 |= VXGE_HW_TIM_CFG1_INT_NUM_TIMER_AC;
3969                         else
3970                                 val64 &= ~VXGE_HW_TIM_CFG1_INT_NUM_TIMER_AC;
3971                 }
3972
3973                 if (config->rti.timer_ci_en != VXGE_HW_USE_FLASH_DEFAULT) {
3974                         if (config->rti.timer_ci_en)
3975                                 val64 |= VXGE_HW_TIM_CFG1_INT_NUM_TIMER_CI;
3976                         else
3977                                 val64 &= ~VXGE_HW_TIM_CFG1_INT_NUM_TIMER_CI;
3978                 }
3979
3980                 if (config->rti.urange_a != VXGE_HW_USE_FLASH_DEFAULT) {
3981                         val64 &= ~VXGE_HW_TIM_CFG1_INT_NUM_URNG_A(0x3f);
3982                         val64 |= VXGE_HW_TIM_CFG1_INT_NUM_URNG_A(
3983                                         config->rti.urange_a);
3984                 }
3985
3986                 if (config->rti.urange_b != VXGE_HW_USE_FLASH_DEFAULT) {
3987                         val64 &= ~VXGE_HW_TIM_CFG1_INT_NUM_URNG_B(0x3f);
3988                         val64 |= VXGE_HW_TIM_CFG1_INT_NUM_URNG_B(
3989                                         config->rti.urange_b);
3990                 }
3991
3992                 if (config->rti.urange_c != VXGE_HW_USE_FLASH_DEFAULT) {
3993                         val64 &= ~VXGE_HW_TIM_CFG1_INT_NUM_URNG_C(0x3f);
3994                         val64 |= VXGE_HW_TIM_CFG1_INT_NUM_URNG_C(
3995                                         config->rti.urange_c);
3996                 }
3997
3998                 writeq(val64, &vp_reg->tim_cfg1_int_num[VXGE_HW_VPATH_INTR_RX]);
3999                 val64 = readq(&vp_reg->tim_cfg2_int_num[VXGE_HW_VPATH_INTR_RX]);
4000
4001                 if (config->rti.uec_a != VXGE_HW_USE_FLASH_DEFAULT) {
4002                         val64 &= ~VXGE_HW_TIM_CFG2_INT_NUM_UEC_A(0xffff);
4003                         val64 |= VXGE_HW_TIM_CFG2_INT_NUM_UEC_A(
4004                                                 config->rti.uec_a);
4005                 }
4006
4007                 if (config->rti.uec_b != VXGE_HW_USE_FLASH_DEFAULT) {
4008                         val64 &= ~VXGE_HW_TIM_CFG2_INT_NUM_UEC_B(0xffff);
4009                         val64 |= VXGE_HW_TIM_CFG2_INT_NUM_UEC_B(
4010                                                 config->rti.uec_b);
4011                 }
4012
4013                 if (config->rti.uec_c != VXGE_HW_USE_FLASH_DEFAULT) {
4014                         val64 &= ~VXGE_HW_TIM_CFG2_INT_NUM_UEC_C(0xffff);
4015                         val64 |= VXGE_HW_TIM_CFG2_INT_NUM_UEC_C(
4016                                                 config->rti.uec_c);
4017                 }
4018
4019                 if (config->rti.uec_d != VXGE_HW_USE_FLASH_DEFAULT) {
4020                         val64 &= ~VXGE_HW_TIM_CFG2_INT_NUM_UEC_D(0xffff);
4021                         val64 |= VXGE_HW_TIM_CFG2_INT_NUM_UEC_D(
4022                                                 config->rti.uec_d);
4023                 }
4024
4025                 writeq(val64, &vp_reg->tim_cfg2_int_num[VXGE_HW_VPATH_INTR_RX]);
4026                 val64 = readq(&vp_reg->tim_cfg3_int_num[VXGE_HW_VPATH_INTR_RX]);
4027
4028                 if (config->rti.timer_ri_en != VXGE_HW_USE_FLASH_DEFAULT) {
4029                         if (config->rti.timer_ri_en)
4030                                 val64 |= VXGE_HW_TIM_CFG3_INT_NUM_TIMER_RI;
4031                         else
4032                                 val64 &= ~VXGE_HW_TIM_CFG3_INT_NUM_TIMER_RI;
4033                 }
4034
4035                 if (config->rti.rtimer_val != VXGE_HW_USE_FLASH_DEFAULT) {
4036                         val64 &= ~VXGE_HW_TIM_CFG3_INT_NUM_RTIMER_VAL(
4037                                         0x3ffffff);
4038                         val64 |= VXGE_HW_TIM_CFG3_INT_NUM_RTIMER_VAL(
4039                                         config->rti.rtimer_val);
4040                 }
4041
4042                 if (config->rti.util_sel != VXGE_HW_USE_FLASH_DEFAULT) {
4043                         val64 &= ~VXGE_HW_TIM_CFG3_INT_NUM_UTIL_SEL(0x3f);
4044                         val64 |= VXGE_HW_TIM_CFG3_INT_NUM_UTIL_SEL(
4045                                         config->rti.util_sel);
4046                 }
4047
4048                 if (config->rti.ltimer_val != VXGE_HW_USE_FLASH_DEFAULT) {
4049                         val64 &= ~VXGE_HW_TIM_CFG3_INT_NUM_LTIMER_VAL(
4050                                         0x3ffffff);
4051                         val64 |= VXGE_HW_TIM_CFG3_INT_NUM_LTIMER_VAL(
4052                                         config->rti.ltimer_val);
4053                 }
4054
4055                 writeq(val64, &vp_reg->tim_cfg3_int_num[VXGE_HW_VPATH_INTR_RX]);
4056         }
4057
4058         val64 = 0;
4059         writeq(val64, &vp_reg->tim_cfg1_int_num[VXGE_HW_VPATH_INTR_EINTA]);
4060         writeq(val64, &vp_reg->tim_cfg2_int_num[VXGE_HW_VPATH_INTR_EINTA]);
4061         writeq(val64, &vp_reg->tim_cfg3_int_num[VXGE_HW_VPATH_INTR_EINTA]);
4062         writeq(val64, &vp_reg->tim_cfg1_int_num[VXGE_HW_VPATH_INTR_BMAP]);
4063         writeq(val64, &vp_reg->tim_cfg2_int_num[VXGE_HW_VPATH_INTR_BMAP]);
4064         writeq(val64, &vp_reg->tim_cfg3_int_num[VXGE_HW_VPATH_INTR_BMAP]);
4065
4066         return status;
4067 }
4068
4069 /*
4070  * __vxge_hw_vpath_initialize
4071  * This routine is the final phase of init which initializes the
4072  * registers of the vpath using the configuration passed.
4073  */
4074 enum vxge_hw_status
4075 __vxge_hw_vpath_initialize(struct __vxge_hw_device *hldev, u32 vp_id)
4076 {
4077         u64 val64;
4078         u32 val32;
4079         enum vxge_hw_status status = VXGE_HW_OK;
4080         struct __vxge_hw_virtualpath *vpath;
4081         struct vxge_hw_vpath_reg __iomem *vp_reg;
4082
4083         vpath = &hldev->virtual_paths[vp_id];
4084
4085         if (!(hldev->vpath_assignments & vxge_mBIT(vp_id))) {
4086                 status = VXGE_HW_ERR_VPATH_NOT_AVAILABLE;
4087                 goto exit;
4088         }
4089         vp_reg = vpath->vp_reg;
4090
4091         status =  __vxge_hw_vpath_swapper_set(vpath->vp_reg);
4092
4093         if (status != VXGE_HW_OK)
4094                 goto exit;
4095
4096         status =  __vxge_hw_vpath_mac_configure(hldev, vp_id);
4097
4098         if (status != VXGE_HW_OK)
4099                 goto exit;
4100
4101         status =  __vxge_hw_vpath_kdfc_configure(hldev, vp_id);
4102
4103         if (status != VXGE_HW_OK)
4104                 goto exit;
4105
4106         status = __vxge_hw_vpath_tim_configure(hldev, vp_id);
4107
4108         if (status != VXGE_HW_OK)
4109                 goto exit;
4110
4111         writeq(0, &vp_reg->gendma_int);
4112
4113         val64 = readq(&vp_reg->rtdma_rd_optimization_ctrl);
4114
4115         /* Get MRRS value from device control */
4116         status  = __vxge_hw_vpath_pci_read(vpath, 1, 0x78, &val32);
4117
4118         if (status == VXGE_HW_OK) {
4119                 val32 = (val32 & VXGE_HW_PCI_EXP_DEVCTL_READRQ) >> 12;
4120                 val64 &=
4121                     ~(VXGE_HW_RTDMA_RD_OPTIMIZATION_CTRL_FB_FILL_THRESH(7));
4122                 val64 |=
4123                     VXGE_HW_RTDMA_RD_OPTIMIZATION_CTRL_FB_FILL_THRESH(val32);
4124
4125                 val64 |= VXGE_HW_RTDMA_RD_OPTIMIZATION_CTRL_FB_WAIT_FOR_SPACE;
4126         }
4127
4128         val64 &= ~(VXGE_HW_RTDMA_RD_OPTIMIZATION_CTRL_FB_ADDR_BDRY(7));
4129         val64 |=
4130             VXGE_HW_RTDMA_RD_OPTIMIZATION_CTRL_FB_ADDR_BDRY(
4131                     VXGE_HW_MAX_PAYLOAD_SIZE_512);
4132
4133         val64 |= VXGE_HW_RTDMA_RD_OPTIMIZATION_CTRL_FB_ADDR_BDRY_EN;
4134         writeq(val64, &vp_reg->rtdma_rd_optimization_ctrl);
4135
4136 exit:
4137         return status;
4138 }
4139
4140 /*
4141  * __vxge_hw_vp_initialize - Initialize Virtual Path structure
4142  * This routine is the initial phase of init which resets the vpath and
4143  * initializes the software support structures.
4144  */
4145 enum vxge_hw_status
4146 __vxge_hw_vp_initialize(struct __vxge_hw_device *hldev, u32 vp_id,
4147                         struct vxge_hw_vp_config *config)
4148 {
4149         struct __vxge_hw_virtualpath *vpath;
4150         enum vxge_hw_status status = VXGE_HW_OK;
4151
4152         if (!(hldev->vpath_assignments & vxge_mBIT(vp_id))) {
4153                 status = VXGE_HW_ERR_VPATH_NOT_AVAILABLE;
4154                 goto exit;
4155         }
4156
4157         vpath = &hldev->virtual_paths[vp_id];
4158
4159         vpath->vp_id = vp_id;
4160         vpath->vp_open = VXGE_HW_VP_OPEN;
4161         vpath->hldev = hldev;
4162         vpath->vp_config = config;
4163         vpath->vp_reg = hldev->vpath_reg[vp_id];
4164         vpath->vpmgmt_reg = hldev->vpmgmt_reg[vp_id];
4165
4166         __vxge_hw_vpath_reset(hldev, vp_id);
4167
4168         status = __vxge_hw_vpath_reset_check(vpath);
4169
4170         if (status != VXGE_HW_OK) {
4171                 memset(vpath, 0, sizeof(struct __vxge_hw_virtualpath));
4172                 goto exit;
4173         }
4174
4175         status = __vxge_hw_vpath_mgmt_read(hldev, vpath);
4176
4177         if (status != VXGE_HW_OK) {
4178                 memset(vpath, 0, sizeof(struct __vxge_hw_virtualpath));
4179                 goto exit;
4180         }
4181
4182         INIT_LIST_HEAD(&vpath->vpath_handles);
4183
4184         vpath->sw_stats = &hldev->stats.sw_dev_info_stats.vpath_info[vp_id];
4185
4186         VXGE_HW_DEVICE_TIM_INT_MASK_SET(hldev->tim_int_mask0,
4187                 hldev->tim_int_mask1, vp_id);
4188
4189         status = __vxge_hw_vpath_initialize(hldev, vp_id);
4190
4191         if (status != VXGE_HW_OK)
4192                 __vxge_hw_vp_terminate(hldev, vp_id);
4193 exit:
4194         return status;
4195 }
4196
4197 /*
4198  * __vxge_hw_vp_terminate - Terminate Virtual Path structure
4199  * This routine closes all channels it opened and freeup memory
4200  */
4201 void
4202 __vxge_hw_vp_terminate(struct __vxge_hw_device *hldev, u32 vp_id)
4203 {
4204         struct __vxge_hw_virtualpath *vpath;
4205
4206         vpath = &hldev->virtual_paths[vp_id];
4207
4208         if (vpath->vp_open == VXGE_HW_VP_NOT_OPEN)
4209                 goto exit;
4210
4211         VXGE_HW_DEVICE_TIM_INT_MASK_RESET(vpath->hldev->tim_int_mask0,
4212                 vpath->hldev->tim_int_mask1, vpath->vp_id);
4213         hldev->stats.hw_dev_info_stats.vpath_info[vpath->vp_id] = NULL;
4214
4215         memset(vpath, 0, sizeof(struct __vxge_hw_virtualpath));
4216 exit:
4217         return;
4218 }
4219
4220 /*
4221  * vxge_hw_vpath_mtu_set - Set MTU.
4222  * Set new MTU value. Example, to use jumbo frames:
4223  * vxge_hw_vpath_mtu_set(my_device, 9600);
4224  */
4225 enum vxge_hw_status
4226 vxge_hw_vpath_mtu_set(struct __vxge_hw_vpath_handle *vp, u32 new_mtu)
4227 {
4228         u64 val64;
4229         enum vxge_hw_status status = VXGE_HW_OK;
4230         struct __vxge_hw_virtualpath *vpath;
4231
4232         if (vp == NULL) {
4233                 status = VXGE_HW_ERR_INVALID_HANDLE;
4234                 goto exit;
4235         }
4236         vpath = vp->vpath;
4237
4238         new_mtu += VXGE_HW_MAC_HEADER_MAX_SIZE;
4239
4240         if ((new_mtu < VXGE_HW_MIN_MTU) || (new_mtu > vpath->max_mtu))
4241                 status = VXGE_HW_ERR_INVALID_MTU_SIZE;
4242
4243         val64 = readq(&vpath->vp_reg->rxmac_vcfg0);
4244
4245         val64 &= ~VXGE_HW_RXMAC_VCFG0_RTS_MAX_FRM_LEN(0x3fff);
4246         val64 |= VXGE_HW_RXMAC_VCFG0_RTS_MAX_FRM_LEN(new_mtu);
4247
4248         writeq(val64, &vpath->vp_reg->rxmac_vcfg0);
4249
4250         vpath->vp_config->mtu = new_mtu - VXGE_HW_MAC_HEADER_MAX_SIZE;
4251
4252 exit:
4253         return status;
4254 }
4255
4256 /*
4257  * vxge_hw_vpath_open - Open a virtual path on a given adapter
4258  * This function is used to open access to virtual path of an
4259  * adapter for offload, GRO operations. This function returns
4260  * synchronously.
4261  */
4262 enum vxge_hw_status
4263 vxge_hw_vpath_open(struct __vxge_hw_device *hldev,
4264                    struct vxge_hw_vpath_attr *attr,
4265                    struct __vxge_hw_vpath_handle **vpath_handle)
4266 {
4267         struct __vxge_hw_virtualpath *vpath;
4268         struct __vxge_hw_vpath_handle *vp;
4269         enum vxge_hw_status status;
4270
4271         vpath = &hldev->virtual_paths[attr->vp_id];
4272
4273         if (vpath->vp_open == VXGE_HW_VP_OPEN) {
4274                 status = VXGE_HW_ERR_INVALID_STATE;
4275                 goto vpath_open_exit1;
4276         }
4277
4278         status = __vxge_hw_vp_initialize(hldev, attr->vp_id,
4279                         &hldev->config.vp_config[attr->vp_id]);
4280
4281         if (status != VXGE_HW_OK)
4282                 goto vpath_open_exit1;
4283
4284         vp = (struct __vxge_hw_vpath_handle *)
4285                 vmalloc(sizeof(struct __vxge_hw_vpath_handle));
4286         if (vp == NULL) {
4287                 status = VXGE_HW_ERR_OUT_OF_MEMORY;
4288                 goto vpath_open_exit2;
4289         }
4290
4291         memset(vp, 0, sizeof(struct __vxge_hw_vpath_handle));
4292
4293         vp->vpath = vpath;
4294
4295         if (vpath->vp_config->fifo.enable == VXGE_HW_FIFO_ENABLE) {
4296                 status = __vxge_hw_fifo_create(vp, &attr->fifo_attr);
4297                 if (status != VXGE_HW_OK)
4298                         goto vpath_open_exit6;
4299         }
4300
4301         if (vpath->vp_config->ring.enable == VXGE_HW_RING_ENABLE) {
4302                 status = __vxge_hw_ring_create(vp, &attr->ring_attr);
4303                 if (status != VXGE_HW_OK)
4304                         goto vpath_open_exit7;
4305
4306                 __vxge_hw_vpath_prc_configure(hldev, attr->vp_id);
4307         }
4308
4309         vpath->fifoh->tx_intr_num =
4310                 (attr->vp_id * VXGE_HW_MAX_INTR_PER_VP)  +
4311                         VXGE_HW_VPATH_INTR_TX;
4312
4313         vpath->stats_block = __vxge_hw_blockpool_block_allocate(hldev,
4314                                 VXGE_HW_BLOCK_SIZE);
4315
4316         if (vpath->stats_block == NULL) {
4317                 status = VXGE_HW_ERR_OUT_OF_MEMORY;
4318                 goto vpath_open_exit8;
4319         }
4320
4321         vpath->hw_stats = (struct vxge_hw_vpath_stats_hw_info *)vpath->
4322                         stats_block->memblock;
4323         memset(vpath->hw_stats, 0,
4324                 sizeof(struct vxge_hw_vpath_stats_hw_info));
4325
4326         hldev->stats.hw_dev_info_stats.vpath_info[attr->vp_id] =
4327                                                 vpath->hw_stats;
4328
4329         vpath->hw_stats_sav =
4330                 &hldev->stats.hw_dev_info_stats.vpath_info_sav[attr->vp_id];
4331         memset(vpath->hw_stats_sav, 0,
4332                         sizeof(struct vxge_hw_vpath_stats_hw_info));
4333
4334         writeq(vpath->stats_block->dma_addr, &vpath->vp_reg->stats_cfg);
4335
4336         status = vxge_hw_vpath_stats_enable(vp);
4337         if (status != VXGE_HW_OK)
4338                 goto vpath_open_exit8;
4339
4340         list_add(&vp->item, &vpath->vpath_handles);
4341
4342         hldev->vpaths_deployed |= vxge_mBIT(vpath->vp_id);
4343
4344         *vpath_handle = vp;
4345
4346         attr->fifo_attr.userdata = vpath->fifoh;
4347         attr->ring_attr.userdata = vpath->ringh;
4348
4349         return VXGE_HW_OK;
4350
4351 vpath_open_exit8:
4352         if (vpath->ringh != NULL)
4353                 __vxge_hw_ring_delete(vp);
4354 vpath_open_exit7:
4355         if (vpath->fifoh != NULL)
4356                 __vxge_hw_fifo_delete(vp);
4357 vpath_open_exit6:
4358         vfree(vp);
4359 vpath_open_exit2:
4360         __vxge_hw_vp_terminate(hldev, attr->vp_id);
4361 vpath_open_exit1:
4362
4363         return status;
4364 }
4365
4366 /**
4367  * vxge_hw_vpath_rx_doorbell_post - Close the handle got from previous vpath
4368  * (vpath) open
4369  * @vp: Handle got from previous vpath open
4370  *
4371  * This function is used to close access to virtual path opened
4372  * earlier.
4373  */
4374 void
4375 vxge_hw_vpath_rx_doorbell_init(struct __vxge_hw_vpath_handle *vp)
4376 {
4377         struct __vxge_hw_virtualpath *vpath = NULL;
4378         u64 new_count, val64, val164;
4379         struct __vxge_hw_ring *ring;
4380
4381         vpath = vp->vpath;
4382         ring = vpath->ringh;
4383
4384         new_count = readq(&vpath->vp_reg->rxdmem_size);
4385         new_count &= 0x1fff;
4386         val164 = (VXGE_HW_RXDMEM_SIZE_PRC_RXDMEM_SIZE(new_count));
4387
4388         writeq(VXGE_HW_PRC_RXD_DOORBELL_NEW_QW_CNT(val164),
4389                 &vpath->vp_reg->prc_rxd_doorbell);
4390         readl(&vpath->vp_reg->prc_rxd_doorbell);
4391
4392         val164 /= 2;
4393         val64 = readq(&vpath->vp_reg->prc_cfg6);
4394         val64 = VXGE_HW_PRC_CFG6_RXD_SPAT(val64);
4395         val64 &= 0x1ff;
4396
4397         /*
4398          * Each RxD is of 4 qwords
4399          */
4400         new_count -= (val64 + 1);
4401         val64 = min(val164, new_count) / 4;
4402
4403         ring->rxds_limit = min(ring->rxds_limit, val64);
4404         if (ring->rxds_limit < 4)
4405                 ring->rxds_limit = 4;
4406 }
4407
4408 /*
4409  * vxge_hw_vpath_close - Close the handle got from previous vpath (vpath) open
4410  * This function is used to close access to virtual path opened
4411  * earlier.
4412  */
4413 enum vxge_hw_status vxge_hw_vpath_close(struct __vxge_hw_vpath_handle *vp)
4414 {
4415         struct __vxge_hw_virtualpath *vpath = NULL;
4416         struct __vxge_hw_device *devh = NULL;
4417         u32 vp_id = vp->vpath->vp_id;
4418         u32 is_empty = TRUE;
4419         enum vxge_hw_status status = VXGE_HW_OK;
4420
4421         vpath = vp->vpath;
4422         devh = vpath->hldev;
4423
4424         if (vpath->vp_open == VXGE_HW_VP_NOT_OPEN) {
4425                 status = VXGE_HW_ERR_VPATH_NOT_OPEN;
4426                 goto vpath_close_exit;
4427         }
4428
4429         list_del(&vp->item);
4430
4431         if (!list_empty(&vpath->vpath_handles)) {
4432                 list_add(&vp->item, &vpath->vpath_handles);
4433                 is_empty = FALSE;
4434         }
4435
4436         if (!is_empty) {
4437                 status = VXGE_HW_FAIL;
4438                 goto vpath_close_exit;
4439         }
4440
4441         devh->vpaths_deployed &= ~vxge_mBIT(vp_id);
4442
4443         if (vpath->ringh != NULL)
4444                 __vxge_hw_ring_delete(vp);
4445
4446         if (vpath->fifoh != NULL)
4447                 __vxge_hw_fifo_delete(vp);
4448
4449         if (vpath->stats_block != NULL)
4450                 __vxge_hw_blockpool_block_free(devh, vpath->stats_block);
4451
4452         vfree(vp);
4453
4454         __vxge_hw_vp_terminate(devh, vp_id);
4455
4456         vpath->vp_open = VXGE_HW_VP_NOT_OPEN;
4457
4458 vpath_close_exit:
4459         return status;
4460 }
4461
4462 /*
4463  * vxge_hw_vpath_reset - Resets vpath
4464  * This function is used to request a reset of vpath
4465  */
4466 enum vxge_hw_status vxge_hw_vpath_reset(struct __vxge_hw_vpath_handle *vp)
4467 {
4468         enum vxge_hw_status status;
4469         u32 vp_id;
4470         struct __vxge_hw_virtualpath *vpath = vp->vpath;
4471
4472         vp_id = vpath->vp_id;
4473
4474         if (vpath->vp_open == VXGE_HW_VP_NOT_OPEN) {
4475                 status = VXGE_HW_ERR_VPATH_NOT_OPEN;
4476                 goto exit;
4477         }
4478
4479         status = __vxge_hw_vpath_reset(vpath->hldev, vp_id);
4480         if (status == VXGE_HW_OK)
4481                 vpath->sw_stats->soft_reset_cnt++;
4482 exit:
4483         return status;
4484 }
4485
4486 /*
4487  * vxge_hw_vpath_recover_from_reset - Poll for reset complete and re-initialize.
4488  * This function poll's for the vpath reset completion and re initializes
4489  * the vpath.
4490  */
4491 enum vxge_hw_status
4492 vxge_hw_vpath_recover_from_reset(struct __vxge_hw_vpath_handle *vp)
4493 {
4494         struct __vxge_hw_virtualpath *vpath = NULL;
4495         enum vxge_hw_status status;
4496         struct __vxge_hw_device *hldev;
4497         u32 vp_id;
4498
4499         vp_id = vp->vpath->vp_id;
4500         vpath = vp->vpath;
4501         hldev = vpath->hldev;
4502
4503         if (vpath->vp_open == VXGE_HW_VP_NOT_OPEN) {
4504                 status = VXGE_HW_ERR_VPATH_NOT_OPEN;
4505                 goto exit;
4506         }
4507
4508         status = __vxge_hw_vpath_reset_check(vpath);
4509         if (status != VXGE_HW_OK)
4510                 goto exit;
4511
4512         status = __vxge_hw_vpath_sw_reset(hldev, vp_id);
4513         if (status != VXGE_HW_OK)
4514                 goto exit;
4515
4516         status = __vxge_hw_vpath_initialize(hldev, vp_id);
4517         if (status != VXGE_HW_OK)
4518                 goto exit;
4519
4520         if (vpath->ringh != NULL)
4521                 __vxge_hw_vpath_prc_configure(hldev, vp_id);
4522
4523         memset(vpath->hw_stats, 0,
4524                 sizeof(struct vxge_hw_vpath_stats_hw_info));
4525
4526         memset(vpath->hw_stats_sav, 0,
4527                 sizeof(struct vxge_hw_vpath_stats_hw_info));
4528
4529         writeq(vpath->stats_block->dma_addr,
4530                 &vpath->vp_reg->stats_cfg);
4531
4532         status = vxge_hw_vpath_stats_enable(vp);
4533
4534 exit:
4535         return status;
4536 }
4537
4538 /*
4539  * vxge_hw_vpath_enable - Enable vpath.
4540  * This routine clears the vpath reset thereby enabling a vpath
4541  * to start forwarding frames and generating interrupts.
4542  */
4543 void
4544 vxge_hw_vpath_enable(struct __vxge_hw_vpath_handle *vp)
4545 {
4546         struct __vxge_hw_device *hldev;
4547         u64 val64;
4548
4549         hldev = vp->vpath->hldev;
4550
4551         val64 = VXGE_HW_CMN_RSTHDLR_CFG1_CLR_VPATH_RESET(
4552                 1 << (16 - vp->vpath->vp_id));
4553
4554         __vxge_hw_pio_mem_write32_upper((u32)vxge_bVALn(val64, 0, 32),
4555                 &hldev->common_reg->cmn_rsthdlr_cfg1);
4556 }
4557
4558 /*
4559  * vxge_hw_vpath_stats_enable - Enable vpath h/wstatistics.
4560  * Enable the DMA vpath statistics. The function is to be called to re-enable
4561  * the adapter to update stats into the host memory
4562  */
4563 enum vxge_hw_status
4564 vxge_hw_vpath_stats_enable(struct __vxge_hw_vpath_handle *vp)
4565 {
4566         enum vxge_hw_status status = VXGE_HW_OK;
4567         struct __vxge_hw_virtualpath *vpath;
4568
4569         vpath = vp->vpath;
4570
4571         if (vpath->vp_open == VXGE_HW_VP_NOT_OPEN) {
4572                 status = VXGE_HW_ERR_VPATH_NOT_OPEN;
4573                 goto exit;
4574         }
4575
4576         memcpy(vpath->hw_stats_sav, vpath->hw_stats,
4577                         sizeof(struct vxge_hw_vpath_stats_hw_info));
4578
4579         status = __vxge_hw_vpath_stats_get(vpath, vpath->hw_stats);
4580 exit:
4581         return status;
4582 }
4583
4584 /*
4585  * __vxge_hw_vpath_stats_access - Get the statistics from the given location
4586  *                           and offset and perform an operation
4587  */
4588 enum vxge_hw_status
4589 __vxge_hw_vpath_stats_access(struct __vxge_hw_virtualpath *vpath,
4590                              u32 operation, u32 offset, u64 *stat)
4591 {
4592         u64 val64;
4593         enum vxge_hw_status status = VXGE_HW_OK;
4594         struct vxge_hw_vpath_reg __iomem *vp_reg;
4595
4596         if (vpath->vp_open == VXGE_HW_VP_NOT_OPEN) {
4597                 status = VXGE_HW_ERR_VPATH_NOT_OPEN;
4598                 goto vpath_stats_access_exit;
4599         }
4600
4601         vp_reg = vpath->vp_reg;
4602
4603         val64 =  VXGE_HW_XMAC_STATS_ACCESS_CMD_OP(operation) |
4604                  VXGE_HW_XMAC_STATS_ACCESS_CMD_STROBE |
4605                  VXGE_HW_XMAC_STATS_ACCESS_CMD_OFFSET_SEL(offset);
4606
4607         status = __vxge_hw_pio_mem_write64(val64,
4608                                 &vp_reg->xmac_stats_access_cmd,
4609                                 VXGE_HW_XMAC_STATS_ACCESS_CMD_STROBE,
4610                                 vpath->hldev->config.device_poll_millis);
4611
4612         if ((status == VXGE_HW_OK) && (operation == VXGE_HW_STATS_OP_READ))
4613                 *stat = readq(&vp_reg->xmac_stats_access_data);
4614         else
4615                 *stat = 0;
4616
4617 vpath_stats_access_exit:
4618         return status;
4619 }
4620
4621 /*
4622  * __vxge_hw_vpath_xmac_tx_stats_get - Get the TX Statistics of a vpath
4623  */
4624 enum vxge_hw_status
4625 __vxge_hw_vpath_xmac_tx_stats_get(
4626         struct __vxge_hw_virtualpath *vpath,
4627         struct vxge_hw_xmac_vpath_tx_stats *vpath_tx_stats)
4628 {
4629         u64 *val64;
4630         int i;
4631         u32 offset = VXGE_HW_STATS_VPATH_TX_OFFSET;
4632         enum vxge_hw_status status = VXGE_HW_OK;
4633
4634         val64 = (u64 *) vpath_tx_stats;
4635
4636         if (vpath->vp_open == VXGE_HW_VP_NOT_OPEN) {
4637                 status = VXGE_HW_ERR_VPATH_NOT_OPEN;
4638                 goto exit;
4639         }
4640
4641         for (i = 0; i < sizeof(struct vxge_hw_xmac_vpath_tx_stats) / 8; i++) {
4642                 status = __vxge_hw_vpath_stats_access(vpath,
4643                                         VXGE_HW_STATS_OP_READ,
4644                                         offset, val64);
4645                 if (status != VXGE_HW_OK)
4646                         goto exit;
4647                 offset++;
4648                 val64++;
4649         }
4650 exit:
4651         return status;
4652 }
4653
4654 /*
4655  * __vxge_hw_vpath_xmac_rx_stats_get - Get the RX Statistics of a vpath
4656  */
4657 enum vxge_hw_status
4658 __vxge_hw_vpath_xmac_rx_stats_get(struct __vxge_hw_virtualpath *vpath,
4659                         struct vxge_hw_xmac_vpath_rx_stats *vpath_rx_stats)
4660 {
4661         u64 *val64;
4662         enum vxge_hw_status status = VXGE_HW_OK;
4663         int i;
4664         u32 offset = VXGE_HW_STATS_VPATH_RX_OFFSET;
4665         val64 = (u64 *) vpath_rx_stats;
4666
4667         if (vpath->vp_open == VXGE_HW_VP_NOT_OPEN) {
4668                 status = VXGE_HW_ERR_VPATH_NOT_OPEN;
4669                 goto exit;
4670         }
4671         for (i = 0; i < sizeof(struct vxge_hw_xmac_vpath_rx_stats) / 8; i++) {
4672                 status = __vxge_hw_vpath_stats_access(vpath,
4673                                         VXGE_HW_STATS_OP_READ,
4674                                         offset >> 3, val64);
4675                 if (status != VXGE_HW_OK)
4676                         goto exit;
4677
4678                 offset += 8;
4679                 val64++;
4680         }
4681 exit:
4682         return status;
4683 }
4684
4685 /*
4686  * __vxge_hw_vpath_stats_get - Get the vpath hw statistics.
4687  */
4688 enum vxge_hw_status __vxge_hw_vpath_stats_get(
4689                         struct __vxge_hw_virtualpath *vpath,
4690                         struct vxge_hw_vpath_stats_hw_info *hw_stats)
4691 {
4692         u64 val64;
4693         enum vxge_hw_status status = VXGE_HW_OK;
4694         struct vxge_hw_vpath_reg __iomem *vp_reg;
4695
4696         if (vpath->vp_open == VXGE_HW_VP_NOT_OPEN) {
4697                 status = VXGE_HW_ERR_VPATH_NOT_OPEN;
4698                 goto exit;
4699         }
4700         vp_reg = vpath->vp_reg;
4701
4702         val64 = readq(&vp_reg->vpath_debug_stats0);
4703         hw_stats->ini_num_mwr_sent =
4704                 (u32)VXGE_HW_VPATH_DEBUG_STATS0_GET_INI_NUM_MWR_SENT(val64);
4705
4706         val64 = readq(&vp_reg->vpath_debug_stats1);
4707         hw_stats->ini_num_mrd_sent =
4708                 (u32)VXGE_HW_VPATH_DEBUG_STATS1_GET_INI_NUM_MRD_SENT(val64);
4709
4710         val64 = readq(&vp_reg->vpath_debug_stats2);
4711         hw_stats->ini_num_cpl_rcvd =
4712                 (u32)VXGE_HW_VPATH_DEBUG_STATS2_GET_INI_NUM_CPL_RCVD(val64);
4713
4714         val64 = readq(&vp_reg->vpath_debug_stats3);
4715         hw_stats->ini_num_mwr_byte_sent =
4716                 VXGE_HW_VPATH_DEBUG_STATS3_GET_INI_NUM_MWR_BYTE_SENT(val64);
4717
4718         val64 = readq(&vp_reg->vpath_debug_stats4);
4719         hw_stats->ini_num_cpl_byte_rcvd =
4720                 VXGE_HW_VPATH_DEBUG_STATS4_GET_INI_NUM_CPL_BYTE_RCVD(val64);
4721
4722         val64 = readq(&vp_reg->vpath_debug_stats5);
4723         hw_stats->wrcrdtarb_xoff =
4724                 (u32)VXGE_HW_VPATH_DEBUG_STATS5_GET_WRCRDTARB_XOFF(val64);
4725
4726         val64 = readq(&vp_reg->vpath_debug_stats6);
4727         hw_stats->rdcrdtarb_xoff =
4728                 (u32)VXGE_HW_VPATH_DEBUG_STATS6_GET_RDCRDTARB_XOFF(val64);
4729
4730         val64 = readq(&vp_reg->vpath_genstats_count01);
4731         hw_stats->vpath_genstats_count0 =
4732         (u32)VXGE_HW_VPATH_GENSTATS_COUNT01_GET_PPIF_VPATH_GENSTATS_COUNT0(
4733                 val64);
4734
4735         val64 = readq(&vp_reg->vpath_genstats_count01);
4736         hw_stats->vpath_genstats_count1 =
4737         (u32)VXGE_HW_VPATH_GENSTATS_COUNT01_GET_PPIF_VPATH_GENSTATS_COUNT1(
4738                 val64);
4739
4740         val64 = readq(&vp_reg->vpath_genstats_count23);
4741         hw_stats->vpath_genstats_count2 =
4742         (u32)VXGE_HW_VPATH_GENSTATS_COUNT23_GET_PPIF_VPATH_GENSTATS_COUNT2(
4743                 val64);
4744
4745         val64 = readq(&vp_reg->vpath_genstats_count01);
4746         hw_stats->vpath_genstats_count3 =
4747         (u32)VXGE_HW_VPATH_GENSTATS_COUNT23_GET_PPIF_VPATH_GENSTATS_COUNT3(
4748                 val64);
4749
4750         val64 = readq(&vp_reg->vpath_genstats_count4);
4751         hw_stats->vpath_genstats_count4 =
4752         (u32)VXGE_HW_VPATH_GENSTATS_COUNT4_GET_PPIF_VPATH_GENSTATS_COUNT4(
4753                 val64);
4754
4755         val64 = readq(&vp_reg->vpath_genstats_count5);
4756         hw_stats->vpath_genstats_count5 =
4757         (u32)VXGE_HW_VPATH_GENSTATS_COUNT5_GET_PPIF_VPATH_GENSTATS_COUNT5(
4758                 val64);
4759
4760         status = __vxge_hw_vpath_xmac_tx_stats_get(vpath, &hw_stats->tx_stats);
4761         if (status != VXGE_HW_OK)
4762                 goto exit;
4763
4764         status = __vxge_hw_vpath_xmac_rx_stats_get(vpath, &hw_stats->rx_stats);
4765         if (status != VXGE_HW_OK)
4766                 goto exit;
4767
4768         VXGE_HW_VPATH_STATS_PIO_READ(
4769                 VXGE_HW_STATS_VPATH_PROG_EVENT_VNUM0_OFFSET);
4770
4771         hw_stats->prog_event_vnum0 =
4772                         (u32)VXGE_HW_STATS_GET_VPATH_PROG_EVENT_VNUM0(val64);
4773
4774         hw_stats->prog_event_vnum1 =
4775                         (u32)VXGE_HW_STATS_GET_VPATH_PROG_EVENT_VNUM1(val64);
4776
4777         VXGE_HW_VPATH_STATS_PIO_READ(
4778                 VXGE_HW_STATS_VPATH_PROG_EVENT_VNUM2_OFFSET);
4779
4780         hw_stats->prog_event_vnum2 =
4781                         (u32)VXGE_HW_STATS_GET_VPATH_PROG_EVENT_VNUM2(val64);
4782
4783         hw_stats->prog_event_vnum3 =
4784                         (u32)VXGE_HW_STATS_GET_VPATH_PROG_EVENT_VNUM3(val64);
4785
4786         val64 = readq(&vp_reg->rx_multi_cast_stats);
4787         hw_stats->rx_multi_cast_frame_discard =
4788                 (u16)VXGE_HW_RX_MULTI_CAST_STATS_GET_FRAME_DISCARD(val64);
4789
4790         val64 = readq(&vp_reg->rx_frm_transferred);
4791         hw_stats->rx_frm_transferred =
4792                 (u32)VXGE_HW_RX_FRM_TRANSFERRED_GET_RX_FRM_TRANSFERRED(val64);
4793
4794         val64 = readq(&vp_reg->rxd_returned);
4795         hw_stats->rxd_returned =
4796                 (u16)VXGE_HW_RXD_RETURNED_GET_RXD_RETURNED(val64);
4797
4798         val64 = readq(&vp_reg->dbg_stats_rx_mpa);
4799         hw_stats->rx_mpa_len_fail_frms =
4800                 (u16)VXGE_HW_DBG_STATS_GET_RX_MPA_LEN_FAIL_FRMS(val64);
4801         hw_stats->rx_mpa_mrk_fail_frms =
4802                 (u16)VXGE_HW_DBG_STATS_GET_RX_MPA_MRK_FAIL_FRMS(val64);
4803         hw_stats->rx_mpa_crc_fail_frms =
4804                 (u16)VXGE_HW_DBG_STATS_GET_RX_MPA_CRC_FAIL_FRMS(val64);
4805
4806         val64 = readq(&vp_reg->dbg_stats_rx_fau);
4807         hw_stats->rx_permitted_frms =
4808                 (u16)VXGE_HW_DBG_STATS_GET_RX_FAU_RX_PERMITTED_FRMS(val64);
4809         hw_stats->rx_vp_reset_discarded_frms =
4810         (u16)VXGE_HW_DBG_STATS_GET_RX_FAU_RX_VP_RESET_DISCARDED_FRMS(val64);
4811         hw_stats->rx_wol_frms =
4812                 (u16)VXGE_HW_DBG_STATS_GET_RX_FAU_RX_WOL_FRMS(val64);
4813
4814         val64 = readq(&vp_reg->tx_vp_reset_discarded_frms);
4815         hw_stats->tx_vp_reset_discarded_frms =
4816         (u16)VXGE_HW_TX_VP_RESET_DISCARDED_FRMS_GET_TX_VP_RESET_DISCARDED_FRMS(
4817                 val64);
4818 exit:
4819         return status;
4820 }
4821
4822 /*
4823  * __vxge_hw_blockpool_create - Create block pool
4824  */
4825
4826 enum vxge_hw_status
4827 __vxge_hw_blockpool_create(struct __vxge_hw_device *hldev,
4828                            struct __vxge_hw_blockpool *blockpool,
4829                            u32 pool_size,
4830                            u32 pool_max)
4831 {
4832         u32 i;
4833         struct __vxge_hw_blockpool_entry *entry = NULL;
4834         void *memblock;
4835         dma_addr_t dma_addr;
4836         struct pci_dev *dma_handle;
4837         struct pci_dev *acc_handle;
4838         enum vxge_hw_status status = VXGE_HW_OK;
4839
4840         if (blockpool == NULL) {
4841                 status = VXGE_HW_FAIL;
4842                 goto blockpool_create_exit;
4843         }
4844
4845         blockpool->hldev = hldev;
4846         blockpool->block_size = VXGE_HW_BLOCK_SIZE;
4847         blockpool->pool_size = 0;
4848         blockpool->pool_max = pool_max;
4849         blockpool->req_out = 0;
4850
4851         INIT_LIST_HEAD(&blockpool->free_block_list);
4852         INIT_LIST_HEAD(&blockpool->free_entry_list);
4853
4854         for (i = 0; i < pool_size + pool_max; i++) {
4855                 entry = kzalloc(sizeof(struct __vxge_hw_blockpool_entry),
4856                                 GFP_KERNEL);
4857                 if (entry == NULL) {
4858                         __vxge_hw_blockpool_destroy(blockpool);
4859                         status = VXGE_HW_ERR_OUT_OF_MEMORY;
4860                         goto blockpool_create_exit;
4861                 }
4862                 list_add(&entry->item, &blockpool->free_entry_list);
4863         }
4864
4865         for (i = 0; i < pool_size; i++) {
4866
4867                 memblock = vxge_os_dma_malloc(
4868                                 hldev->pdev,
4869                                 VXGE_HW_BLOCK_SIZE,
4870                                 &dma_handle,
4871                                 &acc_handle);
4872
4873                 if (memblock == NULL) {
4874                         __vxge_hw_blockpool_destroy(blockpool);
4875                         status = VXGE_HW_ERR_OUT_OF_MEMORY;
4876                         goto blockpool_create_exit;
4877                 }
4878
4879                 dma_addr = pci_map_single(hldev->pdev, memblock,
4880                                 VXGE_HW_BLOCK_SIZE, PCI_DMA_BIDIRECTIONAL);
4881
4882                 if (unlikely(pci_dma_mapping_error(hldev->pdev,
4883                                 dma_addr))) {
4884
4885                         vxge_os_dma_free(hldev->pdev, memblock, &acc_handle);
4886                         __vxge_hw_blockpool_destroy(blockpool);
4887                         status = VXGE_HW_ERR_OUT_OF_MEMORY;
4888                         goto blockpool_create_exit;
4889                 }
4890
4891                 if (!list_empty(&blockpool->free_entry_list))
4892                         entry = (struct __vxge_hw_blockpool_entry *)
4893                                 list_first_entry(&blockpool->free_entry_list,
4894                                         struct __vxge_hw_blockpool_entry,
4895                                         item);
4896
4897                 if (entry == NULL)
4898                         entry =
4899                             kzalloc(sizeof(struct __vxge_hw_blockpool_entry),
4900                                         GFP_KERNEL);
4901                 if (entry != NULL) {
4902                         list_del(&entry->item);
4903                         entry->length = VXGE_HW_BLOCK_SIZE;
4904                         entry->memblock = memblock;
4905                         entry->dma_addr = dma_addr;
4906                         entry->acc_handle = acc_handle;
4907                         entry->dma_handle = dma_handle;
4908                         list_add(&entry->item,
4909                                           &blockpool->free_block_list);
4910                         blockpool->pool_size++;
4911                 } else {
4912                         __vxge_hw_blockpool_destroy(blockpool);
4913                         status = VXGE_HW_ERR_OUT_OF_MEMORY;
4914                         goto blockpool_create_exit;
4915                 }
4916         }
4917
4918 blockpool_create_exit:
4919         return status;
4920 }
4921
4922 /*
4923  * __vxge_hw_blockpool_destroy - Deallocates the block pool
4924  */
4925
4926 void __vxge_hw_blockpool_destroy(struct __vxge_hw_blockpool *blockpool)
4927 {
4928
4929         struct __vxge_hw_device *hldev;
4930         struct list_head *p, *n;
4931         u16 ret;
4932
4933         if (blockpool == NULL) {
4934                 ret = 1;
4935                 goto exit;
4936         }
4937
4938         hldev = blockpool->hldev;
4939
4940         list_for_each_safe(p, n, &blockpool->free_block_list) {
4941
4942                 pci_unmap_single(hldev->pdev,
4943                         ((struct __vxge_hw_blockpool_entry *)p)->dma_addr,
4944                         ((struct __vxge_hw_blockpool_entry *)p)->length,
4945                         PCI_DMA_BIDIRECTIONAL);
4946
4947                 vxge_os_dma_free(hldev->pdev,
4948                         ((struct __vxge_hw_blockpool_entry *)p)->memblock,
4949                         &((struct __vxge_hw_blockpool_entry *) p)->acc_handle);
4950
4951                 list_del(
4952                         &((struct __vxge_hw_blockpool_entry *)p)->item);
4953                 kfree(p);
4954                 blockpool->pool_size--;
4955         }
4956
4957         list_for_each_safe(p, n, &blockpool->free_entry_list) {
4958                 list_del(
4959                         &((struct __vxge_hw_blockpool_entry *)p)->item);
4960                 kfree((void *)p);
4961         }
4962         ret = 0;
4963 exit:
4964         return;
4965 }
4966
4967 /*
4968  * __vxge_hw_blockpool_blocks_add - Request additional blocks
4969  */
4970 static
4971 void __vxge_hw_blockpool_blocks_add(struct __vxge_hw_blockpool *blockpool)
4972 {
4973         u32 nreq = 0, i;
4974
4975         if ((blockpool->pool_size  +  blockpool->req_out) <
4976                 VXGE_HW_MIN_DMA_BLOCK_POOL_SIZE) {
4977                 nreq = VXGE_HW_INCR_DMA_BLOCK_POOL_SIZE;
4978                 blockpool->req_out += nreq;
4979         }
4980
4981         for (i = 0; i < nreq; i++)
4982                 vxge_os_dma_malloc_async(
4983                         ((struct __vxge_hw_device *)blockpool->hldev)->pdev,
4984                         blockpool->hldev, VXGE_HW_BLOCK_SIZE);
4985 }
4986
4987 /*
4988  * __vxge_hw_blockpool_blocks_remove - Free additional blocks
4989  */
4990 static
4991 void __vxge_hw_blockpool_blocks_remove(struct __vxge_hw_blockpool *blockpool)
4992 {
4993         struct list_head *p, *n;
4994
4995         list_for_each_safe(p, n, &blockpool->free_block_list) {
4996
4997                 if (blockpool->pool_size < blockpool->pool_max)
4998                         break;
4999
5000                 pci_unmap_single(
5001                         ((struct __vxge_hw_device *)blockpool->hldev)->pdev,
5002                         ((struct __vxge_hw_blockpool_entry *)p)->dma_addr,
5003                         ((struct __vxge_hw_blockpool_entry *)p)->length,
5004                         PCI_DMA_BIDIRECTIONAL);
5005
5006                 vxge_os_dma_free(
5007                         ((struct __vxge_hw_device *)blockpool->hldev)->pdev,
5008                         ((struct __vxge_hw_blockpool_entry *)p)->memblock,
5009                         &((struct __vxge_hw_blockpool_entry *)p)->acc_handle);
5010
5011                 list_del(&((struct __vxge_hw_blockpool_entry *)p)->item);
5012
5013                 list_add(p, &blockpool->free_entry_list);
5014
5015                 blockpool->pool_size--;
5016
5017         }
5018 }
5019
5020 /*
5021  * vxge_hw_blockpool_block_add - callback for vxge_os_dma_malloc_async
5022  * Adds a block to block pool
5023  */
5024 void vxge_hw_blockpool_block_add(
5025                         struct __vxge_hw_device *devh,
5026                         void *block_addr,
5027                         u32 length,
5028                         struct pci_dev *dma_h,
5029                         struct pci_dev *acc_handle)
5030 {
5031         struct __vxge_hw_blockpool  *blockpool;
5032         struct __vxge_hw_blockpool_entry  *entry = NULL;
5033         dma_addr_t dma_addr;
5034         enum vxge_hw_status status = VXGE_HW_OK;
5035         u32 req_out;
5036
5037         blockpool = &devh->block_pool;
5038
5039         if (block_addr == NULL) {
5040                 blockpool->req_out--;
5041                 status = VXGE_HW_FAIL;
5042                 goto exit;
5043         }
5044
5045         dma_addr = pci_map_single(devh->pdev, block_addr, length,
5046                                 PCI_DMA_BIDIRECTIONAL);
5047
5048         if (unlikely(pci_dma_mapping_error(devh->pdev, dma_addr))) {
5049
5050                 vxge_os_dma_free(devh->pdev, block_addr, &acc_handle);
5051                 blockpool->req_out--;
5052                 status = VXGE_HW_FAIL;
5053                 goto exit;
5054         }
5055
5056
5057         if (!list_empty(&blockpool->free_entry_list))
5058                 entry = (struct __vxge_hw_blockpool_entry *)
5059                         list_first_entry(&blockpool->free_entry_list,
5060                                 struct __vxge_hw_blockpool_entry,
5061                                 item);
5062
5063         if (entry == NULL)
5064                 entry = (struct __vxge_hw_blockpool_entry *)
5065                         vmalloc(sizeof(struct __vxge_hw_blockpool_entry));
5066         else
5067                 list_del(&entry->item);
5068
5069         if (entry != NULL) {
5070                 entry->length = length;
5071                 entry->memblock = block_addr;
5072                 entry->dma_addr = dma_addr;
5073                 entry->acc_handle = acc_handle;
5074                 entry->dma_handle = dma_h;
5075                 list_add(&entry->item, &blockpool->free_block_list);
5076                 blockpool->pool_size++;
5077                 status = VXGE_HW_OK;
5078         } else
5079                 status = VXGE_HW_ERR_OUT_OF_MEMORY;
5080
5081         blockpool->req_out--;
5082
5083         req_out = blockpool->req_out;
5084 exit:
5085         return;
5086 }
5087
5088 /*
5089  * __vxge_hw_blockpool_malloc - Allocate a memory block from pool
5090  * Allocates a block of memory of given size, either from block pool
5091  * or by calling vxge_os_dma_malloc()
5092  */
5093 void *
5094 __vxge_hw_blockpool_malloc(struct __vxge_hw_device *devh, u32 size,
5095                                 struct vxge_hw_mempool_dma *dma_object)
5096 {
5097         struct __vxge_hw_blockpool_entry *entry = NULL;
5098         struct __vxge_hw_blockpool  *blockpool;
5099         void *memblock = NULL;
5100         enum vxge_hw_status status = VXGE_HW_OK;
5101
5102         blockpool = &devh->block_pool;
5103
5104         if (size != blockpool->block_size) {
5105
5106                 memblock = vxge_os_dma_malloc(devh->pdev, size,
5107                                                 &dma_object->handle,
5108                                                 &dma_object->acc_handle);
5109
5110                 if (memblock == NULL) {
5111                         status = VXGE_HW_ERR_OUT_OF_MEMORY;
5112                         goto exit;
5113                 }
5114
5115                 dma_object->addr = pci_map_single(devh->pdev, memblock, size,
5116                                         PCI_DMA_BIDIRECTIONAL);
5117
5118                 if (unlikely(pci_dma_mapping_error(devh->pdev,
5119                                 dma_object->addr))) {
5120                         vxge_os_dma_free(devh->pdev, memblock,
5121                                 &dma_object->acc_handle);
5122                         status = VXGE_HW_ERR_OUT_OF_MEMORY;
5123                         goto exit;
5124                 }
5125
5126         } else {
5127
5128                 if (!list_empty(&blockpool->free_block_list))
5129                         entry = (struct __vxge_hw_blockpool_entry *)
5130                                 list_first_entry(&blockpool->free_block_list,
5131                                         struct __vxge_hw_blockpool_entry,
5132                                         item);
5133
5134                 if (entry != NULL) {
5135                         list_del(&entry->item);
5136                         dma_object->addr = entry->dma_addr;
5137                         dma_object->handle = entry->dma_handle;
5138                         dma_object->acc_handle = entry->acc_handle;
5139                         memblock = entry->memblock;
5140
5141                         list_add(&entry->item,
5142                                 &blockpool->free_entry_list);
5143                         blockpool->pool_size--;
5144                 }
5145
5146                 if (memblock != NULL)
5147                         __vxge_hw_blockpool_blocks_add(blockpool);
5148         }
5149 exit:
5150         return memblock;
5151 }
5152
5153 /*
5154  * __vxge_hw_blockpool_free - Frees the memory allcoated with
5155                                 __vxge_hw_blockpool_malloc
5156  */
5157 void
5158 __vxge_hw_blockpool_free(struct __vxge_hw_device *devh,
5159                         void *memblock, u32 size,
5160                         struct vxge_hw_mempool_dma *dma_object)
5161 {
5162         struct __vxge_hw_blockpool_entry *entry = NULL;
5163         struct __vxge_hw_blockpool  *blockpool;
5164         enum vxge_hw_status status = VXGE_HW_OK;
5165
5166         blockpool = &devh->block_pool;
5167
5168         if (size != blockpool->block_size) {
5169                 pci_unmap_single(devh->pdev, dma_object->addr, size,
5170                         PCI_DMA_BIDIRECTIONAL);
5171                 vxge_os_dma_free(devh->pdev, memblock, &dma_object->acc_handle);
5172         } else {
5173
5174                 if (!list_empty(&blockpool->free_entry_list))
5175                         entry = (struct __vxge_hw_blockpool_entry *)
5176                                 list_first_entry(&blockpool->free_entry_list,
5177                                         struct __vxge_hw_blockpool_entry,
5178                                         item);
5179
5180                 if (entry == NULL)
5181                         entry = (struct __vxge_hw_blockpool_entry *)
5182                                 vmalloc(sizeof(
5183                                         struct __vxge_hw_blockpool_entry));
5184                 else
5185                         list_del(&entry->item);
5186
5187                 if (entry != NULL) {
5188                         entry->length = size;
5189                         entry->memblock = memblock;
5190                         entry->dma_addr = dma_object->addr;
5191                         entry->acc_handle = dma_object->acc_handle;
5192                         entry->dma_handle = dma_object->handle;
5193                         list_add(&entry->item,
5194                                         &blockpool->free_block_list);
5195                         blockpool->pool_size++;
5196                         status = VXGE_HW_OK;
5197                 } else
5198                         status = VXGE_HW_ERR_OUT_OF_MEMORY;
5199
5200                 if (status == VXGE_HW_OK)
5201                         __vxge_hw_blockpool_blocks_remove(blockpool);
5202         }
5203
5204         return;
5205 }
5206
5207 /*
5208  * __vxge_hw_blockpool_block_allocate - Allocates a block from block pool
5209  * This function allocates a block from block pool or from the system
5210  */
5211 struct __vxge_hw_blockpool_entry *
5212 __vxge_hw_blockpool_block_allocate(struct __vxge_hw_device *devh, u32 size)
5213 {
5214         struct __vxge_hw_blockpool_entry *entry = NULL;
5215         struct __vxge_hw_blockpool  *blockpool;
5216
5217         blockpool = &devh->block_pool;
5218
5219         if (size == blockpool->block_size) {
5220
5221                 if (!list_empty(&blockpool->free_block_list))
5222                         entry = (struct __vxge_hw_blockpool_entry *)
5223                                 list_first_entry(&blockpool->free_block_list,
5224                                         struct __vxge_hw_blockpool_entry,
5225                                         item);
5226
5227                 if (entry != NULL) {
5228                         list_del(&entry->item);
5229                         blockpool->pool_size--;
5230                 }
5231         }
5232
5233         if (entry != NULL)
5234                 __vxge_hw_blockpool_blocks_add(blockpool);
5235
5236         return entry;
5237 }
5238
5239 /*
5240  * __vxge_hw_blockpool_block_free - Frees a block from block pool
5241  * @devh: Hal device
5242  * @entry: Entry of block to be freed
5243  *
5244  * This function frees a block from block pool
5245  */
5246 void
5247 __vxge_hw_blockpool_block_free(struct __vxge_hw_device *devh,
5248                         struct __vxge_hw_blockpool_entry *entry)
5249 {
5250         struct __vxge_hw_blockpool  *blockpool;
5251
5252         blockpool = &devh->block_pool;
5253
5254         if (entry->length == blockpool->block_size) {
5255                 list_add(&entry->item, &blockpool->free_block_list);
5256                 blockpool->pool_size++;
5257         }
5258
5259         __vxge_hw_blockpool_blocks_remove(blockpool);
5260
5261         return;
5262 }