projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
[PATCH] IB/ipath: memory management cleanups
[linux-2.6-block.git] / drivers / infiniband / hw / ipath / ipath_driver.c
CommitLineData
7bb206e3 1/*
759d5768 2 * Copyright (c) 2006 QLogic, Inc. All rights reserved.
7bb206e3
BS		 3 * Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
4 *
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
10 *
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
13 * conditions are met:
14 *
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer.
18 *
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
32 */
33
34#include <linux/spinlock.h>
35#include <linux/idr.h>
36#include <linux/pci.h>
37#include <linux/delay.h>
38#include <linux/netdevice.h>
39#include <linux/vmalloc.h>
40
41#include "ipath_kernel.h"
42#include "ips_common.h"
43#include "ipath_layer.h"
44
45static void ipath_update_pio_bufs(struct ipath_devdata *);
46
47const char *ipath_get_unit_name(int unit)
48{
49 static char iname[16];
50 snprintf(iname, sizeof iname, "infinipath%u", unit);
51 return iname;
52}
53
54EXPORT_SYMBOL_GPL(ipath_get_unit_name);
55
759d5768 56#define DRIVER_LOAD_MSG "QLogic " IPATH_DRV_NAME " loaded: "
7bb206e3
BS		 57#define PFX IPATH_DRV_NAME ": "
58
59/*
60 * The size has to be longer than this string, so we can append
61 * board/chip information to it in the init code.
62 */
63const char ipath_core_version[] = IPATH_IDSTR "\n";
64
65static struct idr unit_table;
66DEFINE_SPINLOCK(ipath_devs_lock);
67LIST_HEAD(ipath_dev_list);
68
69wait_queue_head_t ipath_sma_state_wait;
70
71unsigned ipath_debug = __IPATH_INFO;
72
73module_param_named(debug, ipath_debug, uint, S_IWUSR | S_IRUGO);
74MODULE_PARM_DESC(debug, "mask for debug prints");
75EXPORT_SYMBOL_GPL(ipath_debug);
76
77MODULE_LICENSE("GPL");
759d5768
BS		 78MODULE_AUTHOR("QLogic <support@pathscale.com>");
79MODULE_DESCRIPTION("QLogic InfiniPath driver");
7bb206e3
BS		 80
81const char *ipath_ibcstatus_str[] = {
82 "Disabled",
83 "LinkUp",
84 "PollActive",
85 "PollQuiet",
86 "SleepDelay",
87 "SleepQuiet",
88 "LState6", /* unused */
89 "LState7", /* unused */
90 "CfgDebounce",
91 "CfgRcvfCfg",
92 "CfgWaitRmt",
93 "CfgIdle",
94 "RecovRetrain",
95 "LState0xD", /* unused */
96 "RecovWaitRmt",
97 "RecovIdle",
98};
99
100/*
101 * These variables are initialized in the chip-specific files
102 * but are defined here.
103 */
104u16 ipath_gpio_sda_num, ipath_gpio_scl_num;
105u64 ipath_gpio_sda, ipath_gpio_scl;
106u64 infinipath_i_bitsextant;
107ipath_err_t infinipath_e_bitsextant, infinipath_hwe_bitsextant;
108u32 infinipath_i_rcvavail_mask, infinipath_i_rcvurg_mask;
109
110static void __devexit ipath_remove_one(struct pci_dev *);
111static int __devinit ipath_init_one(struct pci_dev *,
112 const struct pci_device_id *);
113
114/* Only needed for registration, nothing else needs this info */
115#define PCI_VENDOR_ID_PATHSCALE 0x1fc1
116#define PCI_DEVICE_ID_INFINIPATH_HT 0xd
117#define PCI_DEVICE_ID_INFINIPATH_PE800 0x10
118
119static const struct pci_device_id ipath_pci_tbl[] = {
6f4bb3d8
RD		 120 { PCI_DEVICE(PCI_VENDOR_ID_PATHSCALE, PCI_DEVICE_ID_INFINIPATH_HT) },
121 { PCI_DEVICE(PCI_VENDOR_ID_PATHSCALE, PCI_DEVICE_ID_INFINIPATH_PE800) },
122 { 0, }
7bb206e3
BS		 123};
124
125MODULE_DEVICE_TABLE(pci, ipath_pci_tbl);
126
127static struct pci_driver ipath_driver = {
128 .name = IPATH_DRV_NAME,
129 .probe = ipath_init_one,
130 .remove = __devexit_p(ipath_remove_one),
131 .id_table = ipath_pci_tbl,
132};
133
7bb206e3
BS		 134
135static inline void read_bars(struct ipath_devdata *dd, struct pci_dev *dev,
136 u32 *bar0, u32 *bar1)
137{
138 int ret;
139
140 ret = pci_read_config_dword(dev, PCI_BASE_ADDRESS_0, bar0);
141 if (ret)
142 ipath_dev_err(dd, "failed to read bar0 before enable: "
143 "error %d\n", -ret);
144
145 ret = pci_read_config_dword(dev, PCI_BASE_ADDRESS_1, bar1);
146 if (ret)
147 ipath_dev_err(dd, "failed to read bar1 before enable: "
148 "error %d\n", -ret);
149
150 ipath_dbg("Read bar0 %x bar1 %x\n", *bar0, *bar1);
151}
152
153static void ipath_free_devdata(struct pci_dev *pdev,
154 struct ipath_devdata *dd)
155{
156 unsigned long flags;
157
158 pci_set_drvdata(pdev, NULL);
159
160 if (dd->ipath_unit != -1) {
161 spin_lock_irqsave(&ipath_devs_lock, flags);
162 idr_remove(&unit_table, dd->ipath_unit);
163 list_del(&dd->ipath_list);
164 spin_unlock_irqrestore(&ipath_devs_lock, flags);
165 }
06993ca6 166 vfree(dd);
7bb206e3
BS		 167}
168
169static struct ipath_devdata *ipath_alloc_devdata(struct pci_dev *pdev)
170{
171 unsigned long flags;
172 struct ipath_devdata *dd;
7bb206e3
BS		 173 int ret;
174
175 if (!idr_pre_get(&unit_table, GFP_KERNEL)) {
176 dd = ERR_PTR(-ENOMEM);
177 goto bail;
178 }
179
06993ca6 180 dd = vmalloc(sizeof(*dd));
7bb206e3
BS		 181 if (!dd) {
182 dd = ERR_PTR(-ENOMEM);
183 goto bail;
184 }
06993ca6 185 memset(dd, 0, sizeof(*dd));
7bb206e3
BS		 186 dd->ipath_unit = -1;
187
188 spin_lock_irqsave(&ipath_devs_lock, flags);
189
190 ret = idr_get_new(&unit_table, dd, &dd->ipath_unit);
191 if (ret < 0) {
192 printk(KERN_ERR IPATH_DRV_NAME
193 ": Could not allocate unit ID: error %d\n", -ret);
194 ipath_free_devdata(pdev, dd);
195 dd = ERR_PTR(ret);
196 goto bail_unlock;
197 }
198
199 dd->pcidev = pdev;
200 pci_set_drvdata(pdev, dd);
201
202 list_add(&dd->ipath_list, &ipath_dev_list);
203
204bail_unlock:
205 spin_unlock_irqrestore(&ipath_devs_lock, flags);
206
207bail:
208 return dd;
209}
210
211static inline struct ipath_devdata *__ipath_lookup(int unit)
212{
213 return idr_find(&unit_table, unit);
214}
215
216struct ipath_devdata *ipath_lookup(int unit)
217{
218 struct ipath_devdata *dd;
219 unsigned long flags;
220
221 spin_lock_irqsave(&ipath_devs_lock, flags);
222 dd = __ipath_lookup(unit);
223 spin_unlock_irqrestore(&ipath_devs_lock, flags);
224
225 return dd;
226}
227
228int ipath_count_units(int *npresentp, int *nupp, u32 *maxportsp)
229{
230 int nunits, npresent, nup;
231 struct ipath_devdata *dd;
232 unsigned long flags;
233 u32 maxports;
234
235 nunits = npresent = nup = maxports = 0;
236
237 spin_lock_irqsave(&ipath_devs_lock, flags);
238
239 list_for_each_entry(dd, &ipath_dev_list, ipath_list) {
240 nunits++;
241 if ((dd->ipath_flags & IPATH_PRESENT) && dd->ipath_kregbase)
242 npresent++;
243 if (dd->ipath_lid &&
244 !(dd->ipath_flags & (IPATH_DISABLED | IPATH_LINKDOWN
245 | IPATH_LINKUNK)))
246 nup++;
247 if (dd->ipath_cfgports > maxports)
248 maxports = dd->ipath_cfgports;
249 }
250
251 spin_unlock_irqrestore(&ipath_devs_lock, flags);
252
253 if (npresentp)
254 *npresentp = npresent;
255 if (nupp)
256 *nupp = nup;
257 if (maxportsp)
258 *maxportsp = maxports;
259
260 return nunits;
261}
262
7bb206e3
BS		 263/*
264 * These next two routines are placeholders in case we don't have per-arch
265 * code for controlling write combining. If explicit control of write
266 * combining is not available, performance will probably be awful.
267 */
268
269int __attribute__((weak)) ipath_enable_wc(struct ipath_devdata *dd)
270{
271 return -EOPNOTSUPP;
272}
273
274void __attribute__((weak)) ipath_disable_wc(struct ipath_devdata *dd)
275{
276}
277
278static int __devinit ipath_init_one(struct pci_dev *pdev,
279 const struct pci_device_id *ent)
280{
281 int ret, len, j;
282 struct ipath_devdata *dd;
283 unsigned long long addr;
284 u32 bar0 = 0, bar1 = 0;
285 u8 rev;
286
7bb206e3
BS		 287 dd = ipath_alloc_devdata(pdev);
288 if (IS_ERR(dd)) {
289 ret = PTR_ERR(dd);
290 printk(KERN_ERR IPATH_DRV_NAME
291 ": Could not allocate devdata: error %d\n", -ret);
f37bda92 292 goto bail;
7bb206e3
BS		 293 }
294
295 ipath_cdbg(VERBOSE, "initializing unit #%u\n", dd->ipath_unit);
296
297 read_bars(dd, pdev, &bar0, &bar1);
298
299 ret = pci_enable_device(pdev);
300 if (ret) {
301 /* This can happen iff:
302 *
303 * We did a chip reset, and then failed to reprogram the
304 * BAR, or the chip reset due to an internal error. We then
305 * unloaded the driver and reloaded it.
306 *
307 * Both reset cases set the BAR back to initial state. For
308 * the latter case, the AER sticky error bit at offset 0x718
309 * should be set, but the Linux kernel doesn't yet know
310 * about that, it appears. If the original BAR was retained
311 * in the kernel data structures, this may be OK.
312 */
313 ipath_dev_err(dd, "enable unit %d failed: error %d\n",
314 dd->ipath_unit, -ret);
315 goto bail_devdata;
316 }
317 addr = pci_resource_start(pdev, 0);
318 len = pci_resource_len(pdev, 0);
319 ipath_cdbg(VERBOSE, "regbase (0) %llx len %d irq %x, vend %x/%x "
320 "driver_data %lx\n", addr, len, pdev->irq, ent->vendor,
321 ent->device, ent->driver_data);
322
323 read_bars(dd, pdev, &bar0, &bar1);
324
325 if (!bar1 && !(bar0 & ~0xf)) {
326 if (addr) {
327 dev_info(&pdev->dev, "BAR is 0 (probable RESET), "
328 "rewriting as %llx\n", addr);
329 ret = pci_write_config_dword(
330 pdev, PCI_BASE_ADDRESS_0, addr);
331 if (ret) {
332 ipath_dev_err(dd, "rewrite of BAR0 "
333 "failed: err %d\n", -ret);
334 goto bail_disable;
335 }
336 ret = pci_write_config_dword(
337 pdev, PCI_BASE_ADDRESS_1, addr >> 32);
338 if (ret) {
339 ipath_dev_err(dd, "rewrite of BAR1 "
340 "failed: err %d\n", -ret);
341 goto bail_disable;
342 }
343 } else {
344 ipath_dev_err(dd, "BAR is 0 (probable RESET), "
345 "not usable until reboot\n");
346 ret = -ENODEV;
347 goto bail_disable;
348 }
349 }
350
351 ret = pci_request_regions(pdev, IPATH_DRV_NAME);
352 if (ret) {
353 dev_info(&pdev->dev, "pci_request_regions unit %u fails: "
354 "err %d\n", dd->ipath_unit, -ret);
355 goto bail_disable;
356 }
357
358 ret = pci_set_dma_mask(pdev, DMA_64BIT_MASK);
359 if (ret) {
68dd43a1
BS		 360 /*
361 * if the 64 bit setup fails, try 32 bit. Some systems
362 * do not setup 64 bit maps on systems with 2GB or less
363 * memory installed.
364 */
365 ret = pci_set_dma_mask(pdev, DMA_32BIT_MASK);
366 if (ret) {
b1d8865a
BS		 367 dev_info(&pdev->dev,
368 "Unable to set DMA mask for unit %u: %d\n",
369 dd->ipath_unit, ret);
68dd43a1
BS		 370 goto bail_regions;
371 }
b1d8865a 372 else {
68dd43a1 373 ipath_dbg("No 64bit DMA mask, used 32 bit mask\n");
b1d8865a
BS		 374 ret = pci_set_consistent_dma_mask(pdev, DMA_32BIT_MASK);
375 if (ret)
376 dev_info(&pdev->dev,
377 "Unable to set DMA consistent mask "
378 "for unit %u: %d\n",
379 dd->ipath_unit, ret);
380
381 }
382 }
383 else {
384 ret = pci_set_consistent_dma_mask(pdev, DMA_64BIT_MASK);
385 if (ret)
386 dev_info(&pdev->dev,
387 "Unable to set DMA consistent mask "
388 "for unit %u: %d\n",
389 dd->ipath_unit, ret);
7bb206e3
BS		 390 }
391
392 pci_set_master(pdev);
393
394 /*
395 * Save BARs to rewrite after device reset. Save all 64 bits of
396 * BAR, just in case.
397 */
398 dd->ipath_pcibar0 = addr;
399 dd->ipath_pcibar1 = addr >> 32;
400 dd->ipath_deviceid = ent->device; /* save for later use */
401 dd->ipath_vendorid = ent->vendor;
402
403 /* setup the chip-specific functions, as early as possible. */
404 switch (ent->device) {
405 case PCI_DEVICE_ID_INFINIPATH_HT:
406 ipath_init_ht400_funcs(dd);
407 break;
408 case PCI_DEVICE_ID_INFINIPATH_PE800:
409 ipath_init_pe800_funcs(dd);
410 break;
411 default:
759d5768 412 ipath_dev_err(dd, "Found unknown QLogic deviceid 0x%x, "
7bb206e3
BS		 413 "failing\n", ent->device);
414 return -ENODEV;
415 }
416
417 for (j = 0; j < 6; j++) {
418 if (!pdev->resource[j].start)
419 continue;
e29419ff
GKH		 420 ipath_cdbg(VERBOSE, "BAR %d start %llx, end %llx, len %llx\n",
421 j, (unsigned long long)pdev->resource[j].start,
422 (unsigned long long)pdev->resource[j].end,
423 (unsigned long long)pci_resource_len(pdev, j));
7bb206e3
BS		 424 }
425
426 if (!addr) {
427 ipath_dev_err(dd, "No valid address in BAR 0!\n");
428 ret = -ENODEV;
429 goto bail_regions;
430 }
431
432 dd->ipath_deviceid = ent->device; /* save for later use */
433 dd->ipath_vendorid = ent->vendor;
434
435 ret = pci_read_config_byte(pdev, PCI_REVISION_ID, &rev);
436 if (ret) {
437 ipath_dev_err(dd, "Failed to read PCI revision ID unit "
438 "%u: err %d\n", dd->ipath_unit, -ret);
439 goto bail_regions; /* shouldn't ever happen */
440 }
441 dd->ipath_pcirev = rev;
442
443 dd->ipath_kregbase = ioremap_nocache(addr, len);
444
445 if (!dd->ipath_kregbase) {
446 ipath_dbg("Unable to map io addr %llx to kvirt, failing\n",
447 addr);
448 ret = -ENOMEM;
449 goto bail_iounmap;
450 }
451 dd->ipath_kregend = (u64 __iomem *)
452 ((void __iomem *)dd->ipath_kregbase + len);
453 dd->ipath_physaddr = addr; /* used for io_remap, etc. */
454 /* for user mmap */
b35f004d
BS		 455 ipath_cdbg(VERBOSE, "mapped io addr %llx to kregbase %p\n",
456 addr, dd->ipath_kregbase);
7bb206e3
BS		 457
458 /*
459 * clear ipath_flags here instead of in ipath_init_chip as it is set
460 * by ipath_setup_htconfig.
461 */
462 dd->ipath_flags = 0;
463
464 if (dd->ipath_f_bus(dd, pdev))
465 ipath_dev_err(dd, "Failed to setup config space; "
466 "continuing anyway\n");
467
468 /*
469 * set up our interrupt handler; SA_SHIRQ probably not needed,
470 * since MSI interrupts shouldn't be shared but won't hurt for now.
471 * check 0 irq after we return from chip-specific bus setup, since
472 * that can affect this due to setup
473 */
474 if (!pdev->irq)
475 ipath_dev_err(dd, "irq is 0, BIOS error? Interrupts won't "
476 "work\n");
477 else {
478 ret = request_irq(pdev->irq, ipath_intr, SA_SHIRQ,
479 IPATH_DRV_NAME, dd);
480 if (ret) {
481 ipath_dev_err(dd, "Couldn't setup irq handler, "
482 "irq=%u: %d\n", pdev->irq, ret);
483 goto bail_iounmap;
484 }
485 }
486
487 ret = ipath_init_chip(dd, 0); /* do the chip-specific init */
488 if (ret)
489 goto bail_iounmap;
490
491 ret = ipath_enable_wc(dd);
492
493 if (ret) {
494 ipath_dev_err(dd, "Write combining not enabled "
495 "(err %d): performance may be poor\n",
496 -ret);
497 ret = 0;
498 }
499
500 ipath_device_create_group(&pdev->dev, dd);
501 ipathfs_add_device(dd);
502 ipath_user_add(dd);
a2acb2ff 503 ipath_diag_add(dd);
7bb206e3
BS		 504 ipath_layer_add(dd);
505
506 goto bail;
507
508bail_iounmap:
509 iounmap((volatile void __iomem *) dd->ipath_kregbase);
510
511bail_regions:
512 pci_release_regions(pdev);
513
514bail_disable:
515 pci_disable_device(pdev);
516
517bail_devdata:
518 ipath_free_devdata(pdev, dd);
519
7bb206e3
BS		 520bail:
521 return ret;
522}
523
524static void __devexit ipath_remove_one(struct pci_dev *pdev)
525{
526 struct ipath_devdata *dd;
527
528 ipath_cdbg(VERBOSE, "removing, pdev=%p\n", pdev);
529 if (!pdev)
530 return;
531
532 dd = pci_get_drvdata(pdev);
a2acb2ff
BS		 533 ipath_layer_remove(dd);
534 ipath_diag_remove(dd);
535 ipath_user_remove(dd);
7bb206e3
BS		 536 ipathfs_remove_device(dd);
537 ipath_device_remove_group(&pdev->dev, dd);
538 ipath_cdbg(VERBOSE, "Releasing pci memory regions, dd %p, "
539 "unit %u\n", dd, (u32) dd->ipath_unit);
540 if (dd->ipath_kregbase) {
541 ipath_cdbg(VERBOSE, "Unmapping kregbase %p\n",
542 dd->ipath_kregbase);
543 iounmap((volatile void __iomem *) dd->ipath_kregbase);
544 dd->ipath_kregbase = NULL;
545 }
546 pci_release_regions(pdev);
547 ipath_cdbg(VERBOSE, "calling pci_disable_device\n");
548 pci_disable_device(pdev);
549
550 ipath_free_devdata(pdev, dd);
7bb206e3
BS		 551}
552
553/* general driver use */
554DEFINE_MUTEX(ipath_mutex);
555
556static DEFINE_SPINLOCK(ipath_pioavail_lock);
557
558/**
559 * ipath_disarm_piobufs - cancel a range of PIO buffers
560 * @dd: the infinipath device
561 * @first: the first PIO buffer to cancel
562 * @cnt: the number of PIO buffers to cancel
563 *
564 * cancel a range of PIO buffers, used when they might be armed, but
565 * not triggered. Used at init to ensure buffer state, and also user
566 * process close, in case it died while writing to a PIO buffer
567 * Also after errors.
568 */
569void ipath_disarm_piobufs(struct ipath_devdata *dd, unsigned first,
570 unsigned cnt)
571{
572 unsigned i, last = first + cnt;
573 u64 sendctrl, sendorig;
574
575 ipath_cdbg(PKT, "disarm %u PIObufs first=%u\n", cnt, first);
576 sendorig = dd->ipath_sendctrl | INFINIPATH_S_DISARM;
577 for (i = first; i < last; i++) {
578 sendctrl = sendorig |
579 (i << INFINIPATH_S_DISARMPIOBUF_SHIFT);
580 ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl,
581 sendctrl);
582 }
583
584 /*
585 * Write it again with current value, in case ipath_sendctrl changed
586 * while we were looping; no critical bits that would require
587 * locking.
588 *
589 * Write a 0, and then the original value, reading scratch in
590 * between. This seems to avoid a chip timing race that causes
591 * pioavail updates to memory to stop.
592 */
593 ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl,
594 0);
595 sendorig = ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
596 ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl,
597 dd->ipath_sendctrl);
598}
599
600/**
601 * ipath_wait_linkstate - wait for an IB link state change to occur
602 * @dd: the infinipath device
603 * @state: the state to wait for
604 * @msecs: the number of milliseconds to wait
605 *
606 * wait up to msecs milliseconds for IB link state change to occur for
607 * now, take the easy polling route. Currently used only by
608 * ipath_layer_set_linkstate. Returns 0 if state reached, otherwise
609 * -ETIMEDOUT state can have multiple states set, for any of several
610 * transitions.
611 */
612int ipath_wait_linkstate(struct ipath_devdata *dd, u32 state, int msecs)
613{
614 dd->ipath_sma_state_wanted = state;
615 wait_event_interruptible_timeout(ipath_sma_state_wait,
616 (dd->ipath_flags & state),
617 msecs_to_jiffies(msecs));
618 dd->ipath_sma_state_wanted = 0;
619
620 if (!(dd->ipath_flags & state)) {
621 u64 val;
622 ipath_cdbg(SMA, "Didn't reach linkstate %s within %u ms\n",
623 /* test INIT ahead of DOWN, both can be set */
624 (state & IPATH_LINKINIT) ? "INIT" :
625 ((state & IPATH_LINKDOWN) ? "DOWN" :
626 ((state & IPATH_LINKARMED) ? "ARM" : "ACTIVE")),
627 msecs);
628 val = ipath_read_kreg64(dd, dd->ipath_kregs->kr_ibcstatus);
629 ipath_cdbg(VERBOSE, "ibcc=%llx ibcstatus=%llx (%s)\n",
630 (unsigned long long) ipath_read_kreg64(
631 dd, dd->ipath_kregs->kr_ibcctrl),
632 (unsigned long long) val,
633 ipath_ibcstatus_str[val & 0xf]);
634 }
635 return (dd->ipath_flags & state) ? 0 : -ETIMEDOUT;
636}
637
638void ipath_decode_err(char *buf, size_t blen, ipath_err_t err)
639{
640 *buf = '\0';
641 if (err & INFINIPATH_E_RHDRLEN)
642 strlcat(buf, "rhdrlen ", blen);
643 if (err & INFINIPATH_E_RBADTID)
644 strlcat(buf, "rbadtid ", blen);
645 if (err & INFINIPATH_E_RBADVERSION)
646 strlcat(buf, "rbadversion ", blen);
647 if (err & INFINIPATH_E_RHDR)
648 strlcat(buf, "rhdr ", blen);
649 if (err & INFINIPATH_E_RLONGPKTLEN)
650 strlcat(buf, "rlongpktlen ", blen);
651 if (err & INFINIPATH_E_RSHORTPKTLEN)
652 strlcat(buf, "rshortpktlen ", blen);
653 if (err & INFINIPATH_E_RMAXPKTLEN)
654 strlcat(buf, "rmaxpktlen ", blen);
655 if (err & INFINIPATH_E_RMINPKTLEN)
656 strlcat(buf, "rminpktlen ", blen);
657 if (err & INFINIPATH_E_RFORMATERR)
658 strlcat(buf, "rformaterr ", blen);
659 if (err & INFINIPATH_E_RUNSUPVL)
660 strlcat(buf, "runsupvl ", blen);
661 if (err & INFINIPATH_E_RUNEXPCHAR)
662 strlcat(buf, "runexpchar ", blen);
663 if (err & INFINIPATH_E_RIBFLOW)
664 strlcat(buf, "ribflow ", blen);
665 if (err & INFINIPATH_E_REBP)
666 strlcat(buf, "EBP ", blen);
667 if (err & INFINIPATH_E_SUNDERRUN)
668 strlcat(buf, "sunderrun ", blen);
669 if (err & INFINIPATH_E_SPIOARMLAUNCH)
670 strlcat(buf, "spioarmlaunch ", blen);
671 if (err & INFINIPATH_E_SUNEXPERRPKTNUM)
672 strlcat(buf, "sunexperrpktnum ", blen);
673 if (err & INFINIPATH_E_SDROPPEDDATAPKT)
674 strlcat(buf, "sdroppeddatapkt ", blen);
675 if (err & INFINIPATH_E_SDROPPEDSMPPKT)
676 strlcat(buf, "sdroppedsmppkt ", blen);
677 if (err & INFINIPATH_E_SMAXPKTLEN)
678 strlcat(buf, "smaxpktlen ", blen);
679 if (err & INFINIPATH_E_SMINPKTLEN)
680 strlcat(buf, "sminpktlen ", blen);
681 if (err & INFINIPATH_E_SUNSUPVL)
682 strlcat(buf, "sunsupVL ", blen);
683 if (err & INFINIPATH_E_SPKTLEN)
684 strlcat(buf, "spktlen ", blen);
685 if (err & INFINIPATH_E_INVALIDADDR)
686 strlcat(buf, "invalidaddr ", blen);
687 if (err & INFINIPATH_E_RICRC)
688 strlcat(buf, "CRC ", blen);
689 if (err & INFINIPATH_E_RVCRC)
690 strlcat(buf, "VCRC ", blen);
691 if (err & INFINIPATH_E_RRCVEGRFULL)
692 strlcat(buf, "rcvegrfull ", blen);
693 if (err & INFINIPATH_E_RRCVHDRFULL)
694 strlcat(buf, "rcvhdrfull ", blen);
695 if (err & INFINIPATH_E_IBSTATUSCHANGED)
696 strlcat(buf, "ibcstatuschg ", blen);
697 if (err & INFINIPATH_E_RIBLOSTLINK)
698 strlcat(buf, "riblostlink ", blen);
699 if (err & INFINIPATH_E_HARDWARE)
700 strlcat(buf, "hardware ", blen);
701 if (err & INFINIPATH_E_RESET)
702 strlcat(buf, "reset ", blen);
703}
704
705/**
706 * get_rhf_errstring - decode RHF errors
707 * @err: the err number
708 * @msg: the output buffer
709 * @len: the length of the output buffer
710 *
711 * only used one place now, may want more later
712 */
713static void get_rhf_errstring(u32 err, char *msg, size_t len)
714{
715 /* if no errors, and so don't need to check what's first */
716 *msg = '\0';
717
718 if (err & INFINIPATH_RHF_H_ICRCERR)
719 strlcat(msg, "icrcerr ", len);
720 if (err & INFINIPATH_RHF_H_VCRCERR)
721 strlcat(msg, "vcrcerr ", len);
722 if (err & INFINIPATH_RHF_H_PARITYERR)
723 strlcat(msg, "parityerr ", len);
724 if (err & INFINIPATH_RHF_H_LENERR)
725 strlcat(msg, "lenerr ", len);
726 if (err & INFINIPATH_RHF_H_MTUERR)
727 strlcat(msg, "mtuerr ", len);
728 if (err & INFINIPATH_RHF_H_IHDRERR)
729 /* infinipath hdr checksum error */
730 strlcat(msg, "ipathhdrerr ", len);
731 if (err & INFINIPATH_RHF_H_TIDERR)
732 strlcat(msg, "tiderr ", len);
733 if (err & INFINIPATH_RHF_H_MKERR)
734 /* bad port, offset, etc. */
735 strlcat(msg, "invalid ipathhdr ", len);
736 if (err & INFINIPATH_RHF_H_IBERR)
737 strlcat(msg, "iberr ", len);
738 if (err & INFINIPATH_RHF_L_SWA)
739 strlcat(msg, "swA ", len);
740 if (err & INFINIPATH_RHF_L_SWB)
741 strlcat(msg, "swB ", len);
742}
743
744/**
745 * ipath_get_egrbuf - get an eager buffer
746 * @dd: the infinipath device
747 * @bufnum: the eager buffer to get
748 * @err: unused
749 *
750 * must only be called if ipath_pd[port] is known to be allocated
751 */
752static inline void *ipath_get_egrbuf(struct ipath_devdata *dd, u32 bufnum,
753 int err)
754{
755 return dd->ipath_port0_skbs ?
756 (void *)dd->ipath_port0_skbs[bufnum]->data : NULL;
757}
758
759/**
760 * ipath_alloc_skb - allocate an skb and buffer with possible constraints
761 * @dd: the infinipath device
762 * @gfp_mask: the sk_buff SFP mask
763 */
764struct sk_buff *ipath_alloc_skb(struct ipath_devdata *dd,
765 gfp_t gfp_mask)
766{
767 struct sk_buff *skb;
768 u32 len;
769
770 /*
771 * Only fully supported way to handle this is to allocate lots
772 * extra, align as needed, and then do skb_reserve(). That wastes
773 * a lot of memory... I'll have to hack this into infinipath_copy
774 * also.
775 */
776
777 /*
778 * We need 4 extra bytes for unaligned transfer copying
779 */
780 if (dd->ipath_flags & IPATH_4BYTE_TID) {
781 /* we need a 4KB multiple alignment, and there is no way
782 * to do it except to allocate extra and then skb_reserve
783 * enough to bring it up to the right alignment.
784 */
785 len = dd->ipath_ibmaxlen + 4 + (1 << 11) - 1;
786 }
787 else
788 len = dd->ipath_ibmaxlen + 4;
789 skb = __dev_alloc_skb(len, gfp_mask);
790 if (!skb) {
791 ipath_dev_err(dd, "Failed to allocate skbuff, length %u\n",
792 len);
793 goto bail;
794 }
795 if (dd->ipath_flags & IPATH_4BYTE_TID) {
796 u32 una = ((1 << 11) - 1) & (unsigned long)(skb->data + 4);
797 if (una)
798 skb_reserve(skb, 4 + (1 << 11) - una);
799 else
800 skb_reserve(skb, 4);
801 } else
802 skb_reserve(skb, 4);
803
804bail:
805 return skb;
806}
807
808/**
809 * ipath_rcv_layer - receive a packet for the layered (ethernet) driver
810 * @dd: the infinipath device
811 * @etail: the sk_buff number
812 * @tlen: the total packet length
813 * @hdr: the ethernet header
814 *
815 * Separate routine for better overall optimization
816 */
817static void ipath_rcv_layer(struct ipath_devdata *dd, u32 etail,
818 u32 tlen, struct ether_header *hdr)
819{
820 u32 elen;
821 u8 pad, *bthbytes;
822 struct sk_buff *skb, *nskb;
823
824 if (dd->ipath_port0_skbs && hdr->sub_opcode == OPCODE_ENCAP) {
825 /*
826 * Allocate a new sk_buff to replace the one we give
827 * to the network stack.
828 */
829 nskb = ipath_alloc_skb(dd, GFP_ATOMIC);
830 if (!nskb) {
831 /* count OK packets that we drop */
832 ipath_stats.sps_krdrops++;
833 return;
834 }
835
836 bthbytes = (u8 *) hdr->bth;
837 pad = (bthbytes[1] >> 4) & 3;
838 /* +CRC32 */
839 elen = tlen - (sizeof(*hdr) + pad + sizeof(u32));
840
841 skb = dd->ipath_port0_skbs[etail];
842 dd->ipath_port0_skbs[etail] = nskb;
843 skb_put(skb, elen);
844
845 dd->ipath_f_put_tid(dd, etail + (u64 __iomem *)
846 ((char __iomem *) dd->ipath_kregbase
847 + dd->ipath_rcvegrbase), 0,
848 virt_to_phys(nskb->data));
849
850 __ipath_layer_rcv(dd, hdr, skb);
851
852 /* another ether packet received */
853 ipath_stats.sps_ether_rpkts++;
854 }
855 else if (hdr->sub_opcode == OPCODE_LID_ARP)
856 __ipath_layer_rcv_lid(dd, hdr);
857}
858
859/*
860 * ipath_kreceive - receive a packet
861 * @dd: the infinipath device
862 *
863 * called from interrupt handler for errors or receive interrupt
864 */
865void ipath_kreceive(struct ipath_devdata *dd)
866{
867 u64 *rc;
868 void *ebuf;
869 const u32 rsize = dd->ipath_rcvhdrentsize; /* words */
870 const u32 maxcnt = dd->ipath_rcvhdrcnt * rsize; /* words */
871 u32 etail = -1, l, hdrqtail;
872 struct ips_message_header *hdr;
873 u32 eflags, i, etype, tlen, pkttot = 0;
874 static u64 totcalls; /* stats, may eventually remove */
875 char emsg[128];
876
877 if (!dd->ipath_hdrqtailptr) {
878 ipath_dev_err(dd,
879 "hdrqtailptr not set, can't do receives\n");
880 goto bail;
881 }
882
883 /* There is already a thread processing this queue. */
884 if (test_and_set_bit(0, &dd->ipath_rcv_pending))
885 goto bail;
886
887 if (dd->ipath_port0head ==
888 (u32)le64_to_cpu(*dd->ipath_hdrqtailptr))
889 goto done;
890
891gotmore:
892 /*
893 * read only once at start. If in flood situation, this helps
894 * performance slightly. If more arrive while we are processing,
895 * we'll come back here and do them
896 */
897 hdrqtail = (u32)le64_to_cpu(*dd->ipath_hdrqtailptr);
898
899 for (i = 0, l = dd->ipath_port0head; l != hdrqtail; i++) {
900 u32 qp;
901 u8 *bthbytes;
902
903 rc = (u64 *) (dd->ipath_pd[0]->port_rcvhdrq + (l << 2));
904 hdr = (struct ips_message_header *)&rc[1];
905 /*
906 * could make a network order version of IPATH_KD_QP, and
907 * do the obvious shift before masking to speed this up.
908 */
909 qp = ntohl(hdr->bth[1]) & 0xffffff;
910 bthbytes = (u8 *) hdr->bth;
911
912 eflags = ips_get_hdr_err_flags((__le32 *) rc);
913 etype = ips_get_rcv_type((__le32 *) rc);
914 /* total length */
915 tlen = ips_get_length_in_bytes((__le32 *) rc);
916 ebuf = NULL;
917 if (etype != RCVHQ_RCV_TYPE_EXPECTED) {
918 /*
919 * it turns out that the chips uses an eager buffer
920 * for all non-expected packets, whether it "needs"
921 * one or not. So always get the index, but don't
922 * set ebuf (so we try to copy data) unless the
923 * length requires it.
924 */
925 etail = ips_get_index((__le32 *) rc);
926 if (tlen > sizeof(*hdr) ||
927 etype == RCVHQ_RCV_TYPE_NON_KD)
928 ebuf = ipath_get_egrbuf(dd, etail, 0);
929 }
930
931 /*
932 * both tiderr and ipathhdrerr are set for all plain IB
933 * packets; only ipathhdrerr should be set.
934 */
935
936 if (etype != RCVHQ_RCV_TYPE_NON_KD && etype !=
937 RCVHQ_RCV_TYPE_ERROR && ips_get_ipath_ver(
938 hdr->iph.ver_port_tid_offset) !=
939 IPS_PROTO_VERSION) {
940 ipath_cdbg(PKT, "Bad InfiniPath protocol version "
941 "%x\n", etype);
942 }
943
944 if (eflags & ~(INFINIPATH_RHF_H_TIDERR |
945 INFINIPATH_RHF_H_IHDRERR)) {
946 get_rhf_errstring(eflags, emsg, sizeof emsg);
947 ipath_cdbg(PKT, "RHFerrs %x hdrqtail=%x typ=%u "
948 "tlen=%x opcode=%x egridx=%x: %s\n",
949 eflags, l, etype, tlen, bthbytes[0],
950 ips_get_index((__le32 *) rc), emsg);
951 } else if (etype == RCVHQ_RCV_TYPE_NON_KD) {
952 int ret = __ipath_verbs_rcv(dd, rc + 1,
953 ebuf, tlen);
954 if (ret == -ENODEV)
955 ipath_cdbg(VERBOSE,
956 "received IB packet, "
957 "not SMA (QP=%x)\n", qp);
958 } else if (etype == RCVHQ_RCV_TYPE_EAGER) {
959 if (qp == IPATH_KD_QP &&
960 bthbytes[0] == ipath_layer_rcv_opcode &&
961 ebuf)
962 ipath_rcv_layer(dd, etail, tlen,
963 (struct ether_header *)hdr);
964 else
965 ipath_cdbg(PKT, "typ %x, opcode %x (eager, "
966 "qp=%x), len %x; ignored\n",
967 etype, bthbytes[0], qp, tlen);
968 }
969 else if (etype == RCVHQ_RCV_TYPE_EXPECTED)
970 ipath_dbg("Bug: Expected TID, opcode %x; ignored\n",
971 be32_to_cpu(hdr->bth[0]) & 0xff);
972 else if (eflags & (INFINIPATH_RHF_H_TIDERR |
973 INFINIPATH_RHF_H_IHDRERR)) {
974 /*
975 * This is a type 3 packet, only the LRH is in the
976 * rcvhdrq, the rest of the header is in the eager
977 * buffer.
978 */
979 u8 opcode;
980 if (ebuf) {
981 bthbytes = (u8 *) ebuf;
982 opcode = *bthbytes;
983 }
984 else
985 opcode = 0;
986 get_rhf_errstring(eflags, emsg, sizeof emsg);
987 ipath_dbg("Err %x (%s), opcode %x, egrbuf %x, "
988 "len %x\n", eflags, emsg, opcode, etail,
989 tlen);
990 } else {
991 /*
992 * error packet, type of error unknown.
993 * Probably type 3, but we don't know, so don't
994 * even try to print the opcode, etc.
995 */
996 ipath_dbg("Error Pkt, but no eflags! egrbuf %x, "
997 "len %x\nhdrq@%lx;hdrq+%x rhf: %llx; "
998 "hdr %llx %llx %llx %llx %llx\n",
999 etail, tlen, (unsigned long) rc, l,
1000 (unsigned long long) rc[0],
1001 (unsigned long long) rc[1],
1002 (unsigned long long) rc[2],
1003 (unsigned long long) rc[3],
1004 (unsigned long long) rc[4],
1005 (unsigned long long) rc[5]);
1006 }
1007 l += rsize;
1008 if (l >= maxcnt)
1009 l = 0;
1010 /*
1011 * update for each packet, to help prevent overflows if we
1012 * have lots of packets.
1013 */
1014 (void)ipath_write_ureg(dd, ur_rcvhdrhead,
1015 dd->ipath_rhdrhead_intr_off | l, 0);
1016 if (etype != RCVHQ_RCV_TYPE_EXPECTED)
1017 (void)ipath_write_ureg(dd, ur_rcvegrindexhead,
1018 etail, 0);
1019 }
1020
1021 pkttot += i;
1022
1023 dd->ipath_port0head = l;
1024
1025 if (hdrqtail != (u32)le64_to_cpu(*dd->ipath_hdrqtailptr))
1026 /* more arrived while we handled first batch */
1027 goto gotmore;
1028
1029 if (pkttot > ipath_stats.sps_maxpkts_call)
1030 ipath_stats.sps_maxpkts_call = pkttot;
1031 ipath_stats.sps_port0pkts += pkttot;
1032 ipath_stats.sps_avgpkts_call =
1033 ipath_stats.sps_port0pkts / ++totcalls;
1034
1035done:
1036 clear_bit(0, &dd->ipath_rcv_pending);
1037 smp_mb__after_clear_bit();
1038
1039bail:;
1040}
1041
1042/**
1043 * ipath_update_pio_bufs - update shadow copy of the PIO availability map
1044 * @dd: the infinipath device
1045 *
1046 * called whenever our local copy indicates we have run out of send buffers
1047 * NOTE: This can be called from interrupt context by some code
1048 * and from non-interrupt context by ipath_getpiobuf().
1049 */
1050
1051static void ipath_update_pio_bufs(struct ipath_devdata *dd)
1052{
1053 unsigned long flags;
1054 int i;
1055 const unsigned piobregs = (unsigned)dd->ipath_pioavregs;
1056
1057 /* If the generation (check) bits have changed, then we update the
1058 * busy bit for the corresponding PIO buffer. This algorithm will
1059 * modify positions to the value they already have in some cases
1060 * (i.e., no change), but it's faster than changing only the bits
1061 * that have changed.
1062 *
1063 * We would like to do this atomicly, to avoid spinlocks in the
1064 * critical send path, but that's not really possible, given the
1065 * type of changes, and that this routine could be called on
1066 * multiple cpu's simultaneously, so we lock in this routine only,
1067 * to avoid conflicting updates; all we change is the shadow, and
1068 * it's a single 64 bit memory location, so by definition the update
1069 * is atomic in terms of what other cpu's can see in testing the
1070 * bits. The spin_lock overhead isn't too bad, since it only
1071 * happens when all buffers are in use, so only cpu overhead, not
1072 * latency or bandwidth is affected.
1073 */
1074#define _IPATH_ALL_CHECKBITS 0x5555555555555555ULL
1075 if (!dd->ipath_pioavailregs_dma) {
1076 ipath_dbg("Update shadow pioavail, but regs_dma NULL!\n");
1077 return;
1078 }
1079 if (ipath_debug & __IPATH_VERBDBG) {
1080 /* only if packet debug and verbose */
1081 volatile __le64 *dma = dd->ipath_pioavailregs_dma;
1082 unsigned long *shadow = dd->ipath_pioavailshadow;
1083
1084 ipath_cdbg(PKT, "Refill avail, dma0=%llx shad0=%lx, "
1085 "d1=%llx s1=%lx, d2=%llx s2=%lx, d3=%llx "
1086 "s3=%lx\n",
1087 (unsigned long long) le64_to_cpu(dma[0]),
1088 shadow[0],
1089 (unsigned long long) le64_to_cpu(dma[1]),
1090 shadow[1],
1091 (unsigned long long) le64_to_cpu(dma[2]),
1092 shadow[2],
1093 (unsigned long long) le64_to_cpu(dma[3]),
1094 shadow[3]);
1095 if (piobregs > 4)
1096 ipath_cdbg(
1097 PKT, "2nd group, dma4=%llx shad4=%lx, "
1098 "d5=%llx s5=%lx, d6=%llx s6=%lx, "
1099 "d7=%llx s7=%lx\n",
1100 (unsigned long long) le64_to_cpu(dma[4]),
1101 shadow[4],
1102 (unsigned long long) le64_to_cpu(dma[5]),
1103 shadow[5],
1104 (unsigned long long) le64_to_cpu(dma[6]),
1105 shadow[6],
1106 (unsigned long long) le64_to_cpu(dma[7]),
1107 shadow[7]);
1108 }
1109 spin_lock_irqsave(&ipath_pioavail_lock, flags);
1110 for (i = 0; i < piobregs; i++) {
1111 u64 pchbusy, pchg, piov, pnew;
1112 /*
1113 * Chip Errata: bug 6641; even and odd qwords>3 are swapped
1114 */
1115 if (i > 3) {
1116 if (i & 1)
1117 piov = le64_to_cpu(
1118 dd->ipath_pioavailregs_dma[i - 1]);
1119 else
1120 piov = le64_to_cpu(
1121 dd->ipath_pioavailregs_dma[i + 1]);
1122 } else
1123 piov = le64_to_cpu(dd->ipath_pioavailregs_dma[i]);
1124 pchg = _IPATH_ALL_CHECKBITS &
1125 ~(dd->ipath_pioavailshadow[i] ^ piov);
1126 pchbusy = pchg << INFINIPATH_SENDPIOAVAIL_BUSY_SHIFT;
1127 if (pchg && (pchbusy & dd->ipath_pioavailshadow[i])) {
1128 pnew = dd->ipath_pioavailshadow[i] & ~pchbusy;
1129 pnew |= piov & pchbusy;
1130 dd->ipath_pioavailshadow[i] = pnew;
1131 }
1132 }
1133 spin_unlock_irqrestore(&ipath_pioavail_lock, flags);
1134}
1135
1136/**
1137 * ipath_setrcvhdrsize - set the receive header size
1138 * @dd: the infinipath device
1139 * @rhdrsize: the receive header size
1140 *
1141 * called from user init code, and also layered driver init
1142 */
1143int ipath_setrcvhdrsize(struct ipath_devdata *dd, unsigned rhdrsize)
1144{
1145 int ret = 0;
1146
1147 if (dd->ipath_flags & IPATH_RCVHDRSZ_SET) {
1148 if (dd->ipath_rcvhdrsize != rhdrsize) {
1149 dev_info(&dd->pcidev->dev,
1150 "Error: can't set protocol header "
1151 "size %u, already %u\n",
1152 rhdrsize, dd->ipath_rcvhdrsize);
1153 ret = -EAGAIN;
1154 } else
1155 ipath_cdbg(VERBOSE, "Reuse same protocol header "
1156 "size %u\n", dd->ipath_rcvhdrsize);
1157 } else if (rhdrsize > (dd->ipath_rcvhdrentsize -
1158 (sizeof(u64) / sizeof(u32)))) {
1159 ipath_dbg("Error: can't set protocol header size %u "
1160 "(> max %u)\n", rhdrsize,
1161 dd->ipath_rcvhdrentsize -
1162 (u32) (sizeof(u64) / sizeof(u32)));
1163 ret = -EOVERFLOW;
1164 } else {
1165 dd->ipath_flags |= IPATH_RCVHDRSZ_SET;
1166 dd->ipath_rcvhdrsize = rhdrsize;
1167 ipath_write_kreg(dd, dd->ipath_kregs->kr_rcvhdrsize,
1168 dd->ipath_rcvhdrsize);
1169 ipath_cdbg(VERBOSE, "Set protocol header size to %u\n",
1170 dd->ipath_rcvhdrsize);
1171 }
1172 return ret;
1173}
1174
1175/**
1176 * ipath_getpiobuf - find an available pio buffer
1177 * @dd: the infinipath device
1178 * @pbufnum: the buffer number is placed here
1179 *
1180 * do appropriate marking as busy, etc.
1181 * returns buffer number if one found (>=0), negative number is error.
1182 * Used by ipath_sma_send_pkt and ipath_layer_send
1183 */
1184u32 __iomem *ipath_getpiobuf(struct ipath_devdata *dd, u32 * pbufnum)
1185{
1186 int i, j, starti, updated = 0;
1187 unsigned piobcnt, iter;
1188 unsigned long flags;
1189 unsigned long *shadow = dd->ipath_pioavailshadow;
1190 u32 __iomem *buf;
1191
1192 piobcnt = (unsigned)(dd->ipath_piobcnt2k
1193 + dd->ipath_piobcnt4k);
1194 starti = dd->ipath_lastport_piobuf;
1195 iter = piobcnt - starti;
1196 if (dd->ipath_upd_pio_shadow) {
1197 /*
1198 * Minor optimization. If we had no buffers on last call,
1199 * start out by doing the update; continue and do scan even
1200 * if no buffers were updated, to be paranoid
1201 */
1202 ipath_update_pio_bufs(dd);
1203 /* we scanned here, don't do it at end of scan */
1204 updated = 1;
1205 i = starti;
1206 } else
1207 i = dd->ipath_lastpioindex;
1208
1209rescan:
1210 /*
1211 * while test_and_set_bit() is atomic, we do that and then the
1212 * change_bit(), and the pair is not. See if this is the cause
1213 * of the remaining armlaunch errors.
1214 */
1215 spin_lock_irqsave(&ipath_pioavail_lock, flags);
1216 for (j = 0; j < iter; j++, i++) {
1217 if (i >= piobcnt)
1218 i = starti;
1219 /*
1220 * To avoid bus lock overhead, we first find a candidate
1221 * buffer, then do the test and set, and continue if that
1222 * fails.
1223 */
1224 if (test_bit((2 * i) + 1, shadow) ||
1225 test_and_set_bit((2 * i) + 1, shadow))
1226 continue;
1227 /* flip generation bit */
1228 change_bit(2 * i, shadow);
1229 break;
1230 }
1231 spin_unlock_irqrestore(&ipath_pioavail_lock, flags);
1232
1233 if (j == iter) {
1234 volatile __le64 *dma = dd->ipath_pioavailregs_dma;
1235
1236 /*
1237 * first time through; shadow exhausted, but may be real
1238 * buffers available, so go see; if any updated, rescan
1239 * (once)
1240 */
1241 if (!updated) {
1242 ipath_update_pio_bufs(dd);
1243 updated = 1;
1244 i = starti;
1245 goto rescan;
1246 }
1247 dd->ipath_upd_pio_shadow = 1;
1248 /*
1249 * not atomic, but if we lose one once in a while, that's OK
1250 */
1251 ipath_stats.sps_nopiobufs++;
1252 if (!(++dd->ipath_consec_nopiobuf % 100000)) {
1253 ipath_dbg(
1254 "%u pio sends with no bufavail; dmacopy: "
1255 "%llx %llx %llx %llx; shadow: "
1256 "%lx %lx %lx %lx\n",
1257 dd->ipath_consec_nopiobuf,
1258 (unsigned long long) le64_to_cpu(dma[0]),
1259 (unsigned long long) le64_to_cpu(dma[1]),
1260 (unsigned long long) le64_to_cpu(dma[2]),
1261 (unsigned long long) le64_to_cpu(dma[3]),
1262 shadow[0], shadow[1], shadow[2],
1263 shadow[3]);
1264 /*
1265 * 4 buffers per byte, 4 registers above, cover rest
1266 * below
1267 */
1268 if ((dd->ipath_piobcnt2k + dd->ipath_piobcnt4k) >
1269 (sizeof(shadow[0]) * 4 * 4))
1270 ipath_dbg("2nd group: dmacopy: %llx %llx "
1271 "%llx %llx; shadow: %lx %lx "
1272 "%lx %lx\n",
1273 (unsigned long long)
1274 le64_to_cpu(dma[4]),
1275 (unsigned long long)
1276 le64_to_cpu(dma[5]),
1277 (unsigned long long)
1278 le64_to_cpu(dma[6]),
1279 (unsigned long long)
1280 le64_to_cpu(dma[7]),
1281 shadow[4], shadow[5],
1282 shadow[6], shadow[7]);
1283 }
1284 buf = NULL;
1285 goto bail;
1286 }
1287
1288 if (updated)
1289 /*
1290 * ran out of bufs, now some (at least this one we just
1291 * got) are now available, so tell the layered driver.
1292 */
1293 __ipath_layer_intr(dd, IPATH_LAYER_INT_SEND_CONTINUE);
1294
1295 /*
1296 * set next starting place. Since it's just an optimization,
1297 * it doesn't matter who wins on this, so no locking
1298 */
1299 dd->ipath_lastpioindex = i + 1;
1300 if (dd->ipath_upd_pio_shadow)
1301 dd->ipath_upd_pio_shadow = 0;
1302 if (dd->ipath_consec_nopiobuf)
1303 dd->ipath_consec_nopiobuf = 0;
1304 if (i < dd->ipath_piobcnt2k)
1305 buf = (u32 __iomem *) (dd->ipath_pio2kbase +
1306 i * dd->ipath_palign);
1307 else
1308 buf = (u32 __iomem *)
1309 (dd->ipath_pio4kbase +
1310 (i - dd->ipath_piobcnt2k) * dd->ipath_4kalign);
1311 ipath_cdbg(VERBOSE, "Return piobuf%u %uk @ %p\n",
1312 i, (i < dd->ipath_piobcnt2k) ? 2 : 4, buf);
1313 if (pbufnum)
1314 *pbufnum = i;
1315
1316bail:
1317 return buf;
1318}
1319
1320/**
1321 * ipath_create_rcvhdrq - create a receive header queue
1322 * @dd: the infinipath device
1323 * @pd: the port data
1324 *
f37bda92
BS		 1325 * this must be contiguous memory (from an i/o perspective), and must be
1326 * DMA'able (which means for some systems, it will go through an IOMMU,
1327 * or be forced into a low address range).
7bb206e3
BS		 1328 */
1329int ipath_create_rcvhdrq(struct ipath_devdata *dd,
1330 struct ipath_portdata *pd)
1331{
f37bda92 1332 int ret = 0;
7bb206e3 1333
7bb206e3 1334 if (!pd->port_rcvhdrq) {
f37bda92 1335 dma_addr_t phys_hdrqtail;
7bb206e3 1336 gfp_t gfp_flags = GFP_USER | __GFP_COMP;
f37bda92
BS		 1337 int amt = ALIGN(dd->ipath_rcvhdrcnt * dd->ipath_rcvhdrentsize *
1338 sizeof(u32), PAGE_SIZE);
7bb206e3
BS		 1339
1340 pd->port_rcvhdrq = dma_alloc_coherent(
1341 &dd->pcidev->dev, amt, &pd->port_rcvhdrq_phys,
1342 gfp_flags);
1343
1344 if (!pd->port_rcvhdrq) {
1345 ipath_dev_err(dd, "attempt to allocate %d bytes "
1346 "for port %u rcvhdrq failed\n",
1347 amt, pd->port_port);
1348 ret = -ENOMEM;
1349 goto bail;
1350 }
f37bda92
BS		 1351 pd->port_rcvhdrtail_kvaddr = dma_alloc_coherent(
1352 &dd->pcidev->dev, PAGE_SIZE, &phys_hdrqtail, GFP_KERNEL);
1353 if (!pd->port_rcvhdrtail_kvaddr) {
1354 ipath_dev_err(dd, "attempt to allocate 1 page "
1355 "for port %u rcvhdrqtailaddr failed\n",
1356 pd->port_port);
1357 ret = -ENOMEM;
1358 goto bail;
1359 }
1360 pd->port_rcvhdrqtailaddr_phys = phys_hdrqtail;
7bb206e3
BS		 1361
1362 pd->port_rcvhdrq_size = amt;
1363
1364 ipath_cdbg(VERBOSE, "%d pages at %p (phys %lx) size=%lu "
1365 "for port %u rcvhdr Q\n",
1366 amt >> PAGE_SHIFT, pd->port_rcvhdrq,
1367 (unsigned long) pd->port_rcvhdrq_phys,
1368 (unsigned long) pd->port_rcvhdrq_size,
1369 pd->port_port);
f37bda92
BS		 1370
1371 ipath_cdbg(VERBOSE, "port %d hdrtailaddr, %llx physical\n",
1372 pd->port_port,
1373 (unsigned long long) phys_hdrqtail);
7bb206e3 1374 }
f37bda92
BS		 1375 else
1376 ipath_cdbg(VERBOSE, "reuse port %d rcvhdrq @%p %llx phys; "
1377 "hdrtailaddr@%p %llx physical\n",
1378 pd->port_port, pd->port_rcvhdrq,
1379 pd->port_rcvhdrq_phys, pd->port_rcvhdrtail_kvaddr,
1380 (unsigned long long)pd->port_rcvhdrqtailaddr_phys);
1381
1382 /* clear for security and sanity on each use */
1383 memset(pd->port_rcvhdrq, 0, pd->port_rcvhdrq_size);
1384 memset((void *)pd->port_rcvhdrtail_kvaddr, 0, PAGE_SIZE);
7bb206e3
BS		 1385
1386 /*
1387 * tell chip each time we init it, even if we are re-using previous
f37bda92 1388 * memory (we zero the register at process close)
7bb206e3 1389 */
f37bda92
BS		 1390 ipath_write_kreg_port(dd, dd->ipath_kregs->kr_rcvhdrtailaddr,
1391 pd->port_port, pd->port_rcvhdrqtailaddr_phys);
7bb206e3
BS		 1392 ipath_write_kreg_port(dd, dd->ipath_kregs->kr_rcvhdraddr,
1393 pd->port_port, pd->port_rcvhdrq_phys);
1394
1395 ret = 0;
1396bail:
1397 return ret;
1398}
1399
1400int ipath_waitfor_complete(struct ipath_devdata *dd, ipath_kreg reg_id,
1401 u64 bits_to_wait_for, u64 * valp)
1402{
1403 unsigned long timeout;
1404 u64 lastval, val;
1405 int ret;
1406
1407 lastval = ipath_read_kreg64(dd, reg_id);
1408 /* wait a ridiculously long time */
1409 timeout = jiffies + msecs_to_jiffies(5);
1410 do {
1411 val = ipath_read_kreg64(dd, reg_id);
1412 /* set so they have something, even on failures. */
1413 *valp = val;
1414 if ((val & bits_to_wait_for) == bits_to_wait_for) {
1415 ret = 0;
1416 break;
1417 }
1418 if (val != lastval)
1419 ipath_cdbg(VERBOSE, "Changed from %llx to %llx, "
1420 "waiting for %llx bits\n",
1421 (unsigned long long) lastval,
1422 (unsigned long long) val,
1423 (unsigned long long) bits_to_wait_for);
1424 cond_resched();
1425 if (time_after(jiffies, timeout)) {
1426 ipath_dbg("Didn't get bits %llx in register 0x%x, "
1427 "got %llx\n",
1428 (unsigned long long) bits_to_wait_for,
1429 reg_id, (unsigned long long) *valp);
1430 ret = -ENODEV;
1431 break;
1432 }
1433 } while (1);
1434
1435 return ret;
1436}
1437
1438/**
1439 * ipath_waitfor_mdio_cmdready - wait for last command to complete
1440 * @dd: the infinipath device
1441 *
1442 * Like ipath_waitfor_complete(), but we wait for the CMDVALID bit to go
1443 * away indicating the last command has completed. It doesn't return data
1444 */
1445int ipath_waitfor_mdio_cmdready(struct ipath_devdata *dd)
1446{
1447 unsigned long timeout;
1448 u64 val;
1449 int ret;
1450
1451 /* wait a ridiculously long time */
1452 timeout = jiffies + msecs_to_jiffies(5);
1453 do {
1454 val = ipath_read_kreg64(dd, dd->ipath_kregs->kr_mdio);
1455 if (!(val & IPATH_MDIO_CMDVALID)) {
1456 ret = 0;
1457 break;
1458 }
1459 cond_resched();
1460 if (time_after(jiffies, timeout)) {
1461 ipath_dbg("CMDVALID stuck in mdio reg? (%llx)\n",
1462 (unsigned long long) val);
1463 ret = -ENODEV;
1464 break;
1465 }
1466 } while (1);
1467
1468 return ret;
1469}
1470
1471void ipath_set_ib_lstate(struct ipath_devdata *dd, int which)
1472{
1473 static const char *what[4] = {
1474 [0] = "DOWN",
1475 [INFINIPATH_IBCC_LINKCMD_INIT] = "INIT",
1476 [INFINIPATH_IBCC_LINKCMD_ARMED] = "ARMED",
1477 [INFINIPATH_IBCC_LINKCMD_ACTIVE] = "ACTIVE"
1478 };
f37bda92
BS		 1479 int linkcmd = (which >> INFINIPATH_IBCC_LINKCMD_SHIFT) &
1480 INFINIPATH_IBCC_LINKCMD_MASK;
1481
7bb206e3
BS		 1482 ipath_cdbg(SMA, "Trying to move unit %u to %s, current ltstate "
1483 "is %s\n", dd->ipath_unit,
f37bda92 1484 what[linkcmd],
7bb206e3
BS		 1485 ipath_ibcstatus_str[
1486 (ipath_read_kreg64
1487 (dd, dd->ipath_kregs->kr_ibcstatus) >>
1488 INFINIPATH_IBCS_LINKTRAININGSTATE_SHIFT) &
1489 INFINIPATH_IBCS_LINKTRAININGSTATE_MASK]);
f37bda92
BS		 1490 /* flush all queued sends when going to DOWN or INIT, to be sure that
1491 * they don't block SMA and other MAD packets */
1492 if (!linkcmd || linkcmd == INFINIPATH_IBCC_LINKCMD_INIT) {
1493 ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl,
1494 INFINIPATH_S_ABORT);
1495 ipath_disarm_piobufs(dd, dd->ipath_lastport_piobuf,
1496 (unsigned)(dd->ipath_piobcnt2k +
1497 dd->ipath_piobcnt4k) -
1498 dd->ipath_lastport_piobuf);
1499 }
7bb206e3
BS		 1500
1501 ipath_write_kreg(dd, dd->ipath_kregs->kr_ibcctrl,
1502 dd->ipath_ibcctrl | which);
1503}
1504
1505/**
1506 * ipath_read_kreg64_port - read a device's per-port 64-bit kernel register
1507 * @dd: the infinipath device
1508 * @regno: the register number to read
1509 * @port: the port containing the register
1510 *
1511 * Registers that vary with the chip implementation constants (port)
1512 * use this routine.
1513 */
1514u64 ipath_read_kreg64_port(const struct ipath_devdata *dd, ipath_kreg regno,
1515 unsigned port)
1516{
1517 u16 where;
1518
1519 if (port < dd->ipath_portcnt &&
1520 (regno == dd->ipath_kregs->kr_rcvhdraddr ||
1521 regno == dd->ipath_kregs->kr_rcvhdrtailaddr))
1522 where = regno + port;
1523 else
1524 where = -1;
1525
1526 return ipath_read_kreg64(dd, where);
1527}
1528
1529/**
1530 * ipath_write_kreg_port - write a device's per-port 64-bit kernel register
1531 * @dd: the infinipath device
1532 * @regno: the register number to write
1533 * @port: the port containing the register
1534 * @value: the value to write
1535 *
1536 * Registers that vary with the chip implementation constants (port)
1537 * use this routine.
1538 */
1539void ipath_write_kreg_port(const struct ipath_devdata *dd, ipath_kreg regno,
1540 unsigned port, u64 value)
1541{
1542 u16 where;
1543
1544 if (port < dd->ipath_portcnt &&
1545 (regno == dd->ipath_kregs->kr_rcvhdraddr ||
1546 regno == dd->ipath_kregs->kr_rcvhdrtailaddr))
1547 where = regno + port;
1548 else
1549 where = -1;
1550
1551 ipath_write_kreg(dd, where, value);
1552}
1553
1554/**
1555 * ipath_shutdown_device - shut down a device
1556 * @dd: the infinipath device
1557 *
1558 * This is called to make the device quiet when we are about to
1559 * unload the driver, and also when the device is administratively
1560 * disabled. It does not free any data structures.
1561 * Everything it does has to be setup again by ipath_init_chip(dd,1)
1562 */
1563void ipath_shutdown_device(struct ipath_devdata *dd)
1564{
1565 u64 val;
1566
1567 ipath_dbg("Shutting down the device\n");
1568
1569 dd->ipath_flags |= IPATH_LINKUNK;
1570 dd->ipath_flags &= ~(IPATH_INITTED | IPATH_LINKDOWN |
1571 IPATH_LINKINIT | IPATH_LINKARMED |
1572 IPATH_LINKACTIVE);
1573 *dd->ipath_statusp &= ~(IPATH_STATUS_IB_CONF |
1574 IPATH_STATUS_IB_READY);
1575
1576 /* mask interrupts, but not errors */
1577 ipath_write_kreg(dd, dd->ipath_kregs->kr_intmask, 0ULL);
1578
1579 dd->ipath_rcvctrl = 0;
1580 ipath_write_kreg(dd, dd->ipath_kregs->kr_rcvctrl,
1581 dd->ipath_rcvctrl);
1582
1583 /*
1584 * gracefully stop all sends allowing any in progress to trickle out
1585 * first.
1586 */
1587 ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl, 0ULL);
1588 /* flush it */
1589 val = ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
1590 /*
1591 * enough for anything that's going to trickle out to have actually
1592 * done so.
1593 */
1594 udelay(5);
1595
1596 /*
1597 * abort any armed or launched PIO buffers that didn't go. (self
1598 * clearing). Will cause any packet currently being transmitted to
1599 * go out with an EBP, and may also cause a short packet error on
1600 * the receiver.
1601 */
1602 ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl,
1603 INFINIPATH_S_ABORT);
1604
1605 ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKINITCMD_DISABLE <<
1606 INFINIPATH_IBCC_LINKINITCMD_SHIFT);
1607
1608 /*
1609 * we are shutting down, so tell the layered driver. We don't do
1610 * this on just a link state change, much like ethernet, a cable
1611 * unplug, etc. doesn't change driver state
1612 */
1613 ipath_layer_intr(dd, IPATH_LAYER_INT_IF_DOWN);
1614
1615 /* disable IBC */
1616 dd->ipath_control &= ~INFINIPATH_C_LINKENABLE;
1617 ipath_write_kreg(dd, dd->ipath_kregs->kr_control,
a40f55fc 1618 dd->ipath_control | INFINIPATH_C_FREEZEMODE);
7bb206e3
BS		 1619
1620 /*
1621 * clear SerdesEnable and turn the leds off; do this here because
1622 * we are unloading, so don't count on interrupts to move along
1623 * Turn the LEDs off explictly for the same reason.
1624 */
1625 dd->ipath_f_quiet_serdes(dd);
1626 dd->ipath_f_setextled(dd, 0, 0);
1627
1628 if (dd->ipath_stats_timer_active) {
1629 del_timer_sync(&dd->ipath_stats_timer);
1630 dd->ipath_stats_timer_active = 0;
1631 }
1632
1633 /*
1634 * clear all interrupts and errors, so that the next time the driver
1635 * is loaded or device is enabled, we know that whatever is set
1636 * happened while we were unloaded
1637 */
1638 ipath_write_kreg(dd, dd->ipath_kregs->kr_hwerrclear,
1639 ~0ULL & ~INFINIPATH_HWE_MEMBISTFAILED);
1640 ipath_write_kreg(dd, dd->ipath_kregs->kr_errorclear, -1LL);
1641 ipath_write_kreg(dd, dd->ipath_kregs->kr_intclear, -1LL);
1642}
1643
1644/**
1645 * ipath_free_pddata - free a port's allocated data
1646 * @dd: the infinipath device
f37bda92 1647 * @pd: the portdata structure
7bb206e3 1648 *
f37bda92
BS		 1649 * free up any allocated data for a port
1650 * This should not touch anything that would affect a simultaneous
1651 * re-allocation of port data, because it is called after ipath_mutex
1652 * is released (and can be called from reinit as well).
1653 * It should never change any chip state, or global driver state.
1654 * (The only exception to global state is freeing the port0 port0_skbs.)
7bb206e3 1655 */
f37bda92 1656void ipath_free_pddata(struct ipath_devdata *dd, struct ipath_portdata *pd)
7bb206e3 1657{
7bb206e3
BS		 1658 if (!pd)
1659 return;
f37bda92
BS		 1660
1661 if (pd->port_rcvhdrq) {
7bb206e3
BS		 1662 ipath_cdbg(VERBOSE, "free closed port %d rcvhdrq @ %p "
1663 "(size=%lu)\n", pd->port_port, pd->port_rcvhdrq,
1664 (unsigned long) pd->port_rcvhdrq_size);
1665 dma_free_coherent(&dd->pcidev->dev, pd->port_rcvhdrq_size,
1666 pd->port_rcvhdrq, pd->port_rcvhdrq_phys);
1667 pd->port_rcvhdrq = NULL;
f37bda92
BS		 1668 if (pd->port_rcvhdrtail_kvaddr) {
1669 dma_free_coherent(&dd->pcidev->dev, PAGE_SIZE,
1670 (void *)pd->port_rcvhdrtail_kvaddr,
1671 pd->port_rcvhdrqtailaddr_phys);
1672 pd->port_rcvhdrtail_kvaddr = NULL;
1673 }
7bb206e3 1674 }
f37bda92
BS		 1675 if (pd->port_port && pd->port_rcvegrbuf) {
1676 unsigned e;
1677
1678 for (e = 0; e < pd->port_rcvegrbuf_chunks; e++) {
1679 void *base = pd->port_rcvegrbuf[e];
1680 size_t size = pd->port_rcvegrbuf_size;
1681
1682 ipath_cdbg(VERBOSE, "egrbuf free(%p, %lu), "
1683 "chunk %u/%u\n", base,
1684 (unsigned long) size,
1685 e, pd->port_rcvegrbuf_chunks);
1686 dma_free_coherent(&dd->pcidev->dev, size,
1687 base, pd->port_rcvegrbuf_phys[e]);
7bb206e3 1688 }
f37bda92
BS		 1689 vfree(pd->port_rcvegrbuf);
1690 pd->port_rcvegrbuf = NULL;
1691 vfree(pd->port_rcvegrbuf_phys);
1692 pd->port_rcvegrbuf_phys = NULL;
7bb206e3 1693 pd->port_rcvegrbuf_chunks = 0;
f37bda92 1694 } else if (pd->port_port == 0 && dd->ipath_port0_skbs) {
7bb206e3
BS		 1695 unsigned e;
1696 struct sk_buff **skbs = dd->ipath_port0_skbs;
1697
1698 dd->ipath_port0_skbs = NULL;
1699 ipath_cdbg(VERBOSE, "free closed port %d ipath_port0_skbs "
1700 "@ %p\n", pd->port_port, skbs);
1701 for (e = 0; e < dd->ipath_rcvegrcnt; e++)
1702 if (skbs[e])
1703 dev_kfree_skb(skbs[e]);
1704 vfree(skbs);
1705 }
f37bda92
BS		 1706 kfree(pd->port_tid_pg_list);
1707 kfree(pd);
7bb206e3
BS		 1708}
1709
ac2ae4c9 1710static int __init infinipath_init(void)
7bb206e3
BS		 1711{
1712 int ret;
1713
1714 ipath_dbg(KERN_INFO DRIVER_LOAD_MSG "%s", ipath_core_version);
1715
1716 /*
1717 * These must be called before the driver is registered with
1718 * the PCI subsystem.
1719 */
1720 idr_init(&unit_table);
1721 if (!idr_pre_get(&unit_table, GFP_KERNEL)) {
1722 ret = -ENOMEM;
1723 goto bail;
1724 }
1725
1726 ret = pci_register_driver(&ipath_driver);
1727 if (ret < 0) {
1728 printk(KERN_ERR IPATH_DRV_NAME
1729 ": Unable to register driver: error %d\n", -ret);
1730 goto bail_unit;
1731 }
1732
1733 ret = ipath_driver_create_group(&ipath_driver.driver);
1734 if (ret < 0) {
1735 printk(KERN_ERR IPATH_DRV_NAME ": Unable to create driver "
1736 "sysfs entries: error %d\n", -ret);
1737 goto bail_pci;
1738 }
1739
1740 ret = ipath_init_ipathfs();
1741 if (ret < 0) {
1742 printk(KERN_ERR IPATH_DRV_NAME ": Unable to create "
1743 "ipathfs: error %d\n", -ret);
1744 goto bail_group;
1745 }
1746
1747 goto bail;
1748
1749bail_group:
1750 ipath_driver_remove_group(&ipath_driver.driver);
1751
1752bail_pci:
1753 pci_unregister_driver(&ipath_driver);
1754
1755bail_unit:
1756 idr_destroy(&unit_table);
1757
1758bail:
1759 return ret;
1760}
1761
1762static void cleanup_device(struct ipath_devdata *dd)
1763{
1764 int port;
1765
1766 ipath_shutdown_device(dd);
1767
1768 if (*dd->ipath_statusp & IPATH_STATUS_CHIP_PRESENT) {
1769 /* can't do anything more with chip; needs re-init */
1770 *dd->ipath_statusp &= ~IPATH_STATUS_CHIP_PRESENT;
1771 if (dd->ipath_kregbase) {
1772 /*
1773 * if we haven't already cleaned up before these are
1774 * to ensure any register reads/writes "fail" until
1775 * re-init
1776 */
1777 dd->ipath_kregbase = NULL;
7bb206e3
BS		 1778 dd->ipath_uregbase = 0;
1779 dd->ipath_sregbase = 0;
1780 dd->ipath_cregbase = 0;
1781 dd->ipath_kregsize = 0;
1782 }
1783 ipath_disable_wc(dd);
1784 }
1785
1786 if (dd->ipath_pioavailregs_dma) {
1787 dma_free_coherent(&dd->pcidev->dev, PAGE_SIZE,
1788 (void *) dd->ipath_pioavailregs_dma,
1789 dd->ipath_pioavailregs_phys);
1790 dd->ipath_pioavailregs_dma = NULL;
1791 }
1792
1793 if (dd->ipath_pageshadow) {
1794 struct page **tmpp = dd->ipath_pageshadow;
1795 int i, cnt = 0;
1796
1797 ipath_cdbg(VERBOSE, "Unlocking any expTID pages still "
1798 "locked\n");
1799 for (port = 0; port < dd->ipath_cfgports; port++) {
1800 int port_tidbase = port * dd->ipath_rcvtidcnt;
1801 int maxtid = port_tidbase + dd->ipath_rcvtidcnt;
1802 for (i = port_tidbase; i < maxtid; i++) {
1803 if (!tmpp[i])
1804 continue;
1805 ipath_release_user_pages(&tmpp[i], 1);
1806 tmpp[i] = NULL;
1807 cnt++;
1808 }
1809 }
1810 if (cnt) {
1811 ipath_stats.sps_pageunlocks += cnt;
1812 ipath_cdbg(VERBOSE, "There were still %u expTID "
1813 "entries locked\n", cnt);
1814 }
1815 if (ipath_stats.sps_pagelocks ||
1816 ipath_stats.sps_pageunlocks)
1817 ipath_cdbg(VERBOSE, "%llu pages locked, %llu "
1818 "unlocked via ipath_m{un}lock\n",
1819 (unsigned long long)
1820 ipath_stats.sps_pagelocks,
1821 (unsigned long long)
1822 ipath_stats.sps_pageunlocks);
1823
1824 ipath_cdbg(VERBOSE, "Free shadow page tid array at %p\n",
1825 dd->ipath_pageshadow);
1826 vfree(dd->ipath_pageshadow);
1827 dd->ipath_pageshadow = NULL;
1828 }
1829
1830 /*
1831 * free any resources still in use (usually just kernel ports)
f37bda92
BS		 1832 * at unload; we do for portcnt, not cfgports, because cfgports
1833 * could have changed while we were loaded.
7bb206e3 1834 */
f37bda92
BS		 1835 for (port = 0; port < dd->ipath_portcnt; port++) {
1836 struct ipath_portdata *pd = dd->ipath_pd[port];
1837 dd->ipath_pd[port] = NULL;
1838 ipath_free_pddata(dd, pd);
1839 }
7bb206e3
BS		 1840 kfree(dd->ipath_pd);
1841 /*
1842 * debuggability, in case some cleanup path tries to use it
1843 * after this
1844 */
1845 dd->ipath_pd = NULL;
1846}
1847
1848static void __exit infinipath_cleanup(void)
1849{
1850 struct ipath_devdata *dd, *tmp;
1851 unsigned long flags;
1852
1853 ipath_exit_ipathfs();
1854
1855 ipath_driver_remove_group(&ipath_driver.driver);
1856
1857 spin_lock_irqsave(&ipath_devs_lock, flags);
1858
1859 /*
1860 * turn off rcv, send, and interrupts for all ports, all drivers
1861 * should also hard reset the chip here?
1862 * free up port 0 (kernel) rcvhdr, egr bufs, and eventually tid bufs
1863 * for all versions of the driver, if they were allocated
1864 */
1865 list_for_each_entry_safe(dd, tmp, &ipath_dev_list, ipath_list) {
1866 spin_unlock_irqrestore(&ipath_devs_lock, flags);
1867
1868 if (dd->ipath_kregbase)
1869 cleanup_device(dd);
1870
1871 if (dd->pcidev) {
1872 if (dd->pcidev->irq) {
1873 ipath_cdbg(VERBOSE,
1874 "unit %u free_irq of irq %x\n",
1875 dd->ipath_unit, dd->pcidev->irq);
1876 free_irq(dd->pcidev->irq, dd);
1877 } else
1878 ipath_dbg("irq is 0, not doing free_irq "
1879 "for unit %u\n", dd->ipath_unit);
7bb206e3 1880
b0ff7c20
BS		 1881 /*
1882 * we check for NULL here, because it's outside
1883 * the kregbase check, and we need to call it
1884 * after the free_irq. Thus it's possible that
1885 * the function pointers were never initialized.
1886 */
1887 if (dd->ipath_f_cleanup)
1888 /* clean up chip-specific stuff */
1889 dd->ipath_f_cleanup(dd);
7bb206e3 1890
b0ff7c20
BS		 1891 dd->pcidev = NULL;
1892 }
7bb206e3
BS		 1893 spin_lock_irqsave(&ipath_devs_lock, flags);
1894 }
1895
1896 spin_unlock_irqrestore(&ipath_devs_lock, flags);
1897
1898 ipath_cdbg(VERBOSE, "Unregistering pci driver\n");
1899 pci_unregister_driver(&ipath_driver);
1900
1901 idr_destroy(&unit_table);
1902}
1903
1904/**
1905 * ipath_reset_device - reset the chip if possible
1906 * @unit: the device to reset
1907 *
1908 * Whether or not reset is successful, we attempt to re-initialize the chip
1909 * (that is, much like a driver unload/reload). We clear the INITTED flag
1910 * so that the various entry points will fail until we reinitialize. For
1911 * now, we only allow this if no user ports are open that use chip resources
1912 */
1913int ipath_reset_device(int unit)
1914{
1915 int ret, i;
1916 struct ipath_devdata *dd = ipath_lookup(unit);
1917
1918 if (!dd) {
1919 ret = -ENODEV;
1920 goto bail;
1921 }
1922
1923 dev_info(&dd->pcidev->dev, "Reset on unit %u requested\n", unit);
1924
1925 if (!dd->ipath_kregbase || !(dd->ipath_flags & IPATH_PRESENT)) {
1926 dev_info(&dd->pcidev->dev, "Invalid unit number %u or "
1927 "not initialized or not present\n", unit);
1928 ret = -ENXIO;
1929 goto bail;
1930 }
1931
1932 if (dd->ipath_pd)
23e86a45 1933 for (i = 1; i < dd->ipath_cfgports; i++) {
7bb206e3
BS		 1934 if (dd->ipath_pd[i] && dd->ipath_pd[i]->port_cnt) {
1935 ipath_dbg("unit %u port %d is in use "
1936 "(PID %u cmd %s), can't reset\n",
1937 unit, i,
1938 dd->ipath_pd[i]->port_pid,
1939 dd->ipath_pd[i]->port_comm);
1940 ret = -EBUSY;
1941 goto bail;
1942 }
1943 }
1944
1945 dd->ipath_flags &= ~IPATH_INITTED;
1946 ret = dd->ipath_f_reset(dd);
1947 if (ret != 1)
1948 ipath_dbg("reset was not successful\n");
1949 ipath_dbg("Trying to reinitialize unit %u after reset attempt\n",
1950 unit);
1951 ret = ipath_init_chip(dd, 1);
1952 if (ret)
1953 ipath_dev_err(dd, "Reinitialize unit %u after "
1954 "reset failed with %d\n", unit, ret);
1955 else
1956 dev_info(&dd->pcidev->dev, "Reinitialized unit %u after "
1957 "resetting\n", unit);
1958
1959bail:
1960 return ret;
1961}
1962
1963module_init(infinipath_init);
1964module_exit(infinipath_cleanup);
Linux 6.x block layer and io_uring tree(s)