2 * Copyright (c) 2004 Topspin Communications. All rights reserved.
3 * Copyright (c) 2005 Intel Corporation. All rights reserved.
4 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
5 * Copyright (c) 2005 Voltaire, Inc. All rights reserved.
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * OpenIB.org BSD license below:
13 * Redistribution and use in source and binary forms, with or
14 * without modification, are permitted provided that the following
17 * - Redistributions of source code must retain the above
18 * copyright notice, this list of conditions and the following
21 * - Redistributions in binary form must reproduce the above
22 * copyright notice, this list of conditions and the following
23 * disclaimer in the documentation and/or other materials
24 * provided with the distribution.
26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
36 #include <linux/module.h>
37 #include <linux/errno.h>
38 #include <linux/slab.h>
39 #include <linux/workqueue.h>
40 #include <linux/netdevice.h>
41 #include <net/addrconf.h>
43 #include <rdma/ib_cache.h>
45 #include "core_priv.h"
47 struct ib_pkey_cache {
52 struct ib_update_work {
53 struct work_struct work;
54 struct ib_device *device;
56 bool enforce_security;
62 enum gid_attr_find_mask {
63 GID_ATTR_FIND_MASK_GID = 1UL << 0,
64 GID_ATTR_FIND_MASK_NETDEV = 1UL << 1,
65 GID_ATTR_FIND_MASK_DEFAULT = 1UL << 2,
66 GID_ATTR_FIND_MASK_GID_TYPE = 1UL << 3,
69 enum gid_table_entry_state {
70 GID_TABLE_ENTRY_INVALID = 1,
71 GID_TABLE_ENTRY_VALID = 2,
73 * Indicates that entry is pending to be removed, there may
74 * be active users of this GID entry.
75 * When last user of the GID entry releases reference to it,
76 * GID entry is detached from the table.
78 GID_TABLE_ENTRY_PENDING_DEL = 3,
81 struct ib_gid_table_entry {
83 struct work_struct del_work;
84 struct ib_gid_attr attr;
86 enum gid_table_entry_state state;
91 /* In RoCE, adding a GID to the table requires:
92 * (a) Find if this GID is already exists.
93 * (b) Find a free space.
94 * (c) Write the new GID
96 * Delete requires different set of operations:
101 /* Any writer to data_vec must hold this lock and the write side of
102 * rwlock. Readers must hold only rwlock. All writers must be in a
106 /* rwlock protects data_vec[ix]->state and entry pointer.
109 struct ib_gid_table_entry **data_vec;
110 /* bit field, each bit indicates the index of default GID */
111 u32 default_gid_indices;
114 static void dispatch_gid_change_event(struct ib_device *ib_dev, u8 port)
116 struct ib_event event;
118 event.device = ib_dev;
119 event.element.port_num = port;
120 event.event = IB_EVENT_GID_CHANGE;
122 ib_dispatch_event(&event);
125 static const char * const gid_type_str[] = {
126 [IB_GID_TYPE_IB] = "IB/RoCE v1",
127 [IB_GID_TYPE_ROCE_UDP_ENCAP] = "RoCE v2",
130 const char *ib_cache_gid_type_str(enum ib_gid_type gid_type)
132 if (gid_type < ARRAY_SIZE(gid_type_str) && gid_type_str[gid_type])
133 return gid_type_str[gid_type];
135 return "Invalid GID type";
137 EXPORT_SYMBOL(ib_cache_gid_type_str);
139 /** rdma_is_zero_gid - Check if given GID is zero or not.
141 * Returns true if given GID is zero, returns false otherwise.
143 bool rdma_is_zero_gid(const union ib_gid *gid)
145 return !memcmp(gid, &zgid, sizeof(*gid));
147 EXPORT_SYMBOL(rdma_is_zero_gid);
149 /** is_gid_index_default - Check if a given index belongs to
150 * reserved default GIDs or not.
151 * @table: GID table pointer
152 * @index: Index to check in GID table
153 * Returns true if index is one of the reserved default GID index otherwise
156 static bool is_gid_index_default(const struct ib_gid_table *table,
159 return index < 32 && (BIT(index) & table->default_gid_indices);
162 int ib_cache_gid_parse_type_str(const char *buf)
172 if (buf[len - 1] == '\n')
175 for (i = 0; i < ARRAY_SIZE(gid_type_str); ++i)
176 if (gid_type_str[i] && !strncmp(buf, gid_type_str[i], len) &&
177 len == strlen(gid_type_str[i])) {
184 EXPORT_SYMBOL(ib_cache_gid_parse_type_str);
186 static struct ib_gid_table *rdma_gid_table(struct ib_device *device, u8 port)
188 return device->cache.ports[port - rdma_start_port(device)].gid;
191 static bool is_gid_entry_free(const struct ib_gid_table_entry *entry)
196 static bool is_gid_entry_valid(const struct ib_gid_table_entry *entry)
198 return entry && entry->state == GID_TABLE_ENTRY_VALID;
201 static void schedule_free_gid(struct kref *kref)
203 struct ib_gid_table_entry *entry =
204 container_of(kref, struct ib_gid_table_entry, kref);
206 queue_work(ib_wq, &entry->del_work);
209 static void free_gid_entry_locked(struct ib_gid_table_entry *entry)
211 struct ib_device *device = entry->attr.device;
212 u8 port_num = entry->attr.port_num;
213 struct ib_gid_table *table = rdma_gid_table(device, port_num);
215 pr_debug("%s device=%s port=%d index=%d gid %pI6\n", __func__,
216 device->name, port_num, entry->attr.index,
217 entry->attr.gid.raw);
219 if (rdma_cap_roce_gid_table(device, port_num) &&
220 entry->state != GID_TABLE_ENTRY_INVALID)
221 device->del_gid(&entry->attr, &entry->context);
223 write_lock_irq(&table->rwlock);
226 * The only way to avoid overwriting NULL in table is
227 * by comparing if it is same entry in table or not!
228 * If new entry in table is added by the time we free here,
229 * don't overwrite the table entry.
231 if (entry == table->data_vec[entry->attr.index])
232 table->data_vec[entry->attr.index] = NULL;
233 /* Now this index is ready to be allocated */
234 write_unlock_irq(&table->rwlock);
236 if (entry->attr.ndev)
237 dev_put(entry->attr.ndev);
241 static void free_gid_entry(struct kref *kref)
243 struct ib_gid_table_entry *entry =
244 container_of(kref, struct ib_gid_table_entry, kref);
246 free_gid_entry_locked(entry);
250 * free_gid_work - Release reference to the GID entry
251 * @work: Work structure to refer to GID entry which needs to be
254 * free_gid_work() frees the entry from the HCA's hardware table
255 * if provider supports it. It releases reference to netdevice.
257 static void free_gid_work(struct work_struct *work)
259 struct ib_gid_table_entry *entry =
260 container_of(work, struct ib_gid_table_entry, del_work);
261 struct ib_device *device = entry->attr.device;
262 u8 port_num = entry->attr.port_num;
263 struct ib_gid_table *table = rdma_gid_table(device, port_num);
265 mutex_lock(&table->lock);
266 free_gid_entry_locked(entry);
267 mutex_unlock(&table->lock);
270 static struct ib_gid_table_entry *
271 alloc_gid_entry(const struct ib_gid_attr *attr)
273 struct ib_gid_table_entry *entry;
275 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
278 kref_init(&entry->kref);
279 memcpy(&entry->attr, attr, sizeof(*attr));
280 if (entry->attr.ndev)
281 dev_hold(entry->attr.ndev);
282 INIT_WORK(&entry->del_work, free_gid_work);
283 entry->state = GID_TABLE_ENTRY_INVALID;
287 static void store_gid_entry(struct ib_gid_table *table,
288 struct ib_gid_table_entry *entry)
290 entry->state = GID_TABLE_ENTRY_VALID;
292 pr_debug("%s device=%s port=%d index=%d gid %pI6\n", __func__,
293 entry->attr.device->name, entry->attr.port_num,
294 entry->attr.index, entry->attr.gid.raw);
296 lockdep_assert_held(&table->lock);
297 write_lock_irq(&table->rwlock);
298 table->data_vec[entry->attr.index] = entry;
299 write_unlock_irq(&table->rwlock);
302 static void get_gid_entry(struct ib_gid_table_entry *entry)
304 kref_get(&entry->kref);
307 static void put_gid_entry(struct ib_gid_table_entry *entry)
309 kref_put(&entry->kref, schedule_free_gid);
312 static void put_gid_entry_locked(struct ib_gid_table_entry *entry)
314 kref_put(&entry->kref, free_gid_entry);
317 static int add_roce_gid(struct ib_gid_table_entry *entry)
319 const struct ib_gid_attr *attr = &entry->attr;
323 pr_err("%s NULL netdev device=%s port=%d index=%d\n",
324 __func__, attr->device->name, attr->port_num,
328 if (rdma_cap_roce_gid_table(attr->device, attr->port_num)) {
329 ret = attr->device->add_gid(attr, &entry->context);
331 pr_err("%s GID add failed device=%s port=%d index=%d\n",
332 __func__, attr->device->name, attr->port_num,
341 * del_gid - Delete GID table entry
343 * @ib_dev: IB device whose GID entry to be deleted
344 * @port: Port number of the IB device
345 * @table: GID table of the IB device for a port
346 * @ix: GID entry index to delete
349 static void del_gid(struct ib_device *ib_dev, u8 port,
350 struct ib_gid_table *table, int ix)
352 struct ib_gid_table_entry *entry;
354 lockdep_assert_held(&table->lock);
356 pr_debug("%s device=%s port=%d index=%d gid %pI6\n", __func__,
357 ib_dev->name, port, ix,
358 table->data_vec[ix]->attr.gid.raw);
360 write_lock_irq(&table->rwlock);
361 entry = table->data_vec[ix];
362 entry->state = GID_TABLE_ENTRY_PENDING_DEL;
364 * For non RoCE protocol, GID entry slot is ready to use.
366 if (!rdma_protocol_roce(ib_dev, port))
367 table->data_vec[ix] = NULL;
368 write_unlock_irq(&table->rwlock);
370 put_gid_entry_locked(entry);
374 * add_modify_gid - Add or modify GID table entry
376 * @table: GID table in which GID to be added or modified
377 * @attr: Attributes of the GID
379 * Returns 0 on success or appropriate error code. It accepts zero
380 * GID addition for non RoCE ports for HCA's who report them as valid
381 * GID. However such zero GIDs are not added to the cache.
383 static int add_modify_gid(struct ib_gid_table *table,
384 const struct ib_gid_attr *attr)
386 struct ib_gid_table_entry *entry;
390 * Invalidate any old entry in the table to make it safe to write to
393 if (is_gid_entry_valid(table->data_vec[attr->index]))
394 del_gid(attr->device, attr->port_num, table, attr->index);
397 * Some HCA's report multiple GID entries with only one valid GID, and
398 * leave other unused entries as the zero GID. Convert zero GIDs to
399 * empty table entries instead of storing them.
401 if (rdma_is_zero_gid(&attr->gid))
404 entry = alloc_gid_entry(attr);
408 if (rdma_protocol_roce(attr->device, attr->port_num)) {
409 ret = add_roce_gid(entry);
414 store_gid_entry(table, entry);
418 put_gid_entry(entry);
422 /* rwlock should be read locked, or lock should be held */
423 static int find_gid(struct ib_gid_table *table, const union ib_gid *gid,
424 const struct ib_gid_attr *val, bool default_gid,
425 unsigned long mask, int *pempty)
429 int empty = pempty ? -1 : 0;
431 while (i < table->sz && (found < 0 || empty < 0)) {
432 struct ib_gid_table_entry *data = table->data_vec[i];
433 struct ib_gid_attr *attr;
438 /* find_gid() is used during GID addition where it is expected
439 * to return a free entry slot which is not duplicate.
440 * Free entry slot is requested and returned if pempty is set,
441 * so lookup free slot only if requested.
443 if (pempty && empty < 0) {
444 if (is_gid_entry_free(data) &&
446 is_gid_index_default(table, curr_index)) {
448 * Found an invalid (free) entry; allocate it.
449 * If default GID is requested, then our
450 * found slot must be one of the DEFAULT
451 * reserved slots or we fail.
452 * This ensures that only DEFAULT reserved
453 * slots are used for default property GIDs.
460 * Additionally find_gid() is used to find valid entry during
461 * lookup operation; so ignore the entries which are marked as
462 * pending for removal and the entries which are marked as
465 if (!is_gid_entry_valid(data))
472 if (mask & GID_ATTR_FIND_MASK_GID_TYPE &&
473 attr->gid_type != val->gid_type)
476 if (mask & GID_ATTR_FIND_MASK_GID &&
477 memcmp(gid, &data->attr.gid, sizeof(*gid)))
480 if (mask & GID_ATTR_FIND_MASK_NETDEV &&
481 attr->ndev != val->ndev)
484 if (mask & GID_ATTR_FIND_MASK_DEFAULT &&
485 is_gid_index_default(table, curr_index) != default_gid)
497 static void make_default_gid(struct net_device *dev, union ib_gid *gid)
499 gid->global.subnet_prefix = cpu_to_be64(0xfe80000000000000LL);
500 addrconf_ifid_eui48(&gid->raw[8], dev);
503 static int __ib_cache_gid_add(struct ib_device *ib_dev, u8 port,
504 union ib_gid *gid, struct ib_gid_attr *attr,
505 unsigned long mask, bool default_gid)
507 struct ib_gid_table *table;
512 /* Do not allow adding zero GID in support of
513 * IB spec version 1.3 section 4.1.1 point (6) and
514 * section 12.7.10 and section 12.7.20
516 if (rdma_is_zero_gid(gid))
519 table = rdma_gid_table(ib_dev, port);
521 mutex_lock(&table->lock);
523 ix = find_gid(table, gid, attr, default_gid, mask, &empty);
531 attr->device = ib_dev;
533 attr->port_num = port;
535 ret = add_modify_gid(table, attr);
537 dispatch_gid_change_event(ib_dev, port);
540 mutex_unlock(&table->lock);
542 pr_warn("%s: unable to add gid %pI6 error=%d\n",
543 __func__, gid->raw, ret);
547 int ib_cache_gid_add(struct ib_device *ib_dev, u8 port,
548 union ib_gid *gid, struct ib_gid_attr *attr)
550 struct net_device *idev;
554 if (ib_dev->get_netdev) {
555 idev = ib_dev->get_netdev(ib_dev, port);
556 if (idev && attr->ndev != idev) {
557 union ib_gid default_gid;
559 /* Adding default GIDs in not permitted */
560 make_default_gid(idev, &default_gid);
561 if (!memcmp(gid, &default_gid, sizeof(*gid))) {
570 mask = GID_ATTR_FIND_MASK_GID |
571 GID_ATTR_FIND_MASK_GID_TYPE |
572 GID_ATTR_FIND_MASK_NETDEV;
574 ret = __ib_cache_gid_add(ib_dev, port, gid, attr, mask, false);
579 _ib_cache_gid_del(struct ib_device *ib_dev, u8 port,
580 union ib_gid *gid, struct ib_gid_attr *attr,
581 unsigned long mask, bool default_gid)
583 struct ib_gid_table *table;
587 table = rdma_gid_table(ib_dev, port);
589 mutex_lock(&table->lock);
591 ix = find_gid(table, gid, attr, default_gid, mask, NULL);
597 del_gid(ib_dev, port, table, ix);
598 dispatch_gid_change_event(ib_dev, port);
601 mutex_unlock(&table->lock);
603 pr_debug("%s: can't delete gid %pI6 error=%d\n",
604 __func__, gid->raw, ret);
608 int ib_cache_gid_del(struct ib_device *ib_dev, u8 port,
609 union ib_gid *gid, struct ib_gid_attr *attr)
611 unsigned long mask = GID_ATTR_FIND_MASK_GID |
612 GID_ATTR_FIND_MASK_GID_TYPE |
613 GID_ATTR_FIND_MASK_DEFAULT |
614 GID_ATTR_FIND_MASK_NETDEV;
616 return _ib_cache_gid_del(ib_dev, port, gid, attr, mask, false);
619 int ib_cache_gid_del_all_netdev_gids(struct ib_device *ib_dev, u8 port,
620 struct net_device *ndev)
622 struct ib_gid_table *table;
624 bool deleted = false;
626 table = rdma_gid_table(ib_dev, port);
628 mutex_lock(&table->lock);
630 for (ix = 0; ix < table->sz; ix++) {
631 if (is_gid_entry_valid(table->data_vec[ix]) &&
632 table->data_vec[ix]->attr.ndev == ndev) {
633 del_gid(ib_dev, port, table, ix);
638 mutex_unlock(&table->lock);
641 dispatch_gid_change_event(ib_dev, port);
647 * rdma_find_gid_by_port - Returns the GID entry attributes when it finds
648 * a valid GID entry for given search parameters. It searches for the specified
649 * GID value in the local software cache.
650 * @device: The device to query.
651 * @gid: The GID value to search for.
652 * @gid_type: The GID type to search for.
653 * @port_num: The port number of the device where the GID value should be
655 * @ndev: In RoCE, the net device of the device. NULL means ignore.
657 * Returns sgid attributes if the GID is found with valid reference or
658 * returns ERR_PTR for the error.
659 * The caller must invoke rdma_put_gid_attr() to release the reference.
661 const struct ib_gid_attr *
662 rdma_find_gid_by_port(struct ib_device *ib_dev,
663 const union ib_gid *gid,
664 enum ib_gid_type gid_type,
665 u8 port, struct net_device *ndev)
668 struct ib_gid_table *table;
669 unsigned long mask = GID_ATTR_FIND_MASK_GID |
670 GID_ATTR_FIND_MASK_GID_TYPE;
671 struct ib_gid_attr val = {.ndev = ndev, .gid_type = gid_type};
672 const struct ib_gid_attr *attr;
675 if (!rdma_is_port_valid(ib_dev, port))
676 return ERR_PTR(-ENOENT);
678 table = rdma_gid_table(ib_dev, port);
681 mask |= GID_ATTR_FIND_MASK_NETDEV;
683 read_lock_irqsave(&table->rwlock, flags);
684 local_index = find_gid(table, gid, &val, false, mask, NULL);
685 if (local_index >= 0) {
686 get_gid_entry(table->data_vec[local_index]);
687 attr = &table->data_vec[local_index]->attr;
688 read_unlock_irqrestore(&table->rwlock, flags);
692 read_unlock_irqrestore(&table->rwlock, flags);
693 return ERR_PTR(-ENOENT);
695 EXPORT_SYMBOL(rdma_find_gid_by_port);
698 * rdma_find_gid_by_filter - Returns the GID table attribute where a
699 * specified GID value occurs
700 * @device: The device to query.
701 * @gid: The GID value to search for.
702 * @port: The port number of the device where the GID value could be
704 * @filter: The filter function is executed on any matching GID in the table.
705 * If the filter function returns true, the corresponding index is returned,
706 * otherwise, we continue searching the GID table. It's guaranteed that
707 * while filter is executed, ndev field is valid and the structure won't
708 * change. filter is executed in an atomic context. filter must not be NULL.
710 * rdma_find_gid_by_filter() searches for the specified GID value
711 * of which the filter function returns true in the port's GID table.
714 const struct ib_gid_attr *rdma_find_gid_by_filter(
715 struct ib_device *ib_dev, const union ib_gid *gid, u8 port,
716 bool (*filter)(const union ib_gid *gid, const struct ib_gid_attr *,
720 const struct ib_gid_attr *res = ERR_PTR(-ENOENT);
721 struct ib_gid_table *table;
725 if (!rdma_is_port_valid(ib_dev, port))
726 return ERR_PTR(-EINVAL);
728 table = rdma_gid_table(ib_dev, port);
730 read_lock_irqsave(&table->rwlock, flags);
731 for (i = 0; i < table->sz; i++) {
732 struct ib_gid_table_entry *entry = table->data_vec[i];
734 if (!is_gid_entry_valid(entry))
737 if (memcmp(gid, &entry->attr.gid, sizeof(*gid)))
740 if (filter(gid, &entry->attr, context)) {
741 get_gid_entry(entry);
746 read_unlock_irqrestore(&table->rwlock, flags);
750 static struct ib_gid_table *alloc_gid_table(int sz)
752 struct ib_gid_table *table = kzalloc(sizeof(*table), GFP_KERNEL);
757 table->data_vec = kcalloc(sz, sizeof(*table->data_vec), GFP_KERNEL);
758 if (!table->data_vec)
761 mutex_init(&table->lock);
764 rwlock_init(&table->rwlock);
772 static void release_gid_table(struct ib_device *device, u8 port,
773 struct ib_gid_table *table)
781 for (i = 0; i < table->sz; i++) {
782 if (is_gid_entry_free(table->data_vec[i]))
784 if (kref_read(&table->data_vec[i]->kref) > 1) {
785 pr_err("GID entry ref leak for %s (index %d) ref=%d\n",
787 kref_read(&table->data_vec[i]->kref));
794 kfree(table->data_vec);
798 static void cleanup_gid_table_port(struct ib_device *ib_dev, u8 port,
799 struct ib_gid_table *table)
802 bool deleted = false;
807 mutex_lock(&table->lock);
808 for (i = 0; i < table->sz; ++i) {
809 if (is_gid_entry_valid(table->data_vec[i])) {
810 del_gid(ib_dev, port, table, i);
814 mutex_unlock(&table->lock);
817 dispatch_gid_change_event(ib_dev, port);
820 void ib_cache_gid_set_default_gid(struct ib_device *ib_dev, u8 port,
821 struct net_device *ndev,
822 unsigned long gid_type_mask,
823 enum ib_cache_gid_default_mode mode)
825 union ib_gid gid = { };
826 struct ib_gid_attr gid_attr;
827 unsigned int gid_type;
830 mask = GID_ATTR_FIND_MASK_GID_TYPE |
831 GID_ATTR_FIND_MASK_DEFAULT |
832 GID_ATTR_FIND_MASK_NETDEV;
833 memset(&gid_attr, 0, sizeof(gid_attr));
834 gid_attr.ndev = ndev;
836 for (gid_type = 0; gid_type < IB_GID_TYPE_SIZE; ++gid_type) {
837 if (1UL << gid_type & ~gid_type_mask)
840 gid_attr.gid_type = gid_type;
842 if (mode == IB_CACHE_GID_DEFAULT_MODE_SET) {
843 make_default_gid(ndev, &gid);
844 __ib_cache_gid_add(ib_dev, port, &gid,
845 &gid_attr, mask, true);
846 } else if (mode == IB_CACHE_GID_DEFAULT_MODE_DELETE) {
847 _ib_cache_gid_del(ib_dev, port, &gid,
848 &gid_attr, mask, true);
853 static void gid_table_reserve_default(struct ib_device *ib_dev, u8 port,
854 struct ib_gid_table *table)
857 unsigned long roce_gid_type_mask;
858 unsigned int num_default_gids;
860 roce_gid_type_mask = roce_gid_type_mask_support(ib_dev, port);
861 num_default_gids = hweight_long(roce_gid_type_mask);
862 /* Reserve starting indices for default GIDs */
863 for (i = 0; i < num_default_gids && i < table->sz; i++)
864 table->default_gid_indices |= BIT(i);
868 static void gid_table_release_one(struct ib_device *ib_dev)
870 struct ib_gid_table *table;
873 for (port = 0; port < ib_dev->phys_port_cnt; port++) {
874 table = ib_dev->cache.ports[port].gid;
875 release_gid_table(ib_dev, port, table);
876 ib_dev->cache.ports[port].gid = NULL;
880 static int _gid_table_setup_one(struct ib_device *ib_dev)
883 struct ib_gid_table *table;
885 for (port = 0; port < ib_dev->phys_port_cnt; port++) {
886 u8 rdma_port = port + rdma_start_port(ib_dev);
888 table = alloc_gid_table(
889 ib_dev->port_immutable[rdma_port].gid_tbl_len);
891 goto rollback_table_setup;
893 gid_table_reserve_default(ib_dev, rdma_port, table);
894 ib_dev->cache.ports[port].gid = table;
898 rollback_table_setup:
899 gid_table_release_one(ib_dev);
903 static void gid_table_cleanup_one(struct ib_device *ib_dev)
905 struct ib_gid_table *table;
908 for (port = 0; port < ib_dev->phys_port_cnt; port++) {
909 table = ib_dev->cache.ports[port].gid;
910 cleanup_gid_table_port(ib_dev, port + rdma_start_port(ib_dev),
915 static int gid_table_setup_one(struct ib_device *ib_dev)
919 err = _gid_table_setup_one(ib_dev);
924 rdma_roce_rescan_device(ib_dev);
930 * rdma_query_gid - Read the GID content from the GID software cache
931 * @device: Device to query the GID
932 * @port_num: Port number of the device
933 * @index: Index of the GID table entry to read
934 * @gid: Pointer to GID where to store the entry's GID
936 * rdma_query_gid() only reads the GID entry content for requested device,
937 * port and index. It reads for IB, RoCE and iWarp link layers. It doesn't
938 * hold any reference to the GID table entry in the HCA or software cache.
940 * Returns 0 on success or appropriate error code.
943 int rdma_query_gid(struct ib_device *device, u8 port_num,
944 int index, union ib_gid *gid)
946 struct ib_gid_table *table;
950 if (!rdma_is_port_valid(device, port_num))
953 table = rdma_gid_table(device, port_num);
954 read_lock_irqsave(&table->rwlock, flags);
956 if (index < 0 || index >= table->sz ||
957 !is_gid_entry_valid(table->data_vec[index]))
960 memcpy(gid, &table->data_vec[index]->attr.gid, sizeof(*gid));
964 read_unlock_irqrestore(&table->rwlock, flags);
967 EXPORT_SYMBOL(rdma_query_gid);
970 * rdma_find_gid - Returns SGID attributes if the matching GID is found.
971 * @device: The device to query.
972 * @gid: The GID value to search for.
973 * @gid_type: The GID type to search for.
974 * @ndev: In RoCE, the net device of the device. NULL means ignore.
976 * rdma_find_gid() searches for the specified GID value in the software cache.
978 * Returns GID attributes if a valid GID is found or returns ERR_PTR for the
979 * error. The caller must invoke rdma_put_gid_attr() to release the reference.
982 const struct ib_gid_attr *rdma_find_gid(struct ib_device *device,
983 const union ib_gid *gid,
984 enum ib_gid_type gid_type,
985 struct net_device *ndev)
987 unsigned long mask = GID_ATTR_FIND_MASK_GID |
988 GID_ATTR_FIND_MASK_GID_TYPE;
989 struct ib_gid_attr gid_attr_val = {.ndev = ndev, .gid_type = gid_type};
993 mask |= GID_ATTR_FIND_MASK_NETDEV;
995 for (p = 0; p < device->phys_port_cnt; p++) {
996 struct ib_gid_table *table;
1000 table = device->cache.ports[p].gid;
1001 read_lock_irqsave(&table->rwlock, flags);
1002 index = find_gid(table, gid, &gid_attr_val, false, mask, NULL);
1004 const struct ib_gid_attr *attr;
1006 get_gid_entry(table->data_vec[index]);
1007 attr = &table->data_vec[index]->attr;
1008 read_unlock_irqrestore(&table->rwlock, flags);
1011 read_unlock_irqrestore(&table->rwlock, flags);
1014 return ERR_PTR(-ENOENT);
1016 EXPORT_SYMBOL(rdma_find_gid);
1018 int ib_get_cached_pkey(struct ib_device *device,
1023 struct ib_pkey_cache *cache;
1024 unsigned long flags;
1027 if (!rdma_is_port_valid(device, port_num))
1030 read_lock_irqsave(&device->cache.lock, flags);
1032 cache = device->cache.ports[port_num - rdma_start_port(device)].pkey;
1034 if (index < 0 || index >= cache->table_len)
1037 *pkey = cache->table[index];
1039 read_unlock_irqrestore(&device->cache.lock, flags);
1043 EXPORT_SYMBOL(ib_get_cached_pkey);
1045 int ib_get_cached_subnet_prefix(struct ib_device *device,
1049 unsigned long flags;
1052 if (!rdma_is_port_valid(device, port_num))
1055 p = port_num - rdma_start_port(device);
1056 read_lock_irqsave(&device->cache.lock, flags);
1057 *sn_pfx = device->cache.ports[p].subnet_prefix;
1058 read_unlock_irqrestore(&device->cache.lock, flags);
1062 EXPORT_SYMBOL(ib_get_cached_subnet_prefix);
1064 int ib_find_cached_pkey(struct ib_device *device,
1069 struct ib_pkey_cache *cache;
1070 unsigned long flags;
1073 int partial_ix = -1;
1075 if (!rdma_is_port_valid(device, port_num))
1078 read_lock_irqsave(&device->cache.lock, flags);
1080 cache = device->cache.ports[port_num - rdma_start_port(device)].pkey;
1084 for (i = 0; i < cache->table_len; ++i)
1085 if ((cache->table[i] & 0x7fff) == (pkey & 0x7fff)) {
1086 if (cache->table[i] & 0x8000) {
1094 if (ret && partial_ix >= 0) {
1095 *index = partial_ix;
1099 read_unlock_irqrestore(&device->cache.lock, flags);
1103 EXPORT_SYMBOL(ib_find_cached_pkey);
1105 int ib_find_exact_cached_pkey(struct ib_device *device,
1110 struct ib_pkey_cache *cache;
1111 unsigned long flags;
1115 if (!rdma_is_port_valid(device, port_num))
1118 read_lock_irqsave(&device->cache.lock, flags);
1120 cache = device->cache.ports[port_num - rdma_start_port(device)].pkey;
1124 for (i = 0; i < cache->table_len; ++i)
1125 if (cache->table[i] == pkey) {
1131 read_unlock_irqrestore(&device->cache.lock, flags);
1135 EXPORT_SYMBOL(ib_find_exact_cached_pkey);
1137 int ib_get_cached_lmc(struct ib_device *device,
1141 unsigned long flags;
1144 if (!rdma_is_port_valid(device, port_num))
1147 read_lock_irqsave(&device->cache.lock, flags);
1148 *lmc = device->cache.ports[port_num - rdma_start_port(device)].lmc;
1149 read_unlock_irqrestore(&device->cache.lock, flags);
1153 EXPORT_SYMBOL(ib_get_cached_lmc);
1155 int ib_get_cached_port_state(struct ib_device *device,
1157 enum ib_port_state *port_state)
1159 unsigned long flags;
1162 if (!rdma_is_port_valid(device, port_num))
1165 read_lock_irqsave(&device->cache.lock, flags);
1166 *port_state = device->cache.ports[port_num
1167 - rdma_start_port(device)].port_state;
1168 read_unlock_irqrestore(&device->cache.lock, flags);
1172 EXPORT_SYMBOL(ib_get_cached_port_state);
1175 * rdma_get_gid_attr - Returns GID attributes for a port of a device
1176 * at a requested gid_index, if a valid GID entry exists.
1177 * @device: The device to query.
1178 * @port_num: The port number on the device where the GID value
1180 * @index: Index of the GID table entry whose attributes are to
1183 * rdma_get_gid_attr() acquires reference count of gid attributes from the
1184 * cached GID table. Caller must invoke rdma_put_gid_attr() to release
1185 * reference to gid attribute regardless of link layer.
1187 * Returns pointer to valid gid attribute or ERR_PTR for the appropriate error
1190 const struct ib_gid_attr *
1191 rdma_get_gid_attr(struct ib_device *device, u8 port_num, int index)
1193 const struct ib_gid_attr *attr = ERR_PTR(-EINVAL);
1194 struct ib_gid_table *table;
1195 unsigned long flags;
1197 if (!rdma_is_port_valid(device, port_num))
1198 return ERR_PTR(-EINVAL);
1200 table = rdma_gid_table(device, port_num);
1201 if (index < 0 || index >= table->sz)
1202 return ERR_PTR(-EINVAL);
1204 read_lock_irqsave(&table->rwlock, flags);
1205 if (!is_gid_entry_valid(table->data_vec[index]))
1208 get_gid_entry(table->data_vec[index]);
1209 attr = &table->data_vec[index]->attr;
1211 read_unlock_irqrestore(&table->rwlock, flags);
1214 EXPORT_SYMBOL(rdma_get_gid_attr);
1217 * rdma_put_gid_attr - Release reference to the GID attribute
1218 * @attr: Pointer to the GID attribute whose reference
1219 * needs to be released.
1221 * rdma_put_gid_attr() must be used to release reference whose
1222 * reference is acquired using rdma_get_gid_attr() or any APIs
1223 * which returns a pointer to the ib_gid_attr regardless of link layer
1227 void rdma_put_gid_attr(const struct ib_gid_attr *attr)
1229 struct ib_gid_table_entry *entry =
1230 container_of(attr, struct ib_gid_table_entry, attr);
1232 put_gid_entry(entry);
1234 EXPORT_SYMBOL(rdma_put_gid_attr);
1237 * rdma_hold_gid_attr - Get reference to existing GID attribute
1239 * @attr: Pointer to the GID attribute whose reference
1240 * needs to be taken.
1242 * Increase the reference count to a GID attribute to keep it from being
1243 * freed. Callers are required to already be holding a reference to attribute.
1246 void rdma_hold_gid_attr(const struct ib_gid_attr *attr)
1248 struct ib_gid_table_entry *entry =
1249 container_of(attr, struct ib_gid_table_entry, attr);
1251 get_gid_entry(entry);
1253 EXPORT_SYMBOL(rdma_hold_gid_attr);
1255 static int config_non_roce_gid_cache(struct ib_device *device,
1256 u8 port, int gid_tbl_len)
1258 struct ib_gid_attr gid_attr = {};
1259 struct ib_gid_table *table;
1263 gid_attr.device = device;
1264 gid_attr.port_num = port;
1265 table = rdma_gid_table(device, port);
1267 mutex_lock(&table->lock);
1268 for (i = 0; i < gid_tbl_len; ++i) {
1269 if (!device->query_gid)
1271 ret = device->query_gid(device, port, i, &gid_attr.gid);
1273 pr_warn("query_gid failed (%d) for %s (index %d)\n",
1274 ret, device->name, i);
1278 add_modify_gid(table, &gid_attr);
1281 mutex_unlock(&table->lock);
1285 static void ib_cache_update(struct ib_device *device,
1287 bool enforce_security)
1289 struct ib_port_attr *tprops = NULL;
1290 struct ib_pkey_cache *pkey_cache = NULL, *old_pkey_cache;
1294 if (!rdma_is_port_valid(device, port))
1297 tprops = kmalloc(sizeof *tprops, GFP_KERNEL);
1301 ret = ib_query_port(device, port, tprops);
1303 pr_warn("ib_query_port failed (%d) for %s\n",
1308 if (!rdma_protocol_roce(device, port)) {
1309 ret = config_non_roce_gid_cache(device, port,
1310 tprops->gid_tbl_len);
1315 pkey_cache = kmalloc(struct_size(pkey_cache, table,
1316 tprops->pkey_tbl_len),
1321 pkey_cache->table_len = tprops->pkey_tbl_len;
1323 for (i = 0; i < pkey_cache->table_len; ++i) {
1324 ret = ib_query_pkey(device, port, i, pkey_cache->table + i);
1326 pr_warn("ib_query_pkey failed (%d) for %s (index %d)\n",
1327 ret, device->name, i);
1332 write_lock_irq(&device->cache.lock);
1334 old_pkey_cache = device->cache.ports[port -
1335 rdma_start_port(device)].pkey;
1337 device->cache.ports[port - rdma_start_port(device)].pkey = pkey_cache;
1338 device->cache.ports[port - rdma_start_port(device)].lmc = tprops->lmc;
1339 device->cache.ports[port - rdma_start_port(device)].port_state =
1342 device->cache.ports[port - rdma_start_port(device)].subnet_prefix =
1343 tprops->subnet_prefix;
1344 write_unlock_irq(&device->cache.lock);
1346 if (enforce_security)
1347 ib_security_cache_change(device,
1349 tprops->subnet_prefix);
1351 kfree(old_pkey_cache);
1360 static void ib_cache_task(struct work_struct *_work)
1362 struct ib_update_work *work =
1363 container_of(_work, struct ib_update_work, work);
1365 ib_cache_update(work->device,
1367 work->enforce_security);
1371 static void ib_cache_event(struct ib_event_handler *handler,
1372 struct ib_event *event)
1374 struct ib_update_work *work;
1376 if (event->event == IB_EVENT_PORT_ERR ||
1377 event->event == IB_EVENT_PORT_ACTIVE ||
1378 event->event == IB_EVENT_LID_CHANGE ||
1379 event->event == IB_EVENT_PKEY_CHANGE ||
1380 event->event == IB_EVENT_SM_CHANGE ||
1381 event->event == IB_EVENT_CLIENT_REREGISTER ||
1382 event->event == IB_EVENT_GID_CHANGE) {
1383 work = kmalloc(sizeof *work, GFP_ATOMIC);
1385 INIT_WORK(&work->work, ib_cache_task);
1386 work->device = event->device;
1387 work->port_num = event->element.port_num;
1388 if (event->event == IB_EVENT_PKEY_CHANGE ||
1389 event->event == IB_EVENT_GID_CHANGE)
1390 work->enforce_security = true;
1392 work->enforce_security = false;
1394 queue_work(ib_wq, &work->work);
1399 int ib_cache_setup_one(struct ib_device *device)
1404 rwlock_init(&device->cache.lock);
1406 device->cache.ports =
1407 kcalloc(rdma_end_port(device) - rdma_start_port(device) + 1,
1408 sizeof(*device->cache.ports),
1410 if (!device->cache.ports)
1413 err = gid_table_setup_one(device);
1415 kfree(device->cache.ports);
1416 device->cache.ports = NULL;
1420 for (p = 0; p <= rdma_end_port(device) - rdma_start_port(device); ++p)
1421 ib_cache_update(device, p + rdma_start_port(device), true);
1423 INIT_IB_EVENT_HANDLER(&device->cache.event_handler,
1424 device, ib_cache_event);
1425 ib_register_event_handler(&device->cache.event_handler);
1429 void ib_cache_release_one(struct ib_device *device)
1434 * The release function frees all the cache elements.
1435 * This function should be called as part of freeing
1436 * all the device's resources when the cache could no
1437 * longer be accessed.
1439 for (p = 0; p <= rdma_end_port(device) - rdma_start_port(device); ++p)
1440 kfree(device->cache.ports[p].pkey);
1442 gid_table_release_one(device);
1443 kfree(device->cache.ports);
1446 void ib_cache_cleanup_one(struct ib_device *device)
1448 /* The cleanup function unregisters the event handler,
1449 * waits for all in-progress workqueue elements and cleans
1450 * up the GID cache. This function should be called after
1451 * the device was removed from the devices list and all
1452 * clients were removed, so the cache exists but is
1453 * non-functional and shouldn't be updated anymore.
1455 ib_unregister_event_handler(&device->cache.event_handler);
1456 flush_workqueue(ib_wq);
1457 gid_table_cleanup_one(device);
1460 * Flush the wq second time for any pending GID delete work.
1462 flush_workqueue(ib_wq);