RDMA/restrack: Count references to the verbs objects

Refactor the restrack code to make sure the kref inside the restrack entry
properly kref's the object in which it is embedded. This slight change is
needed for future conversions of MR and QP which are refcounted before the
release and kfree.

The ideal flow from ib_core perspective as follows:
* Allocate ib_* structure with rdma_zalloc_*.
* Set everything that is known to ib_core to that newly created object.
* Initialize kref with restrack help
* Call to driver specific allocation functions.
* Insert into restrack DB
....
* Return and release restrack with restrack_put.

Largely this means a rdma_restrack_new() should be called near allocating
the containing structure.

Link: https://lore.kernel.org/r/20200922091106.2152715-4-leon@kernel.org
Signed-off-by: Leon Romanovsky <leonro@mellanox.com>
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>

diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c
index 22f0b7ccdd6c9ab6817a5e3d7ebd7a07f4e60e17..568ad223d40f49d055ed8a3dad205050e66d9899 100644 (file)
--- a/drivers/infiniband/core/cma.c
+++ b/drivers/infiniband/core/cma.c
@@ -833,8 +833,6 @@ struct rdma_cm_id *__rdma_create_id(struct net *net,
        if (!id_priv)
                return ERR_PTR(-ENOMEM);
 
-       rdma_restrack_set_task(&id_priv->res, caller);
-       id_priv->res.type = RDMA_RESTRACK_CM_ID;
        id_priv->state = RDMA_CM_IDLE;
        id_priv->id.context = context;
        id_priv->id.event_handler = event_handler;
@@ -854,6 +852,9 @@ struct rdma_cm_id *__rdma_create_id(struct net *net,
        id_priv->id.route.addr.dev_addr.net = get_net(net);
        id_priv->seq_num &= 0x00ffffff;
 
+       rdma_restrack_new(&id_priv->res, RDMA_RESTRACK_CM_ID);
+       rdma_restrack_set_task(&id_priv->res, caller);
+
        return &id_priv->id;
 }
 EXPORT_SYMBOL(__rdma_create_id);

diff --git a/drivers/infiniband/core/core_priv.h b/drivers/infiniband/core/core_priv.h
index a1e6a67b2c4a73118590e70934baed238ec65faf..2ad276cd9781bb75e1e4e0ff2216f241358f4432 100644 (file)
--- a/drivers/infiniband/core/core_priv.h
+++ b/drivers/infiniband/core/core_priv.h
@@ -44,6 +44,7 @@
 #include <rdma/ib_mad.h>
 #include <rdma/restrack.h>
 #include "mad_priv.h"
+#include "restrack.h"
 
 /* Total number of ports combined across all struct ib_devices's */
 #define RDMA_MAX_PORTS 8192
@@ -352,6 +353,7 @@ static inline struct ib_qp *_ib_create_qp(struct ib_device *dev,
        INIT_LIST_HEAD(&qp->rdma_mrs);
        INIT_LIST_HEAD(&qp->sig_mrs);
 
+       rdma_restrack_new(&qp->res, RDMA_RESTRACK_QP);
        /*
         * We don't track XRC QPs for now, because they don't have PD
         * and more importantly they are created internaly by driver,
@@ -359,7 +361,6 @@ static inline struct ib_qp *_ib_create_qp(struct ib_device *dev,
         */
        is_xrc = qp_type == IB_QPT_XRC_INI || qp_type == IB_QPT_XRC_TGT;
        if ((qp_type < IB_QPT_MAX && !is_xrc) || qp_type == IB_QPT_DRIVER) {
-               qp->res.type = RDMA_RESTRACK_QP;
                if (uobj)
                        rdma_restrack_uadd(&qp->res);
                else

diff --git a/drivers/infiniband/core/counters.c b/drivers/infiniband/core/counters.c
index 63616688044285bda23b1e4e55cd005a326c5c96..c059de99d19c59dee8fca57af14bf38499d8c8f5 100644 (file)
--- a/drivers/infiniband/core/counters.c
+++ b/drivers/infiniband/core/counters.c
@@ -80,8 +80,9 @@ static struct rdma_counter *rdma_counter_alloc(struct ib_device *dev, u8 port,
 
        counter->device    = dev;
        counter->port      = port;
-       counter->res.type  = RDMA_RESTRACK_COUNTER;
-       counter->stats     = dev->ops.counter_alloc_stats(counter);
+
+       rdma_restrack_new(&counter->res, RDMA_RESTRACK_COUNTER);
+       counter->stats = dev->ops.counter_alloc_stats(counter);
        if (!counter->stats)
                goto err_stats;
 
@@ -107,6 +108,7 @@ err_mode:
        mutex_unlock(&port_counter->lock);
        kfree(counter->stats);
 err_stats:
+       rdma_restrack_put(&counter->res);
        kfree(counter);
        return NULL;
 }

diff --git a/drivers/infiniband/core/cq.c b/drivers/infiniband/core/cq.c
index 19e36e52181beddff54f194b0f3c2d3af7b4efef..2c3ff8dda717d8a30eb77739eb73118a80e3a051 100644 (file)
--- a/drivers/infiniband/core/cq.c
+++ b/drivers/infiniband/core/cq.c
@@ -235,15 +235,13 @@ struct ib_cq *__ib_alloc_cq(struct ib_device *dev, void *private, int nr_cqe,
        if (!cq->wc)
                goto out_free_cq;
 
-       cq->res.type = RDMA_RESTRACK_CQ;
+       rdma_restrack_new(&cq->res, RDMA_RESTRACK_CQ);
        rdma_restrack_set_task(&cq->res, caller);
 
        ret = dev->ops.create_cq(cq, &cq_attr, NULL);
        if (ret)
                goto out_free_wc;
 
-       rdma_restrack_kadd(&cq->res);
-
        rdma_dim_init(cq);
 
        switch (cq->poll_ctx) {
@@ -269,14 +267,15 @@ struct ib_cq *__ib_alloc_cq(struct ib_device *dev, void *private, int nr_cqe,
                goto out_destroy_cq;
        }
 
+       rdma_restrack_kadd(&cq->res);
        trace_cq_alloc(cq, nr_cqe, comp_vector, poll_ctx);
        return cq;
 
 out_destroy_cq:
        rdma_dim_destroy(cq);
-       rdma_restrack_del(&cq->res);
        cq->device->ops.destroy_cq(cq, NULL);
 out_free_wc:
+       rdma_restrack_put(&cq->res);
        kfree(cq->wc);
 out_free_cq:
        kfree(cq);

diff --git a/drivers/infiniband/core/restrack.c b/drivers/infiniband/core/restrack.c
index 62fbb0ae9cb4a66a24c921f27cbf98460e95a199..22c658e8c157ffed30c4e43da9cb69421c34218f 100644 (file)
--- a/drivers/infiniband/core/restrack.c
+++ b/drivers/infiniband/core/restrack.c
@@ -202,6 +202,21 @@ void rdma_restrack_attach_task(struct rdma_restrack_entry *res,
        res->task = task;
 }
 
+/**
+ * rdma_restrack_new() - Initializes new restrack entry to allow _put() interface
+ * to release memory in fully automatic way.
+ * @res - Entry to initialize
+ * @type - REstrack type
+ */
+void rdma_restrack_new(struct rdma_restrack_entry *res,
+                      enum rdma_restrack_type type)
+{
+       kref_init(&res->kref);
+       init_completion(&res->comp);
+       res->type = type;
+}
+EXPORT_SYMBOL(rdma_restrack_new);
+
 static void rdma_restrack_add(struct rdma_restrack_entry *res)
 {
        struct ib_device *dev = res_to_dev(res);
@@ -213,8 +228,6 @@ static void rdma_restrack_add(struct rdma_restrack_entry *res)
 
        rt = &dev->res[res->type];
 
-       kref_init(&res->kref);
-       init_completion(&res->comp);
        if (res->type == RDMA_RESTRACK_QP) {
                /* Special case to ensure that LQPN points to right QP */
                struct ib_qp *qp = container_of(res, struct ib_qp, res);
@@ -305,6 +318,10 @@ static void restrack_release(struct kref *kref)
        struct rdma_restrack_entry *res;
 
        res = container_of(kref, struct rdma_restrack_entry, kref);
+       if (res->task) {
+               put_task_struct(res->task);
+               res->task = NULL;
+       }
        complete(&res->comp);
 }
 
@@ -314,14 +331,23 @@ int rdma_restrack_put(struct rdma_restrack_entry *res)
 }
 EXPORT_SYMBOL(rdma_restrack_put);
 
+/**
+ * rdma_restrack_del() - delete object from the reource tracking database
+ * @res:  resource entry
+ */
 void rdma_restrack_del(struct rdma_restrack_entry *res)
 {
        struct rdma_restrack_entry *old;
        struct rdma_restrack_root *rt;
        struct ib_device *dev;
 
-       if (!res->valid)
-               goto out;
+       if (!res->valid) {
+               if (res->task) {
+                       put_task_struct(res->task);
+                       res->task = NULL;
+               }
+               return;
+       }
 
        dev = res_to_dev(res);
        if (WARN_ON(!dev))
@@ -330,16 +356,12 @@ void rdma_restrack_del(struct rdma_restrack_entry *res)
        rt = &dev->res[res->type];
 
        old = xa_erase(&rt->xa, res->id);
+       if (res->type == RDMA_RESTRACK_MR || res->type == RDMA_RESTRACK_QP)
+               return;
        WARN_ON(old != res);
        res->valid = false;
 
        rdma_restrack_put(res);
        wait_for_completion(&res->comp);
-
-out:
-       if (res->task) {
-               put_task_struct(res->task);
-               res->task = NULL;
-       }
 }
 EXPORT_SYMBOL(rdma_restrack_del);

diff --git a/drivers/infiniband/core/restrack.h b/drivers/infiniband/core/restrack.h
index d084e5f8984911460399af83df0a2447171c42b2..16006a5e7408e689787faa9fdf86aff4acab4f0d 100644 (file)
--- a/drivers/infiniband/core/restrack.h
+++ b/drivers/infiniband/core/restrack.h
@@ -25,6 +25,9 @@ struct rdma_restrack_root {
 
 int rdma_restrack_init(struct ib_device *dev);
 void rdma_restrack_clean(struct ib_device *dev);
+void rdma_restrack_del(struct rdma_restrack_entry *res);
+void rdma_restrack_new(struct rdma_restrack_entry *res,
+                      enum rdma_restrack_type type);
 void rdma_restrack_attach_task(struct rdma_restrack_entry *res,
                               struct task_struct *task);
 #endif /* _RDMA_CORE_RESTRACK_H_ */

diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c
index e5c0784e151d960a40c174c2ff642fe10ea1f657..6ddd26548da8f6d2592d40f97828768557ee98d9 100644 (file)
--- a/drivers/infiniband/core/uverbs_cmd.c
+++ b/drivers/infiniband/core/uverbs_cmd.c
@@ -218,10 +218,11 @@ int ib_alloc_ucontext(struct uverbs_attr_bundle *attrs)
        if (!ucontext)
                return -ENOMEM;
 
-       ucontext->res.type = RDMA_RESTRACK_CTX;
        ucontext->device = ib_dev;
        ucontext->ufile = ufile;
        xa_init_flags(&ucontext->mmap_xa, XA_FLAGS_ALLOC);
+
+       rdma_restrack_new(&ucontext->res, RDMA_RESTRACK_CTX);
        attrs->context = ucontext;
        return 0;
 }
@@ -313,6 +314,7 @@ static int ib_uverbs_get_context(struct uverbs_attr_bundle *attrs)
 err_uobj:
        rdma_alloc_abort_uobject(uobj, attrs, false);
 err_ucontext:
+       rdma_restrack_put(&attrs->context->res);
        kfree(attrs->context);
        attrs->context = NULL;
        return ret;
@@ -439,8 +441,8 @@ static int ib_uverbs_alloc_pd(struct uverbs_attr_bundle *attrs)
        pd->device  = ib_dev;
        pd->uobject = uobj;
        atomic_set(&pd->usecnt, 0);
-       pd->res.type = RDMA_RESTRACK_PD;
 
+       rdma_restrack_new(&pd->res, RDMA_RESTRACK_PD);
        ret = ib_dev->ops.alloc_pd(pd, &attrs->driver_udata);
        if (ret)
                goto err_alloc;
@@ -453,6 +455,7 @@ static int ib_uverbs_alloc_pd(struct uverbs_attr_bundle *attrs)
        return uverbs_response(attrs, &resp, sizeof(resp));
 
 err_alloc:
+       rdma_restrack_put(&pd->res);
        kfree(pd);
 err:
        uobj_alloc_abort(uobj, attrs);
@@ -742,8 +745,9 @@ static int ib_uverbs_reg_mr(struct uverbs_attr_bundle *attrs)
        mr->sig_attrs = NULL;
        mr->uobject = uobj;
        atomic_inc(&pd->usecnt);
-       mr->res.type = RDMA_RESTRACK_MR;
        mr->iova = cmd.hca_va;
+
+       rdma_restrack_new(&mr->res, RDMA_RESTRACK_MR);
        rdma_restrack_uadd(&mr->res);
 
        uobj->object = mr;
@@ -1002,8 +1006,8 @@ static int create_cq(struct uverbs_attr_bundle *attrs,
        cq->event_handler = ib_uverbs_cq_event_handler;
        cq->cq_context    = ev_file ? &ev_file->ev_queue : NULL;
        atomic_set(&cq->usecnt, 0);
-       cq->res.type = RDMA_RESTRACK_CQ;
 
+       rdma_restrack_new(&cq->res, RDMA_RESTRACK_CQ);
        ret = ib_dev->ops.create_cq(cq, &attr, &attrs->driver_udata);
        if (ret)
                goto err_free;
@@ -1021,6 +1025,7 @@ static int create_cq(struct uverbs_attr_bundle *attrs,
        return uverbs_response(attrs, &resp, sizeof(resp));
 
 err_free:
+       rdma_restrack_put(&cq->res);
        kfree(cq);
 err_file:
        if (ev_file)

diff --git a/drivers/infiniband/core/uverbs_std_types_cq.c b/drivers/infiniband/core/uverbs_std_types_cq.c
index b1c7dacc02de1508481087047312ed0d0b4ebb34..3a5fd6c9ba722a4df8a31d80f731edcefd7272ff 100644 (file)
--- a/drivers/infiniband/core/uverbs_std_types_cq.c
+++ b/drivers/infiniband/core/uverbs_std_types_cq.c
@@ -33,6 +33,7 @@
 #include <rdma/uverbs_std_types.h>
 #include "rdma_core.h"
 #include "uverbs.h"
+#include "restrack.h"
 
 static int uverbs_free_cq(struct ib_uobject *uobject,
                          enum rdma_remove_reason why,
@@ -123,8 +124,8 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)(
        cq->event_handler = ib_uverbs_cq_event_handler;
        cq->cq_context    = ev_file ? &ev_file->ev_queue : NULL;
        atomic_set(&cq->usecnt, 0);
-       cq->res.type = RDMA_RESTRACK_CQ;
 
+       rdma_restrack_new(&cq->res, RDMA_RESTRACK_CQ);
        ret = ib_dev->ops.create_cq(cq, &attr, &attrs->driver_udata);
        if (ret)
                goto err_free;
@@ -139,6 +140,7 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)(
        return ret;
 
 err_free:
+       rdma_restrack_put(&cq->res);
        kfree(cq);
 err_event_file:
        if (obj->uevent.event_file)

diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c
index d0c3767b0f9f74298ddf982a896fb8b047a405e5..2607c410b09f6b77a1f3a45d978f23fa5c404cb9 100644 (file)
--- a/drivers/infiniband/core/verbs.c
+++ b/drivers/infiniband/core/verbs.c
@@ -272,11 +272,12 @@ struct ib_pd *__ib_alloc_pd(struct ib_device *device, unsigned int flags,
        atomic_set(&pd->usecnt, 0);
        pd->flags = flags;
 
-       pd->res.type = RDMA_RESTRACK_PD;
+       rdma_restrack_new(&pd->res, RDMA_RESTRACK_PD);
        rdma_restrack_set_task(&pd->res, caller);
 
        ret = device->ops.alloc_pd(pd, NULL);
        if (ret) {
+               rdma_restrack_put(&pd->res);
                kfree(pd);
                return ERR_PTR(ret);
        }
@@ -1996,11 +1997,13 @@ struct ib_cq *__ib_create_cq(struct ib_device *device,
        cq->event_handler = event_handler;
        cq->cq_context = cq_context;
        atomic_set(&cq->usecnt, 0);
-       cq->res.type = RDMA_RESTRACK_CQ;
+
+       rdma_restrack_new(&cq->res, RDMA_RESTRACK_CQ);
        rdma_restrack_set_task(&cq->res, caller);
 
        ret = device->ops.create_cq(cq, cq_attr, NULL);
        if (ret) {
+               rdma_restrack_put(&cq->res);
                kfree(cq);
                return ERR_PTR(ret);
        }
@@ -2076,7 +2079,8 @@ struct ib_mr *ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
        mr->pd = pd;
        mr->dm = NULL;
        atomic_inc(&pd->usecnt);
-       mr->res.type = RDMA_RESTRACK_MR;
+
+       rdma_restrack_new(&mr->res, RDMA_RESTRACK_MR);
        rdma_restrack_kadd(&mr->res);
 
        return mr;
@@ -2156,11 +2160,11 @@ struct ib_mr *ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
        mr->uobject = NULL;
        atomic_inc(&pd->usecnt);
        mr->need_inval = false;
-       mr->res.type = RDMA_RESTRACK_MR;
-       rdma_restrack_kadd(&mr->res);
        mr->type = mr_type;
        mr->sig_attrs = NULL;
 
+       rdma_restrack_new(&mr->res, RDMA_RESTRACK_MR);
+       rdma_restrack_kadd(&mr->res);
 out:
        trace_mr_alloc(pd, mr_type, max_num_sg, mr);
        return mr;
@@ -2216,11 +2220,11 @@ struct ib_mr *ib_alloc_mr_integrity(struct ib_pd *pd,
        mr->uobject = NULL;
        atomic_inc(&pd->usecnt);
        mr->need_inval = false;
-       mr->res.type = RDMA_RESTRACK_MR;
-       rdma_restrack_kadd(&mr->res);
        mr->type = IB_MR_TYPE_INTEGRITY;
        mr->sig_attrs = sig_attrs;
 
+       rdma_restrack_new(&mr->res, RDMA_RESTRACK_MR);
+       rdma_restrack_kadd(&mr->res);
 out:
        trace_mr_integ_alloc(pd, max_num_data_sg, max_num_meta_sg, mr);
        return mr;

diff --git a/include/rdma/restrack.h b/include/rdma/restrack.h
index 7682d1bcf78950a514606c0a0def556d6e8f405c..d7c16623793912ec73a5acafa6a2d6fb7af5db83 100644 (file)
--- a/include/rdma/restrack.h
+++ b/include/rdma/restrack.h
@@ -110,13 +110,6 @@ int rdma_restrack_count(struct ib_device *dev,
 void rdma_restrack_kadd(struct rdma_restrack_entry *res);
 void rdma_restrack_uadd(struct rdma_restrack_entry *res);
 
-/**
- * rdma_restrack_del() - delete object from the reource tracking database
- * @res:  resource entry
- * @type: actual type of object to operate
- */
-void rdma_restrack_del(struct rdma_restrack_entry *res);
-
 /**
  * rdma_is_kernel_res() - check the owner of resource
  * @res:  resource entry