RDMA/umem: Avoid synchronize_srcu in the ODP MR destruction path

synchronize_rcu is slow enough that it should be avoided on the syscall
path when user space is destroying MRs. After all the rework we can now
trivially do this by having call_srcu kfree the per_mm.

Signed-off-by: Jason Gunthorpe <jgg@mellanox.com>
Signed-off-by: Leon Romanovsky <leonro@mellanox.com>
Signed-off-by: Doug Ledford <dledford@redhat.com>

diff --git a/drivers/infiniband/core/umem_odp.c b/drivers/infiniband/core/umem_odp.c
index d7b6422b961164603277429a59ddf6f4830295f1..2b4c5e7dd5a173c270e131016e40fcb892e04d70 100644 (file)
--- a/drivers/infiniband/core/umem_odp.c
+++ b/drivers/infiniband/core/umem_odp.c
@@ -307,6 +307,11 @@ found:
        return 0;
 }
 
+static void free_per_mm(struct rcu_head *rcu)
+{
+       kfree(container_of(rcu, struct ib_ucontext_per_mm, rcu));
+}
+
 void put_per_mm(struct ib_umem_odp *umem_odp)
 {
        struct ib_ucontext_per_mm *per_mm = umem_odp->per_mm;
@@ -334,9 +339,10 @@ void put_per_mm(struct ib_umem_odp *umem_odp)
        per_mm->active = false;
        up_write(&per_mm->umem_rwsem);
 
-       mmu_notifier_unregister(&per_mm->mn, per_mm->mm);
+       WARN_ON(!RB_EMPTY_ROOT(&per_mm->umem_tree.rb_root));
+       mmu_notifier_unregister_no_release(&per_mm->mn, per_mm->mm);
        put_pid(per_mm->tgid);
-       kfree(per_mm);
+       mmu_notifier_call_srcu(&per_mm->rcu, free_per_mm);
 }
 
 struct ib_umem_odp *ib_alloc_odp_umem(struct ib_ucontext_per_mm *per_mm,

diff --git a/include/rdma/ib_umem_odp.h b/include/rdma/ib_umem_odp.h
index ec05c82ead7af30fc80634d7536a50fa4983658e..0b1446fe2fab514fe8e5fa6d76b4aa1a490d6940 100644 (file)
--- a/include/rdma/ib_umem_odp.h
+++ b/include/rdma/ib_umem_odp.h
@@ -99,6 +99,7 @@ struct ib_ucontext_per_mm {
        unsigned int odp_mrs_count;
 
        struct list_head ucontext_list;
+       struct rcu_head rcu;
 };
 
 int ib_umem_odp_get(struct ib_umem_odp *umem_odp, int access);