Merge tag 'for-linus-hmm' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma

[linux-2.6-block.git] / drivers / infiniband / hw / mlx5 / odp.c

diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c
index 0a59912a4cef640067472319da83d7123f6eb5d1..dd26e7acb37e4b331890d7ab929389b0d76e3fe7 100644 (file)
--- a/drivers/infiniband/hw/mlx5/odp.c
+++ b/drivers/infiniband/hw/mlx5/odp.c
@@ -184,7 +184,7 @@ void mlx5_odp_populate_klm(struct mlx5_klm *pklm, size_t offset,
        for (i = 0; i < nentries; i++, pklm++) {
                pklm->bcount = cpu_to_be32(MLX5_IMR_MTT_SIZE);
                va = (offset + i) * MLX5_IMR_MTT_SIZE;
-               if (odp && odp->umem.address == va) {
+               if (odp && ib_umem_start(odp) == va) {
                        struct mlx5_ib_mr *mtt = odp->private;
 
                        pklm->key = cpu_to_be32(mtt->ibmr.lkey);
@@ -206,7 +206,7 @@ static void mr_leaf_free_action(struct work_struct *work)
        mr->parent = NULL;
        synchronize_srcu(&mr->dev->mr_srcu);
 
-       ib_umem_release(&odp->umem);
+       ib_umem_odp_release(odp);
        if (imr->live)
                mlx5_ib_update_xlt(imr, idx, 1, 0,
                                   MLX5_IB_UPD_XLT_INDIRECT |
@@ -386,7 +386,7 @@ static void mlx5_ib_page_fault_resume(struct mlx5_ib_dev *dev,
 }
 
 static struct mlx5_ib_mr *implicit_mr_alloc(struct ib_pd *pd,
-                                           struct ib_umem *umem,
+                                           struct ib_umem_odp *umem_odp,
                                            bool ksm, int access_flags)
 {
        struct mlx5_ib_dev *dev = to_mdev(pd->device);
@@ -404,7 +404,7 @@ static struct mlx5_ib_mr *implicit_mr_alloc(struct ib_pd *pd,
        mr->dev = dev;
        mr->access_flags = access_flags;
        mr->mmkey.iova = 0;
-       mr->umem = umem;
+       mr->umem = &umem_odp->umem;
 
        if (ksm) {
                err = mlx5_ib_update_xlt(mr, 0,
@@ -464,18 +464,17 @@ next_mr:
                if (nentries)
                        nentries++;
        } else {
-               odp = ib_alloc_odp_umem(odp_mr, addr,
-                                       MLX5_IMR_MTT_SIZE);
+               odp = ib_umem_odp_alloc_child(odp_mr, addr, MLX5_IMR_MTT_SIZE);
                if (IS_ERR(odp)) {
                        mutex_unlock(&odp_mr->umem_mutex);
                        return ERR_CAST(odp);
                }
 
-               mtt = implicit_mr_alloc(mr->ibmr.pd, &odp->umem, 0,
+               mtt = implicit_mr_alloc(mr->ibmr.pd, odp, 0,
                                        mr->access_flags);
                if (IS_ERR(mtt)) {
                        mutex_unlock(&odp_mr->umem_mutex);
-                       ib_umem_release(&odp->umem);
+                       ib_umem_odp_release(odp);
                        return ERR_CAST(mtt);
                }
 
@@ -497,7 +496,7 @@ next_mr:
        addr += MLX5_IMR_MTT_SIZE;
        if (unlikely(addr < io_virt + bcnt)) {
                odp = odp_next(odp);
-               if (odp && odp->umem.address != addr)
+               if (odp && ib_umem_start(odp) != addr)
                        odp = NULL;
                goto next_mr;
        }
@@ -521,19 +520,19 @@ struct mlx5_ib_mr *mlx5_ib_alloc_implicit_mr(struct mlx5_ib_pd *pd,
                                             int access_flags)
 {
        struct mlx5_ib_mr *imr;
-       struct ib_umem *umem;
+       struct ib_umem_odp *umem_odp;
 
-       umem = ib_umem_get(udata, 0, 0, access_flags, 0);
-       if (IS_ERR(umem))
-               return ERR_CAST(umem);
+       umem_odp = ib_umem_odp_alloc_implicit(udata, access_flags);
+       if (IS_ERR(umem_odp))
+               return ERR_CAST(umem_odp);
 
-       imr = implicit_mr_alloc(&pd->ibpd, umem, 1, access_flags);
+       imr = implicit_mr_alloc(&pd->ibpd, umem_odp, 1, access_flags);
        if (IS_ERR(imr)) {
-               ib_umem_release(umem);
+               ib_umem_odp_release(umem_odp);
                return ERR_CAST(imr);
        }
 
-       imr->umem = umem;
+       imr->umem = &umem_odp->umem;
        init_waitqueue_head(&imr->q_leaf_free);
        atomic_set(&imr->num_leaf_free, 0);
        atomic_set(&imr->num_pending_prefetch, 0);
@@ -541,34 +540,31 @@ struct mlx5_ib_mr *mlx5_ib_alloc_implicit_mr(struct mlx5_ib_pd *pd,
        return imr;
 }
 
-static int mr_leaf_free(struct ib_umem_odp *umem_odp, u64 start, u64 end,
-                       void *cookie)
+void mlx5_ib_free_implicit_mr(struct mlx5_ib_mr *imr)
 {
-       struct mlx5_ib_mr *mr = umem_odp->private, *imr = cookie;
-
-       if (mr->parent != imr)
-               return 0;
-
-       ib_umem_odp_unmap_dma_pages(umem_odp, ib_umem_start(umem_odp),
-                                   ib_umem_end(umem_odp));
+       struct ib_ucontext_per_mm *per_mm = mr_to_per_mm(imr);
+       struct rb_node *node;
 
-       if (umem_odp->dying)
-               return 0;
+       down_read(&per_mm->umem_rwsem);
+       for (node = rb_first_cached(&per_mm->umem_tree); node;
+            node = rb_next(node)) {
+               struct ib_umem_odp *umem_odp =
+                       rb_entry(node, struct ib_umem_odp, interval_tree.rb);
+               struct mlx5_ib_mr *mr = umem_odp->private;
 
-       WRITE_ONCE(umem_odp->dying, 1);
-       atomic_inc(&imr->num_leaf_free);
-       schedule_work(&umem_odp->work);
+               if (mr->parent != imr)
+                       continue;
 
-       return 0;
-}
+               ib_umem_odp_unmap_dma_pages(umem_odp, ib_umem_start(umem_odp),
+                                           ib_umem_end(umem_odp));
 
-void mlx5_ib_free_implicit_mr(struct mlx5_ib_mr *imr)
-{
-       struct ib_ucontext_per_mm *per_mm = mr_to_per_mm(imr);
+               if (umem_odp->dying)
+                       continue;
 
-       down_read(&per_mm->umem_rwsem);
-       rbt_ib_umem_for_each_in_range(&per_mm->umem_tree, 0, ULLONG_MAX,
-                                     mr_leaf_free, true, imr);
+               WRITE_ONCE(umem_odp->dying, 1);
+               atomic_inc(&imr->num_leaf_free);
+               schedule_work(&umem_odp->work);
+       }
        up_read(&per_mm->umem_rwsem);
 
        wait_event(imr->q_leaf_free, !atomic_read(&imr->num_leaf_free));
@@ -589,7 +585,7 @@ static int pagefault_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr,
        struct ib_umem_odp *odp;
        size_t size;
 
-       if (!odp_mr->page_list) {
+       if (odp_mr->is_implicit_odp) {
                odp = implicit_mr_get_data(mr, io_virt, bcnt);
 
                if (IS_ERR(odp))
@@ -607,7 +603,7 @@ next_mr:
        start_idx = (io_virt - (mr->mmkey.iova & page_mask)) >> page_shift;
        access_mask = ODP_READ_ALLOWED_BIT;
 
-       if (prefetch && !downgrade && !mr->umem->writable) {
+       if (prefetch && !downgrade && !odp->umem.writable) {
                /* prefetch with write-access must
                 * be supported by the MR
                 */
@@ -615,7 +611,7 @@ next_mr:
                goto out;
        }
 
-       if (mr->umem->writable && !downgrade)
+       if (odp->umem.writable && !downgrade)
                access_mask |= ODP_WRITE_ALLOWED_BIT;
 
        current_seq = READ_ONCE(odp->notifiers_seq);
@@ -625,8 +621,8 @@ next_mr:
         */
        smp_rmb();
 
-       ret = ib_umem_odp_map_dma_pages(to_ib_umem_odp(mr->umem), io_virt, size,
-                                       access_mask, current_seq);
+       ret = ib_umem_odp_map_dma_pages(odp, io_virt, size, access_mask,
+                                       current_seq);
 
        if (ret < 0)
                goto out;
@@ -634,8 +630,7 @@ next_mr:
        np = ret;
 
        mutex_lock(&odp->umem_mutex);
-       if (!ib_umem_mmu_notifier_retry(to_ib_umem_odp(mr->umem),
-                                       current_seq)) {
+       if (!ib_umem_mmu_notifier_retry(odp, current_seq)) {
                /*
                 * No need to check whether the MTTs really belong to
                 * this MR, since ib_umem_odp_map_dma_pages already
@@ -668,7 +663,7 @@ next_mr:
 
                io_virt += size;
                next = odp_next(odp);
-               if (unlikely(!next || next->umem.address != io_virt)) {
+               if (unlikely(!next || ib_umem_start(next) != io_virt)) {
                        mlx5_ib_dbg(dev, "next implicit leaf removed at 0x%llx. got %p\n",
                                    io_virt, next);
                        return -EAGAIN;
@@ -1618,6 +1613,7 @@ void mlx5_odp_init_mr_cache_entry(struct mlx5_cache_ent *ent)
 
 static const struct ib_device_ops mlx5_ib_dev_odp_ops = {
        .advise_mr = mlx5_ib_advise_mr,
+       .invalidate_range = mlx5_ib_invalidate_range,
 };
 
 int mlx5_ib_odp_init_one(struct mlx5_ib_dev *dev)