struct rb_node *node;
down_read(&per_mm->umem_rwsem);
- if (!per_mm->active)
+ if (!per_mm->mn.users)
goto out;
for (node = rb_first_cached(&per_mm->umem_tree); node;
*/
ib_umem_notifier_start_account(umem_odp);
complete_all(&umem_odp->notifier_completion);
- umem_odp->umem.context->device->ops.invalidate_range(
+ umem_odp->umem.ibdev->ops.invalidate_range(
umem_odp, ib_umem_start(umem_odp),
ib_umem_end(umem_odp));
}
u64 start, u64 end, void *cookie)
{
ib_umem_notifier_start_account(item);
- item->umem.context->device->ops.invalidate_range(item, start, end);
+ item->umem.ibdev->ops.invalidate_range(item, start, end);
return 0;
}
else if (!down_read_trylock(&per_mm->umem_rwsem))
return -EAGAIN;
- if (!per_mm->active) {
+ if (!per_mm->mn.users) {
up_read(&per_mm->umem_rwsem);
/*
- * At this point active is permanently set and visible to this
+ * At this point users is permanently zero and visible to this
* CPU without a lock, that fact is relied on to skip the unlock
* in range_end.
*/
struct ib_ucontext_per_mm *per_mm =
container_of(mn, struct ib_ucontext_per_mm, mn);
- if (unlikely(!per_mm->active))
+ if (unlikely(!per_mm->mn.users))
return;
rbt_ib_umem_for_each_in_range(&per_mm->umem_tree, range->start,
up_read(&per_mm->umem_rwsem);
}
-static const struct mmu_notifier_ops ib_umem_notifiers = {
- .release = ib_umem_notifier_release,
- .invalidate_range_start = ib_umem_notifier_invalidate_range_start,
- .invalidate_range_end = ib_umem_notifier_invalidate_range_end,
-};
-
-static void remove_umem_from_per_mm(struct ib_umem_odp *umem_odp)
-{
- struct ib_ucontext_per_mm *per_mm = umem_odp->per_mm;
-
- down_write(&per_mm->umem_rwsem);
- interval_tree_remove(&umem_odp->interval_tree, &per_mm->umem_tree);
- complete_all(&umem_odp->notifier_completion);
- up_write(&per_mm->umem_rwsem);
-}
-
-static struct ib_ucontext_per_mm *alloc_per_mm(struct ib_ucontext *ctx,
- struct mm_struct *mm)
+static struct mmu_notifier *ib_umem_alloc_notifier(struct mm_struct *mm)
{
struct ib_ucontext_per_mm *per_mm;
- int ret;
per_mm = kzalloc(sizeof(*per_mm), GFP_KERNEL);
if (!per_mm)
return ERR_PTR(-ENOMEM);
- per_mm->context = ctx;
- per_mm->mm = mm;
per_mm->umem_tree = RB_ROOT_CACHED;
init_rwsem(&per_mm->umem_rwsem);
- per_mm->active = true;
+ WARN_ON(mm != current->mm);
rcu_read_lock();
per_mm->tgid = get_task_pid(current->group_leader, PIDTYPE_PID);
rcu_read_unlock();
-
- WARN_ON(mm != current->mm);
-
- per_mm->mn.ops = &ib_umem_notifiers;
- ret = mmu_notifier_register(&per_mm->mn, per_mm->mm);
- if (ret) {
- dev_err(&ctx->device->dev,
- "Failed to register mmu_notifier %d\n", ret);
- goto out_pid;
- }
-
- list_add(&per_mm->ucontext_list, &ctx->per_mm_list);
- return per_mm;
-
-out_pid:
- put_pid(per_mm->tgid);
- kfree(per_mm);
- return ERR_PTR(ret);
-}
-
-static struct ib_ucontext_per_mm *get_per_mm(struct ib_umem_odp *umem_odp)
-{
- struct ib_ucontext *ctx = umem_odp->umem.context;
- struct ib_ucontext_per_mm *per_mm;
-
- lockdep_assert_held(&ctx->per_mm_list_lock);
-
- /*
- * Generally speaking we expect only one or two per_mm in this list,
- * so no reason to optimize this search today.
- */
- list_for_each_entry(per_mm, &ctx->per_mm_list, ucontext_list) {
- if (per_mm->mm == umem_odp->umem.owning_mm)
- return per_mm;
- }
-
- return alloc_per_mm(ctx, umem_odp->umem.owning_mm);
-}
-
-static void free_per_mm(struct rcu_head *rcu)
-{
- kfree(container_of(rcu, struct ib_ucontext_per_mm, rcu));
+ return &per_mm->mn;
}
-static void put_per_mm(struct ib_umem_odp *umem_odp)
+static void ib_umem_free_notifier(struct mmu_notifier *mn)
{
- struct ib_ucontext_per_mm *per_mm = umem_odp->per_mm;
- struct ib_ucontext *ctx = umem_odp->umem.context;
- bool need_free;
-
- mutex_lock(&ctx->per_mm_list_lock);
- umem_odp->per_mm = NULL;
- per_mm->odp_mrs_count--;
- need_free = per_mm->odp_mrs_count == 0;
- if (need_free)
- list_del(&per_mm->ucontext_list);
- mutex_unlock(&ctx->per_mm_list_lock);
-
- if (!need_free)
- return;
-
- /*
- * NOTE! mmu_notifier_unregister() can happen between a start/end
- * callback, resulting in an start/end, and thus an unbalanced
- * lock. This doesn't really matter to us since we are about to kfree
- * the memory that holds the lock, however LOCKDEP doesn't like this.
- */
- down_write(&per_mm->umem_rwsem);
- per_mm->active = false;
- up_write(&per_mm->umem_rwsem);
+ struct ib_ucontext_per_mm *per_mm =
+ container_of(mn, struct ib_ucontext_per_mm, mn);
WARN_ON(!RB_EMPTY_ROOT(&per_mm->umem_tree.rb_root));
- mmu_notifier_unregister_no_release(&per_mm->mn, per_mm->mm);
+
put_pid(per_mm->tgid);
- mmu_notifier_call_srcu(&per_mm->rcu, free_per_mm);
+ kfree(per_mm);
}
-static inline int ib_init_umem_odp(struct ib_umem_odp *umem_odp,
- struct ib_ucontext_per_mm *per_mm)
+static const struct mmu_notifier_ops ib_umem_notifiers = {
+ .release = ib_umem_notifier_release,
+ .invalidate_range_start = ib_umem_notifier_invalidate_range_start,
+ .invalidate_range_end = ib_umem_notifier_invalidate_range_end,
+ .alloc_notifier = ib_umem_alloc_notifier,
+ .free_notifier = ib_umem_free_notifier,
+};
+
+static inline int ib_init_umem_odp(struct ib_umem_odp *umem_odp)
{
- struct ib_ucontext *ctx = umem_odp->umem.context;
+ struct ib_ucontext_per_mm *per_mm;
+ struct mmu_notifier *mn;
int ret;
umem_odp->umem.is_odp = 1;
}
}
- mutex_lock(&ctx->per_mm_list_lock);
- if (!per_mm) {
- per_mm = get_per_mm(umem_odp);
- if (IS_ERR(per_mm)) {
- ret = PTR_ERR(per_mm);
- goto out_unlock;
- }
+ mn = mmu_notifier_get(&ib_umem_notifiers, umem_odp->umem.owning_mm);
+ if (IS_ERR(mn)) {
+ ret = PTR_ERR(mn);
+ goto out_dma_list;
}
- umem_odp->per_mm = per_mm;
- per_mm->odp_mrs_count++;
- mutex_unlock(&ctx->per_mm_list_lock);
+ umem_odp->per_mm = per_mm =
+ container_of(mn, struct ib_ucontext_per_mm, mn);
mutex_init(&umem_odp->umem_mutex);
init_completion(&umem_odp->notifier_completion);
return 0;
-out_unlock:
- mutex_unlock(&ctx->per_mm_list_lock);
+out_dma_list:
kvfree(umem_odp->dma_list);
out_page_list:
kvfree(umem_odp->page_list);
if (!umem_odp)
return ERR_PTR(-ENOMEM);
umem = &umem_odp->umem;
- umem->context = context;
+ umem->ibdev = context->device;
umem->writable = ib_access_writable(access);
umem->owning_mm = current->mm;
umem_odp->is_implicit_odp = 1;
umem_odp->page_shift = PAGE_SHIFT;
- ret = ib_init_umem_odp(umem_odp, NULL);
+ ret = ib_init_umem_odp(umem_odp);
if (ret) {
kfree(umem_odp);
return ERR_PTR(ret);
if (!odp_data)
return ERR_PTR(-ENOMEM);
umem = &odp_data->umem;
- umem->context = root->umem.context;
+ umem->ibdev = root->umem.ibdev;
umem->length = size;
umem->address = addr;
umem->writable = root->umem.writable;
umem->owning_mm = root->umem.owning_mm;
odp_data->page_shift = PAGE_SHIFT;
- ret = ib_init_umem_odp(odp_data, root->per_mm);
+ ret = ib_init_umem_odp(odp_data);
if (ret) {
kfree(odp_data);
return ERR_PTR(ret);
if (!umem_odp)
return ERR_PTR(-ENOMEM);
- umem_odp->umem.context = context;
+ umem_odp->umem.ibdev = context->device;
umem_odp->umem.length = size;
umem_odp->umem.address = addr;
umem_odp->umem.writable = ib_access_writable(access);
up_read(&mm->mmap_sem);
}
- ret = ib_init_umem_odp(umem_odp, NULL);
+ ret = ib_init_umem_odp(umem_odp);
if (ret)
goto err_free;
return umem_odp;
void ib_umem_odp_release(struct ib_umem_odp *umem_odp)
{
+ struct ib_ucontext_per_mm *per_mm = umem_odp->per_mm;
+
/*
* Ensure that no more pages are mapped in the umem.
*
if (!umem_odp->is_implicit_odp) {
ib_umem_odp_unmap_dma_pages(umem_odp, ib_umem_start(umem_odp),
ib_umem_end(umem_odp));
- remove_umem_from_per_mm(umem_odp);
kvfree(umem_odp->dma_list);
kvfree(umem_odp->page_list);
}
- put_per_mm(umem_odp);
+
+ down_write(&per_mm->umem_rwsem);
+ if (!umem_odp->is_implicit_odp) {
+ interval_tree_remove(&umem_odp->interval_tree,
+ &per_mm->umem_tree);
+ complete_all(&umem_odp->notifier_completion);
+ }
+ /*
+ * NOTE! mmu_notifier_unregister() can happen between a start/end
+ * callback, resulting in a missing end, and thus an unbalanced
+ * lock. This doesn't really matter to us since we are about to kfree
+ * the memory that holds the lock, however LOCKDEP doesn't like this.
+ * Thus we call the mmu_notifier_put under the rwsem and test the
+ * internal users count to reliably see if we are past this point.
+ */
+ mmu_notifier_put(&per_mm->mn);
+ up_write(&per_mm->umem_rwsem);
+
mmdrop(umem_odp->umem.owning_mm);
kfree(umem_odp);
}
u64 access_mask,
unsigned long current_seq)
{
- struct ib_ucontext *context = umem_odp->umem.context;
- struct ib_device *dev = context->device;
+ struct ib_device *dev = umem_odp->umem.ibdev;
dma_addr_t dma_addr;
int remove_existing_mapping = 0;
int ret = 0;
{
int idx;
u64 addr;
- struct ib_device *dev = umem_odp->umem.context->device;
+ struct ib_device *dev = umem_odp->umem.ibdev;
virt = max_t(u64, virt, ib_umem_start(umem_odp));
bound = min_t(u64, bound, ib_umem_end(umem_odp));
projects / linux-2.6-block.git / blobdiff