return buffer;
}
- static int binder_update_page_range(struct binder_alloc *alloc, int allocate,
- void __user *start, void __user *end)
+ static inline void
+ binder_set_installed_page(struct binder_lru_page *lru_page,
+ struct page *page)
+ {
+ /* Pairs with acquire in binder_get_installed_page() */
+ smp_store_release(&lru_page->page_ptr, page);
+ }
+
+ static inline struct page *
+ binder_get_installed_page(struct binder_lru_page *lru_page)
+ {
+ /* Pairs with release in binder_set_installed_page() */
+ return smp_load_acquire(&lru_page->page_ptr);
+ }
+
+ static void binder_lru_freelist_add(struct binder_alloc *alloc,
+ unsigned long start, unsigned long end)
{
- void __user *page_addr;
- unsigned long user_page_addr;
struct binder_lru_page *page;
- struct vm_area_struct *vma = NULL;
- struct mm_struct *mm = NULL;
- bool need_mm = false;
+ unsigned long page_addr;
- binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC,
- "%d: %s pages %pK-%pK\n", alloc->pid,
- allocate ? "allocate" : "free", start, end);
+ trace_binder_update_page_range(alloc, false, start, end);
- if (end <= start)
- return 0;
+ for (page_addr = start; page_addr < end; page_addr += PAGE_SIZE) {
+ size_t index;
+ int ret;
- trace_binder_update_page_range(alloc, allocate, start, end);
+ index = (page_addr - alloc->buffer) / PAGE_SIZE;
+ page = &alloc->pages[index];
- if (allocate == 0)
- goto free_range;
+ if (!binder_get_installed_page(page))
+ continue;
- for (page_addr = start; page_addr < end; page_addr += PAGE_SIZE) {
- page = &alloc->pages[(page_addr - alloc->buffer) / PAGE_SIZE];
- if (!page->page_ptr) {
- need_mm = true;
- break;
- }
+ trace_binder_free_lru_start(alloc, index);
+
- ret = list_lru_add(&binder_freelist, &page->lru);
++ ret = list_lru_add_obj(&binder_freelist, &page->lru);
+ WARN_ON(!ret);
+
+ trace_binder_free_lru_end(alloc, index);
}
+ }
+
+ static int binder_install_single_page(struct binder_alloc *alloc,
+ struct binder_lru_page *lru_page,
+ unsigned long addr)
+ {
+ struct page *page;
+ int ret = 0;
- if (need_mm && mmget_not_zero(alloc->mm))
- mm = alloc->mm;
+ if (!mmget_not_zero(alloc->mm))
+ return -ESRCH;
- if (mm) {
- mmap_write_lock(mm);
- vma = alloc->vma;
+ /*
+ * Protected with mmap_sem in write mode as multiple tasks
+ * might race to install the same page.
+ */
+ mmap_write_lock(alloc->mm);
+ if (binder_get_installed_page(lru_page))
+ goto out;
+
+ if (!alloc->vma) {
+ pr_err("%d: %s failed, no vma\n", alloc->pid, __func__);
+ ret = -ESRCH;
+ goto out;
}
- if (!vma && need_mm) {
- binder_alloc_debug(BINDER_DEBUG_USER_ERROR,
- "%d: binder_alloc_buf failed to map pages in userspace, no vma\n",
- alloc->pid);
- goto err_no_vma;
+ page = alloc_page(GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO);
+ if (!page) {
+ pr_err("%d: failed to allocate page\n", alloc->pid);
+ ret = -ENOMEM;
+ goto out;
}
- for (page_addr = start; page_addr < end; page_addr += PAGE_SIZE) {
+ ret = vm_insert_page(alloc->vma, addr, page);
+ if (ret) {
+ pr_err("%d: %s failed to insert page at offset %lx with %d\n",
+ alloc->pid, __func__, addr - alloc->buffer, ret);
+ __free_page(page);
+ ret = -ENOMEM;
+ goto out;
+ }
+
+ /* Mark page installation complete and safe to use */
+ binder_set_installed_page(lru_page, page);
+ out:
+ mmap_write_unlock(alloc->mm);
+ mmput_async(alloc->mm);
+ return ret;
+ }
+
+ static int binder_install_buffer_pages(struct binder_alloc *alloc,
+ struct binder_buffer *buffer,
+ size_t size)
+ {
+ struct binder_lru_page *page;
+ unsigned long start, final;
+ unsigned long page_addr;
+
+ start = buffer->user_data & PAGE_MASK;
+ final = PAGE_ALIGN(buffer->user_data + size);
+
+ for (page_addr = start; page_addr < final; page_addr += PAGE_SIZE) {
+ unsigned long index;
int ret;
- bool on_lru;
- size_t index;
index = (page_addr - alloc->buffer) / PAGE_SIZE;
page = &alloc->pages[index];
index = (page_addr - alloc->buffer) / PAGE_SIZE;
page = &alloc->pages[index];
- trace_binder_free_lru_start(alloc, index);
+ if (page->page_ptr) {
+ trace_binder_alloc_lru_start(alloc, index);
- ret = list_lru_add_obj(&binder_alloc_lru, &page->lru);
- WARN_ON(!ret);
- on_lru = list_lru_del(&binder_freelist, &page->lru);
++ on_lru = list_lru_del_obj(&binder_freelist, &page->lru);
+ WARN_ON(!on_lru);
- trace_binder_free_lru_end(alloc, index);
- if (page_addr == start)
- break;
- continue;
-
- err_vm_insert_page_failed:
- __free_page(page->page_ptr);
- page->page_ptr = NULL;
- err_alloc_page_failed:
- err_page_ptr_cleared:
- if (page_addr == start)
- break;
- }
- err_no_vma:
- if (mm) {
- mmap_write_unlock(mm);
- mmput(mm);
+ trace_binder_alloc_lru_end(alloc, index);
+ continue;
+ }
+
+ if (index + 1 > alloc->pages_high)
+ alloc->pages_high = index + 1;
}
- return vma ? -ENOMEM : -ESRCH;
}
static inline void binder_alloc_set_vma(struct binder_alloc *alloc,
if (!alloc->pages[i].page_ptr)
continue;
- on_lru = list_lru_del_obj(&binder_alloc_lru,
- on_lru = list_lru_del(&binder_freelist,
-- &alloc->pages[i].lru);
++ on_lru = list_lru_del_obj(&binder_freelist,
++ &alloc->pages[i].lru);
page_addr = alloc->buffer + i * PAGE_SIZE;
binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC,
- "%s: %d: page %d at %pK %s\n",
- __func__, alloc->pid, i, page_addr,
+ "%s: %d: page %d %s\n",
+ __func__, alloc->pid, i,
on_lru ? "on lru" : "active");
__free_page(alloc->pages[i].page_ptr);
page_count++;