epoll: ep_unregister_pollwait() can use the freed pwq->whead

[linux-2.6-block.git] / fs / eventpoll.c

diff --git a/fs/eventpoll.c b/fs/eventpoll.c
index 34bbfc6dd8dceca3b174348bde94867dada0b981..ea54cdef04ddc94d282c22f8c2b721169ee77293 100644 (file)
--- a/fs/eventpoll.c
+++ b/fs/eventpoll.c
@@ -320,6 +320,11 @@ static inline int ep_is_linked(struct list_head *p)
        return !list_empty(p);
 }
 
+static inline struct eppoll_entry *ep_pwq_from_wait(wait_queue_t *p)
+{
+       return container_of(p, struct eppoll_entry, wait);
+}
+
 /* Get the "struct epitem" from a wait queue pointer */
 static inline struct epitem *ep_item_from_wait(wait_queue_t *p)
 {
@@ -467,6 +472,18 @@ static void ep_poll_safewake(wait_queue_head_t *wq)
        put_cpu();
 }
 
+static void ep_remove_wait_queue(struct eppoll_entry *pwq)
+{
+       wait_queue_head_t *whead;
+
+       rcu_read_lock();
+       /* If it is cleared by POLLFREE, it should be rcu-safe */
+       whead = rcu_dereference(pwq->whead);
+       if (whead)
+               remove_wait_queue(whead, &pwq->wait);
+       rcu_read_unlock();
+}
+
 /*
  * This function unregisters poll callbacks from the associated file
  * descriptor.  Must be called with "mtx" held (or "epmutex" if called from
@@ -481,7 +498,7 @@ static void ep_unregister_pollwait(struct eventpoll *ep, struct epitem *epi)
                pwq = list_first_entry(lsthead, struct eppoll_entry, llink);
 
                list_del(&pwq->llink);
-               remove_wait_queue(pwq->whead, &pwq->wait);
+               ep_remove_wait_queue(pwq);
                kmem_cache_free(pwq_cache, pwq);
        }
 }
@@ -842,9 +859,16 @@ static int ep_poll_callback(wait_queue_t *wait, unsigned mode, int sync, void *k
        struct epitem *epi = ep_item_from_wait(wait);
        struct eventpoll *ep = epi->ep;
 
-       /* the caller holds eppoll_entry->whead->lock */
-       if ((unsigned long)key & POLLFREE)
+       if ((unsigned long)key & POLLFREE) {
+               ep_pwq_from_wait(wait)->whead = NULL;
+               /*
+                * whead = NULL above can race with ep_remove_wait_queue()
+                * which can do another remove_wait_queue() after us, so we
+                * can't use __remove_wait_queue(). whead->lock is held by
+                * the caller.
+                */
                list_del_init(&wait->task_list);
+       }
 
        spin_lock_irqsave(&ep->lock, flags);