kprobes: kretprobe scalability improvement

[linux-2.6-block.git] / kernel / kprobes.c

diff --git a/kernel/kprobes.c b/kernel/kprobes.c
index 0c6185aefaef576c55fb324e6d30a8e2c0050d1f..075a632e6c7c3f93d686a7096244261b6cfa5570 100644 (file)
--- a/kernel/kprobes.c
+++ b/kernel/kprobes.c
@@ -1877,13 +1877,27 @@ static struct notifier_block kprobe_exceptions_nb = {
 #ifdef CONFIG_KRETPROBES
 
 #if !defined(CONFIG_KRETPROBE_ON_RETHOOK)
+
+/* callbacks for objpool of kretprobe instances */
+static int kretprobe_init_inst(void *nod, void *context)
+{
+       struct kretprobe_instance *ri = nod;
+
+       ri->rph = context;
+       return 0;
+}
+static int kretprobe_fini_pool(struct objpool_head *head, void *context)
+{
+       kfree(context);
+       return 0;
+}
+
 static void free_rp_inst_rcu(struct rcu_head *head)
 {
        struct kretprobe_instance *ri = container_of(head, struct kretprobe_instance, rcu);
+       struct kretprobe_holder *rph = ri->rph;
 
-       if (refcount_dec_and_test(&ri->rph->ref))
-               kfree(ri->rph);
-       kfree(ri);
+       objpool_drop(ri, &rph->pool);
 }
 NOKPROBE_SYMBOL(free_rp_inst_rcu);
 
@@ -1892,7 +1906,7 @@ static void recycle_rp_inst(struct kretprobe_instance *ri)
        struct kretprobe *rp = get_kretprobe(ri);
 
        if (likely(rp))
-               freelist_add(&ri->freelist, &rp->freelist);
+               objpool_push(ri, &rp->rph->pool);
        else
                call_rcu(&ri->rcu, free_rp_inst_rcu);
 }
@@ -1929,23 +1943,12 @@ NOKPROBE_SYMBOL(kprobe_flush_task);
 
 static inline void free_rp_inst(struct kretprobe *rp)
 {
-       struct kretprobe_instance *ri;
-       struct freelist_node *node;
-       int count = 0;
-
-       node = rp->freelist.head;
-       while (node) {
-               ri = container_of(node, struct kretprobe_instance, freelist);
-               node = node->next;
-
-               kfree(ri);
-               count++;
-       }
+       struct kretprobe_holder *rph = rp->rph;
 
-       if (refcount_sub_and_test(count, &rp->rph->ref)) {
-               kfree(rp->rph);
-               rp->rph = NULL;
-       }
+       if (!rph)
+               return;
+       rp->rph = NULL;
+       objpool_fini(&rph->pool);
 }
 
 /* This assumes the 'tsk' is the current task or the is not running. */
@@ -2087,19 +2090,17 @@ NOKPROBE_SYMBOL(__kretprobe_trampoline_handler)
 static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs)
 {
        struct kretprobe *rp = container_of(p, struct kretprobe, kp);
+       struct kretprobe_holder *rph = rp->rph;
        struct kretprobe_instance *ri;
-       struct freelist_node *fn;
 
-       fn = freelist_try_get(&rp->freelist);
-       if (!fn) {
+       ri = objpool_pop(&rph->pool);
+       if (!ri) {
                rp->nmissed++;
                return 0;
        }
 
-       ri = container_of(fn, struct kretprobe_instance, freelist);
-
        if (rp->entry_handler && rp->entry_handler(ri, regs)) {
-               freelist_add(&ri->freelist, &rp->freelist);
+               objpool_push(ri, &rph->pool);
                return 0;
        }
 
@@ -2193,7 +2194,6 @@ int kprobe_on_func_entry(kprobe_opcode_t *addr, const char *sym, unsigned long o
 int register_kretprobe(struct kretprobe *rp)
 {
        int ret;
-       struct kretprobe_instance *inst;
        int i;
        void *addr;
 
@@ -2227,19 +2227,12 @@ int register_kretprobe(struct kretprobe *rp)
                rp->maxactive = max_t(unsigned int, 10, 2*num_possible_cpus());
 
 #ifdef CONFIG_KRETPROBE_ON_RETHOOK
-       rp->rh = rethook_alloc((void *)rp, kretprobe_rethook_handler);
-       if (!rp->rh)
-               return -ENOMEM;
+       rp->rh = rethook_alloc((void *)rp, kretprobe_rethook_handler,
+                               sizeof(struct kretprobe_instance) +
+                               rp->data_size, rp->maxactive);
+       if (IS_ERR(rp->rh))
+               return PTR_ERR(rp->rh);
 
-       for (i = 0; i < rp->maxactive; i++) {
-               inst = kzalloc(struct_size(inst, data, rp->data_size), GFP_KERNEL);
-               if (inst == NULL) {
-                       rethook_free(rp->rh);
-                       rp->rh = NULL;
-                       return -ENOMEM;
-               }
-               rethook_add_node(rp->rh, &inst->node);
-       }
        rp->nmissed = 0;
        /* Establish function entry probe point */
        ret = register_kprobe(&rp->kp);
@@ -2248,24 +2241,18 @@ int register_kretprobe(struct kretprobe *rp)
                rp->rh = NULL;
        }
 #else  /* !CONFIG_KRETPROBE_ON_RETHOOK */
-       rp->freelist.head = NULL;
        rp->rph = kzalloc(sizeof(struct kretprobe_holder), GFP_KERNEL);
        if (!rp->rph)
                return -ENOMEM;
 
-       rp->rph->rp = rp;
-       for (i = 0; i < rp->maxactive; i++) {
-               inst = kzalloc(struct_size(inst, data, rp->data_size), GFP_KERNEL);
-               if (inst == NULL) {
-                       refcount_set(&rp->rph->ref, i);
-                       free_rp_inst(rp);
-                       return -ENOMEM;
-               }
-               inst->rph = rp->rph;
-               freelist_add(&inst->freelist, &rp->freelist);
+       if (objpool_init(&rp->rph->pool, rp->maxactive, rp->data_size +
+                       sizeof(struct kretprobe_instance), GFP_KERNEL,
+                       rp->rph, kretprobe_init_inst, kretprobe_fini_pool)) {
+               kfree(rp->rph);
+               rp->rph = NULL;
+               return -ENOMEM;
        }
-       refcount_set(&rp->rph->ref, i);
-
+       rp->rph->rp = rp;
        rp->nmissed = 0;
        /* Establish function entry probe point */
        ret = register_kprobe(&rp->kp);