ftrace: Use manual free after synchronize_sched() not call_rcu_sched()

[~andy/linux] / kernel / trace / ftrace.c

diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
index 98ca94a4181967800df7a2b858c594347617866f..25770824598f8fc5b498d2134715680a404b915c 100644 (file)
--- a/kernel/trace/ftrace.c
+++ b/kernel/trace/ftrace.c
@@ -1068,7 +1068,7 @@ struct ftrace_func_probe {
        unsigned long           flags;
        unsigned long           ip;
        void                    *data;
-       struct rcu_head         rcu;
+       struct list_head        free_list;
 };
 
 struct ftrace_func_entry {
@@ -2978,28 +2978,27 @@ static void __disable_ftrace_function_probe(void)
 }
 
 
-static void ftrace_free_entry_rcu(struct rcu_head *rhp)
+static void ftrace_free_entry(struct ftrace_func_probe *entry)
 {
-       struct ftrace_func_probe *entry =
-               container_of(rhp, struct ftrace_func_probe, rcu);
-
        if (entry->ops->free)
-               entry->ops->free(&entry->data);
+               entry->ops->free(entry->ops, entry->ip, &entry->data);
        kfree(entry);
 }
 
-
 int
 register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
                              void *data)
 {
        struct ftrace_func_probe *entry;
+       struct ftrace_hash **orig_hash = &trace_probe_ops.filter_hash;
+       struct ftrace_hash *hash;
        struct ftrace_page *pg;
        struct dyn_ftrace *rec;
        int type, len, not;
        unsigned long key;
        int count = 0;
        char *search;
+       int ret;
 
        type = filter_parse_regex(glob, strlen(glob), &search, &not);
        len = strlen(search);
@@ -3010,8 +3009,16 @@ register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
 
        mutex_lock(&ftrace_lock);
 
-       if (unlikely(ftrace_disabled))
+       hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);
+       if (!hash) {
+               count = -ENOMEM;
+               goto out_unlock;
+       }
+
+       if (unlikely(ftrace_disabled)) {
+               count = -ENODEV;
                goto out_unlock;
+       }
 
        do_for_each_ftrace_rec(pg, rec) {
 
@@ -3035,14 +3042,21 @@ register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
                 * for each function we find. We call the callback
                 * to give the caller an opportunity to do so.
                 */
-               if (ops->callback) {
-                       if (ops->callback(rec->ip, &entry->data) < 0) {
+               if (ops->init) {
+                       if (ops->init(ops, rec->ip, &entry->data) < 0) {
                                /* caller does not like this func */
                                kfree(entry);
                                continue;
                        }
                }
 
+               ret = enter_record(hash, rec, 0);
+               if (ret < 0) {
+                       kfree(entry);
+                       count = ret;
+                       goto out_unlock;
+               }
+
                entry->ops = ops;
                entry->ip = rec->ip;
 
@@ -3050,10 +3064,16 @@ register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
                hlist_add_head_rcu(&entry->node, &ftrace_func_hash[key]);
 
        } while_for_each_ftrace_rec();
+
+       ret = ftrace_hash_move(&trace_probe_ops, 1, orig_hash, hash);
+       if (ret < 0)
+               count = ret;
+
        __enable_ftrace_function_probe();
 
  out_unlock:
        mutex_unlock(&ftrace_lock);
+       free_ftrace_hash(hash);
 
        return count;
 }
@@ -3067,7 +3087,12 @@ static void
 __unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
                                  void *data, int flags)
 {
+       struct ftrace_func_entry *rec_entry;
        struct ftrace_func_probe *entry;
+       struct ftrace_func_probe *p;
+       struct ftrace_hash **orig_hash = &trace_probe_ops.filter_hash;
+       struct list_head free_list;
+       struct ftrace_hash *hash;
        struct hlist_node *n, *tmp;
        char str[KSYM_SYMBOL_LEN];
        int type = MATCH_FULL;
@@ -3088,6 +3113,14 @@ __unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
        }
 
        mutex_lock(&ftrace_lock);
+
+       hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);
+       if (!hash)
+               /* Hmm, should report this somehow */
+               goto out_unlock;
+
+       INIT_LIST_HEAD(&free_list);
+
        for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
                struct hlist_head *hhd = &ftrace_func_hash[i];
 
@@ -3108,12 +3141,30 @@ __unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
                                        continue;
                        }
 
-                       hlist_del(&entry->node);
-                       call_rcu(&entry->rcu, ftrace_free_entry_rcu);
+                       rec_entry = ftrace_lookup_ip(hash, entry->ip);
+                       /* It is possible more than one entry had this ip */
+                       if (rec_entry)
+                               free_hash_entry(hash, rec_entry);
+
+                       hlist_del_rcu(&entry->node);
+                       list_add(&entry->free_list, &free_list);
                }
        }
        __disable_ftrace_function_probe();
+       /*
+        * Remove after the disable is called. Otherwise, if the last
+        * probe is removed, a null hash means *all enabled*.
+        */
+       ftrace_hash_move(&trace_probe_ops, 1, orig_hash, hash);
+       synchronize_sched();
+       list_for_each_entry_safe(entry, p, &free_list, free_list) {
+               list_del(&entry->free_list);
+               ftrace_free_entry(entry);
+       }
+               
+ out_unlock:
        mutex_unlock(&ftrace_lock);
+       free_ftrace_hash(hash);
 }
 
 void