2 * Kprobes-based tracing events
4 * Created by Masami Hiramatsu <mhiramat@redhat.com>
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20 #include <linux/module.h>
21 #include <linux/uaccess.h>
23 #include "trace_probe.h"
25 #define KPROBE_EVENT_SYSTEM "kprobes"
28 * Kprobe event core functions
31 struct list_head list;
32 struct kretprobe rp; /* Use rp.kp for kprobe use */
34 const char *symbol; /* symbol name */
35 struct trace_probe tp;
38 struct event_file_link {
39 struct ftrace_event_file *file;
40 struct list_head list;
43 #define SIZEOF_TRACE_KPROBE(n) \
44 (offsetof(struct trace_kprobe, tp.args) + \
45 (sizeof(struct probe_arg) * (n)))
48 static __kprobes bool trace_kprobe_is_return(struct trace_kprobe *tk)
50 return tk->rp.handler != NULL;
53 static __kprobes const char *trace_kprobe_symbol(struct trace_kprobe *tk)
55 return tk->symbol ? tk->symbol : "unknown";
58 static __kprobes unsigned long trace_kprobe_offset(struct trace_kprobe *tk)
60 return tk->rp.kp.offset;
63 static __kprobes bool trace_kprobe_has_gone(struct trace_kprobe *tk)
65 return !!(kprobe_gone(&tk->rp.kp));
68 static __kprobes bool trace_kprobe_within_module(struct trace_kprobe *tk,
71 int len = strlen(mod->name);
72 const char *name = trace_kprobe_symbol(tk);
73 return strncmp(mod->name, name, len) == 0 && name[len] == ':';
76 static __kprobes bool trace_kprobe_is_on_module(struct trace_kprobe *tk)
78 return !!strchr(trace_kprobe_symbol(tk), ':');
81 static int register_kprobe_event(struct trace_kprobe *tk);
82 static int unregister_kprobe_event(struct trace_kprobe *tk);
84 static DEFINE_MUTEX(probe_lock);
85 static LIST_HEAD(probe_list);
87 static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs);
88 static int kretprobe_dispatcher(struct kretprobe_instance *ri,
89 struct pt_regs *regs);
92 * Allocate new trace_probe and initialize it (including kprobes).
94 static struct trace_kprobe *alloc_trace_kprobe(const char *group,
99 int nargs, bool is_return)
101 struct trace_kprobe *tk;
104 tk = kzalloc(SIZEOF_TRACE_KPROBE(nargs), GFP_KERNEL);
109 tk->symbol = kstrdup(symbol, GFP_KERNEL);
112 tk->rp.kp.symbol_name = tk->symbol;
113 tk->rp.kp.offset = offs;
115 tk->rp.kp.addr = addr;
118 tk->rp.handler = kretprobe_dispatcher;
120 tk->rp.kp.pre_handler = kprobe_dispatcher;
122 if (!event || !is_good_name(event)) {
127 tk->tp.call.class = &tk->tp.class;
128 tk->tp.call.name = kstrdup(event, GFP_KERNEL);
129 if (!tk->tp.call.name)
132 if (!group || !is_good_name(group)) {
137 tk->tp.class.system = kstrdup(group, GFP_KERNEL);
138 if (!tk->tp.class.system)
141 INIT_LIST_HEAD(&tk->list);
142 INIT_LIST_HEAD(&tk->tp.files);
145 kfree(tk->tp.call.name);
151 static void free_trace_kprobe(struct trace_kprobe *tk)
155 for (i = 0; i < tk->tp.nr_args; i++)
156 traceprobe_free_probe_arg(&tk->tp.args[i]);
158 kfree(tk->tp.call.class->system);
159 kfree(tk->tp.call.name);
164 static struct trace_kprobe *find_trace_kprobe(const char *event,
167 struct trace_kprobe *tk;
169 list_for_each_entry(tk, &probe_list, list)
170 if (strcmp(tk->tp.call.name, event) == 0 &&
171 strcmp(tk->tp.call.class->system, group) == 0)
178 * if the file is NULL, enable "perf" handler, or enable "trace" handler.
181 enable_trace_kprobe(struct trace_kprobe *tk, struct ftrace_event_file *file)
186 struct event_file_link *link;
188 link = kmalloc(sizeof(*link), GFP_KERNEL);
195 list_add_tail_rcu(&link->list, &tk->tp.files);
197 tk->tp.flags |= TP_FLAG_TRACE;
199 tk->tp.flags |= TP_FLAG_PROFILE;
201 if (trace_probe_is_registered(&tk->tp) && !trace_kprobe_has_gone(tk)) {
202 if (trace_kprobe_is_return(tk))
203 ret = enable_kretprobe(&tk->rp);
205 ret = enable_kprobe(&tk->rp.kp);
211 static struct event_file_link *
212 find_event_file_link(struct trace_probe *tp, struct ftrace_event_file *file)
214 struct event_file_link *link;
216 list_for_each_entry(link, &tp->files, list)
217 if (link->file == file)
224 * Disable trace_probe
225 * if the file is NULL, disable "perf" handler, or disable "trace" handler.
228 disable_trace_kprobe(struct trace_kprobe *tk, struct ftrace_event_file *file)
230 struct event_file_link *link = NULL;
235 link = find_event_file_link(&tk->tp, file);
241 list_del_rcu(&link->list);
243 if (!list_empty(&tk->tp.files))
246 tk->tp.flags &= ~TP_FLAG_TRACE;
248 tk->tp.flags &= ~TP_FLAG_PROFILE;
250 if (!trace_probe_is_enabled(&tk->tp) && trace_probe_is_registered(&tk->tp)) {
251 if (trace_kprobe_is_return(tk))
252 disable_kretprobe(&tk->rp);
254 disable_kprobe(&tk->rp.kp);
260 * Synchronize with kprobe_trace_func/kretprobe_trace_func
261 * to ensure disabled (all running handlers are finished).
262 * This is not only for kfree(), but also the caller,
263 * trace_remove_event_call() supposes it for releasing
264 * event_call related objects, which will be accessed in
265 * the kprobe_trace_func/kretprobe_trace_func.
268 kfree(link); /* Ignored if link == NULL */
274 /* Internal register function - just handle k*probes and flags */
275 static int __register_trace_kprobe(struct trace_kprobe *tk)
279 if (trace_probe_is_registered(&tk->tp))
282 for (i = 0; i < tk->tp.nr_args; i++)
283 traceprobe_update_arg(&tk->tp.args[i]);
285 /* Set/clear disabled flag according to tp->flag */
286 if (trace_probe_is_enabled(&tk->tp))
287 tk->rp.kp.flags &= ~KPROBE_FLAG_DISABLED;
289 tk->rp.kp.flags |= KPROBE_FLAG_DISABLED;
291 if (trace_kprobe_is_return(tk))
292 ret = register_kretprobe(&tk->rp);
294 ret = register_kprobe(&tk->rp.kp);
297 tk->tp.flags |= TP_FLAG_REGISTERED;
299 pr_warning("Could not insert probe at %s+%lu: %d\n",
300 trace_kprobe_symbol(tk), trace_kprobe_offset(tk), ret);
301 if (ret == -ENOENT && trace_kprobe_is_on_module(tk)) {
302 pr_warning("This probe might be able to register after"
303 "target module is loaded. Continue.\n");
305 } else if (ret == -EILSEQ) {
306 pr_warning("Probing address(0x%p) is not an "
307 "instruction boundary.\n",
316 /* Internal unregister function - just handle k*probes and flags */
317 static void __unregister_trace_kprobe(struct trace_kprobe *tk)
319 if (trace_probe_is_registered(&tk->tp)) {
320 if (trace_kprobe_is_return(tk))
321 unregister_kretprobe(&tk->rp);
323 unregister_kprobe(&tk->rp.kp);
324 tk->tp.flags &= ~TP_FLAG_REGISTERED;
325 /* Cleanup kprobe for reuse */
326 if (tk->rp.kp.symbol_name)
327 tk->rp.kp.addr = NULL;
331 /* Unregister a trace_probe and probe_event: call with locking probe_lock */
332 static int unregister_trace_kprobe(struct trace_kprobe *tk)
334 /* Enabled event can not be unregistered */
335 if (trace_probe_is_enabled(&tk->tp))
338 /* Will fail if probe is being used by ftrace or perf */
339 if (unregister_kprobe_event(tk))
342 __unregister_trace_kprobe(tk);
348 /* Register a trace_probe and probe_event */
349 static int register_trace_kprobe(struct trace_kprobe *tk)
351 struct trace_kprobe *old_tk;
354 mutex_lock(&probe_lock);
356 /* Delete old (same name) event if exist */
357 old_tk = find_trace_kprobe(tk->tp.call.name, tk->tp.call.class->system);
359 ret = unregister_trace_kprobe(old_tk);
362 free_trace_kprobe(old_tk);
365 /* Register new event */
366 ret = register_kprobe_event(tk);
368 pr_warning("Failed to register probe event(%d)\n", ret);
372 /* Register k*probe */
373 ret = __register_trace_kprobe(tk);
375 unregister_kprobe_event(tk);
377 list_add_tail(&tk->list, &probe_list);
380 mutex_unlock(&probe_lock);
384 /* Module notifier call back, checking event on the module */
385 static int trace_kprobe_module_callback(struct notifier_block *nb,
386 unsigned long val, void *data)
388 struct module *mod = data;
389 struct trace_kprobe *tk;
392 if (val != MODULE_STATE_COMING)
395 /* Update probes on coming module */
396 mutex_lock(&probe_lock);
397 list_for_each_entry(tk, &probe_list, list) {
398 if (trace_kprobe_within_module(tk, mod)) {
399 /* Don't need to check busy - this should have gone. */
400 __unregister_trace_kprobe(tk);
401 ret = __register_trace_kprobe(tk);
403 pr_warning("Failed to re-register probe %s on"
405 tk->tp.call.name, mod->name, ret);
408 mutex_unlock(&probe_lock);
413 static struct notifier_block trace_kprobe_module_nb = {
414 .notifier_call = trace_kprobe_module_callback,
415 .priority = 1 /* Invoked after kprobe module callback */
418 static int create_trace_kprobe(int argc, char **argv)
422 * - Add kprobe: p[:[GRP/]EVENT] [MOD:]KSYM[+OFFS]|KADDR [FETCHARGS]
423 * - Add kretprobe: r[:[GRP/]EVENT] [MOD:]KSYM[+0] [FETCHARGS]
425 * $retval : fetch return value
426 * $stack : fetch stack address
427 * $stackN : fetch Nth of stack (N:0-)
428 * @ADDR : fetch memory at ADDR (ADDR should be in kernel)
429 * @SYM[+|-offs] : fetch memory at SYM +|- offs (SYM is a data symbol)
430 * %REG : fetch register REG
431 * Dereferencing memory fetch:
432 * +|-offs(ARG) : fetch memory at ARG +|- offs address.
433 * Alias name of args:
434 * NAME=FETCHARG : set NAME as alias of FETCHARG.
436 * FETCHARG:TYPE : use TYPE instead of unsigned long.
438 struct trace_kprobe *tk;
440 bool is_return = false, is_delete = false;
441 char *symbol = NULL, *event = NULL, *group = NULL;
443 unsigned long offset = 0;
445 char buf[MAX_EVENT_NAME_LEN];
447 /* argc must be >= 1 */
448 if (argv[0][0] == 'p')
450 else if (argv[0][0] == 'r')
452 else if (argv[0][0] == '-')
455 pr_info("Probe definition must be started with 'p', 'r' or"
460 if (argv[0][1] == ':') {
462 if (strchr(event, '/')) {
464 event = strchr(group, '/') + 1;
466 if (strlen(group) == 0) {
467 pr_info("Group name is not specified\n");
471 if (strlen(event) == 0) {
472 pr_info("Event name is not specified\n");
477 group = KPROBE_EVENT_SYSTEM;
481 pr_info("Delete command needs an event name.\n");
484 mutex_lock(&probe_lock);
485 tk = find_trace_kprobe(event, group);
487 mutex_unlock(&probe_lock);
488 pr_info("Event %s/%s doesn't exist.\n", group, event);
491 /* delete an event */
492 ret = unregister_trace_kprobe(tk);
494 free_trace_kprobe(tk);
495 mutex_unlock(&probe_lock);
500 pr_info("Probe point is not specified.\n");
503 if (isdigit(argv[1][0])) {
505 pr_info("Return probe point must be a symbol.\n");
508 /* an address specified */
509 ret = kstrtoul(&argv[1][0], 0, (unsigned long *)&addr);
511 pr_info("Failed to parse address.\n");
515 /* a symbol specified */
517 /* TODO: support .init module functions */
518 ret = traceprobe_split_symbol_offset(symbol, &offset);
520 pr_info("Failed to parse symbol.\n");
523 if (offset && is_return) {
524 pr_info("Return probe must be used without offset.\n");
528 argc -= 2; argv += 2;
532 /* Make a new event name */
534 snprintf(buf, MAX_EVENT_NAME_LEN, "%c_%s_%ld",
535 is_return ? 'r' : 'p', symbol, offset);
537 snprintf(buf, MAX_EVENT_NAME_LEN, "%c_0x%p",
538 is_return ? 'r' : 'p', addr);
541 tk = alloc_trace_kprobe(group, event, addr, symbol, offset, argc,
544 pr_info("Failed to allocate trace_probe.(%d)\n",
549 /* parse arguments */
551 for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) {
552 struct probe_arg *parg = &tk->tp.args[i];
554 /* Increment count for freeing args in error case */
557 /* Parse argument name */
558 arg = strchr(argv[i], '=');
561 parg->name = kstrdup(argv[i], GFP_KERNEL);
564 /* If argument name is omitted, set "argN" */
565 snprintf(buf, MAX_EVENT_NAME_LEN, "arg%d", i + 1);
566 parg->name = kstrdup(buf, GFP_KERNEL);
570 pr_info("Failed to allocate argument[%d] name.\n", i);
575 if (!is_good_name(parg->name)) {
576 pr_info("Invalid argument[%d] name: %s\n",
582 if (traceprobe_conflict_field_name(parg->name,
584 pr_info("Argument[%d] name '%s' conflicts with "
585 "another field.\n", i, argv[i]);
590 /* Parse fetch argument */
591 ret = traceprobe_parse_probe_arg(arg, &tk->tp.size, parg,
594 pr_info("Parse error at argument[%d]. (%d)\n", i, ret);
599 ret = register_trace_kprobe(tk);
605 free_trace_kprobe(tk);
609 static int release_all_trace_kprobes(void)
611 struct trace_kprobe *tk;
614 mutex_lock(&probe_lock);
615 /* Ensure no probe is in use. */
616 list_for_each_entry(tk, &probe_list, list)
617 if (trace_probe_is_enabled(&tk->tp)) {
621 /* TODO: Use batch unregistration */
622 while (!list_empty(&probe_list)) {
623 tk = list_entry(probe_list.next, struct trace_kprobe, list);
624 ret = unregister_trace_kprobe(tk);
627 free_trace_kprobe(tk);
631 mutex_unlock(&probe_lock);
636 /* Probes listing interfaces */
637 static void *probes_seq_start(struct seq_file *m, loff_t *pos)
639 mutex_lock(&probe_lock);
640 return seq_list_start(&probe_list, *pos);
643 static void *probes_seq_next(struct seq_file *m, void *v, loff_t *pos)
645 return seq_list_next(v, &probe_list, pos);
648 static void probes_seq_stop(struct seq_file *m, void *v)
650 mutex_unlock(&probe_lock);
653 static int probes_seq_show(struct seq_file *m, void *v)
655 struct trace_kprobe *tk = v;
658 seq_printf(m, "%c", trace_kprobe_is_return(tk) ? 'r' : 'p');
659 seq_printf(m, ":%s/%s", tk->tp.call.class->system, tk->tp.call.name);
662 seq_printf(m, " 0x%p", tk->rp.kp.addr);
663 else if (tk->rp.kp.offset)
664 seq_printf(m, " %s+%u", trace_kprobe_symbol(tk),
667 seq_printf(m, " %s", trace_kprobe_symbol(tk));
669 for (i = 0; i < tk->tp.nr_args; i++)
670 seq_printf(m, " %s=%s", tk->tp.args[i].name, tk->tp.args[i].comm);
676 static const struct seq_operations probes_seq_op = {
677 .start = probes_seq_start,
678 .next = probes_seq_next,
679 .stop = probes_seq_stop,
680 .show = probes_seq_show
683 static int probes_open(struct inode *inode, struct file *file)
687 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
688 ret = release_all_trace_kprobes();
693 return seq_open(file, &probes_seq_op);
696 static ssize_t probes_write(struct file *file, const char __user *buffer,
697 size_t count, loff_t *ppos)
699 return traceprobe_probes_write(file, buffer, count, ppos,
700 create_trace_kprobe);
703 static const struct file_operations kprobe_events_ops = {
704 .owner = THIS_MODULE,
708 .release = seq_release,
709 .write = probes_write,
712 /* Probes profiling interfaces */
713 static int probes_profile_seq_show(struct seq_file *m, void *v)
715 struct trace_kprobe *tk = v;
717 seq_printf(m, " %-44s %15lu %15lu\n", tk->tp.call.name, tk->nhit,
723 static const struct seq_operations profile_seq_op = {
724 .start = probes_seq_start,
725 .next = probes_seq_next,
726 .stop = probes_seq_stop,
727 .show = probes_profile_seq_show
730 static int profile_open(struct inode *inode, struct file *file)
732 return seq_open(file, &profile_seq_op);
735 static const struct file_operations kprobe_profile_ops = {
736 .owner = THIS_MODULE,
737 .open = profile_open,
740 .release = seq_release,
743 /* Sum up total data length for dynamic arraies (strings) */
744 static __kprobes int __get_data_size(struct trace_probe *tp,
745 struct pt_regs *regs)
750 for (i = 0; i < tp->nr_args; i++)
751 if (unlikely(tp->args[i].fetch_size.fn)) {
752 call_fetch(&tp->args[i].fetch_size, regs, &len);
759 /* Store the value of each argument */
760 static __kprobes void store_trace_args(int ent_size, struct trace_probe *tp,
761 struct pt_regs *regs,
762 u8 *data, int maxlen)
766 u32 *dl; /* Data (relative) location */
768 for (i = 0; i < tp->nr_args; i++) {
769 if (unlikely(tp->args[i].fetch_size.fn)) {
771 * First, we set the relative location and
772 * maximum data length to *dl
774 dl = (u32 *)(data + tp->args[i].offset);
775 *dl = make_data_rloc(maxlen, end - tp->args[i].offset);
776 /* Then try to fetch string or dynamic array data */
777 call_fetch(&tp->args[i].fetch, regs, dl);
778 /* Reduce maximum length */
779 end += get_rloc_len(*dl);
780 maxlen -= get_rloc_len(*dl);
781 /* Trick here, convert data_rloc to data_loc */
782 *dl = convert_rloc_to_loc(*dl,
783 ent_size + tp->args[i].offset);
785 /* Just fetching data normally */
786 call_fetch(&tp->args[i].fetch, regs,
787 data + tp->args[i].offset);
792 static __kprobes void
793 __kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs,
794 struct ftrace_event_file *ftrace_file)
796 struct kprobe_trace_entry_head *entry;
797 struct ring_buffer_event *event;
798 struct ring_buffer *buffer;
800 unsigned long irq_flags;
801 struct ftrace_event_call *call = &tk->tp.call;
803 WARN_ON(call != ftrace_file->event_call);
805 if (test_bit(FTRACE_EVENT_FL_SOFT_DISABLED_BIT, &ftrace_file->flags))
808 local_save_flags(irq_flags);
809 pc = preempt_count();
811 dsize = __get_data_size(&tk->tp, regs);
812 size = sizeof(*entry) + tk->tp.size + dsize;
814 event = trace_event_buffer_lock_reserve(&buffer, ftrace_file,
816 size, irq_flags, pc);
820 entry = ring_buffer_event_data(event);
821 entry->ip = (unsigned long)tk->rp.kp.addr;
822 store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize);
824 if (!filter_check_discard(ftrace_file, entry, buffer, event))
825 trace_buffer_unlock_commit_regs(buffer, event,
826 irq_flags, pc, regs);
829 static __kprobes void
830 kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs)
832 struct event_file_link *link;
834 list_for_each_entry_rcu(link, &tk->tp.files, list)
835 __kprobe_trace_func(tk, regs, link->file);
838 /* Kretprobe handler */
839 static __kprobes void
840 __kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
841 struct pt_regs *regs,
842 struct ftrace_event_file *ftrace_file)
844 struct kretprobe_trace_entry_head *entry;
845 struct ring_buffer_event *event;
846 struct ring_buffer *buffer;
848 unsigned long irq_flags;
849 struct ftrace_event_call *call = &tk->tp.call;
851 WARN_ON(call != ftrace_file->event_call);
853 if (test_bit(FTRACE_EVENT_FL_SOFT_DISABLED_BIT, &ftrace_file->flags))
856 local_save_flags(irq_flags);
857 pc = preempt_count();
859 dsize = __get_data_size(&tk->tp, regs);
860 size = sizeof(*entry) + tk->tp.size + dsize;
862 event = trace_event_buffer_lock_reserve(&buffer, ftrace_file,
864 size, irq_flags, pc);
868 entry = ring_buffer_event_data(event);
869 entry->func = (unsigned long)tk->rp.kp.addr;
870 entry->ret_ip = (unsigned long)ri->ret_addr;
871 store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize);
873 if (!filter_check_discard(ftrace_file, entry, buffer, event))
874 trace_buffer_unlock_commit_regs(buffer, event,
875 irq_flags, pc, regs);
878 static __kprobes void
879 kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
880 struct pt_regs *regs)
882 struct event_file_link *link;
884 list_for_each_entry_rcu(link, &tk->tp.files, list)
885 __kretprobe_trace_func(tk, ri, regs, link->file);
888 /* Event entry printers */
889 static enum print_line_t
890 print_kprobe_event(struct trace_iterator *iter, int flags,
891 struct trace_event *event)
893 struct kprobe_trace_entry_head *field;
894 struct trace_seq *s = &iter->seq;
895 struct trace_probe *tp;
899 field = (struct kprobe_trace_entry_head *)iter->ent;
900 tp = container_of(event, struct trace_probe, call.event);
902 if (!trace_seq_printf(s, "%s: (", tp->call.name))
905 if (!seq_print_ip_sym(s, field->ip, flags | TRACE_ITER_SYM_OFFSET))
908 if (!trace_seq_puts(s, ")"))
911 data = (u8 *)&field[1];
912 for (i = 0; i < tp->nr_args; i++)
913 if (!tp->args[i].type->print(s, tp->args[i].name,
914 data + tp->args[i].offset, field))
917 if (!trace_seq_puts(s, "\n"))
920 return TRACE_TYPE_HANDLED;
922 return TRACE_TYPE_PARTIAL_LINE;
925 static enum print_line_t
926 print_kretprobe_event(struct trace_iterator *iter, int flags,
927 struct trace_event *event)
929 struct kretprobe_trace_entry_head *field;
930 struct trace_seq *s = &iter->seq;
931 struct trace_probe *tp;
935 field = (struct kretprobe_trace_entry_head *)iter->ent;
936 tp = container_of(event, struct trace_probe, call.event);
938 if (!trace_seq_printf(s, "%s: (", tp->call.name))
941 if (!seq_print_ip_sym(s, field->ret_ip, flags | TRACE_ITER_SYM_OFFSET))
944 if (!trace_seq_puts(s, " <- "))
947 if (!seq_print_ip_sym(s, field->func, flags & ~TRACE_ITER_SYM_OFFSET))
950 if (!trace_seq_puts(s, ")"))
953 data = (u8 *)&field[1];
954 for (i = 0; i < tp->nr_args; i++)
955 if (!tp->args[i].type->print(s, tp->args[i].name,
956 data + tp->args[i].offset, field))
959 if (!trace_seq_puts(s, "\n"))
962 return TRACE_TYPE_HANDLED;
964 return TRACE_TYPE_PARTIAL_LINE;
968 static int kprobe_event_define_fields(struct ftrace_event_call *event_call)
971 struct kprobe_trace_entry_head field;
972 struct trace_kprobe *tk = (struct trace_kprobe *)event_call->data;
974 DEFINE_FIELD(unsigned long, ip, FIELD_STRING_IP, 0);
975 /* Set argument names as fields */
976 for (i = 0; i < tk->tp.nr_args; i++) {
977 struct probe_arg *parg = &tk->tp.args[i];
979 ret = trace_define_field(event_call, parg->type->fmttype,
981 sizeof(field) + parg->offset,
983 parg->type->is_signed,
991 static int kretprobe_event_define_fields(struct ftrace_event_call *event_call)
994 struct kretprobe_trace_entry_head field;
995 struct trace_kprobe *tk = (struct trace_kprobe *)event_call->data;
997 DEFINE_FIELD(unsigned long, func, FIELD_STRING_FUNC, 0);
998 DEFINE_FIELD(unsigned long, ret_ip, FIELD_STRING_RETIP, 0);
999 /* Set argument names as fields */
1000 for (i = 0; i < tk->tp.nr_args; i++) {
1001 struct probe_arg *parg = &tk->tp.args[i];
1003 ret = trace_define_field(event_call, parg->type->fmttype,
1005 sizeof(field) + parg->offset,
1007 parg->type->is_signed,
1015 static int __set_print_fmt(struct trace_kprobe *tk, char *buf, int len)
1020 const char *fmt, *arg;
1022 if (!trace_kprobe_is_return(tk)) {
1024 arg = "REC->" FIELD_STRING_IP;
1026 fmt = "(%lx <- %lx)";
1027 arg = "REC->" FIELD_STRING_FUNC ", REC->" FIELD_STRING_RETIP;
1030 /* When len=0, we just calculate the needed length */
1031 #define LEN_OR_ZERO (len ? len - pos : 0)
1033 pos += snprintf(buf + pos, LEN_OR_ZERO, "\"%s", fmt);
1035 for (i = 0; i < tk->tp.nr_args; i++) {
1036 pos += snprintf(buf + pos, LEN_OR_ZERO, " %s=%s",
1037 tk->tp.args[i].name, tk->tp.args[i].type->fmt);
1040 pos += snprintf(buf + pos, LEN_OR_ZERO, "\", %s", arg);
1042 for (i = 0; i < tk->tp.nr_args; i++) {
1043 if (strcmp(tk->tp.args[i].type->name, "string") == 0)
1044 pos += snprintf(buf + pos, LEN_OR_ZERO,
1046 tk->tp.args[i].name);
1048 pos += snprintf(buf + pos, LEN_OR_ZERO, ", REC->%s",
1049 tk->tp.args[i].name);
1054 /* return the length of print_fmt */
1058 static int set_print_fmt(struct trace_kprobe *tk)
1063 /* First: called with 0 length to calculate the needed length */
1064 len = __set_print_fmt(tk, NULL, 0);
1065 print_fmt = kmalloc(len + 1, GFP_KERNEL);
1069 /* Second: actually write the @print_fmt */
1070 __set_print_fmt(tk, print_fmt, len + 1);
1071 tk->tp.call.print_fmt = print_fmt;
1076 #ifdef CONFIG_PERF_EVENTS
1078 /* Kprobe profile handler */
1079 static __kprobes void
1080 kprobe_perf_func(struct trace_kprobe *tk, struct pt_regs *regs)
1082 struct ftrace_event_call *call = &tk->tp.call;
1083 struct kprobe_trace_entry_head *entry;
1084 struct hlist_head *head;
1085 int size, __size, dsize;
1088 head = this_cpu_ptr(call->perf_events);
1089 if (hlist_empty(head))
1092 dsize = __get_data_size(&tk->tp, regs);
1093 __size = sizeof(*entry) + tk->tp.size + dsize;
1094 size = ALIGN(__size + sizeof(u32), sizeof(u64));
1095 size -= sizeof(u32);
1097 entry = perf_trace_buf_prepare(size, call->event.type, regs, &rctx);
1101 entry->ip = (unsigned long)tk->rp.kp.addr;
1102 memset(&entry[1], 0, dsize);
1103 store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize);
1104 perf_trace_buf_submit(entry, size, rctx, 0, 1, regs, head, NULL);
1107 /* Kretprobe profile handler */
1108 static __kprobes void
1109 kretprobe_perf_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
1110 struct pt_regs *regs)
1112 struct ftrace_event_call *call = &tk->tp.call;
1113 struct kretprobe_trace_entry_head *entry;
1114 struct hlist_head *head;
1115 int size, __size, dsize;
1118 head = this_cpu_ptr(call->perf_events);
1119 if (hlist_empty(head))
1122 dsize = __get_data_size(&tk->tp, regs);
1123 __size = sizeof(*entry) + tk->tp.size + dsize;
1124 size = ALIGN(__size + sizeof(u32), sizeof(u64));
1125 size -= sizeof(u32);
1127 entry = perf_trace_buf_prepare(size, call->event.type, regs, &rctx);
1131 entry->func = (unsigned long)tk->rp.kp.addr;
1132 entry->ret_ip = (unsigned long)ri->ret_addr;
1133 store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize);
1134 perf_trace_buf_submit(entry, size, rctx, 0, 1, regs, head, NULL);
1136 #endif /* CONFIG_PERF_EVENTS */
1139 * called by perf_trace_init() or __ftrace_set_clr_event() under event_mutex.
1141 * kprobe_trace_self_tests_init() does enable_trace_probe/disable_trace_probe
1142 * lockless, but we can't race with this __init function.
1145 int kprobe_register(struct ftrace_event_call *event,
1146 enum trace_reg type, void *data)
1148 struct trace_kprobe *tk = (struct trace_kprobe *)event->data;
1149 struct ftrace_event_file *file = data;
1152 case TRACE_REG_REGISTER:
1153 return enable_trace_kprobe(tk, file);
1154 case TRACE_REG_UNREGISTER:
1155 return disable_trace_kprobe(tk, file);
1157 #ifdef CONFIG_PERF_EVENTS
1158 case TRACE_REG_PERF_REGISTER:
1159 return enable_trace_kprobe(tk, NULL);
1160 case TRACE_REG_PERF_UNREGISTER:
1161 return disable_trace_kprobe(tk, NULL);
1162 case TRACE_REG_PERF_OPEN:
1163 case TRACE_REG_PERF_CLOSE:
1164 case TRACE_REG_PERF_ADD:
1165 case TRACE_REG_PERF_DEL:
1173 int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs)
1175 struct trace_kprobe *tk = container_of(kp, struct trace_kprobe, rp.kp);
1179 if (tk->tp.flags & TP_FLAG_TRACE)
1180 kprobe_trace_func(tk, regs);
1181 #ifdef CONFIG_PERF_EVENTS
1182 if (tk->tp.flags & TP_FLAG_PROFILE)
1183 kprobe_perf_func(tk, regs);
1185 return 0; /* We don't tweek kernel, so just return 0 */
1189 int kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs)
1191 struct trace_kprobe *tk = container_of(ri->rp, struct trace_kprobe, rp);
1195 if (tk->tp.flags & TP_FLAG_TRACE)
1196 kretprobe_trace_func(tk, ri, regs);
1197 #ifdef CONFIG_PERF_EVENTS
1198 if (tk->tp.flags & TP_FLAG_PROFILE)
1199 kretprobe_perf_func(tk, ri, regs);
1201 return 0; /* We don't tweek kernel, so just return 0 */
1204 static struct trace_event_functions kretprobe_funcs = {
1205 .trace = print_kretprobe_event
1208 static struct trace_event_functions kprobe_funcs = {
1209 .trace = print_kprobe_event
1212 static int register_kprobe_event(struct trace_kprobe *tk)
1214 struct ftrace_event_call *call = &tk->tp.call;
1217 /* Initialize ftrace_event_call */
1218 INIT_LIST_HEAD(&call->class->fields);
1219 if (trace_kprobe_is_return(tk)) {
1220 call->event.funcs = &kretprobe_funcs;
1221 call->class->define_fields = kretprobe_event_define_fields;
1223 call->event.funcs = &kprobe_funcs;
1224 call->class->define_fields = kprobe_event_define_fields;
1226 if (set_print_fmt(tk) < 0)
1228 ret = register_ftrace_event(&call->event);
1230 kfree(call->print_fmt);
1234 call->class->reg = kprobe_register;
1236 ret = trace_add_event_call(call);
1238 pr_info("Failed to register kprobe event: %s\n", call->name);
1239 kfree(call->print_fmt);
1240 unregister_ftrace_event(&call->event);
1245 static int unregister_kprobe_event(struct trace_kprobe *tk)
1249 /* tp->event is unregistered in trace_remove_event_call() */
1250 ret = trace_remove_event_call(&tk->tp.call);
1252 kfree(tk->tp.call.print_fmt);
1256 /* Make a debugfs interface for controlling probe points */
1257 static __init int init_kprobe_trace(void)
1259 struct dentry *d_tracer;
1260 struct dentry *entry;
1262 if (register_module_notifier(&trace_kprobe_module_nb))
1265 d_tracer = tracing_init_dentry();
1269 entry = debugfs_create_file("kprobe_events", 0644, d_tracer,
1270 NULL, &kprobe_events_ops);
1272 /* Event list interface */
1274 pr_warning("Could not create debugfs "
1275 "'kprobe_events' entry\n");
1277 /* Profile interface */
1278 entry = debugfs_create_file("kprobe_profile", 0444, d_tracer,
1279 NULL, &kprobe_profile_ops);
1282 pr_warning("Could not create debugfs "
1283 "'kprobe_profile' entry\n");
1286 fs_initcall(init_kprobe_trace);
1289 #ifdef CONFIG_FTRACE_STARTUP_TEST
1292 * The "__used" keeps gcc from removing the function symbol
1293 * from the kallsyms table.
1295 static __used int kprobe_trace_selftest_target(int a1, int a2, int a3,
1296 int a4, int a5, int a6)
1298 return a1 + a2 + a3 + a4 + a5 + a6;
1301 static struct ftrace_event_file *
1302 find_trace_probe_file(struct trace_kprobe *tk, struct trace_array *tr)
1304 struct ftrace_event_file *file;
1306 list_for_each_entry(file, &tr->events, list)
1307 if (file->event_call == &tk->tp.call)
1314 * Nobody but us can call enable_trace_kprobe/disable_trace_kprobe at this
1315 * stage, we can do this lockless.
1317 static __init int kprobe_trace_self_tests_init(void)
1320 int (*target)(int, int, int, int, int, int);
1321 struct trace_kprobe *tk;
1322 struct ftrace_event_file *file;
1324 target = kprobe_trace_selftest_target;
1326 pr_info("Testing kprobe tracing: ");
1328 ret = traceprobe_command("p:testprobe kprobe_trace_selftest_target "
1329 "$stack $stack0 +0($stack)",
1330 create_trace_kprobe);
1331 if (WARN_ON_ONCE(ret)) {
1332 pr_warn("error on probing function entry.\n");
1335 /* Enable trace point */
1336 tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM);
1337 if (WARN_ON_ONCE(tk == NULL)) {
1338 pr_warn("error on getting new probe.\n");
1341 file = find_trace_probe_file(tk, top_trace_array());
1342 if (WARN_ON_ONCE(file == NULL)) {
1343 pr_warn("error on getting probe file.\n");
1346 enable_trace_kprobe(tk, file);
1350 ret = traceprobe_command("r:testprobe2 kprobe_trace_selftest_target "
1351 "$retval", create_trace_kprobe);
1352 if (WARN_ON_ONCE(ret)) {
1353 pr_warn("error on probing function return.\n");
1356 /* Enable trace point */
1357 tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM);
1358 if (WARN_ON_ONCE(tk == NULL)) {
1359 pr_warn("error on getting 2nd new probe.\n");
1362 file = find_trace_probe_file(tk, top_trace_array());
1363 if (WARN_ON_ONCE(file == NULL)) {
1364 pr_warn("error on getting probe file.\n");
1367 enable_trace_kprobe(tk, file);
1374 ret = target(1, 2, 3, 4, 5, 6);
1376 /* Disable trace points before removing it */
1377 tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM);
1378 if (WARN_ON_ONCE(tk == NULL)) {
1379 pr_warn("error on getting test probe.\n");
1382 file = find_trace_probe_file(tk, top_trace_array());
1383 if (WARN_ON_ONCE(file == NULL)) {
1384 pr_warn("error on getting probe file.\n");
1387 disable_trace_kprobe(tk, file);
1390 tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM);
1391 if (WARN_ON_ONCE(tk == NULL)) {
1392 pr_warn("error on getting 2nd test probe.\n");
1395 file = find_trace_probe_file(tk, top_trace_array());
1396 if (WARN_ON_ONCE(file == NULL)) {
1397 pr_warn("error on getting probe file.\n");
1400 disable_trace_kprobe(tk, file);
1403 ret = traceprobe_command("-:testprobe", create_trace_kprobe);
1404 if (WARN_ON_ONCE(ret)) {
1405 pr_warn("error on deleting a probe.\n");
1409 ret = traceprobe_command("-:testprobe2", create_trace_kprobe);
1410 if (WARN_ON_ONCE(ret)) {
1411 pr_warn("error on deleting a probe.\n");
1416 release_all_trace_kprobes();
1418 pr_cont("NG: Some tests are failed. Please check them.\n");
1424 late_initcall(kprobe_trace_self_tests_init);