From ac99c58c9e56967037382e31f865b72b10127965 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Sun, 15 Mar 2009 22:10:35 +0100 Subject: [PATCH 1/7] tracing/syscalls: fix missing release of tracing Impact: fix 'stuck' syscall tracer The syscall tracer uses a refcounter to enable several users simultaneously. But the refcounter did not behave correctly and always restored its value to 0 after calling start_syscall_tracing(). Therefore, stop_syscall_tracing() couldn't release correctly the tasks from tracing. Also the tracer forgot to reset the buffer when it is released. Drop the pointless refcount decrement on start_syscall_tracing() and reset the buffer when we release the tracer. This fixes two reported issue: - when we switch from syscall tracer to another tracer, syscall tracing continued. - incorrect use of the refcount. Reported-by: Andrew Morton Reported-by: Ingo Molnar Signed-off-by: Frederic Weisbecker LKML-Reference: <1237151439-6755-1-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar --- kernel/trace/trace_syscalls.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/kernel/trace/trace_syscalls.c b/kernel/trace/trace_syscalls.c index c72e599230ff..c5fc1d8880f6 100644 --- a/kernel/trace/trace_syscalls.c +++ b/kernel/trace/trace_syscalls.c @@ -96,8 +96,9 @@ void start_ftrace_syscalls(void) unsigned long flags; struct task_struct *g, *t; + /* Don't enable the flag on the tasks twice */ if (atomic_inc_return(&refcount) != 1) - goto out; + return; arch_init_ftrace_syscalls(); read_lock_irqsave(&tasklist_lock, flags); @@ -107,8 +108,6 @@ void start_ftrace_syscalls(void) } while_each_thread(g, t); read_unlock_irqrestore(&tasklist_lock, flags); -out: - atomic_dec(&refcount); } void stop_ftrace_syscalls(void) @@ -116,8 +115,9 @@ void stop_ftrace_syscalls(void) unsigned long flags; struct task_struct *g, *t; + /* There are perhaps still some users */ if (atomic_dec_return(&refcount)) - goto out; + return; read_lock_irqsave(&tasklist_lock, flags); @@ -126,8 +126,6 @@ void stop_ftrace_syscalls(void) } while_each_thread(g, t); read_unlock_irqrestore(&tasklist_lock, flags); -out: - atomic_inc(&refcount); } void ftrace_syscall_enter(struct pt_regs *regs) @@ -201,6 +199,7 @@ static int init_syscall_tracer(struct trace_array *tr) static void reset_syscall_tracer(struct trace_array *tr) { stop_ftrace_syscalls(); + tracing_reset_online_cpus(tr); } static struct trace_event syscall_enter_event = { From 6404434525bb9f8f2239998f30fd7c93f2efa5b3 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Sun, 15 Mar 2009 22:10:36 +0100 Subject: [PATCH 2/7] tracing/syscalls: various cleanups Impact: cleanup - Drop unused cpu variable - Fix some errors on comments Signed-off-by: Frederic Weisbecker LKML-Reference: <1237151439-6755-2-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar --- kernel/trace/trace_syscalls.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/kernel/trace/trace_syscalls.c b/kernel/trace/trace_syscalls.c index c5fc1d8880f6..26f9a8679d3d 100644 --- a/kernel/trace/trace_syscalls.c +++ b/kernel/trace/trace_syscalls.c @@ -7,7 +7,7 @@ static atomic_t refcount; -/* Our two options */ +/* Option to display the parameters types */ enum { TRACE_SYSCALLS_OPT_TYPES = 0x1, }; @@ -18,7 +18,7 @@ static struct tracer_opt syscalls_opts[] = { }; static struct tracer_flags syscalls_flags = { - .val = 0, /* By default: no args types */ + .val = 0, /* By default: no parameters types */ .opts = syscalls_opts }; @@ -135,12 +135,9 @@ void ftrace_syscall_enter(struct pt_regs *regs) struct ring_buffer_event *event; int size; int syscall_nr; - int cpu; syscall_nr = syscall_get_nr(current, regs); - cpu = raw_smp_processor_id(); - sys_data = syscall_nr_to_meta(syscall_nr); if (!sys_data) return; @@ -166,12 +163,9 @@ void ftrace_syscall_exit(struct pt_regs *regs) struct syscall_metadata *sys_data; struct ring_buffer_event *event; int syscall_nr; - int cpu; syscall_nr = syscall_get_nr(current, regs); - cpu = raw_smp_processor_id(); - sys_data = syscall_nr_to_meta(syscall_nr); if (!sys_data) return; From 5be71b61f17b0e3bc8ad0b1a1b7b53ab7d574ebb Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Sun, 15 Mar 2009 22:10:37 +0100 Subject: [PATCH 3/7] tracing/syscalls: protect thread flag toggling from races Impact: fix syscall tracer enable/disable race The current thread flag toggling is racy as shown in the following scenario: - task A is the last user of syscall tracing, it releases the TIF_SYSCALL_FTRACE on each tasks - at the same time task B start syscall tracing. refcount == 0 so it sets up TIF_SYSCALL_FTRACE on each tasks. The effect of the mixup is unpredictable. So this fix adds a mutex on {start,stop}_syscall_tracing(). Reported-by: Andrew Morton Reported-by: Ingo Molnar LKML-Reference: <1237151439-6755-3-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar --- kernel/trace/trace_syscalls.c | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/kernel/trace/trace_syscalls.c b/kernel/trace/trace_syscalls.c index 26f9a8679d3d..a2a3af29c943 100644 --- a/kernel/trace/trace_syscalls.c +++ b/kernel/trace/trace_syscalls.c @@ -5,7 +5,11 @@ #include "trace_output.h" #include "trace.h" -static atomic_t refcount; +/* Keep a counter of the syscall tracing users */ +static int refcount; + +/* Prevent from races on thread flags toggling */ +static DEFINE_MUTEX(syscall_trace_lock); /* Option to display the parameters types */ enum { @@ -96,9 +100,11 @@ void start_ftrace_syscalls(void) unsigned long flags; struct task_struct *g, *t; + mutex_lock(&syscall_trace_lock); + /* Don't enable the flag on the tasks twice */ - if (atomic_inc_return(&refcount) != 1) - return; + if (++refcount != 1) + goto unlock; arch_init_ftrace_syscalls(); read_lock_irqsave(&tasklist_lock, flags); @@ -108,6 +114,9 @@ void start_ftrace_syscalls(void) } while_each_thread(g, t); read_unlock_irqrestore(&tasklist_lock, flags); + +unlock: + mutex_unlock(&syscall_trace_lock); } void stop_ftrace_syscalls(void) @@ -115,9 +124,11 @@ void stop_ftrace_syscalls(void) unsigned long flags; struct task_struct *g, *t; + mutex_lock(&syscall_trace_lock); + /* There are perhaps still some users */ - if (atomic_dec_return(&refcount)) - return; + if (--refcount) + goto unlock; read_lock_irqsave(&tasklist_lock, flags); @@ -126,6 +137,9 @@ void stop_ftrace_syscalls(void) } while_each_thread(g, t); read_unlock_irqrestore(&tasklist_lock, flags); + +unlock: + mutex_unlock(&syscall_trace_lock); } void ftrace_syscall_enter(struct pt_regs *regs) From 0ea1c4156bf9e2eb370cc5c6fa6eb112bd844dec Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Sun, 15 Mar 2009 22:10:38 +0100 Subject: [PATCH 4/7] tracing/syscalls: select kallsysms Syscall tracing must select kallsysms. The arch code builds a table to find the syscall metadata by syscall number. It needs the syscalls names resolution from the symbol table to know which name found on the syscalls metadatas match a function pointer from the arch sys_call_table. Reported-by: Andrew Morton Reported-by: Ingo Molnar Signed-off-by: Frederic Weisbecker LKML-Reference: <1237151439-6755-4-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar --- kernel/trace/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig index 95a0ad191f19..b0a46f889659 100644 --- a/kernel/trace/Kconfig +++ b/kernel/trace/Kconfig @@ -182,6 +182,7 @@ config FTRACE_SYSCALLS bool "Trace syscalls" depends on HAVE_FTRACE_SYSCALLS select TRACING + select KALLSYMS help Basic tracer to catch the syscall entry and exit events. From 59f586db98919d7d9c43527b26c8de1cdf9ed912 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Sun, 15 Mar 2009 22:10:39 +0100 Subject: [PATCH 5/7] tracing/core: fix missing mutex unlock on tracing_set_tracer() Impact: fix possible locking imbalance In case of ring buffer resize failure, tracing_set_tracer forgot to release trace_types_lock. Fix it. Signed-off-by: Frederic Weisbecker LKML-Reference: <1237151439-6755-5-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar --- kernel/trace/trace.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index efe3202c0209..c0cf946d42f5 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -2494,7 +2494,7 @@ static int tracing_set_tracer(const char *buf) if (!ring_buffer_expanded) { ret = tracing_resize_ring_buffer(trace_buf_size); if (ret < 0) - return ret; + goto out; ret = 0; } From ac1d52d0b85854958c7e78c8006e39aadb6ce4b8 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Mon, 16 Mar 2009 00:32:41 +0100 Subject: [PATCH 6/7] tracing/ftrace: fix double calls to tracing_start() Impact: fix a warning during preemptirqsoff selftests When the preemptirqsoff selftest fails, we see the following warning: [ 6.050000] Testing tracer preemptirqsoff: .. no entries found .. ------------[ cut here ]------------ [ 6.060000] WARNING: at kernel/trace/trace.c:688 tracing_start+0x67/0xd3() [ 6.060000] Modules linked in: [ 6.060000] Pid: 1, comm: swapper Tainted: G [ 6.060000] Call Trace: [ 6.060000] [] warn_slowpath+0xb1/0x100 [ 6.060000] [] ? trace_preempt_on+0x35/0x4b [ 6.060000] [] ? tracing_start+0x31/0xd3 [ 6.060000] [] ? tracing_start+0x31/0xd3 [ 6.060000] [] ? __lock_acquired+0xe6/0x1f2 [ 6.060000] [] ? tracing_start+0x31/0xd3 [ 6.060000] [] tracing_start+0x67/0xd3 [ 6.060000] [] ? irqsoff_tracer_reset+0x2d/0x57 [ 6.060000] [] trace_selftest_startup_preemptirqsoff+0x1c8/0x1f1 [ 6.060000] [] register_tracer+0x12f/0x241 [ 6.060000] [] ? init_irqsoff_tracer+0x0/0x53 [ 6.060000] [] init_irqsoff_tracer+0x3b/0x53 This is because in fail case, the preemptirqsoff tracer selftest calls twice the tracing_start() function: int trace_selftest_startup_preemptirqsoff(struct tracer *trace, struct trace_array *tr) { if (!ret && !count) { printk(KERN_CONT ".. no entries found .."); ret = -1; tracing_start(); <----- goto out; } [...] out: trace->reset(tr); tracing_start(); <------ tracing_max_latency = save_max; return ret; } Since it is well handled in the out path, we don't need the conditional one. Reported-by: Ingo Molnar Signed-off-by: Frederic Weisbecker LKML-Reference: <1237159961-7447-1-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar --- kernel/trace/trace_selftest.c | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/kernel/trace/trace_selftest.c b/kernel/trace/trace_selftest.c index f907a2b29028..a2ca6f0fef9b 100644 --- a/kernel/trace/trace_selftest.c +++ b/kernel/trace/trace_selftest.c @@ -414,7 +414,7 @@ trace_selftest_startup_preemptirqsoff(struct tracer *trace, struct trace_array * ret = tracer_init(trace, tr); if (ret) { warn_failed_init_tracer(trace, ret); - goto out; + goto out_no_start; } /* reset the max latency */ @@ -432,21 +432,16 @@ trace_selftest_startup_preemptirqsoff(struct tracer *trace, struct trace_array * tracing_stop(); /* check both trace buffers */ ret = trace_test_buffer(tr, NULL); - if (ret) { - tracing_start(); + if (ret) goto out; - } ret = trace_test_buffer(&max_tr, &count); - if (ret) { - tracing_start(); + if (ret) goto out; - } if (!ret && !count) { printk(KERN_CONT ".. no entries found .."); ret = -1; - tracing_start(); goto out; } @@ -475,9 +470,10 @@ trace_selftest_startup_preemptirqsoff(struct tracer *trace, struct trace_array * goto out; } - out: - trace->reset(tr); +out: tracing_start(); +out_no_start: + trace->reset(tr); tracing_max_latency = save_max; return ret; From 2fc1dfbe17e7705c55b7a99da995fa565e26f151 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Mon, 16 Mar 2009 01:45:03 +0100 Subject: [PATCH 7/7] tracing/core: fix early free of cpumasks Impact: fix crashes when tracing cpumasks While ring-buffer allocation, the cpumasks are allocated too, including the tracing cpumask and the per-cpu file mask handler. But these cpumasks are freed accidentally just after. Fix it. Reported-by: Ingo Molnar Signed-off-by: Frederic Weisbecker LKML-Reference: <1237164303-11476-1-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar --- kernel/trace/trace.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index c0cf946d42f5..ae32d3b99b4b 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -4125,7 +4125,8 @@ __init static int tracer_alloc_buffers(void) &trace_panic_notifier); register_die_notifier(&trace_die_notifier); - ret = 0; + + return 0; out_free_cpumask: free_cpumask_var(tracing_reader_cpumask);