Merge branch 'tip/perf/core-2' of git://git.kernel.org/pub/scm/linux/kernel/git/roste...
[~shefty/rdma-dev.git] / kernel / trace / trace.c
1 /*
2  * ring buffer based function tracer
3  *
4  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5  * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
6  *
7  * Originally taken from the RT patch by:
8  *    Arnaldo Carvalho de Melo <acme@redhat.com>
9  *
10  * Based on code from the latency_tracer, that is:
11  *  Copyright (C) 2004-2006 Ingo Molnar
12  *  Copyright (C) 2004 Nadia Yvette Chambers
13  */
14 #include <linux/ring_buffer.h>
15 #include <generated/utsrelease.h>
16 #include <linux/stacktrace.h>
17 #include <linux/writeback.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/notifier.h>
21 #include <linux/irqflags.h>
22 #include <linux/irq_work.h>
23 #include <linux/debugfs.h>
24 #include <linux/pagemap.h>
25 #include <linux/hardirq.h>
26 #include <linux/linkage.h>
27 #include <linux/uaccess.h>
28 #include <linux/kprobes.h>
29 #include <linux/ftrace.h>
30 #include <linux/module.h>
31 #include <linux/percpu.h>
32 #include <linux/splice.h>
33 #include <linux/kdebug.h>
34 #include <linux/string.h>
35 #include <linux/rwsem.h>
36 #include <linux/slab.h>
37 #include <linux/ctype.h>
38 #include <linux/init.h>
39 #include <linux/poll.h>
40 #include <linux/nmi.h>
41 #include <linux/fs.h>
42
43 #include "trace.h"
44 #include "trace_output.h"
45
46 /*
47  * On boot up, the ring buffer is set to the minimum size, so that
48  * we do not waste memory on systems that are not using tracing.
49  */
50 int ring_buffer_expanded;
51
52 /*
53  * We need to change this state when a selftest is running.
54  * A selftest will lurk into the ring-buffer to count the
55  * entries inserted during the selftest although some concurrent
56  * insertions into the ring-buffer such as trace_printk could occurred
57  * at the same time, giving false positive or negative results.
58  */
59 static bool __read_mostly tracing_selftest_running;
60
61 /*
62  * If a tracer is running, we do not want to run SELFTEST.
63  */
64 bool __read_mostly tracing_selftest_disabled;
65
66 /* For tracers that don't implement custom flags */
67 static struct tracer_opt dummy_tracer_opt[] = {
68         { }
69 };
70
71 static struct tracer_flags dummy_tracer_flags = {
72         .val = 0,
73         .opts = dummy_tracer_opt
74 };
75
76 static int dummy_set_flag(u32 old_flags, u32 bit, int set)
77 {
78         return 0;
79 }
80
81 /*
82  * To prevent the comm cache from being overwritten when no
83  * tracing is active, only save the comm when a trace event
84  * occurred.
85  */
86 static DEFINE_PER_CPU(bool, trace_cmdline_save);
87
88 /*
89  * When a reader is waiting for data, then this variable is
90  * set to true.
91  */
92 static bool trace_wakeup_needed;
93
94 static struct irq_work trace_work_wakeup;
95
96 /*
97  * Kill all tracing for good (never come back).
98  * It is initialized to 1 but will turn to zero if the initialization
99  * of the tracer is successful. But that is the only place that sets
100  * this back to zero.
101  */
102 static int tracing_disabled = 1;
103
104 DEFINE_PER_CPU(int, ftrace_cpu_disabled);
105
106 cpumask_var_t __read_mostly     tracing_buffer_mask;
107
108 /*
109  * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
110  *
111  * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
112  * is set, then ftrace_dump is called. This will output the contents
113  * of the ftrace buffers to the console.  This is very useful for
114  * capturing traces that lead to crashes and outputing it to a
115  * serial console.
116  *
117  * It is default off, but you can enable it with either specifying
118  * "ftrace_dump_on_oops" in the kernel command line, or setting
119  * /proc/sys/kernel/ftrace_dump_on_oops
120  * Set 1 if you want to dump buffers of all CPUs
121  * Set 2 if you want to dump the buffer of the CPU that triggered oops
122  */
123
124 enum ftrace_dump_mode ftrace_dump_on_oops;
125
126 static int tracing_set_tracer(const char *buf);
127
128 #define MAX_TRACER_SIZE         100
129 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
130 static char *default_bootup_tracer;
131
132 static int __init set_cmdline_ftrace(char *str)
133 {
134         strncpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
135         default_bootup_tracer = bootup_tracer_buf;
136         /* We are using ftrace early, expand it */
137         ring_buffer_expanded = 1;
138         return 1;
139 }
140 __setup("ftrace=", set_cmdline_ftrace);
141
142 static int __init set_ftrace_dump_on_oops(char *str)
143 {
144         if (*str++ != '=' || !*str) {
145                 ftrace_dump_on_oops = DUMP_ALL;
146                 return 1;
147         }
148
149         if (!strcmp("orig_cpu", str)) {
150                 ftrace_dump_on_oops = DUMP_ORIG;
151                 return 1;
152         }
153
154         return 0;
155 }
156 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
157
158
159 static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
160 static char *trace_boot_options __initdata;
161
162 static int __init set_trace_boot_options(char *str)
163 {
164         strncpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
165         trace_boot_options = trace_boot_options_buf;
166         return 0;
167 }
168 __setup("trace_options=", set_trace_boot_options);
169
170 unsigned long long ns2usecs(cycle_t nsec)
171 {
172         nsec += 500;
173         do_div(nsec, 1000);
174         return nsec;
175 }
176
177 /*
178  * The global_trace is the descriptor that holds the tracing
179  * buffers for the live tracing. For each CPU, it contains
180  * a link list of pages that will store trace entries. The
181  * page descriptor of the pages in the memory is used to hold
182  * the link list by linking the lru item in the page descriptor
183  * to each of the pages in the buffer per CPU.
184  *
185  * For each active CPU there is a data field that holds the
186  * pages for the buffer for that CPU. Each CPU has the same number
187  * of pages allocated for its buffer.
188  */
189 static struct trace_array       global_trace;
190
191 static DEFINE_PER_CPU(struct trace_array_cpu, global_trace_cpu);
192
193 int filter_current_check_discard(struct ring_buffer *buffer,
194                                  struct ftrace_event_call *call, void *rec,
195                                  struct ring_buffer_event *event)
196 {
197         return filter_check_discard(call, rec, buffer, event);
198 }
199 EXPORT_SYMBOL_GPL(filter_current_check_discard);
200
201 cycle_t ftrace_now(int cpu)
202 {
203         u64 ts;
204
205         /* Early boot up does not have a buffer yet */
206         if (!global_trace.buffer)
207                 return trace_clock_local();
208
209         ts = ring_buffer_time_stamp(global_trace.buffer, cpu);
210         ring_buffer_normalize_time_stamp(global_trace.buffer, cpu, &ts);
211
212         return ts;
213 }
214
215 /*
216  * The max_tr is used to snapshot the global_trace when a maximum
217  * latency is reached. Some tracers will use this to store a maximum
218  * trace while it continues examining live traces.
219  *
220  * The buffers for the max_tr are set up the same as the global_trace.
221  * When a snapshot is taken, the link list of the max_tr is swapped
222  * with the link list of the global_trace and the buffers are reset for
223  * the global_trace so the tracing can continue.
224  */
225 static struct trace_array       max_tr;
226
227 static DEFINE_PER_CPU(struct trace_array_cpu, max_tr_data);
228
229 int tracing_is_enabled(void)
230 {
231         return tracing_is_on();
232 }
233
234 /*
235  * trace_buf_size is the size in bytes that is allocated
236  * for a buffer. Note, the number of bytes is always rounded
237  * to page size.
238  *
239  * This number is purposely set to a low number of 16384.
240  * If the dump on oops happens, it will be much appreciated
241  * to not have to wait for all that output. Anyway this can be
242  * boot time and run time configurable.
243  */
244 #define TRACE_BUF_SIZE_DEFAULT  1441792UL /* 16384 * 88 (sizeof(entry)) */
245
246 static unsigned long            trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
247
248 /* trace_types holds a link list of available tracers. */
249 static struct tracer            *trace_types __read_mostly;
250
251 /* current_trace points to the tracer that is currently active */
252 static struct tracer            *current_trace __read_mostly;
253
254 /*
255  * trace_types_lock is used to protect the trace_types list.
256  */
257 static DEFINE_MUTEX(trace_types_lock);
258
259 /*
260  * serialize the access of the ring buffer
261  *
262  * ring buffer serializes readers, but it is low level protection.
263  * The validity of the events (which returns by ring_buffer_peek() ..etc)
264  * are not protected by ring buffer.
265  *
266  * The content of events may become garbage if we allow other process consumes
267  * these events concurrently:
268  *   A) the page of the consumed events may become a normal page
269  *      (not reader page) in ring buffer, and this page will be rewrited
270  *      by events producer.
271  *   B) The page of the consumed events may become a page for splice_read,
272  *      and this page will be returned to system.
273  *
274  * These primitives allow multi process access to different cpu ring buffer
275  * concurrently.
276  *
277  * These primitives don't distinguish read-only and read-consume access.
278  * Multi read-only access are also serialized.
279  */
280
281 #ifdef CONFIG_SMP
282 static DECLARE_RWSEM(all_cpu_access_lock);
283 static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
284
285 static inline void trace_access_lock(int cpu)
286 {
287         if (cpu == TRACE_PIPE_ALL_CPU) {
288                 /* gain it for accessing the whole ring buffer. */
289                 down_write(&all_cpu_access_lock);
290         } else {
291                 /* gain it for accessing a cpu ring buffer. */
292
293                 /* Firstly block other trace_access_lock(TRACE_PIPE_ALL_CPU). */
294                 down_read(&all_cpu_access_lock);
295
296                 /* Secondly block other access to this @cpu ring buffer. */
297                 mutex_lock(&per_cpu(cpu_access_lock, cpu));
298         }
299 }
300
301 static inline void trace_access_unlock(int cpu)
302 {
303         if (cpu == TRACE_PIPE_ALL_CPU) {
304                 up_write(&all_cpu_access_lock);
305         } else {
306                 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
307                 up_read(&all_cpu_access_lock);
308         }
309 }
310
311 static inline void trace_access_lock_init(void)
312 {
313         int cpu;
314
315         for_each_possible_cpu(cpu)
316                 mutex_init(&per_cpu(cpu_access_lock, cpu));
317 }
318
319 #else
320
321 static DEFINE_MUTEX(access_lock);
322
323 static inline void trace_access_lock(int cpu)
324 {
325         (void)cpu;
326         mutex_lock(&access_lock);
327 }
328
329 static inline void trace_access_unlock(int cpu)
330 {
331         (void)cpu;
332         mutex_unlock(&access_lock);
333 }
334
335 static inline void trace_access_lock_init(void)
336 {
337 }
338
339 #endif
340
341 /* trace_wait is a waitqueue for tasks blocked on trace_poll */
342 static DECLARE_WAIT_QUEUE_HEAD(trace_wait);
343
344 /* trace_flags holds trace_options default values */
345 unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
346         TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
347         TRACE_ITER_GRAPH_TIME | TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE |
348         TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS;
349
350 static int trace_stop_count;
351 static DEFINE_RAW_SPINLOCK(tracing_start_lock);
352
353 /**
354  * trace_wake_up - wake up tasks waiting for trace input
355  *
356  * Schedules a delayed work to wake up any task that is blocked on the
357  * trace_wait queue. These is used with trace_poll for tasks polling the
358  * trace.
359  */
360 static void trace_wake_up(struct irq_work *work)
361 {
362         wake_up_all(&trace_wait);
363
364 }
365
366 /**
367  * tracing_on - enable tracing buffers
368  *
369  * This function enables tracing buffers that may have been
370  * disabled with tracing_off.
371  */
372 void tracing_on(void)
373 {
374         if (global_trace.buffer)
375                 ring_buffer_record_on(global_trace.buffer);
376         /*
377          * This flag is only looked at when buffers haven't been
378          * allocated yet. We don't really care about the race
379          * between setting this flag and actually turning
380          * on the buffer.
381          */
382         global_trace.buffer_disabled = 0;
383 }
384 EXPORT_SYMBOL_GPL(tracing_on);
385
386 /**
387  * tracing_off - turn off tracing buffers
388  *
389  * This function stops the tracing buffers from recording data.
390  * It does not disable any overhead the tracers themselves may
391  * be causing. This function simply causes all recording to
392  * the ring buffers to fail.
393  */
394 void tracing_off(void)
395 {
396         if (global_trace.buffer)
397                 ring_buffer_record_off(global_trace.buffer);
398         /*
399          * This flag is only looked at when buffers haven't been
400          * allocated yet. We don't really care about the race
401          * between setting this flag and actually turning
402          * on the buffer.
403          */
404         global_trace.buffer_disabled = 1;
405 }
406 EXPORT_SYMBOL_GPL(tracing_off);
407
408 /**
409  * tracing_is_on - show state of ring buffers enabled
410  */
411 int tracing_is_on(void)
412 {
413         if (global_trace.buffer)
414                 return ring_buffer_record_is_on(global_trace.buffer);
415         return !global_trace.buffer_disabled;
416 }
417 EXPORT_SYMBOL_GPL(tracing_is_on);
418
419 static int __init set_buf_size(char *str)
420 {
421         unsigned long buf_size;
422
423         if (!str)
424                 return 0;
425         buf_size = memparse(str, &str);
426         /* nr_entries can not be zero */
427         if (buf_size == 0)
428                 return 0;
429         trace_buf_size = buf_size;
430         return 1;
431 }
432 __setup("trace_buf_size=", set_buf_size);
433
434 static int __init set_tracing_thresh(char *str)
435 {
436         unsigned long threshold;
437         int ret;
438
439         if (!str)
440                 return 0;
441         ret = kstrtoul(str, 0, &threshold);
442         if (ret < 0)
443                 return 0;
444         tracing_thresh = threshold * 1000;
445         return 1;
446 }
447 __setup("tracing_thresh=", set_tracing_thresh);
448
449 unsigned long nsecs_to_usecs(unsigned long nsecs)
450 {
451         return nsecs / 1000;
452 }
453
454 /* These must match the bit postions in trace_iterator_flags */
455 static const char *trace_options[] = {
456         "print-parent",
457         "sym-offset",
458         "sym-addr",
459         "verbose",
460         "raw",
461         "hex",
462         "bin",
463         "block",
464         "stacktrace",
465         "trace_printk",
466         "ftrace_preempt",
467         "branch",
468         "annotate",
469         "userstacktrace",
470         "sym-userobj",
471         "printk-msg-only",
472         "context-info",
473         "latency-format",
474         "sleep-time",
475         "graph-time",
476         "record-cmd",
477         "overwrite",
478         "disable_on_free",
479         "irq-info",
480         "markers",
481         NULL
482 };
483
484 static struct {
485         u64 (*func)(void);
486         const char *name;
487         int in_ns;              /* is this clock in nanoseconds? */
488 } trace_clocks[] = {
489         { trace_clock_local,    "local",        1 },
490         { trace_clock_global,   "global",       1 },
491         { trace_clock_counter,  "counter",      0 },
492         ARCH_TRACE_CLOCKS
493 };
494
495 int trace_clock_id;
496
497 /*
498  * trace_parser_get_init - gets the buffer for trace parser
499  */
500 int trace_parser_get_init(struct trace_parser *parser, int size)
501 {
502         memset(parser, 0, sizeof(*parser));
503
504         parser->buffer = kmalloc(size, GFP_KERNEL);
505         if (!parser->buffer)
506                 return 1;
507
508         parser->size = size;
509         return 0;
510 }
511
512 /*
513  * trace_parser_put - frees the buffer for trace parser
514  */
515 void trace_parser_put(struct trace_parser *parser)
516 {
517         kfree(parser->buffer);
518 }
519
520 /*
521  * trace_get_user - reads the user input string separated by  space
522  * (matched by isspace(ch))
523  *
524  * For each string found the 'struct trace_parser' is updated,
525  * and the function returns.
526  *
527  * Returns number of bytes read.
528  *
529  * See kernel/trace/trace.h for 'struct trace_parser' details.
530  */
531 int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
532         size_t cnt, loff_t *ppos)
533 {
534         char ch;
535         size_t read = 0;
536         ssize_t ret;
537
538         if (!*ppos)
539                 trace_parser_clear(parser);
540
541         ret = get_user(ch, ubuf++);
542         if (ret)
543                 goto out;
544
545         read++;
546         cnt--;
547
548         /*
549          * The parser is not finished with the last write,
550          * continue reading the user input without skipping spaces.
551          */
552         if (!parser->cont) {
553                 /* skip white space */
554                 while (cnt && isspace(ch)) {
555                         ret = get_user(ch, ubuf++);
556                         if (ret)
557                                 goto out;
558                         read++;
559                         cnt--;
560                 }
561
562                 /* only spaces were written */
563                 if (isspace(ch)) {
564                         *ppos += read;
565                         ret = read;
566                         goto out;
567                 }
568
569                 parser->idx = 0;
570         }
571
572         /* read the non-space input */
573         while (cnt && !isspace(ch)) {
574                 if (parser->idx < parser->size - 1)
575                         parser->buffer[parser->idx++] = ch;
576                 else {
577                         ret = -EINVAL;
578                         goto out;
579                 }
580                 ret = get_user(ch, ubuf++);
581                 if (ret)
582                         goto out;
583                 read++;
584                 cnt--;
585         }
586
587         /* We either got finished input or we have to wait for another call. */
588         if (isspace(ch)) {
589                 parser->buffer[parser->idx] = 0;
590                 parser->cont = false;
591         } else {
592                 parser->cont = true;
593                 parser->buffer[parser->idx++] = ch;
594         }
595
596         *ppos += read;
597         ret = read;
598
599 out:
600         return ret;
601 }
602
603 ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
604 {
605         int len;
606         int ret;
607
608         if (!cnt)
609                 return 0;
610
611         if (s->len <= s->readpos)
612                 return -EBUSY;
613
614         len = s->len - s->readpos;
615         if (cnt > len)
616                 cnt = len;
617         ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
618         if (ret == cnt)
619                 return -EFAULT;
620
621         cnt -= ret;
622
623         s->readpos += cnt;
624         return cnt;
625 }
626
627 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
628 {
629         int len;
630
631         if (s->len <= s->readpos)
632                 return -EBUSY;
633
634         len = s->len - s->readpos;
635         if (cnt > len)
636                 cnt = len;
637         memcpy(buf, s->buffer + s->readpos, cnt);
638
639         s->readpos += cnt;
640         return cnt;
641 }
642
643 /*
644  * ftrace_max_lock is used to protect the swapping of buffers
645  * when taking a max snapshot. The buffers themselves are
646  * protected by per_cpu spinlocks. But the action of the swap
647  * needs its own lock.
648  *
649  * This is defined as a arch_spinlock_t in order to help
650  * with performance when lockdep debugging is enabled.
651  *
652  * It is also used in other places outside the update_max_tr
653  * so it needs to be defined outside of the
654  * CONFIG_TRACER_MAX_TRACE.
655  */
656 static arch_spinlock_t ftrace_max_lock =
657         (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
658
659 unsigned long __read_mostly     tracing_thresh;
660
661 #ifdef CONFIG_TRACER_MAX_TRACE
662 unsigned long __read_mostly     tracing_max_latency;
663
664 /*
665  * Copy the new maximum trace into the separate maximum-trace
666  * structure. (this way the maximum trace is permanently saved,
667  * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
668  */
669 static void
670 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
671 {
672         struct trace_array_cpu *data = tr->data[cpu];
673         struct trace_array_cpu *max_data;
674
675         max_tr.cpu = cpu;
676         max_tr.time_start = data->preempt_timestamp;
677
678         max_data = max_tr.data[cpu];
679         max_data->saved_latency = tracing_max_latency;
680         max_data->critical_start = data->critical_start;
681         max_data->critical_end = data->critical_end;
682
683         memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
684         max_data->pid = tsk->pid;
685         max_data->uid = task_uid(tsk);
686         max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
687         max_data->policy = tsk->policy;
688         max_data->rt_priority = tsk->rt_priority;
689
690         /* record this tasks comm */
691         tracing_record_cmdline(tsk);
692 }
693
694 /**
695  * update_max_tr - snapshot all trace buffers from global_trace to max_tr
696  * @tr: tracer
697  * @tsk: the task with the latency
698  * @cpu: The cpu that initiated the trace.
699  *
700  * Flip the buffers between the @tr and the max_tr and record information
701  * about which task was the cause of this latency.
702  */
703 void
704 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
705 {
706         struct ring_buffer *buf = tr->buffer;
707
708         if (trace_stop_count)
709                 return;
710
711         WARN_ON_ONCE(!irqs_disabled());
712         if (!current_trace->use_max_tr) {
713                 WARN_ON_ONCE(1);
714                 return;
715         }
716         arch_spin_lock(&ftrace_max_lock);
717
718         tr->buffer = max_tr.buffer;
719         max_tr.buffer = buf;
720
721         __update_max_tr(tr, tsk, cpu);
722         arch_spin_unlock(&ftrace_max_lock);
723 }
724
725 /**
726  * update_max_tr_single - only copy one trace over, and reset the rest
727  * @tr - tracer
728  * @tsk - task with the latency
729  * @cpu - the cpu of the buffer to copy.
730  *
731  * Flip the trace of a single CPU buffer between the @tr and the max_tr.
732  */
733 void
734 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
735 {
736         int ret;
737
738         if (trace_stop_count)
739                 return;
740
741         WARN_ON_ONCE(!irqs_disabled());
742         if (!current_trace->use_max_tr) {
743                 WARN_ON_ONCE(1);
744                 return;
745         }
746
747         arch_spin_lock(&ftrace_max_lock);
748
749         ret = ring_buffer_swap_cpu(max_tr.buffer, tr->buffer, cpu);
750
751         if (ret == -EBUSY) {
752                 /*
753                  * We failed to swap the buffer due to a commit taking
754                  * place on this CPU. We fail to record, but we reset
755                  * the max trace buffer (no one writes directly to it)
756                  * and flag that it failed.
757                  */
758                 trace_array_printk(&max_tr, _THIS_IP_,
759                         "Failed to swap buffers due to commit in progress\n");
760         }
761
762         WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
763
764         __update_max_tr(tr, tsk, cpu);
765         arch_spin_unlock(&ftrace_max_lock);
766 }
767 #endif /* CONFIG_TRACER_MAX_TRACE */
768
769 static void default_wait_pipe(struct trace_iterator *iter)
770 {
771         DEFINE_WAIT(wait);
772
773         prepare_to_wait(&trace_wait, &wait, TASK_INTERRUPTIBLE);
774
775         /*
776          * The events can happen in critical sections where
777          * checking a work queue can cause deadlocks.
778          * After adding a task to the queue, this flag is set
779          * only to notify events to try to wake up the queue
780          * using irq_work.
781          *
782          * We don't clear it even if the buffer is no longer
783          * empty. The flag only causes the next event to run
784          * irq_work to do the work queue wake up. The worse
785          * that can happen if we race with !trace_empty() is that
786          * an event will cause an irq_work to try to wake up
787          * an empty queue.
788          *
789          * There's no reason to protect this flag either, as
790          * the work queue and irq_work logic will do the necessary
791          * synchronization for the wake ups. The only thing
792          * that is necessary is that the wake up happens after
793          * a task has been queued. It's OK for spurious wake ups.
794          */
795         trace_wakeup_needed = true;
796
797         if (trace_empty(iter))
798                 schedule();
799
800         finish_wait(&trace_wait, &wait);
801 }
802
803 /**
804  * register_tracer - register a tracer with the ftrace system.
805  * @type - the plugin for the tracer
806  *
807  * Register a new plugin tracer.
808  */
809 int register_tracer(struct tracer *type)
810 {
811         struct tracer *t;
812         int ret = 0;
813
814         if (!type->name) {
815                 pr_info("Tracer must have a name\n");
816                 return -1;
817         }
818
819         if (strlen(type->name) >= MAX_TRACER_SIZE) {
820                 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
821                 return -1;
822         }
823
824         mutex_lock(&trace_types_lock);
825
826         tracing_selftest_running = true;
827
828         for (t = trace_types; t; t = t->next) {
829                 if (strcmp(type->name, t->name) == 0) {
830                         /* already found */
831                         pr_info("Tracer %s already registered\n",
832                                 type->name);
833                         ret = -1;
834                         goto out;
835                 }
836         }
837
838         if (!type->set_flag)
839                 type->set_flag = &dummy_set_flag;
840         if (!type->flags)
841                 type->flags = &dummy_tracer_flags;
842         else
843                 if (!type->flags->opts)
844                         type->flags->opts = dummy_tracer_opt;
845         if (!type->wait_pipe)
846                 type->wait_pipe = default_wait_pipe;
847
848
849 #ifdef CONFIG_FTRACE_STARTUP_TEST
850         if (type->selftest && !tracing_selftest_disabled) {
851                 struct tracer *saved_tracer = current_trace;
852                 struct trace_array *tr = &global_trace;
853
854                 /*
855                  * Run a selftest on this tracer.
856                  * Here we reset the trace buffer, and set the current
857                  * tracer to be this tracer. The tracer can then run some
858                  * internal tracing to verify that everything is in order.
859                  * If we fail, we do not register this tracer.
860                  */
861                 tracing_reset_online_cpus(tr);
862
863                 current_trace = type;
864
865                 /* If we expanded the buffers, make sure the max is expanded too */
866                 if (ring_buffer_expanded && type->use_max_tr)
867                         ring_buffer_resize(max_tr.buffer, trace_buf_size,
868                                                 RING_BUFFER_ALL_CPUS);
869
870                 /* the test is responsible for initializing and enabling */
871                 pr_info("Testing tracer %s: ", type->name);
872                 ret = type->selftest(type, tr);
873                 /* the test is responsible for resetting too */
874                 current_trace = saved_tracer;
875                 if (ret) {
876                         printk(KERN_CONT "FAILED!\n");
877                         /* Add the warning after printing 'FAILED' */
878                         WARN_ON(1);
879                         goto out;
880                 }
881                 /* Only reset on passing, to avoid touching corrupted buffers */
882                 tracing_reset_online_cpus(tr);
883
884                 /* Shrink the max buffer again */
885                 if (ring_buffer_expanded && type->use_max_tr)
886                         ring_buffer_resize(max_tr.buffer, 1,
887                                                 RING_BUFFER_ALL_CPUS);
888
889                 printk(KERN_CONT "PASSED\n");
890         }
891 #endif
892
893         type->next = trace_types;
894         trace_types = type;
895
896  out:
897         tracing_selftest_running = false;
898         mutex_unlock(&trace_types_lock);
899
900         if (ret || !default_bootup_tracer)
901                 goto out_unlock;
902
903         if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
904                 goto out_unlock;
905
906         printk(KERN_INFO "Starting tracer '%s'\n", type->name);
907         /* Do we want this tracer to start on bootup? */
908         tracing_set_tracer(type->name);
909         default_bootup_tracer = NULL;
910         /* disable other selftests, since this will break it. */
911         tracing_selftest_disabled = 1;
912 #ifdef CONFIG_FTRACE_STARTUP_TEST
913         printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
914                type->name);
915 #endif
916
917  out_unlock:
918         return ret;
919 }
920
921 void tracing_reset(struct trace_array *tr, int cpu)
922 {
923         struct ring_buffer *buffer = tr->buffer;
924
925         ring_buffer_record_disable(buffer);
926
927         /* Make sure all commits have finished */
928         synchronize_sched();
929         ring_buffer_reset_cpu(buffer, cpu);
930
931         ring_buffer_record_enable(buffer);
932 }
933
934 void tracing_reset_online_cpus(struct trace_array *tr)
935 {
936         struct ring_buffer *buffer = tr->buffer;
937         int cpu;
938
939         ring_buffer_record_disable(buffer);
940
941         /* Make sure all commits have finished */
942         synchronize_sched();
943
944         tr->time_start = ftrace_now(tr->cpu);
945
946         for_each_online_cpu(cpu)
947                 ring_buffer_reset_cpu(buffer, cpu);
948
949         ring_buffer_record_enable(buffer);
950 }
951
952 void tracing_reset_current(int cpu)
953 {
954         tracing_reset(&global_trace, cpu);
955 }
956
957 void tracing_reset_current_online_cpus(void)
958 {
959         tracing_reset_online_cpus(&global_trace);
960 }
961
962 #define SAVED_CMDLINES 128
963 #define NO_CMDLINE_MAP UINT_MAX
964 static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
965 static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
966 static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
967 static int cmdline_idx;
968 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
969
970 /* temporary disable recording */
971 static atomic_t trace_record_cmdline_disabled __read_mostly;
972
973 static void trace_init_cmdlines(void)
974 {
975         memset(&map_pid_to_cmdline, NO_CMDLINE_MAP, sizeof(map_pid_to_cmdline));
976         memset(&map_cmdline_to_pid, NO_CMDLINE_MAP, sizeof(map_cmdline_to_pid));
977         cmdline_idx = 0;
978 }
979
980 int is_tracing_stopped(void)
981 {
982         return trace_stop_count;
983 }
984
985 /**
986  * ftrace_off_permanent - disable all ftrace code permanently
987  *
988  * This should only be called when a serious anomally has
989  * been detected.  This will turn off the function tracing,
990  * ring buffers, and other tracing utilites. It takes no
991  * locks and can be called from any context.
992  */
993 void ftrace_off_permanent(void)
994 {
995         tracing_disabled = 1;
996         ftrace_stop();
997         tracing_off_permanent();
998 }
999
1000 /**
1001  * tracing_start - quick start of the tracer
1002  *
1003  * If tracing is enabled but was stopped by tracing_stop,
1004  * this will start the tracer back up.
1005  */
1006 void tracing_start(void)
1007 {
1008         struct ring_buffer *buffer;
1009         unsigned long flags;
1010
1011         if (tracing_disabled)
1012                 return;
1013
1014         raw_spin_lock_irqsave(&tracing_start_lock, flags);
1015         if (--trace_stop_count) {
1016                 if (trace_stop_count < 0) {
1017                         /* Someone screwed up their debugging */
1018                         WARN_ON_ONCE(1);
1019                         trace_stop_count = 0;
1020                 }
1021                 goto out;
1022         }
1023
1024         /* Prevent the buffers from switching */
1025         arch_spin_lock(&ftrace_max_lock);
1026
1027         buffer = global_trace.buffer;
1028         if (buffer)
1029                 ring_buffer_record_enable(buffer);
1030
1031         buffer = max_tr.buffer;
1032         if (buffer)
1033                 ring_buffer_record_enable(buffer);
1034
1035         arch_spin_unlock(&ftrace_max_lock);
1036
1037         ftrace_start();
1038  out:
1039         raw_spin_unlock_irqrestore(&tracing_start_lock, flags);
1040 }
1041
1042 /**
1043  * tracing_stop - quick stop of the tracer
1044  *
1045  * Light weight way to stop tracing. Use in conjunction with
1046  * tracing_start.
1047  */
1048 void tracing_stop(void)
1049 {
1050         struct ring_buffer *buffer;
1051         unsigned long flags;
1052
1053         ftrace_stop();
1054         raw_spin_lock_irqsave(&tracing_start_lock, flags);
1055         if (trace_stop_count++)
1056                 goto out;
1057
1058         /* Prevent the buffers from switching */
1059         arch_spin_lock(&ftrace_max_lock);
1060
1061         buffer = global_trace.buffer;
1062         if (buffer)
1063                 ring_buffer_record_disable(buffer);
1064
1065         buffer = max_tr.buffer;
1066         if (buffer)
1067                 ring_buffer_record_disable(buffer);
1068
1069         arch_spin_unlock(&ftrace_max_lock);
1070
1071  out:
1072         raw_spin_unlock_irqrestore(&tracing_start_lock, flags);
1073 }
1074
1075 void trace_stop_cmdline_recording(void);
1076
1077 static void trace_save_cmdline(struct task_struct *tsk)
1078 {
1079         unsigned pid, idx;
1080
1081         if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
1082                 return;
1083
1084         /*
1085          * It's not the end of the world if we don't get
1086          * the lock, but we also don't want to spin
1087          * nor do we want to disable interrupts,
1088          * so if we miss here, then better luck next time.
1089          */
1090         if (!arch_spin_trylock(&trace_cmdline_lock))
1091                 return;
1092
1093         idx = map_pid_to_cmdline[tsk->pid];
1094         if (idx == NO_CMDLINE_MAP) {
1095                 idx = (cmdline_idx + 1) % SAVED_CMDLINES;
1096
1097                 /*
1098                  * Check whether the cmdline buffer at idx has a pid
1099                  * mapped. We are going to overwrite that entry so we
1100                  * need to clear the map_pid_to_cmdline. Otherwise we
1101                  * would read the new comm for the old pid.
1102                  */
1103                 pid = map_cmdline_to_pid[idx];
1104                 if (pid != NO_CMDLINE_MAP)
1105                         map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
1106
1107                 map_cmdline_to_pid[idx] = tsk->pid;
1108                 map_pid_to_cmdline[tsk->pid] = idx;
1109
1110                 cmdline_idx = idx;
1111         }
1112
1113         memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
1114
1115         arch_spin_unlock(&trace_cmdline_lock);
1116 }
1117
1118 void trace_find_cmdline(int pid, char comm[])
1119 {
1120         unsigned map;
1121
1122         if (!pid) {
1123                 strcpy(comm, "<idle>");
1124                 return;
1125         }
1126
1127         if (WARN_ON_ONCE(pid < 0)) {
1128                 strcpy(comm, "<XXX>");
1129                 return;
1130         }
1131
1132         if (pid > PID_MAX_DEFAULT) {
1133                 strcpy(comm, "<...>");
1134                 return;
1135         }
1136
1137         preempt_disable();
1138         arch_spin_lock(&trace_cmdline_lock);
1139         map = map_pid_to_cmdline[pid];
1140         if (map != NO_CMDLINE_MAP)
1141                 strcpy(comm, saved_cmdlines[map]);
1142         else
1143                 strcpy(comm, "<...>");
1144
1145         arch_spin_unlock(&trace_cmdline_lock);
1146         preempt_enable();
1147 }
1148
1149 void tracing_record_cmdline(struct task_struct *tsk)
1150 {
1151         if (atomic_read(&trace_record_cmdline_disabled) || !tracing_is_on())
1152                 return;
1153
1154         if (!__this_cpu_read(trace_cmdline_save))
1155                 return;
1156
1157         __this_cpu_write(trace_cmdline_save, false);
1158
1159         trace_save_cmdline(tsk);
1160 }
1161
1162 void
1163 tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1164                              int pc)
1165 {
1166         struct task_struct *tsk = current;
1167
1168         entry->preempt_count            = pc & 0xff;
1169         entry->pid                      = (tsk) ? tsk->pid : 0;
1170         entry->padding                  = 0;
1171         entry->flags =
1172 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1173                 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
1174 #else
1175                 TRACE_FLAG_IRQS_NOSUPPORT |
1176 #endif
1177                 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1178                 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
1179                 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
1180 }
1181 EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
1182
1183 struct ring_buffer_event *
1184 trace_buffer_lock_reserve(struct ring_buffer *buffer,
1185                           int type,
1186                           unsigned long len,
1187                           unsigned long flags, int pc)
1188 {
1189         struct ring_buffer_event *event;
1190
1191         event = ring_buffer_lock_reserve(buffer, len);
1192         if (event != NULL) {
1193                 struct trace_entry *ent = ring_buffer_event_data(event);
1194
1195                 tracing_generic_entry_update(ent, flags, pc);
1196                 ent->type = type;
1197         }
1198
1199         return event;
1200 }
1201
1202 void
1203 __buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
1204 {
1205         __this_cpu_write(trace_cmdline_save, true);
1206         if (trace_wakeup_needed) {
1207                 trace_wakeup_needed = false;
1208                 /* irq_work_queue() supplies it's own memory barriers */
1209                 irq_work_queue(&trace_work_wakeup);
1210         }
1211         ring_buffer_unlock_commit(buffer, event);
1212 }
1213
1214 static inline void
1215 __trace_buffer_unlock_commit(struct ring_buffer *buffer,
1216                              struct ring_buffer_event *event,
1217                              unsigned long flags, int pc)
1218 {
1219         __buffer_unlock_commit(buffer, event);
1220
1221         ftrace_trace_stack(buffer, flags, 6, pc);
1222         ftrace_trace_userstack(buffer, flags, pc);
1223 }
1224
1225 void trace_buffer_unlock_commit(struct ring_buffer *buffer,
1226                                 struct ring_buffer_event *event,
1227                                 unsigned long flags, int pc)
1228 {
1229         __trace_buffer_unlock_commit(buffer, event, flags, pc);
1230 }
1231 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit);
1232
1233 struct ring_buffer_event *
1234 trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1235                                   int type, unsigned long len,
1236                                   unsigned long flags, int pc)
1237 {
1238         *current_rb = global_trace.buffer;
1239         return trace_buffer_lock_reserve(*current_rb,
1240                                          type, len, flags, pc);
1241 }
1242 EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
1243
1244 void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
1245                                         struct ring_buffer_event *event,
1246                                         unsigned long flags, int pc)
1247 {
1248         __trace_buffer_unlock_commit(buffer, event, flags, pc);
1249 }
1250 EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
1251
1252 void trace_buffer_unlock_commit_regs(struct ring_buffer *buffer,
1253                                      struct ring_buffer_event *event,
1254                                      unsigned long flags, int pc,
1255                                      struct pt_regs *regs)
1256 {
1257         __buffer_unlock_commit(buffer, event);
1258
1259         ftrace_trace_stack_regs(buffer, flags, 0, pc, regs);
1260         ftrace_trace_userstack(buffer, flags, pc);
1261 }
1262 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit_regs);
1263
1264 void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1265                                          struct ring_buffer_event *event)
1266 {
1267         ring_buffer_discard_commit(buffer, event);
1268 }
1269 EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
1270
1271 void
1272 trace_function(struct trace_array *tr,
1273                unsigned long ip, unsigned long parent_ip, unsigned long flags,
1274                int pc)
1275 {
1276         struct ftrace_event_call *call = &event_function;
1277         struct ring_buffer *buffer = tr->buffer;
1278         struct ring_buffer_event *event;
1279         struct ftrace_entry *entry;
1280
1281         /* If we are reading the ring buffer, don't trace */
1282         if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
1283                 return;
1284
1285         event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
1286                                           flags, pc);
1287         if (!event)
1288                 return;
1289         entry   = ring_buffer_event_data(event);
1290         entry->ip                       = ip;
1291         entry->parent_ip                = parent_ip;
1292
1293         if (!filter_check_discard(call, entry, buffer, event))
1294                 __buffer_unlock_commit(buffer, event);
1295 }
1296
1297 void
1298 ftrace(struct trace_array *tr, struct trace_array_cpu *data,
1299        unsigned long ip, unsigned long parent_ip, unsigned long flags,
1300        int pc)
1301 {
1302         if (likely(!atomic_read(&data->disabled)))
1303                 trace_function(tr, ip, parent_ip, flags, pc);
1304 }
1305
1306 #ifdef CONFIG_STACKTRACE
1307
1308 #define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1309 struct ftrace_stack {
1310         unsigned long           calls[FTRACE_STACK_MAX_ENTRIES];
1311 };
1312
1313 static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
1314 static DEFINE_PER_CPU(int, ftrace_stack_reserve);
1315
1316 static void __ftrace_trace_stack(struct ring_buffer *buffer,
1317                                  unsigned long flags,
1318                                  int skip, int pc, struct pt_regs *regs)
1319 {
1320         struct ftrace_event_call *call = &event_kernel_stack;
1321         struct ring_buffer_event *event;
1322         struct stack_entry *entry;
1323         struct stack_trace trace;
1324         int use_stack;
1325         int size = FTRACE_STACK_ENTRIES;
1326
1327         trace.nr_entries        = 0;
1328         trace.skip              = skip;
1329
1330         /*
1331          * Since events can happen in NMIs there's no safe way to
1332          * use the per cpu ftrace_stacks. We reserve it and if an interrupt
1333          * or NMI comes in, it will just have to use the default
1334          * FTRACE_STACK_SIZE.
1335          */
1336         preempt_disable_notrace();
1337
1338         use_stack = ++__get_cpu_var(ftrace_stack_reserve);
1339         /*
1340          * We don't need any atomic variables, just a barrier.
1341          * If an interrupt comes in, we don't care, because it would
1342          * have exited and put the counter back to what we want.
1343          * We just need a barrier to keep gcc from moving things
1344          * around.
1345          */
1346         barrier();
1347         if (use_stack == 1) {
1348                 trace.entries           = &__get_cpu_var(ftrace_stack).calls[0];
1349                 trace.max_entries       = FTRACE_STACK_MAX_ENTRIES;
1350
1351                 if (regs)
1352                         save_stack_trace_regs(regs, &trace);
1353                 else
1354                         save_stack_trace(&trace);
1355
1356                 if (trace.nr_entries > size)
1357                         size = trace.nr_entries;
1358         } else
1359                 /* From now on, use_stack is a boolean */
1360                 use_stack = 0;
1361
1362         size *= sizeof(unsigned long);
1363
1364         event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1365                                           sizeof(*entry) + size, flags, pc);
1366         if (!event)
1367                 goto out;
1368         entry = ring_buffer_event_data(event);
1369
1370         memset(&entry->caller, 0, size);
1371
1372         if (use_stack)
1373                 memcpy(&entry->caller, trace.entries,
1374                        trace.nr_entries * sizeof(unsigned long));
1375         else {
1376                 trace.max_entries       = FTRACE_STACK_ENTRIES;
1377                 trace.entries           = entry->caller;
1378                 if (regs)
1379                         save_stack_trace_regs(regs, &trace);
1380                 else
1381                         save_stack_trace(&trace);
1382         }
1383
1384         entry->size = trace.nr_entries;
1385
1386         if (!filter_check_discard(call, entry, buffer, event))
1387                 __buffer_unlock_commit(buffer, event);
1388
1389  out:
1390         /* Again, don't let gcc optimize things here */
1391         barrier();
1392         __get_cpu_var(ftrace_stack_reserve)--;
1393         preempt_enable_notrace();
1394
1395 }
1396
1397 void ftrace_trace_stack_regs(struct ring_buffer *buffer, unsigned long flags,
1398                              int skip, int pc, struct pt_regs *regs)
1399 {
1400         if (!(trace_flags & TRACE_ITER_STACKTRACE))
1401                 return;
1402
1403         __ftrace_trace_stack(buffer, flags, skip, pc, regs);
1404 }
1405
1406 void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
1407                         int skip, int pc)
1408 {
1409         if (!(trace_flags & TRACE_ITER_STACKTRACE))
1410                 return;
1411
1412         __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
1413 }
1414
1415 void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1416                    int pc)
1417 {
1418         __ftrace_trace_stack(tr->buffer, flags, skip, pc, NULL);
1419 }
1420
1421 /**
1422  * trace_dump_stack - record a stack back trace in the trace buffer
1423  */
1424 void trace_dump_stack(void)
1425 {
1426         unsigned long flags;
1427
1428         if (tracing_disabled || tracing_selftest_running)
1429                 return;
1430
1431         local_save_flags(flags);
1432
1433         /* skipping 3 traces, seems to get us at the caller of this function */
1434         __ftrace_trace_stack(global_trace.buffer, flags, 3, preempt_count(), NULL);
1435 }
1436
1437 static DEFINE_PER_CPU(int, user_stack_count);
1438
1439 void
1440 ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
1441 {
1442         struct ftrace_event_call *call = &event_user_stack;
1443         struct ring_buffer_event *event;
1444         struct userstack_entry *entry;
1445         struct stack_trace trace;
1446
1447         if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1448                 return;
1449
1450         /*
1451          * NMIs can not handle page faults, even with fix ups.
1452          * The save user stack can (and often does) fault.
1453          */
1454         if (unlikely(in_nmi()))
1455                 return;
1456
1457         /*
1458          * prevent recursion, since the user stack tracing may
1459          * trigger other kernel events.
1460          */
1461         preempt_disable();
1462         if (__this_cpu_read(user_stack_count))
1463                 goto out;
1464
1465         __this_cpu_inc(user_stack_count);
1466
1467         event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
1468                                           sizeof(*entry), flags, pc);
1469         if (!event)
1470                 goto out_drop_count;
1471         entry   = ring_buffer_event_data(event);
1472
1473         entry->tgid             = current->tgid;
1474         memset(&entry->caller, 0, sizeof(entry->caller));
1475
1476         trace.nr_entries        = 0;
1477         trace.max_entries       = FTRACE_STACK_ENTRIES;
1478         trace.skip              = 0;
1479         trace.entries           = entry->caller;
1480
1481         save_stack_trace_user(&trace);
1482         if (!filter_check_discard(call, entry, buffer, event))
1483                 __buffer_unlock_commit(buffer, event);
1484
1485  out_drop_count:
1486         __this_cpu_dec(user_stack_count);
1487  out:
1488         preempt_enable();
1489 }
1490
1491 #ifdef UNUSED
1492 static void __trace_userstack(struct trace_array *tr, unsigned long flags)
1493 {
1494         ftrace_trace_userstack(tr, flags, preempt_count());
1495 }
1496 #endif /* UNUSED */
1497
1498 #endif /* CONFIG_STACKTRACE */
1499
1500 /* created for use with alloc_percpu */
1501 struct trace_buffer_struct {
1502         char buffer[TRACE_BUF_SIZE];
1503 };
1504
1505 static struct trace_buffer_struct *trace_percpu_buffer;
1506 static struct trace_buffer_struct *trace_percpu_sirq_buffer;
1507 static struct trace_buffer_struct *trace_percpu_irq_buffer;
1508 static struct trace_buffer_struct *trace_percpu_nmi_buffer;
1509
1510 /*
1511  * The buffer used is dependent on the context. There is a per cpu
1512  * buffer for normal context, softirq contex, hard irq context and
1513  * for NMI context. Thise allows for lockless recording.
1514  *
1515  * Note, if the buffers failed to be allocated, then this returns NULL
1516  */
1517 static char *get_trace_buf(void)
1518 {
1519         struct trace_buffer_struct *percpu_buffer;
1520         struct trace_buffer_struct *buffer;
1521
1522         /*
1523          * If we have allocated per cpu buffers, then we do not
1524          * need to do any locking.
1525          */
1526         if (in_nmi())
1527                 percpu_buffer = trace_percpu_nmi_buffer;
1528         else if (in_irq())
1529                 percpu_buffer = trace_percpu_irq_buffer;
1530         else if (in_softirq())
1531                 percpu_buffer = trace_percpu_sirq_buffer;
1532         else
1533                 percpu_buffer = trace_percpu_buffer;
1534
1535         if (!percpu_buffer)
1536                 return NULL;
1537
1538         buffer = per_cpu_ptr(percpu_buffer, smp_processor_id());
1539
1540         return buffer->buffer;
1541 }
1542
1543 static int alloc_percpu_trace_buffer(void)
1544 {
1545         struct trace_buffer_struct *buffers;
1546         struct trace_buffer_struct *sirq_buffers;
1547         struct trace_buffer_struct *irq_buffers;
1548         struct trace_buffer_struct *nmi_buffers;
1549
1550         buffers = alloc_percpu(struct trace_buffer_struct);
1551         if (!buffers)
1552                 goto err_warn;
1553
1554         sirq_buffers = alloc_percpu(struct trace_buffer_struct);
1555         if (!sirq_buffers)
1556                 goto err_sirq;
1557
1558         irq_buffers = alloc_percpu(struct trace_buffer_struct);
1559         if (!irq_buffers)
1560                 goto err_irq;
1561
1562         nmi_buffers = alloc_percpu(struct trace_buffer_struct);
1563         if (!nmi_buffers)
1564                 goto err_nmi;
1565
1566         trace_percpu_buffer = buffers;
1567         trace_percpu_sirq_buffer = sirq_buffers;
1568         trace_percpu_irq_buffer = irq_buffers;
1569         trace_percpu_nmi_buffer = nmi_buffers;
1570
1571         return 0;
1572
1573  err_nmi:
1574         free_percpu(irq_buffers);
1575  err_irq:
1576         free_percpu(sirq_buffers);
1577  err_sirq:
1578         free_percpu(buffers);
1579  err_warn:
1580         WARN(1, "Could not allocate percpu trace_printk buffer");
1581         return -ENOMEM;
1582 }
1583
1584 static int buffers_allocated;
1585
1586 void trace_printk_init_buffers(void)
1587 {
1588         if (buffers_allocated)
1589                 return;
1590
1591         if (alloc_percpu_trace_buffer())
1592                 return;
1593
1594         pr_info("ftrace: Allocated trace_printk buffers\n");
1595
1596         /* Expand the buffers to set size */
1597         tracing_update_buffers();
1598
1599         buffers_allocated = 1;
1600
1601         /*
1602          * trace_printk_init_buffers() can be called by modules.
1603          * If that happens, then we need to start cmdline recording
1604          * directly here. If the global_trace.buffer is already
1605          * allocated here, then this was called by module code.
1606          */
1607         if (global_trace.buffer)
1608                 tracing_start_cmdline_record();
1609 }
1610
1611 void trace_printk_start_comm(void)
1612 {
1613         /* Start tracing comms if trace printk is set */
1614         if (!buffers_allocated)
1615                 return;
1616         tracing_start_cmdline_record();
1617 }
1618
1619 static void trace_printk_start_stop_comm(int enabled)
1620 {
1621         if (!buffers_allocated)
1622                 return;
1623
1624         if (enabled)
1625                 tracing_start_cmdline_record();
1626         else
1627                 tracing_stop_cmdline_record();
1628 }
1629
1630 /**
1631  * trace_vbprintk - write binary msg to tracing buffer
1632  *
1633  */
1634 int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
1635 {
1636         struct ftrace_event_call *call = &event_bprint;
1637         struct ring_buffer_event *event;
1638         struct ring_buffer *buffer;
1639         struct trace_array *tr = &global_trace;
1640         struct bprint_entry *entry;
1641         unsigned long flags;
1642         char *tbuffer;
1643         int len = 0, size, pc;
1644
1645         if (unlikely(tracing_selftest_running || tracing_disabled))
1646                 return 0;
1647
1648         /* Don't pollute graph traces with trace_vprintk internals */
1649         pause_graph_tracing();
1650
1651         pc = preempt_count();
1652         preempt_disable_notrace();
1653
1654         tbuffer = get_trace_buf();
1655         if (!tbuffer) {
1656                 len = 0;
1657                 goto out;
1658         }
1659
1660         len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
1661
1662         if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
1663                 goto out;
1664
1665         local_save_flags(flags);
1666         size = sizeof(*entry) + sizeof(u32) * len;
1667         buffer = tr->buffer;
1668         event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
1669                                           flags, pc);
1670         if (!event)
1671                 goto out;
1672         entry = ring_buffer_event_data(event);
1673         entry->ip                       = ip;
1674         entry->fmt                      = fmt;
1675
1676         memcpy(entry->buf, tbuffer, sizeof(u32) * len);
1677         if (!filter_check_discard(call, entry, buffer, event)) {
1678                 __buffer_unlock_commit(buffer, event);
1679                 ftrace_trace_stack(buffer, flags, 6, pc);
1680         }
1681
1682 out:
1683         preempt_enable_notrace();
1684         unpause_graph_tracing();
1685
1686         return len;
1687 }
1688 EXPORT_SYMBOL_GPL(trace_vbprintk);
1689
1690 int trace_array_printk(struct trace_array *tr,
1691                        unsigned long ip, const char *fmt, ...)
1692 {
1693         int ret;
1694         va_list ap;
1695
1696         if (!(trace_flags & TRACE_ITER_PRINTK))
1697                 return 0;
1698
1699         va_start(ap, fmt);
1700         ret = trace_array_vprintk(tr, ip, fmt, ap);
1701         va_end(ap);
1702         return ret;
1703 }
1704
1705 int trace_array_vprintk(struct trace_array *tr,
1706                         unsigned long ip, const char *fmt, va_list args)
1707 {
1708         struct ftrace_event_call *call = &event_print;
1709         struct ring_buffer_event *event;
1710         struct ring_buffer *buffer;
1711         int len = 0, size, pc;
1712         struct print_entry *entry;
1713         unsigned long flags;
1714         char *tbuffer;
1715
1716         if (tracing_disabled || tracing_selftest_running)
1717                 return 0;
1718
1719         /* Don't pollute graph traces with trace_vprintk internals */
1720         pause_graph_tracing();
1721
1722         pc = preempt_count();
1723         preempt_disable_notrace();
1724
1725
1726         tbuffer = get_trace_buf();
1727         if (!tbuffer) {
1728                 len = 0;
1729                 goto out;
1730         }
1731
1732         len = vsnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
1733         if (len > TRACE_BUF_SIZE)
1734                 goto out;
1735
1736         local_save_flags(flags);
1737         size = sizeof(*entry) + len + 1;
1738         buffer = tr->buffer;
1739         event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
1740                                           flags, pc);
1741         if (!event)
1742                 goto out;
1743         entry = ring_buffer_event_data(event);
1744         entry->ip = ip;
1745
1746         memcpy(&entry->buf, tbuffer, len);
1747         entry->buf[len] = '\0';
1748         if (!filter_check_discard(call, entry, buffer, event)) {
1749                 __buffer_unlock_commit(buffer, event);
1750                 ftrace_trace_stack(buffer, flags, 6, pc);
1751         }
1752  out:
1753         preempt_enable_notrace();
1754         unpause_graph_tracing();
1755
1756         return len;
1757 }
1758
1759 int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
1760 {
1761         return trace_array_vprintk(&global_trace, ip, fmt, args);
1762 }
1763 EXPORT_SYMBOL_GPL(trace_vprintk);
1764
1765 static void trace_iterator_increment(struct trace_iterator *iter)
1766 {
1767         struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
1768
1769         iter->idx++;
1770         if (buf_iter)
1771                 ring_buffer_read(buf_iter, NULL);
1772 }
1773
1774 static struct trace_entry *
1775 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
1776                 unsigned long *lost_events)
1777 {
1778         struct ring_buffer_event *event;
1779         struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
1780
1781         if (buf_iter)
1782                 event = ring_buffer_iter_peek(buf_iter, ts);
1783         else
1784                 event = ring_buffer_peek(iter->tr->buffer, cpu, ts,
1785                                          lost_events);
1786
1787         if (event) {
1788                 iter->ent_size = ring_buffer_event_length(event);
1789                 return ring_buffer_event_data(event);
1790         }
1791         iter->ent_size = 0;
1792         return NULL;
1793 }
1794
1795 static struct trace_entry *
1796 __find_next_entry(struct trace_iterator *iter, int *ent_cpu,
1797                   unsigned long *missing_events, u64 *ent_ts)
1798 {
1799         struct ring_buffer *buffer = iter->tr->buffer;
1800         struct trace_entry *ent, *next = NULL;
1801         unsigned long lost_events = 0, next_lost = 0;
1802         int cpu_file = iter->cpu_file;
1803         u64 next_ts = 0, ts;
1804         int next_cpu = -1;
1805         int next_size = 0;
1806         int cpu;
1807
1808         /*
1809          * If we are in a per_cpu trace file, don't bother by iterating over
1810          * all cpu and peek directly.
1811          */
1812         if (cpu_file > TRACE_PIPE_ALL_CPU) {
1813                 if (ring_buffer_empty_cpu(buffer, cpu_file))
1814                         return NULL;
1815                 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
1816                 if (ent_cpu)
1817                         *ent_cpu = cpu_file;
1818
1819                 return ent;
1820         }
1821
1822         for_each_tracing_cpu(cpu) {
1823
1824                 if (ring_buffer_empty_cpu(buffer, cpu))
1825                         continue;
1826
1827                 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
1828
1829                 /*
1830                  * Pick the entry with the smallest timestamp:
1831                  */
1832                 if (ent && (!next || ts < next_ts)) {
1833                         next = ent;
1834                         next_cpu = cpu;
1835                         next_ts = ts;
1836                         next_lost = lost_events;
1837                         next_size = iter->ent_size;
1838                 }
1839         }
1840
1841         iter->ent_size = next_size;
1842
1843         if (ent_cpu)
1844                 *ent_cpu = next_cpu;
1845
1846         if (ent_ts)
1847                 *ent_ts = next_ts;
1848
1849         if (missing_events)
1850                 *missing_events = next_lost;
1851
1852         return next;
1853 }
1854
1855 /* Find the next real entry, without updating the iterator itself */
1856 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
1857                                           int *ent_cpu, u64 *ent_ts)
1858 {
1859         return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
1860 }
1861
1862 /* Find the next real entry, and increment the iterator to the next entry */
1863 void *trace_find_next_entry_inc(struct trace_iterator *iter)
1864 {
1865         iter->ent = __find_next_entry(iter, &iter->cpu,
1866                                       &iter->lost_events, &iter->ts);
1867
1868         if (iter->ent)
1869                 trace_iterator_increment(iter);
1870
1871         return iter->ent ? iter : NULL;
1872 }
1873
1874 static void trace_consume(struct trace_iterator *iter)
1875 {
1876         ring_buffer_consume(iter->tr->buffer, iter->cpu, &iter->ts,
1877                             &iter->lost_events);
1878 }
1879
1880 static void *s_next(struct seq_file *m, void *v, loff_t *pos)
1881 {
1882         struct trace_iterator *iter = m->private;
1883         int i = (int)*pos;
1884         void *ent;
1885
1886         WARN_ON_ONCE(iter->leftover);
1887
1888         (*pos)++;
1889
1890         /* can't go backwards */
1891         if (iter->idx > i)
1892                 return NULL;
1893
1894         if (iter->idx < 0)
1895                 ent = trace_find_next_entry_inc(iter);
1896         else
1897                 ent = iter;
1898
1899         while (ent && iter->idx < i)
1900                 ent = trace_find_next_entry_inc(iter);
1901
1902         iter->pos = *pos;
1903
1904         return ent;
1905 }
1906
1907 void tracing_iter_reset(struct trace_iterator *iter, int cpu)
1908 {
1909         struct trace_array *tr = iter->tr;
1910         struct ring_buffer_event *event;
1911         struct ring_buffer_iter *buf_iter;
1912         unsigned long entries = 0;
1913         u64 ts;
1914
1915         tr->data[cpu]->skipped_entries = 0;
1916
1917         buf_iter = trace_buffer_iter(iter, cpu);
1918         if (!buf_iter)
1919                 return;
1920
1921         ring_buffer_iter_reset(buf_iter);
1922
1923         /*
1924          * We could have the case with the max latency tracers
1925          * that a reset never took place on a cpu. This is evident
1926          * by the timestamp being before the start of the buffer.
1927          */
1928         while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
1929                 if (ts >= iter->tr->time_start)
1930                         break;
1931                 entries++;
1932                 ring_buffer_read(buf_iter, NULL);
1933         }
1934
1935         tr->data[cpu]->skipped_entries = entries;
1936 }
1937
1938 /*
1939  * The current tracer is copied to avoid a global locking
1940  * all around.
1941  */
1942 static void *s_start(struct seq_file *m, loff_t *pos)
1943 {
1944         struct trace_iterator *iter = m->private;
1945         static struct tracer *old_tracer;
1946         int cpu_file = iter->cpu_file;
1947         void *p = NULL;
1948         loff_t l = 0;
1949         int cpu;
1950
1951         /* copy the tracer to avoid using a global lock all around */
1952         mutex_lock(&trace_types_lock);
1953         if (unlikely(old_tracer != current_trace && current_trace)) {
1954                 old_tracer = current_trace;
1955                 *iter->trace = *current_trace;
1956         }
1957         mutex_unlock(&trace_types_lock);
1958
1959         atomic_inc(&trace_record_cmdline_disabled);
1960
1961         if (*pos != iter->pos) {
1962                 iter->ent = NULL;
1963                 iter->cpu = 0;
1964                 iter->idx = -1;
1965
1966                 if (cpu_file == TRACE_PIPE_ALL_CPU) {
1967                         for_each_tracing_cpu(cpu)
1968                                 tracing_iter_reset(iter, cpu);
1969                 } else
1970                         tracing_iter_reset(iter, cpu_file);
1971
1972                 iter->leftover = 0;
1973                 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
1974                         ;
1975
1976         } else {
1977                 /*
1978                  * If we overflowed the seq_file before, then we want
1979                  * to just reuse the trace_seq buffer again.
1980                  */
1981                 if (iter->leftover)
1982                         p = iter;
1983                 else {
1984                         l = *pos - 1;
1985                         p = s_next(m, p, &l);
1986                 }
1987         }
1988
1989         trace_event_read_lock();
1990         trace_access_lock(cpu_file);
1991         return p;
1992 }
1993
1994 static void s_stop(struct seq_file *m, void *p)
1995 {
1996         struct trace_iterator *iter = m->private;
1997
1998         atomic_dec(&trace_record_cmdline_disabled);
1999         trace_access_unlock(iter->cpu_file);
2000         trace_event_read_unlock();
2001 }
2002
2003 static void
2004 get_total_entries(struct trace_array *tr, unsigned long *total, unsigned long *entries)
2005 {
2006         unsigned long count;
2007         int cpu;
2008
2009         *total = 0;
2010         *entries = 0;
2011
2012         for_each_tracing_cpu(cpu) {
2013                 count = ring_buffer_entries_cpu(tr->buffer, cpu);
2014                 /*
2015                  * If this buffer has skipped entries, then we hold all
2016                  * entries for the trace and we need to ignore the
2017                  * ones before the time stamp.
2018                  */
2019                 if (tr->data[cpu]->skipped_entries) {
2020                         count -= tr->data[cpu]->skipped_entries;
2021                         /* total is the same as the entries */
2022                         *total += count;
2023                 } else
2024                         *total += count +
2025                                 ring_buffer_overrun_cpu(tr->buffer, cpu);
2026                 *entries += count;
2027         }
2028 }
2029
2030 static void print_lat_help_header(struct seq_file *m)
2031 {
2032         seq_puts(m, "#                  _------=> CPU#            \n");
2033         seq_puts(m, "#                 / _-----=> irqs-off        \n");
2034         seq_puts(m, "#                | / _----=> need-resched    \n");
2035         seq_puts(m, "#                || / _---=> hardirq/softirq \n");
2036         seq_puts(m, "#                ||| / _--=> preempt-depth   \n");
2037         seq_puts(m, "#                |||| /     delay             \n");
2038         seq_puts(m, "#  cmd     pid   ||||| time  |   caller      \n");
2039         seq_puts(m, "#     \\   /      |||||  \\    |   /           \n");
2040 }
2041
2042 static void print_event_info(struct trace_array *tr, struct seq_file *m)
2043 {
2044         unsigned long total;
2045         unsigned long entries;
2046
2047         get_total_entries(tr, &total, &entries);
2048         seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu   #P:%d\n",
2049                    entries, total, num_online_cpus());
2050         seq_puts(m, "#\n");
2051 }
2052
2053 static void print_func_help_header(struct trace_array *tr, struct seq_file *m)
2054 {
2055         print_event_info(tr, m);
2056         seq_puts(m, "#           TASK-PID   CPU#      TIMESTAMP  FUNCTION\n");
2057         seq_puts(m, "#              | |       |          |         |\n");
2058 }
2059
2060 static void print_func_help_header_irq(struct trace_array *tr, struct seq_file *m)
2061 {
2062         print_event_info(tr, m);
2063         seq_puts(m, "#                              _-----=> irqs-off\n");
2064         seq_puts(m, "#                             / _----=> need-resched\n");
2065         seq_puts(m, "#                            | / _---=> hardirq/softirq\n");
2066         seq_puts(m, "#                            || / _--=> preempt-depth\n");
2067         seq_puts(m, "#                            ||| /     delay\n");
2068         seq_puts(m, "#           TASK-PID   CPU#  ||||    TIMESTAMP  FUNCTION\n");
2069         seq_puts(m, "#              | |       |   ||||       |         |\n");
2070 }
2071
2072 void
2073 print_trace_header(struct seq_file *m, struct trace_iterator *iter)
2074 {
2075         unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2076         struct trace_array *tr = iter->tr;
2077         struct trace_array_cpu *data = tr->data[tr->cpu];
2078         struct tracer *type = current_trace;
2079         unsigned long entries;
2080         unsigned long total;
2081         const char *name = "preemption";
2082
2083         if (type)
2084                 name = type->name;
2085
2086         get_total_entries(tr, &total, &entries);
2087
2088         seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
2089                    name, UTS_RELEASE);
2090         seq_puts(m, "# -----------------------------------"
2091                  "---------------------------------\n");
2092         seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
2093                    " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
2094                    nsecs_to_usecs(data->saved_latency),
2095                    entries,
2096                    total,
2097                    tr->cpu,
2098 #if defined(CONFIG_PREEMPT_NONE)
2099                    "server",
2100 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
2101                    "desktop",
2102 #elif defined(CONFIG_PREEMPT)
2103                    "preempt",
2104 #else
2105                    "unknown",
2106 #endif
2107                    /* These are reserved for later use */
2108                    0, 0, 0, 0);
2109 #ifdef CONFIG_SMP
2110         seq_printf(m, " #P:%d)\n", num_online_cpus());
2111 #else
2112         seq_puts(m, ")\n");
2113 #endif
2114         seq_puts(m, "#    -----------------\n");
2115         seq_printf(m, "#    | task: %.16s-%d "
2116                    "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
2117                    data->comm, data->pid,
2118                    from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
2119                    data->policy, data->rt_priority);
2120         seq_puts(m, "#    -----------------\n");
2121
2122         if (data->critical_start) {
2123                 seq_puts(m, "#  => started at: ");
2124                 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
2125                 trace_print_seq(m, &iter->seq);
2126                 seq_puts(m, "\n#  => ended at:   ");
2127                 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
2128                 trace_print_seq(m, &iter->seq);
2129                 seq_puts(m, "\n#\n");
2130         }
2131
2132         seq_puts(m, "#\n");
2133 }
2134
2135 static void test_cpu_buff_start(struct trace_iterator *iter)
2136 {
2137         struct trace_seq *s = &iter->seq;
2138
2139         if (!(trace_flags & TRACE_ITER_ANNOTATE))
2140                 return;
2141
2142         if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
2143                 return;
2144
2145         if (cpumask_test_cpu(iter->cpu, iter->started))
2146                 return;
2147
2148         if (iter->tr->data[iter->cpu]->skipped_entries)
2149                 return;
2150
2151         cpumask_set_cpu(iter->cpu, iter->started);
2152
2153         /* Don't print started cpu buffer for the first entry of the trace */
2154         if (iter->idx > 1)
2155                 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
2156                                 iter->cpu);
2157 }
2158
2159 static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
2160 {
2161         struct trace_seq *s = &iter->seq;
2162         unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2163         struct trace_entry *entry;
2164         struct trace_event *event;
2165
2166         entry = iter->ent;
2167
2168         test_cpu_buff_start(iter);
2169
2170         event = ftrace_find_event(entry->type);
2171
2172         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2173                 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2174                         if (!trace_print_lat_context(iter))
2175                                 goto partial;
2176                 } else {
2177                         if (!trace_print_context(iter))
2178                                 goto partial;
2179                 }
2180         }
2181
2182         if (event)
2183                 return event->funcs->trace(iter, sym_flags, event);
2184
2185         if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
2186                 goto partial;
2187
2188         return TRACE_TYPE_HANDLED;
2189 partial:
2190         return TRACE_TYPE_PARTIAL_LINE;
2191 }
2192
2193 static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
2194 {
2195         struct trace_seq *s = &iter->seq;
2196         struct trace_entry *entry;
2197         struct trace_event *event;
2198
2199         entry = iter->ent;
2200
2201         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2202                 if (!trace_seq_printf(s, "%d %d %llu ",
2203                                       entry->pid, iter->cpu, iter->ts))
2204                         goto partial;
2205         }
2206
2207         event = ftrace_find_event(entry->type);
2208         if (event)
2209                 return event->funcs->raw(iter, 0, event);
2210
2211         if (!trace_seq_printf(s, "%d ?\n", entry->type))
2212                 goto partial;
2213
2214         return TRACE_TYPE_HANDLED;
2215 partial:
2216         return TRACE_TYPE_PARTIAL_LINE;
2217 }
2218
2219 static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
2220 {
2221         struct trace_seq *s = &iter->seq;
2222         unsigned char newline = '\n';
2223         struct trace_entry *entry;
2224         struct trace_event *event;
2225
2226         entry = iter->ent;
2227
2228         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2229                 SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
2230                 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
2231                 SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
2232         }
2233
2234         event = ftrace_find_event(entry->type);
2235         if (event) {
2236                 enum print_line_t ret = event->funcs->hex(iter, 0, event);
2237                 if (ret != TRACE_TYPE_HANDLED)
2238                         return ret;
2239         }
2240
2241         SEQ_PUT_FIELD_RET(s, newline);
2242
2243         return TRACE_TYPE_HANDLED;
2244 }
2245
2246 static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
2247 {
2248         struct trace_seq *s = &iter->seq;
2249         struct trace_entry *entry;
2250         struct trace_event *event;
2251
2252         entry = iter->ent;
2253
2254         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2255                 SEQ_PUT_FIELD_RET(s, entry->pid);
2256                 SEQ_PUT_FIELD_RET(s, iter->cpu);
2257                 SEQ_PUT_FIELD_RET(s, iter->ts);
2258         }
2259
2260         event = ftrace_find_event(entry->type);
2261         return event ? event->funcs->binary(iter, 0, event) :
2262                 TRACE_TYPE_HANDLED;
2263 }
2264
2265 int trace_empty(struct trace_iterator *iter)
2266 {
2267         struct ring_buffer_iter *buf_iter;
2268         int cpu;
2269
2270         /* If we are looking at one CPU buffer, only check that one */
2271         if (iter->cpu_file != TRACE_PIPE_ALL_CPU) {
2272                 cpu = iter->cpu_file;
2273                 buf_iter = trace_buffer_iter(iter, cpu);
2274                 if (buf_iter) {
2275                         if (!ring_buffer_iter_empty(buf_iter))
2276                                 return 0;
2277                 } else {
2278                         if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
2279                                 return 0;
2280                 }
2281                 return 1;
2282         }
2283
2284         for_each_tracing_cpu(cpu) {
2285                 buf_iter = trace_buffer_iter(iter, cpu);
2286                 if (buf_iter) {
2287                         if (!ring_buffer_iter_empty(buf_iter))
2288                                 return 0;
2289                 } else {
2290                         if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
2291                                 return 0;
2292                 }
2293         }
2294
2295         return 1;
2296 }
2297
2298 /*  Called with trace_event_read_lock() held. */
2299 enum print_line_t print_trace_line(struct trace_iterator *iter)
2300 {
2301         enum print_line_t ret;
2302
2303         if (iter->lost_events &&
2304             !trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2305                                  iter->cpu, iter->lost_events))
2306                 return TRACE_TYPE_PARTIAL_LINE;
2307
2308         if (iter->trace && iter->trace->print_line) {
2309                 ret = iter->trace->print_line(iter);
2310                 if (ret != TRACE_TYPE_UNHANDLED)
2311                         return ret;
2312         }
2313
2314         if (iter->ent->type == TRACE_BPRINT &&
2315                         trace_flags & TRACE_ITER_PRINTK &&
2316                         trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2317                 return trace_print_bprintk_msg_only(iter);
2318
2319         if (iter->ent->type == TRACE_PRINT &&
2320                         trace_flags & TRACE_ITER_PRINTK &&
2321                         trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2322                 return trace_print_printk_msg_only(iter);
2323
2324         if (trace_flags & TRACE_ITER_BIN)
2325                 return print_bin_fmt(iter);
2326
2327         if (trace_flags & TRACE_ITER_HEX)
2328                 return print_hex_fmt(iter);
2329
2330         if (trace_flags & TRACE_ITER_RAW)
2331                 return print_raw_fmt(iter);
2332
2333         return print_trace_fmt(iter);
2334 }
2335
2336 void trace_latency_header(struct seq_file *m)
2337 {
2338         struct trace_iterator *iter = m->private;
2339
2340         /* print nothing if the buffers are empty */
2341         if (trace_empty(iter))
2342                 return;
2343
2344         if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2345                 print_trace_header(m, iter);
2346
2347         if (!(trace_flags & TRACE_ITER_VERBOSE))
2348                 print_lat_help_header(m);
2349 }
2350
2351 void trace_default_header(struct seq_file *m)
2352 {
2353         struct trace_iterator *iter = m->private;
2354
2355         if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
2356                 return;
2357
2358         if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2359                 /* print nothing if the buffers are empty */
2360                 if (trace_empty(iter))
2361                         return;
2362                 print_trace_header(m, iter);
2363                 if (!(trace_flags & TRACE_ITER_VERBOSE))
2364                         print_lat_help_header(m);
2365         } else {
2366                 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
2367                         if (trace_flags & TRACE_ITER_IRQ_INFO)
2368                                 print_func_help_header_irq(iter->tr, m);
2369                         else
2370                                 print_func_help_header(iter->tr, m);
2371                 }
2372         }
2373 }
2374
2375 static void test_ftrace_alive(struct seq_file *m)
2376 {
2377         if (!ftrace_is_dead())
2378                 return;
2379         seq_printf(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n");
2380         seq_printf(m, "#          MAY BE MISSING FUNCTION EVENTS\n");
2381 }
2382
2383 static int s_show(struct seq_file *m, void *v)
2384 {
2385         struct trace_iterator *iter = v;
2386         int ret;
2387
2388         if (iter->ent == NULL) {
2389                 if (iter->tr) {
2390                         seq_printf(m, "# tracer: %s\n", iter->trace->name);
2391                         seq_puts(m, "#\n");
2392                         test_ftrace_alive(m);
2393                 }
2394                 if (iter->trace && iter->trace->print_header)
2395                         iter->trace->print_header(m);
2396                 else
2397                         trace_default_header(m);
2398
2399         } else if (iter->leftover) {
2400                 /*
2401                  * If we filled the seq_file buffer earlier, we
2402                  * want to just show it now.
2403                  */
2404                 ret = trace_print_seq(m, &iter->seq);
2405
2406                 /* ret should this time be zero, but you never know */
2407                 iter->leftover = ret;
2408
2409         } else {
2410                 print_trace_line(iter);
2411                 ret = trace_print_seq(m, &iter->seq);
2412                 /*
2413                  * If we overflow the seq_file buffer, then it will
2414                  * ask us for this data again at start up.
2415                  * Use that instead.
2416                  *  ret is 0 if seq_file write succeeded.
2417                  *        -1 otherwise.
2418                  */
2419                 iter->leftover = ret;
2420         }
2421
2422         return 0;
2423 }
2424
2425 static const struct seq_operations tracer_seq_ops = {
2426         .start          = s_start,
2427         .next           = s_next,
2428         .stop           = s_stop,
2429         .show           = s_show,
2430 };
2431
2432 static struct trace_iterator *
2433 __tracing_open(struct inode *inode, struct file *file)
2434 {
2435         long cpu_file = (long) inode->i_private;
2436         struct trace_iterator *iter;
2437         int cpu;
2438
2439         if (tracing_disabled)
2440                 return ERR_PTR(-ENODEV);
2441
2442         iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
2443         if (!iter)
2444                 return ERR_PTR(-ENOMEM);
2445
2446         iter->buffer_iter = kzalloc(sizeof(*iter->buffer_iter) * num_possible_cpus(),
2447                                     GFP_KERNEL);
2448         if (!iter->buffer_iter)
2449                 goto release;
2450
2451         /*
2452          * We make a copy of the current tracer to avoid concurrent
2453          * changes on it while we are reading.
2454          */
2455         mutex_lock(&trace_types_lock);
2456         iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
2457         if (!iter->trace)
2458                 goto fail;
2459
2460         if (current_trace)
2461                 *iter->trace = *current_trace;
2462
2463         if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
2464                 goto fail;
2465
2466         if (current_trace && current_trace->print_max)
2467                 iter->tr = &max_tr;
2468         else
2469                 iter->tr = &global_trace;
2470         iter->pos = -1;
2471         mutex_init(&iter->mutex);
2472         iter->cpu_file = cpu_file;
2473
2474         /* Notify the tracer early; before we stop tracing. */
2475         if (iter->trace && iter->trace->open)
2476                 iter->trace->open(iter);
2477
2478         /* Annotate start of buffers if we had overruns */
2479         if (ring_buffer_overruns(iter->tr->buffer))
2480                 iter->iter_flags |= TRACE_FILE_ANNOTATE;
2481
2482         /* Output in nanoseconds only if we are using a clock in nanoseconds. */
2483         if (trace_clocks[trace_clock_id].in_ns)
2484                 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
2485
2486         /* stop the trace while dumping */
2487         tracing_stop();
2488
2489         if (iter->cpu_file == TRACE_PIPE_ALL_CPU) {
2490                 for_each_tracing_cpu(cpu) {
2491                         iter->buffer_iter[cpu] =
2492                                 ring_buffer_read_prepare(iter->tr->buffer, cpu);
2493                 }
2494                 ring_buffer_read_prepare_sync();
2495                 for_each_tracing_cpu(cpu) {
2496                         ring_buffer_read_start(iter->buffer_iter[cpu]);
2497                         tracing_iter_reset(iter, cpu);
2498                 }
2499         } else {
2500                 cpu = iter->cpu_file;
2501                 iter->buffer_iter[cpu] =
2502                         ring_buffer_read_prepare(iter->tr->buffer, cpu);
2503                 ring_buffer_read_prepare_sync();
2504                 ring_buffer_read_start(iter->buffer_iter[cpu]);
2505                 tracing_iter_reset(iter, cpu);
2506         }
2507
2508         mutex_unlock(&trace_types_lock);
2509
2510         return iter;
2511
2512  fail:
2513         mutex_unlock(&trace_types_lock);
2514         kfree(iter->trace);
2515         kfree(iter->buffer_iter);
2516 release:
2517         seq_release_private(inode, file);
2518         return ERR_PTR(-ENOMEM);
2519 }
2520
2521 int tracing_open_generic(struct inode *inode, struct file *filp)
2522 {
2523         if (tracing_disabled)
2524                 return -ENODEV;
2525
2526         filp->private_data = inode->i_private;
2527         return 0;
2528 }
2529
2530 static int tracing_release(struct inode *inode, struct file *file)
2531 {
2532         struct seq_file *m = file->private_data;
2533         struct trace_iterator *iter;
2534         int cpu;
2535
2536         if (!(file->f_mode & FMODE_READ))
2537                 return 0;
2538
2539         iter = m->private;
2540
2541         mutex_lock(&trace_types_lock);
2542         for_each_tracing_cpu(cpu) {
2543                 if (iter->buffer_iter[cpu])
2544                         ring_buffer_read_finish(iter->buffer_iter[cpu]);
2545         }
2546
2547         if (iter->trace && iter->trace->close)
2548                 iter->trace->close(iter);
2549
2550         /* reenable tracing if it was previously enabled */
2551         tracing_start();
2552         mutex_unlock(&trace_types_lock);
2553
2554         mutex_destroy(&iter->mutex);
2555         free_cpumask_var(iter->started);
2556         kfree(iter->trace);
2557         kfree(iter->buffer_iter);
2558         seq_release_private(inode, file);
2559         return 0;
2560 }
2561
2562 static int tracing_open(struct inode *inode, struct file *file)
2563 {
2564         struct trace_iterator *iter;
2565         int ret = 0;
2566
2567         /* If this file was open for write, then erase contents */
2568         if ((file->f_mode & FMODE_WRITE) &&
2569             (file->f_flags & O_TRUNC)) {
2570                 long cpu = (long) inode->i_private;
2571
2572                 if (cpu == TRACE_PIPE_ALL_CPU)
2573                         tracing_reset_online_cpus(&global_trace);
2574                 else
2575                         tracing_reset(&global_trace, cpu);
2576         }
2577
2578         if (file->f_mode & FMODE_READ) {
2579                 iter = __tracing_open(inode, file);
2580                 if (IS_ERR(iter))
2581                         ret = PTR_ERR(iter);
2582                 else if (trace_flags & TRACE_ITER_LATENCY_FMT)
2583                         iter->iter_flags |= TRACE_FILE_LAT_FMT;
2584         }
2585         return ret;
2586 }
2587
2588 static void *
2589 t_next(struct seq_file *m, void *v, loff_t *pos)
2590 {
2591         struct tracer *t = v;
2592
2593         (*pos)++;
2594
2595         if (t)
2596                 t = t->next;
2597
2598         return t;
2599 }
2600
2601 static void *t_start(struct seq_file *m, loff_t *pos)
2602 {
2603         struct tracer *t;
2604         loff_t l = 0;
2605
2606         mutex_lock(&trace_types_lock);
2607         for (t = trace_types; t && l < *pos; t = t_next(m, t, &l))
2608                 ;
2609
2610         return t;
2611 }
2612
2613 static void t_stop(struct seq_file *m, void *p)
2614 {
2615         mutex_unlock(&trace_types_lock);
2616 }
2617
2618 static int t_show(struct seq_file *m, void *v)
2619 {
2620         struct tracer *t = v;
2621
2622         if (!t)
2623                 return 0;
2624
2625         seq_printf(m, "%s", t->name);
2626         if (t->next)
2627                 seq_putc(m, ' ');
2628         else
2629                 seq_putc(m, '\n');
2630
2631         return 0;
2632 }
2633
2634 static const struct seq_operations show_traces_seq_ops = {
2635         .start          = t_start,
2636         .next           = t_next,
2637         .stop           = t_stop,
2638         .show           = t_show,
2639 };
2640
2641 static int show_traces_open(struct inode *inode, struct file *file)
2642 {
2643         if (tracing_disabled)
2644                 return -ENODEV;
2645
2646         return seq_open(file, &show_traces_seq_ops);
2647 }
2648
2649 static ssize_t
2650 tracing_write_stub(struct file *filp, const char __user *ubuf,
2651                    size_t count, loff_t *ppos)
2652 {
2653         return count;
2654 }
2655
2656 static loff_t tracing_seek(struct file *file, loff_t offset, int origin)
2657 {
2658         if (file->f_mode & FMODE_READ)
2659                 return seq_lseek(file, offset, origin);
2660         else
2661                 return 0;
2662 }
2663
2664 static const struct file_operations tracing_fops = {
2665         .open           = tracing_open,
2666         .read           = seq_read,
2667         .write          = tracing_write_stub,
2668         .llseek         = tracing_seek,
2669         .release        = tracing_release,
2670 };
2671
2672 static const struct file_operations show_traces_fops = {
2673         .open           = show_traces_open,
2674         .read           = seq_read,
2675         .release        = seq_release,
2676         .llseek         = seq_lseek,
2677 };
2678
2679 /*
2680  * Only trace on a CPU if the bitmask is set:
2681  */
2682 static cpumask_var_t tracing_cpumask;
2683
2684 /*
2685  * The tracer itself will not take this lock, but still we want
2686  * to provide a consistent cpumask to user-space:
2687  */
2688 static DEFINE_MUTEX(tracing_cpumask_update_lock);
2689
2690 /*
2691  * Temporary storage for the character representation of the
2692  * CPU bitmask (and one more byte for the newline):
2693  */
2694 static char mask_str[NR_CPUS + 1];
2695
2696 static ssize_t
2697 tracing_cpumask_read(struct file *filp, char __user *ubuf,
2698                      size_t count, loff_t *ppos)
2699 {
2700         int len;
2701
2702         mutex_lock(&tracing_cpumask_update_lock);
2703
2704         len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
2705         if (count - len < 2) {
2706                 count = -EINVAL;
2707                 goto out_err;
2708         }
2709         len += sprintf(mask_str + len, "\n");
2710         count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
2711
2712 out_err:
2713         mutex_unlock(&tracing_cpumask_update_lock);
2714
2715         return count;
2716 }
2717
2718 static ssize_t
2719 tracing_cpumask_write(struct file *filp, const char __user *ubuf,
2720                       size_t count, loff_t *ppos)
2721 {
2722         int err, cpu;
2723         cpumask_var_t tracing_cpumask_new;
2724
2725         if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
2726                 return -ENOMEM;
2727
2728         err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
2729         if (err)
2730                 goto err_unlock;
2731
2732         mutex_lock(&tracing_cpumask_update_lock);
2733
2734         local_irq_disable();
2735         arch_spin_lock(&ftrace_max_lock);
2736         for_each_tracing_cpu(cpu) {
2737                 /*
2738                  * Increase/decrease the disabled counter if we are
2739                  * about to flip a bit in the cpumask:
2740                  */
2741                 if (cpumask_test_cpu(cpu, tracing_cpumask) &&
2742                                 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
2743                         atomic_inc(&global_trace.data[cpu]->disabled);
2744                         ring_buffer_record_disable_cpu(global_trace.buffer, cpu);
2745                 }
2746                 if (!cpumask_test_cpu(cpu, tracing_cpumask) &&
2747                                 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
2748                         atomic_dec(&global_trace.data[cpu]->disabled);
2749                         ring_buffer_record_enable_cpu(global_trace.buffer, cpu);
2750                 }
2751         }
2752         arch_spin_unlock(&ftrace_max_lock);
2753         local_irq_enable();
2754
2755         cpumask_copy(tracing_cpumask, tracing_cpumask_new);
2756
2757         mutex_unlock(&tracing_cpumask_update_lock);
2758         free_cpumask_var(tracing_cpumask_new);
2759
2760         return count;
2761
2762 err_unlock:
2763         free_cpumask_var(tracing_cpumask_new);
2764
2765         return err;
2766 }
2767
2768 static const struct file_operations tracing_cpumask_fops = {
2769         .open           = tracing_open_generic,
2770         .read           = tracing_cpumask_read,
2771         .write          = tracing_cpumask_write,
2772         .llseek         = generic_file_llseek,
2773 };
2774
2775 static int tracing_trace_options_show(struct seq_file *m, void *v)
2776 {
2777         struct tracer_opt *trace_opts;
2778         u32 tracer_flags;
2779         int i;
2780
2781         mutex_lock(&trace_types_lock);
2782         tracer_flags = current_trace->flags->val;
2783         trace_opts = current_trace->flags->opts;
2784
2785         for (i = 0; trace_options[i]; i++) {
2786                 if (trace_flags & (1 << i))
2787                         seq_printf(m, "%s\n", trace_options[i]);
2788                 else
2789                         seq_printf(m, "no%s\n", trace_options[i]);
2790         }
2791
2792         for (i = 0; trace_opts[i].name; i++) {
2793                 if (tracer_flags & trace_opts[i].bit)
2794                         seq_printf(m, "%s\n", trace_opts[i].name);
2795                 else
2796                         seq_printf(m, "no%s\n", trace_opts[i].name);
2797         }
2798         mutex_unlock(&trace_types_lock);
2799
2800         return 0;
2801 }
2802
2803 static int __set_tracer_option(struct tracer *trace,
2804                                struct tracer_flags *tracer_flags,
2805                                struct tracer_opt *opts, int neg)
2806 {
2807         int ret;
2808
2809         ret = trace->set_flag(tracer_flags->val, opts->bit, !neg);
2810         if (ret)
2811                 return ret;
2812
2813         if (neg)
2814                 tracer_flags->val &= ~opts->bit;
2815         else
2816                 tracer_flags->val |= opts->bit;
2817         return 0;
2818 }
2819
2820 /* Try to assign a tracer specific option */
2821 static int set_tracer_option(struct tracer *trace, char *cmp, int neg)
2822 {
2823         struct tracer_flags *tracer_flags = trace->flags;
2824         struct tracer_opt *opts = NULL;
2825         int i;
2826
2827         for (i = 0; tracer_flags->opts[i].name; i++) {
2828                 opts = &tracer_flags->opts[i];
2829
2830                 if (strcmp(cmp, opts->name) == 0)
2831                         return __set_tracer_option(trace, trace->flags,
2832                                                    opts, neg);
2833         }
2834
2835         return -EINVAL;
2836 }
2837
2838 static void set_tracer_flags(unsigned int mask, int enabled)
2839 {
2840         /* do nothing if flag is already set */
2841         if (!!(trace_flags & mask) == !!enabled)
2842                 return;
2843
2844         if (enabled)
2845                 trace_flags |= mask;
2846         else
2847                 trace_flags &= ~mask;
2848
2849         if (mask == TRACE_ITER_RECORD_CMD)
2850                 trace_event_enable_cmd_record(enabled);
2851
2852         if (mask == TRACE_ITER_OVERWRITE)
2853                 ring_buffer_change_overwrite(global_trace.buffer, enabled);
2854
2855         if (mask == TRACE_ITER_PRINTK)
2856                 trace_printk_start_stop_comm(enabled);
2857 }
2858
2859 static int trace_set_options(char *option)
2860 {
2861         char *cmp;
2862         int neg = 0;
2863         int ret = 0;
2864         int i;
2865
2866         cmp = strstrip(option);
2867
2868         if (strncmp(cmp, "no", 2) == 0) {
2869                 neg = 1;
2870                 cmp += 2;
2871         }
2872
2873         for (i = 0; trace_options[i]; i++) {
2874                 if (strcmp(cmp, trace_options[i]) == 0) {
2875                         set_tracer_flags(1 << i, !neg);
2876                         break;
2877                 }
2878         }
2879
2880         /* If no option could be set, test the specific tracer options */
2881         if (!trace_options[i]) {
2882                 mutex_lock(&trace_types_lock);
2883                 ret = set_tracer_option(current_trace, cmp, neg);
2884                 mutex_unlock(&trace_types_lock);
2885         }
2886
2887         return ret;
2888 }
2889
2890 static ssize_t
2891 tracing_trace_options_write(struct file *filp, const char __user *ubuf,
2892                         size_t cnt, loff_t *ppos)
2893 {
2894         char buf[64];
2895
2896         if (cnt >= sizeof(buf))
2897                 return -EINVAL;
2898
2899         if (copy_from_user(&buf, ubuf, cnt))
2900                 return -EFAULT;
2901
2902         trace_set_options(buf);
2903
2904         *ppos += cnt;
2905
2906         return cnt;
2907 }
2908
2909 static int tracing_trace_options_open(struct inode *inode, struct file *file)
2910 {
2911         if (tracing_disabled)
2912                 return -ENODEV;
2913         return single_open(file, tracing_trace_options_show, NULL);
2914 }
2915
2916 static const struct file_operations tracing_iter_fops = {
2917         .open           = tracing_trace_options_open,
2918         .read           = seq_read,
2919         .llseek         = seq_lseek,
2920         .release        = single_release,
2921         .write          = tracing_trace_options_write,
2922 };
2923
2924 static const char readme_msg[] =
2925         "tracing mini-HOWTO:\n\n"
2926         "# mount -t debugfs nodev /sys/kernel/debug\n\n"
2927         "# cat /sys/kernel/debug/tracing/available_tracers\n"
2928         "wakeup wakeup_rt preemptirqsoff preemptoff irqsoff function nop\n\n"
2929         "# cat /sys/kernel/debug/tracing/current_tracer\n"
2930         "nop\n"
2931         "# echo wakeup > /sys/kernel/debug/tracing/current_tracer\n"
2932         "# cat /sys/kernel/debug/tracing/current_tracer\n"
2933         "wakeup\n"
2934         "# cat /sys/kernel/debug/tracing/trace_options\n"
2935         "noprint-parent nosym-offset nosym-addr noverbose\n"
2936         "# echo print-parent > /sys/kernel/debug/tracing/trace_options\n"
2937         "# echo 1 > /sys/kernel/debug/tracing/tracing_on\n"
2938         "# cat /sys/kernel/debug/tracing/trace > /tmp/trace.txt\n"
2939         "# echo 0 > /sys/kernel/debug/tracing/tracing_on\n"
2940 ;
2941
2942 static ssize_t
2943 tracing_readme_read(struct file *filp, char __user *ubuf,
2944                        size_t cnt, loff_t *ppos)
2945 {
2946         return simple_read_from_buffer(ubuf, cnt, ppos,
2947                                         readme_msg, strlen(readme_msg));
2948 }
2949
2950 static const struct file_operations tracing_readme_fops = {
2951         .open           = tracing_open_generic,
2952         .read           = tracing_readme_read,
2953         .llseek         = generic_file_llseek,
2954 };
2955
2956 static ssize_t
2957 tracing_saved_cmdlines_read(struct file *file, char __user *ubuf,
2958                                 size_t cnt, loff_t *ppos)
2959 {
2960         char *buf_comm;
2961         char *file_buf;
2962         char *buf;
2963         int len = 0;
2964         int pid;
2965         int i;
2966
2967         file_buf = kmalloc(SAVED_CMDLINES*(16+TASK_COMM_LEN), GFP_KERNEL);
2968         if (!file_buf)
2969                 return -ENOMEM;
2970
2971         buf_comm = kmalloc(TASK_COMM_LEN, GFP_KERNEL);
2972         if (!buf_comm) {
2973                 kfree(file_buf);
2974                 return -ENOMEM;
2975         }
2976
2977         buf = file_buf;
2978
2979         for (i = 0; i < SAVED_CMDLINES; i++) {
2980                 int r;
2981
2982                 pid = map_cmdline_to_pid[i];
2983                 if (pid == -1 || pid == NO_CMDLINE_MAP)
2984                         continue;
2985
2986                 trace_find_cmdline(pid, buf_comm);
2987                 r = sprintf(buf, "%d %s\n", pid, buf_comm);
2988                 buf += r;
2989                 len += r;
2990         }
2991
2992         len = simple_read_from_buffer(ubuf, cnt, ppos,
2993                                       file_buf, len);
2994
2995         kfree(file_buf);
2996         kfree(buf_comm);
2997
2998         return len;
2999 }
3000
3001 static const struct file_operations tracing_saved_cmdlines_fops = {
3002     .open       = tracing_open_generic,
3003     .read       = tracing_saved_cmdlines_read,
3004     .llseek     = generic_file_llseek,
3005 };
3006
3007 static ssize_t
3008 tracing_set_trace_read(struct file *filp, char __user *ubuf,
3009                        size_t cnt, loff_t *ppos)
3010 {
3011         char buf[MAX_TRACER_SIZE+2];
3012         int r;
3013
3014         mutex_lock(&trace_types_lock);
3015         if (current_trace)
3016                 r = sprintf(buf, "%s\n", current_trace->name);
3017         else
3018                 r = sprintf(buf, "\n");
3019         mutex_unlock(&trace_types_lock);
3020
3021         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3022 }
3023
3024 int tracer_init(struct tracer *t, struct trace_array *tr)
3025 {
3026         tracing_reset_online_cpus(tr);
3027         return t->init(tr);
3028 }
3029
3030 static void set_buffer_entries(struct trace_array *tr, unsigned long val)
3031 {
3032         int cpu;
3033         for_each_tracing_cpu(cpu)
3034                 tr->data[cpu]->entries = val;
3035 }
3036
3037 /* resize @tr's buffer to the size of @size_tr's entries */
3038 static int resize_buffer_duplicate_size(struct trace_array *tr,
3039                                         struct trace_array *size_tr, int cpu_id)
3040 {
3041         int cpu, ret = 0;
3042
3043         if (cpu_id == RING_BUFFER_ALL_CPUS) {
3044                 for_each_tracing_cpu(cpu) {
3045                         ret = ring_buffer_resize(tr->buffer,
3046                                         size_tr->data[cpu]->entries, cpu);
3047                         if (ret < 0)
3048                                 break;
3049                         tr->data[cpu]->entries = size_tr->data[cpu]->entries;
3050                 }
3051         } else {
3052                 ret = ring_buffer_resize(tr->buffer,
3053                                         size_tr->data[cpu_id]->entries, cpu_id);
3054                 if (ret == 0)
3055                         tr->data[cpu_id]->entries =
3056                                 size_tr->data[cpu_id]->entries;
3057         }
3058
3059         return ret;
3060 }
3061
3062 static int __tracing_resize_ring_buffer(unsigned long size, int cpu)
3063 {
3064         int ret;
3065
3066         /*
3067          * If kernel or user changes the size of the ring buffer
3068          * we use the size that was given, and we can forget about
3069          * expanding it later.
3070          */
3071         ring_buffer_expanded = 1;
3072
3073         /* May be called before buffers are initialized */
3074         if (!global_trace.buffer)
3075                 return 0;
3076
3077         ret = ring_buffer_resize(global_trace.buffer, size, cpu);
3078         if (ret < 0)
3079                 return ret;
3080
3081         if (!current_trace->use_max_tr)
3082                 goto out;
3083
3084         ret = ring_buffer_resize(max_tr.buffer, size, cpu);
3085         if (ret < 0) {
3086                 int r = resize_buffer_duplicate_size(&global_trace,
3087                                                      &global_trace, cpu);
3088                 if (r < 0) {
3089                         /*
3090                          * AARGH! We are left with different
3091                          * size max buffer!!!!
3092                          * The max buffer is our "snapshot" buffer.
3093                          * When a tracer needs a snapshot (one of the
3094                          * latency tracers), it swaps the max buffer
3095                          * with the saved snap shot. We succeeded to
3096                          * update the size of the main buffer, but failed to
3097                          * update the size of the max buffer. But when we tried
3098                          * to reset the main buffer to the original size, we
3099                          * failed there too. This is very unlikely to
3100                          * happen, but if it does, warn and kill all
3101                          * tracing.
3102                          */
3103                         WARN_ON(1);
3104                         tracing_disabled = 1;
3105                 }
3106                 return ret;
3107         }
3108
3109         if (cpu == RING_BUFFER_ALL_CPUS)
3110                 set_buffer_entries(&max_tr, size);
3111         else
3112                 max_tr.data[cpu]->entries = size;
3113
3114  out:
3115         if (cpu == RING_BUFFER_ALL_CPUS)
3116                 set_buffer_entries(&global_trace, size);
3117         else
3118                 global_trace.data[cpu]->entries = size;
3119
3120         return ret;
3121 }
3122
3123 static ssize_t tracing_resize_ring_buffer(unsigned long size, int cpu_id)
3124 {
3125         int ret = size;
3126
3127         mutex_lock(&trace_types_lock);
3128
3129         if (cpu_id != RING_BUFFER_ALL_CPUS) {
3130                 /* make sure, this cpu is enabled in the mask */
3131                 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
3132                         ret = -EINVAL;
3133                         goto out;
3134                 }
3135         }
3136
3137         ret = __tracing_resize_ring_buffer(size, cpu_id);
3138         if (ret < 0)
3139                 ret = -ENOMEM;
3140
3141 out:
3142         mutex_unlock(&trace_types_lock);
3143
3144         return ret;
3145 }
3146
3147
3148 /**
3149  * tracing_update_buffers - used by tracing facility to expand ring buffers
3150  *
3151  * To save on memory when the tracing is never used on a system with it
3152  * configured in. The ring buffers are set to a minimum size. But once
3153  * a user starts to use the tracing facility, then they need to grow
3154  * to their default size.
3155  *
3156  * This function is to be called when a tracer is about to be used.
3157  */
3158 int tracing_update_buffers(void)
3159 {
3160         int ret = 0;
3161
3162         mutex_lock(&trace_types_lock);
3163         if (!ring_buffer_expanded)
3164                 ret = __tracing_resize_ring_buffer(trace_buf_size,
3165                                                 RING_BUFFER_ALL_CPUS);
3166         mutex_unlock(&trace_types_lock);
3167
3168         return ret;
3169 }
3170
3171 struct trace_option_dentry;
3172
3173 static struct trace_option_dentry *
3174 create_trace_option_files(struct tracer *tracer);
3175
3176 static void
3177 destroy_trace_option_files(struct trace_option_dentry *topts);
3178
3179 static int tracing_set_tracer(const char *buf)
3180 {
3181         static struct trace_option_dentry *topts;
3182         struct trace_array *tr = &global_trace;
3183         struct tracer *t;
3184         int ret = 0;
3185
3186         mutex_lock(&trace_types_lock);
3187
3188         if (!ring_buffer_expanded) {
3189                 ret = __tracing_resize_ring_buffer(trace_buf_size,
3190                                                 RING_BUFFER_ALL_CPUS);
3191                 if (ret < 0)
3192                         goto out;
3193                 ret = 0;
3194         }
3195
3196         for (t = trace_types; t; t = t->next) {
3197                 if (strcmp(t->name, buf) == 0)
3198                         break;
3199         }
3200         if (!t) {
3201                 ret = -EINVAL;
3202                 goto out;
3203         }
3204         if (t == current_trace)
3205                 goto out;
3206
3207         trace_branch_disable();
3208         if (current_trace && current_trace->reset)
3209                 current_trace->reset(tr);
3210         if (current_trace && current_trace->use_max_tr) {
3211                 /*
3212                  * We don't free the ring buffer. instead, resize it because
3213                  * The max_tr ring buffer has some state (e.g. ring->clock) and
3214                  * we want preserve it.
3215                  */
3216                 ring_buffer_resize(max_tr.buffer, 1, RING_BUFFER_ALL_CPUS);
3217                 set_buffer_entries(&max_tr, 1);
3218         }
3219         destroy_trace_option_files(topts);
3220
3221         current_trace = &nop_trace;
3222
3223         topts = create_trace_option_files(t);
3224         if (t->use_max_tr) {
3225                 /* we need to make per cpu buffer sizes equivalent */
3226                 ret = resize_buffer_duplicate_size(&max_tr, &global_trace,
3227                                                    RING_BUFFER_ALL_CPUS);
3228                 if (ret < 0)
3229                         goto out;
3230         }
3231
3232         if (t->init) {
3233                 ret = tracer_init(t, tr);
3234                 if (ret)
3235                         goto out;
3236         }
3237
3238         current_trace = t;
3239         trace_branch_enable(tr);
3240  out:
3241         mutex_unlock(&trace_types_lock);
3242
3243         return ret;
3244 }
3245
3246 static ssize_t
3247 tracing_set_trace_write(struct file *filp, const char __user *ubuf,
3248                         size_t cnt, loff_t *ppos)
3249 {
3250         char buf[MAX_TRACER_SIZE+1];
3251         int i;
3252         size_t ret;
3253         int err;
3254
3255         ret = cnt;
3256
3257         if (cnt > MAX_TRACER_SIZE)
3258                 cnt = MAX_TRACER_SIZE;
3259
3260         if (copy_from_user(&buf, ubuf, cnt))
3261                 return -EFAULT;
3262
3263         buf[cnt] = 0;
3264
3265         /* strip ending whitespace. */
3266         for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
3267                 buf[i] = 0;
3268
3269         err = tracing_set_tracer(buf);
3270         if (err)
3271                 return err;
3272
3273         *ppos += ret;
3274
3275         return ret;
3276 }
3277
3278 static ssize_t
3279 tracing_max_lat_read(struct file *filp, char __user *ubuf,
3280                      size_t cnt, loff_t *ppos)
3281 {
3282         unsigned long *ptr = filp->private_data;
3283         char buf[64];
3284         int r;
3285
3286         r = snprintf(buf, sizeof(buf), "%ld\n",
3287                      *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
3288         if (r > sizeof(buf))
3289                 r = sizeof(buf);
3290         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3291 }
3292
3293 static ssize_t
3294 tracing_max_lat_write(struct file *filp, const char __user *ubuf,
3295                       size_t cnt, loff_t *ppos)
3296 {
3297         unsigned long *ptr = filp->private_data;
3298         unsigned long val;
3299         int ret;
3300
3301         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3302         if (ret)
3303                 return ret;
3304
3305         *ptr = val * 1000;
3306
3307         return cnt;
3308 }
3309
3310 static int tracing_open_pipe(struct inode *inode, struct file *filp)
3311 {
3312         long cpu_file = (long) inode->i_private;
3313         struct trace_iterator *iter;
3314         int ret = 0;
3315
3316         if (tracing_disabled)
3317                 return -ENODEV;
3318
3319         mutex_lock(&trace_types_lock);
3320
3321         /* create a buffer to store the information to pass to userspace */
3322         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
3323         if (!iter) {
3324                 ret = -ENOMEM;
3325                 goto out;
3326         }
3327
3328         /*
3329          * We make a copy of the current tracer to avoid concurrent
3330          * changes on it while we are reading.
3331          */
3332         iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
3333         if (!iter->trace) {
3334                 ret = -ENOMEM;
3335                 goto fail;
3336         }
3337         if (current_trace)
3338                 *iter->trace = *current_trace;
3339
3340         if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
3341                 ret = -ENOMEM;
3342                 goto fail;
3343         }
3344
3345         /* trace pipe does not show start of buffer */
3346         cpumask_setall(iter->started);
3347
3348         if (trace_flags & TRACE_ITER_LATENCY_FMT)
3349                 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3350
3351         /* Output in nanoseconds only if we are using a clock in nanoseconds. */
3352         if (trace_clocks[trace_clock_id].in_ns)
3353                 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
3354
3355         iter->cpu_file = cpu_file;
3356         iter->tr = &global_trace;
3357         mutex_init(&iter->mutex);
3358         filp->private_data = iter;
3359
3360         if (iter->trace->pipe_open)
3361                 iter->trace->pipe_open(iter);
3362
3363         nonseekable_open(inode, filp);
3364 out:
3365         mutex_unlock(&trace_types_lock);
3366         return ret;
3367
3368 fail:
3369         kfree(iter->trace);
3370         kfree(iter);
3371         mutex_unlock(&trace_types_lock);
3372         return ret;
3373 }
3374
3375 static int tracing_release_pipe(struct inode *inode, struct file *file)
3376 {
3377         struct trace_iterator *iter = file->private_data;
3378
3379         mutex_lock(&trace_types_lock);
3380
3381         if (iter->trace->pipe_close)
3382                 iter->trace->pipe_close(iter);
3383
3384         mutex_unlock(&trace_types_lock);
3385
3386         free_cpumask_var(iter->started);
3387         mutex_destroy(&iter->mutex);
3388         kfree(iter->trace);
3389         kfree(iter);
3390
3391         return 0;
3392 }
3393
3394 static unsigned int
3395 tracing_poll_pipe(struct file *filp, poll_table *poll_table)
3396 {
3397         struct trace_iterator *iter = filp->private_data;
3398
3399         if (trace_flags & TRACE_ITER_BLOCK) {
3400                 /*
3401                  * Always select as readable when in blocking mode
3402                  */
3403                 return POLLIN | POLLRDNORM;
3404         } else {
3405                 if (!trace_empty(iter))
3406                         return POLLIN | POLLRDNORM;
3407                 poll_wait(filp, &trace_wait, poll_table);
3408                 if (!trace_empty(iter))
3409                         return POLLIN | POLLRDNORM;
3410
3411                 return 0;
3412         }
3413 }
3414
3415 /*
3416  * This is a make-shift waitqueue.
3417  * A tracer might use this callback on some rare cases:
3418  *
3419  *  1) the current tracer might hold the runqueue lock when it wakes up
3420  *     a reader, hence a deadlock (sched, function, and function graph tracers)
3421  *  2) the function tracers, trace all functions, we don't want
3422  *     the overhead of calling wake_up and friends
3423  *     (and tracing them too)
3424  *
3425  *     Anyway, this is really very primitive wakeup.
3426  */
3427 void poll_wait_pipe(struct trace_iterator *iter)
3428 {
3429         set_current_state(TASK_INTERRUPTIBLE);
3430         /* sleep for 100 msecs, and try again. */
3431         schedule_timeout(HZ / 10);
3432 }
3433
3434 /* Must be called with trace_types_lock mutex held. */
3435 static int tracing_wait_pipe(struct file *filp)
3436 {
3437         struct trace_iterator *iter = filp->private_data;
3438
3439         while (trace_empty(iter)) {
3440
3441                 if ((filp->f_flags & O_NONBLOCK)) {
3442                         return -EAGAIN;
3443                 }
3444
3445                 mutex_unlock(&iter->mutex);
3446
3447                 iter->trace->wait_pipe(iter);
3448
3449                 mutex_lock(&iter->mutex);
3450
3451                 if (signal_pending(current))
3452                         return -EINTR;
3453
3454                 /*
3455                  * We block until we read something and tracing is enabled.
3456                  * We still block if tracing is disabled, but we have never
3457                  * read anything. This allows a user to cat this file, and
3458                  * then enable tracing. But after we have read something,
3459                  * we give an EOF when tracing is again disabled.
3460                  *
3461                  * iter->pos will be 0 if we haven't read anything.
3462                  */
3463                 if (tracing_is_enabled() && iter->pos)
3464                         break;
3465         }
3466
3467         return 1;
3468 }
3469
3470 /*
3471  * Consumer reader.
3472  */
3473 static ssize_t
3474 tracing_read_pipe(struct file *filp, char __user *ubuf,
3475                   size_t cnt, loff_t *ppos)
3476 {
3477         struct trace_iterator *iter = filp->private_data;
3478         static struct tracer *old_tracer;
3479         ssize_t sret;
3480
3481         /* return any leftover data */
3482         sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3483         if (sret != -EBUSY)
3484                 return sret;
3485
3486         trace_seq_init(&iter->seq);
3487
3488         /* copy the tracer to avoid using a global lock all around */
3489         mutex_lock(&trace_types_lock);
3490         if (unlikely(old_tracer != current_trace && current_trace)) {
3491                 old_tracer = current_trace;
3492                 *iter->trace = *current_trace;
3493         }
3494         mutex_unlock(&trace_types_lock);
3495
3496         /*
3497          * Avoid more than one consumer on a single file descriptor
3498          * This is just a matter of traces coherency, the ring buffer itself
3499          * is protected.
3500          */
3501         mutex_lock(&iter->mutex);
3502         if (iter->trace->read) {
3503                 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
3504                 if (sret)
3505                         goto out;
3506         }
3507
3508 waitagain:
3509         sret = tracing_wait_pipe(filp);
3510         if (sret <= 0)
3511                 goto out;
3512
3513         /* stop when tracing is finished */
3514         if (trace_empty(iter)) {
3515                 sret = 0;
3516                 goto out;
3517         }
3518
3519         if (cnt >= PAGE_SIZE)
3520                 cnt = PAGE_SIZE - 1;
3521
3522         /* reset all but tr, trace, and overruns */
3523         memset(&iter->seq, 0,
3524                sizeof(struct trace_iterator) -
3525                offsetof(struct trace_iterator, seq));
3526         iter->pos = -1;
3527
3528         trace_event_read_lock();
3529         trace_access_lock(iter->cpu_file);
3530         while (trace_find_next_entry_inc(iter) != NULL) {
3531                 enum print_line_t ret;
3532                 int len = iter->seq.len;
3533
3534                 ret = print_trace_line(iter);
3535                 if (ret == TRACE_TYPE_PARTIAL_LINE) {
3536                         /* don't print partial lines */
3537                         iter->seq.len = len;
3538                         break;
3539                 }
3540                 if (ret != TRACE_TYPE_NO_CONSUME)
3541                         trace_consume(iter);
3542
3543                 if (iter->seq.len >= cnt)
3544                         break;
3545
3546                 /*
3547                  * Setting the full flag means we reached the trace_seq buffer
3548                  * size and we should leave by partial output condition above.
3549                  * One of the trace_seq_* functions is not used properly.
3550                  */
3551                 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
3552                           iter->ent->type);
3553         }
3554         trace_access_unlock(iter->cpu_file);
3555         trace_event_read_unlock();
3556