trace.h 66 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122
  1. // SPDX-License-Identifier: GPL-2.0
  2. #ifndef _LINUX_KERNEL_TRACE_H
  3. #define _LINUX_KERNEL_TRACE_H
  4. #include <linux/fs.h>
  5. #include <linux/atomic.h>
  6. #include <linux/sched.h>
  7. #include <linux/clocksource.h>
  8. #include <linux/ring_buffer.h>
  9. #include <linux/mmiotrace.h>
  10. #include <linux/tracepoint.h>
  11. #include <linux/ftrace.h>
  12. #include <linux/trace.h>
  13. #include <linux/hw_breakpoint.h>
  14. #include <linux/trace_seq.h>
  15. #include <linux/trace_events.h>
  16. #include <linux/compiler.h>
  17. #include <linux/glob.h>
  18. #include <linux/irq_work.h>
  19. #include <linux/workqueue.h>
  20. #include <linux/ctype.h>
  21. #ifdef CONFIG_FTRACE_SYSCALLS
  22. #include <asm/unistd.h> /* For NR_SYSCALLS */
  23. #include <asm/syscall.h> /* some archs define it here */
  24. #endif
  25. enum trace_type {
  26. __TRACE_FIRST_TYPE = 0,
  27. TRACE_FN,
  28. TRACE_CTX,
  29. TRACE_WAKE,
  30. TRACE_STACK,
  31. TRACE_PRINT,
  32. TRACE_BPRINT,
  33. TRACE_MMIO_RW,
  34. TRACE_MMIO_MAP,
  35. TRACE_BRANCH,
  36. TRACE_GRAPH_RET,
  37. TRACE_GRAPH_ENT,
  38. TRACE_USER_STACK,
  39. TRACE_BLK,
  40. TRACE_BPUTS,
  41. TRACE_HWLAT,
  42. TRACE_RAW_DATA,
  43. __TRACE_LAST_TYPE,
  44. };
  45. #undef __field
  46. #define __field(type, item) type item;
  47. #undef __field_fn
  48. #define __field_fn(type, item) type item;
  49. #undef __field_struct
  50. #define __field_struct(type, item) __field(type, item)
  51. #undef __field_desc
  52. #define __field_desc(type, container, item)
  53. #undef __field_packed
  54. #define __field_packed(type, container, item)
  55. #undef __array
  56. #define __array(type, item, size) type item[size];
  57. #undef __array_desc
  58. #define __array_desc(type, container, item, size)
  59. #undef __dynamic_array
  60. #define __dynamic_array(type, item) type item[];
  61. #undef F_STRUCT
  62. #define F_STRUCT(args...) args
  63. #undef FTRACE_ENTRY
  64. #define FTRACE_ENTRY(name, struct_name, id, tstruct, print) \
  65. struct struct_name { \
  66. struct trace_entry ent; \
  67. tstruct \
  68. }
  69. #undef FTRACE_ENTRY_DUP
  70. #define FTRACE_ENTRY_DUP(name, name_struct, id, tstruct, printk)
  71. #undef FTRACE_ENTRY_REG
  72. #define FTRACE_ENTRY_REG(name, struct_name, id, tstruct, print, regfn) \
  73. FTRACE_ENTRY(name, struct_name, id, PARAMS(tstruct), PARAMS(print))
  74. #undef FTRACE_ENTRY_PACKED
  75. #define FTRACE_ENTRY_PACKED(name, struct_name, id, tstruct, print) \
  76. FTRACE_ENTRY(name, struct_name, id, PARAMS(tstruct), PARAMS(print)) __packed
  77. #include "trace_entries.h"
  78. /* Use this for memory failure errors */
  79. #define MEM_FAIL(condition, fmt, ...) ({ \
  80. static bool __section(".data.once") __warned; \
  81. int __ret_warn_once = !!(condition); \
  82. \
  83. if (unlikely(__ret_warn_once && !__warned)) { \
  84. __warned = true; \
  85. pr_err("ERROR: " fmt, ##__VA_ARGS__); \
  86. } \
  87. unlikely(__ret_warn_once); \
  88. })
  89. /*
  90. * syscalls are special, and need special handling, this is why
  91. * they are not included in trace_entries.h
  92. */
  93. struct syscall_trace_enter {
  94. struct trace_entry ent;
  95. int nr;
  96. unsigned long args[];
  97. };
  98. struct syscall_trace_exit {
  99. struct trace_entry ent;
  100. int nr;
  101. long ret;
  102. };
  103. struct kprobe_trace_entry_head {
  104. struct trace_entry ent;
  105. unsigned long ip;
  106. };
  107. struct kretprobe_trace_entry_head {
  108. struct trace_entry ent;
  109. unsigned long func;
  110. unsigned long ret_ip;
  111. };
  112. /*
  113. * trace_flag_type is an enumeration that holds different
  114. * states when a trace occurs. These are:
  115. * IRQS_OFF - interrupts were disabled
  116. * IRQS_NOSUPPORT - arch does not support irqs_disabled_flags
  117. * NEED_RESCHED - reschedule is requested
  118. * HARDIRQ - inside an interrupt handler
  119. * SOFTIRQ - inside a softirq handler
  120. */
  121. enum trace_flag_type {
  122. TRACE_FLAG_IRQS_OFF = 0x01,
  123. TRACE_FLAG_IRQS_NOSUPPORT = 0x02,
  124. TRACE_FLAG_NEED_RESCHED = 0x04,
  125. TRACE_FLAG_HARDIRQ = 0x08,
  126. TRACE_FLAG_SOFTIRQ = 0x10,
  127. TRACE_FLAG_PREEMPT_RESCHED = 0x20,
  128. TRACE_FLAG_NMI = 0x40,
  129. };
  130. #define TRACE_BUF_SIZE 1024
  131. struct trace_array;
  132. /*
  133. * The CPU trace array - it consists of thousands of trace entries
  134. * plus some other descriptor data: (for example which task started
  135. * the trace, etc.)
  136. */
  137. struct trace_array_cpu {
  138. atomic_t disabled;
  139. void *buffer_page; /* ring buffer spare */
  140. unsigned long entries;
  141. unsigned long saved_latency;
  142. unsigned long critical_start;
  143. unsigned long critical_end;
  144. unsigned long critical_sequence;
  145. unsigned long nice;
  146. unsigned long policy;
  147. unsigned long rt_priority;
  148. unsigned long skipped_entries;
  149. u64 preempt_timestamp;
  150. pid_t pid;
  151. kuid_t uid;
  152. char comm[TASK_COMM_LEN];
  153. #ifdef CONFIG_FUNCTION_TRACER
  154. int ftrace_ignore_pid;
  155. #endif
  156. bool ignore_pid;
  157. };
  158. struct tracer;
  159. struct trace_option_dentry;
  160. struct array_buffer {
  161. struct trace_array *tr;
  162. struct trace_buffer *buffer;
  163. struct trace_array_cpu __percpu *data;
  164. u64 time_start;
  165. int cpu;
  166. };
  167. #define TRACE_FLAGS_MAX_SIZE 32
  168. struct trace_options {
  169. struct tracer *tracer;
  170. struct trace_option_dentry *topts;
  171. };
  172. struct trace_pid_list {
  173. int pid_max;
  174. unsigned long *pids;
  175. };
  176. enum {
  177. TRACE_PIDS = BIT(0),
  178. TRACE_NO_PIDS = BIT(1),
  179. };
  180. static inline bool pid_type_enabled(int type, struct trace_pid_list *pid_list,
  181. struct trace_pid_list *no_pid_list)
  182. {
  183. /* Return true if the pid list in type has pids */
  184. return ((type & TRACE_PIDS) && pid_list) ||
  185. ((type & TRACE_NO_PIDS) && no_pid_list);
  186. }
  187. static inline bool still_need_pid_events(int type, struct trace_pid_list *pid_list,
  188. struct trace_pid_list *no_pid_list)
  189. {
  190. /*
  191. * Turning off what is in @type, return true if the "other"
  192. * pid list, still has pids in it.
  193. */
  194. return (!(type & TRACE_PIDS) && pid_list) ||
  195. (!(type & TRACE_NO_PIDS) && no_pid_list);
  196. }
  197. typedef bool (*cond_update_fn_t)(struct trace_array *tr, void *cond_data);
  198. /**
  199. * struct cond_snapshot - conditional snapshot data and callback
  200. *
  201. * The cond_snapshot structure encapsulates a callback function and
  202. * data associated with the snapshot for a given tracing instance.
  203. *
  204. * When a snapshot is taken conditionally, by invoking
  205. * tracing_snapshot_cond(tr, cond_data), the cond_data passed in is
  206. * passed in turn to the cond_snapshot.update() function. That data
  207. * can be compared by the update() implementation with the cond_data
  208. * contained within the struct cond_snapshot instance associated with
  209. * the trace_array. Because the tr->max_lock is held throughout the
  210. * update() call, the update() function can directly retrieve the
  211. * cond_snapshot and cond_data associated with the per-instance
  212. * snapshot associated with the trace_array.
  213. *
  214. * The cond_snapshot.update() implementation can save data to be
  215. * associated with the snapshot if it decides to, and returns 'true'
  216. * in that case, or it returns 'false' if the conditional snapshot
  217. * shouldn't be taken.
  218. *
  219. * The cond_snapshot instance is created and associated with the
  220. * user-defined cond_data by tracing_cond_snapshot_enable().
  221. * Likewise, the cond_snapshot instance is destroyed and is no longer
  222. * associated with the trace instance by
  223. * tracing_cond_snapshot_disable().
  224. *
  225. * The method below is required.
  226. *
  227. * @update: When a conditional snapshot is invoked, the update()
  228. * callback function is invoked with the tr->max_lock held. The
  229. * update() implementation signals whether or not to actually
  230. * take the snapshot, by returning 'true' if so, 'false' if no
  231. * snapshot should be taken. Because the max_lock is held for
  232. * the duration of update(), the implementation is safe to
  233. * directly retrieved and save any implementation data it needs
  234. * to in association with the snapshot.
  235. */
  236. struct cond_snapshot {
  237. void *cond_data;
  238. cond_update_fn_t update;
  239. };
  240. /*
  241. * The trace array - an array of per-CPU trace arrays. This is the
  242. * highest level data structure that individual tracers deal with.
  243. * They have on/off state as well:
  244. */
  245. struct trace_array {
  246. struct list_head list;
  247. char *name;
  248. struct array_buffer array_buffer;
  249. #ifdef CONFIG_TRACER_MAX_TRACE
  250. /*
  251. * The max_buffer is used to snapshot the trace when a maximum
  252. * latency is reached, or when the user initiates a snapshot.
  253. * Some tracers will use this to store a maximum trace while
  254. * it continues examining live traces.
  255. *
  256. * The buffers for the max_buffer are set up the same as the array_buffer
  257. * When a snapshot is taken, the buffer of the max_buffer is swapped
  258. * with the buffer of the array_buffer and the buffers are reset for
  259. * the array_buffer so the tracing can continue.
  260. */
  261. struct array_buffer max_buffer;
  262. bool allocated_snapshot;
  263. #endif
  264. #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
  265. unsigned long max_latency;
  266. #ifdef CONFIG_FSNOTIFY
  267. struct dentry *d_max_latency;
  268. struct work_struct fsnotify_work;
  269. struct irq_work fsnotify_irqwork;
  270. #endif
  271. #endif
  272. struct trace_pid_list __rcu *filtered_pids;
  273. struct trace_pid_list __rcu *filtered_no_pids;
  274. /*
  275. * max_lock is used to protect the swapping of buffers
  276. * when taking a max snapshot. The buffers themselves are
  277. * protected by per_cpu spinlocks. But the action of the swap
  278. * needs its own lock.
  279. *
  280. * This is defined as a arch_spinlock_t in order to help
  281. * with performance when lockdep debugging is enabled.
  282. *
  283. * It is also used in other places outside the update_max_tr
  284. * so it needs to be defined outside of the
  285. * CONFIG_TRACER_MAX_TRACE.
  286. */
  287. arch_spinlock_t max_lock;
  288. int buffer_disabled;
  289. #ifdef CONFIG_FTRACE_SYSCALLS
  290. int sys_refcount_enter;
  291. int sys_refcount_exit;
  292. struct trace_event_file __rcu *enter_syscall_files[NR_syscalls];
  293. struct trace_event_file __rcu *exit_syscall_files[NR_syscalls];
  294. #endif
  295. int stop_count;
  296. int clock_id;
  297. int nr_topts;
  298. bool clear_trace;
  299. int buffer_percent;
  300. unsigned int n_err_log_entries;
  301. struct tracer *current_trace;
  302. unsigned int trace_flags;
  303. unsigned char trace_flags_index[TRACE_FLAGS_MAX_SIZE];
  304. unsigned int flags;
  305. raw_spinlock_t start_lock;
  306. struct list_head err_log;
  307. struct dentry *dir;
  308. struct dentry *options;
  309. struct dentry *percpu_dir;
  310. struct dentry *event_dir;
  311. struct trace_options *topts;
  312. struct list_head systems;
  313. struct list_head events;
  314. struct trace_event_file *trace_marker_file;
  315. cpumask_var_t tracing_cpumask; /* only trace on set CPUs */
  316. int ref;
  317. int trace_ref;
  318. #ifdef CONFIG_FUNCTION_TRACER
  319. struct ftrace_ops *ops;
  320. struct trace_pid_list __rcu *function_pids;
  321. struct trace_pid_list __rcu *function_no_pids;
  322. #ifdef CONFIG_DYNAMIC_FTRACE
  323. /* All of these are protected by the ftrace_lock */
  324. struct list_head func_probes;
  325. struct list_head mod_trace;
  326. struct list_head mod_notrace;
  327. #endif
  328. /* function tracing enabled */
  329. int function_enabled;
  330. #endif
  331. int time_stamp_abs_ref;
  332. struct list_head hist_vars;
  333. #ifdef CONFIG_TRACER_SNAPSHOT
  334. struct cond_snapshot *cond_snapshot;
  335. #endif
  336. };
  337. enum {
  338. TRACE_ARRAY_FL_GLOBAL = (1 << 0)
  339. };
  340. extern struct list_head ftrace_trace_arrays;
  341. extern struct mutex trace_types_lock;
  342. extern int trace_array_get(struct trace_array *tr);
  343. extern int tracing_check_open_get_tr(struct trace_array *tr);
  344. extern struct trace_array *trace_array_find(const char *instance);
  345. extern struct trace_array *trace_array_find_get(const char *instance);
  346. extern int tracing_set_time_stamp_abs(struct trace_array *tr, bool abs);
  347. extern int tracing_set_clock(struct trace_array *tr, const char *clockstr);
  348. extern bool trace_clock_in_ns(struct trace_array *tr);
  349. /*
  350. * The global tracer (top) should be the first trace array added,
  351. * but we check the flag anyway.
  352. */
  353. static inline struct trace_array *top_trace_array(void)
  354. {
  355. struct trace_array *tr;
  356. if (list_empty(&ftrace_trace_arrays))
  357. return NULL;
  358. tr = list_entry(ftrace_trace_arrays.prev,
  359. typeof(*tr), list);
  360. WARN_ON(!(tr->flags & TRACE_ARRAY_FL_GLOBAL));
  361. return tr;
  362. }
  363. #define FTRACE_CMP_TYPE(var, type) \
  364. __builtin_types_compatible_p(typeof(var), type *)
  365. #undef IF_ASSIGN
  366. #define IF_ASSIGN(var, entry, etype, id) \
  367. if (FTRACE_CMP_TYPE(var, etype)) { \
  368. var = (typeof(var))(entry); \
  369. WARN_ON(id != 0 && (entry)->type != id); \
  370. break; \
  371. }
  372. /* Will cause compile errors if type is not found. */
  373. extern void __ftrace_bad_type(void);
  374. /*
  375. * The trace_assign_type is a verifier that the entry type is
  376. * the same as the type being assigned. To add new types simply
  377. * add a line with the following format:
  378. *
  379. * IF_ASSIGN(var, ent, type, id);
  380. *
  381. * Where "type" is the trace type that includes the trace_entry
  382. * as the "ent" item. And "id" is the trace identifier that is
  383. * used in the trace_type enum.
  384. *
  385. * If the type can have more than one id, then use zero.
  386. */
  387. #define trace_assign_type(var, ent) \
  388. do { \
  389. IF_ASSIGN(var, ent, struct ftrace_entry, TRACE_FN); \
  390. IF_ASSIGN(var, ent, struct ctx_switch_entry, 0); \
  391. IF_ASSIGN(var, ent, struct stack_entry, TRACE_STACK); \
  392. IF_ASSIGN(var, ent, struct userstack_entry, TRACE_USER_STACK);\
  393. IF_ASSIGN(var, ent, struct print_entry, TRACE_PRINT); \
  394. IF_ASSIGN(var, ent, struct bprint_entry, TRACE_BPRINT); \
  395. IF_ASSIGN(var, ent, struct bputs_entry, TRACE_BPUTS); \
  396. IF_ASSIGN(var, ent, struct hwlat_entry, TRACE_HWLAT); \
  397. IF_ASSIGN(var, ent, struct raw_data_entry, TRACE_RAW_DATA);\
  398. IF_ASSIGN(var, ent, struct trace_mmiotrace_rw, \
  399. TRACE_MMIO_RW); \
  400. IF_ASSIGN(var, ent, struct trace_mmiotrace_map, \
  401. TRACE_MMIO_MAP); \
  402. IF_ASSIGN(var, ent, struct trace_branch, TRACE_BRANCH); \
  403. IF_ASSIGN(var, ent, struct ftrace_graph_ent_entry, \
  404. TRACE_GRAPH_ENT); \
  405. IF_ASSIGN(var, ent, struct ftrace_graph_ret_entry, \
  406. TRACE_GRAPH_RET); \
  407. __ftrace_bad_type(); \
  408. } while (0)
  409. /*
  410. * An option specific to a tracer. This is a boolean value.
  411. * The bit is the bit index that sets its value on the
  412. * flags value in struct tracer_flags.
  413. */
  414. struct tracer_opt {
  415. const char *name; /* Will appear on the trace_options file */
  416. u32 bit; /* Mask assigned in val field in tracer_flags */
  417. };
  418. /*
  419. * The set of specific options for a tracer. Your tracer
  420. * have to set the initial value of the flags val.
  421. */
  422. struct tracer_flags {
  423. u32 val;
  424. struct tracer_opt *opts;
  425. struct tracer *trace;
  426. };
  427. /* Makes more easy to define a tracer opt */
  428. #define TRACER_OPT(s, b) .name = #s, .bit = b
  429. struct trace_option_dentry {
  430. struct tracer_opt *opt;
  431. struct tracer_flags *flags;
  432. struct trace_array *tr;
  433. struct dentry *entry;
  434. };
  435. /**
  436. * struct tracer - a specific tracer and its callbacks to interact with tracefs
  437. * @name: the name chosen to select it on the available_tracers file
  438. * @init: called when one switches to this tracer (echo name > current_tracer)
  439. * @reset: called when one switches to another tracer
  440. * @start: called when tracing is unpaused (echo 1 > tracing_on)
  441. * @stop: called when tracing is paused (echo 0 > tracing_on)
  442. * @update_thresh: called when tracing_thresh is updated
  443. * @open: called when the trace file is opened
  444. * @pipe_open: called when the trace_pipe file is opened
  445. * @close: called when the trace file is released
  446. * @pipe_close: called when the trace_pipe file is released
  447. * @read: override the default read callback on trace_pipe
  448. * @splice_read: override the default splice_read callback on trace_pipe
  449. * @selftest: selftest to run on boot (see trace_selftest.c)
  450. * @print_headers: override the first lines that describe your columns
  451. * @print_line: callback that prints a trace
  452. * @set_flag: signals one of your private flags changed (trace_options file)
  453. * @flags: your private flags
  454. */
  455. struct tracer {
  456. const char *name;
  457. int (*init)(struct trace_array *tr);
  458. void (*reset)(struct trace_array *tr);
  459. void (*start)(struct trace_array *tr);
  460. void (*stop)(struct trace_array *tr);
  461. int (*update_thresh)(struct trace_array *tr);
  462. void (*open)(struct trace_iterator *iter);
  463. void (*pipe_open)(struct trace_iterator *iter);
  464. void (*close)(struct trace_iterator *iter);
  465. void (*pipe_close)(struct trace_iterator *iter);
  466. ssize_t (*read)(struct trace_iterator *iter,
  467. struct file *filp, char __user *ubuf,
  468. size_t cnt, loff_t *ppos);
  469. ssize_t (*splice_read)(struct trace_iterator *iter,
  470. struct file *filp,
  471. loff_t *ppos,
  472. struct pipe_inode_info *pipe,
  473. size_t len,
  474. unsigned int flags);
  475. #ifdef CONFIG_FTRACE_STARTUP_TEST
  476. int (*selftest)(struct tracer *trace,
  477. struct trace_array *tr);
  478. #endif
  479. void (*print_header)(struct seq_file *m);
  480. enum print_line_t (*print_line)(struct trace_iterator *iter);
  481. /* If you handled the flag setting, return 0 */
  482. int (*set_flag)(struct trace_array *tr,
  483. u32 old_flags, u32 bit, int set);
  484. /* Return 0 if OK with change, else return non-zero */
  485. int (*flag_changed)(struct trace_array *tr,
  486. u32 mask, int set);
  487. struct tracer *next;
  488. struct tracer_flags *flags;
  489. int enabled;
  490. bool print_max;
  491. bool allow_instances;
  492. #ifdef CONFIG_TRACER_MAX_TRACE
  493. bool use_max_tr;
  494. #endif
  495. /* True if tracer cannot be enabled in kernel param */
  496. bool noboot;
  497. };
  498. /* Only current can touch trace_recursion */
  499. /*
  500. * For function tracing recursion:
  501. * The order of these bits are important.
  502. *
  503. * When function tracing occurs, the following steps are made:
  504. * If arch does not support a ftrace feature:
  505. * call internal function (uses INTERNAL bits) which calls...
  506. * If callback is registered to the "global" list, the list
  507. * function is called and recursion checks the GLOBAL bits.
  508. * then this function calls...
  509. * The function callback, which can use the FTRACE bits to
  510. * check for recursion.
  511. */
  512. enum {
  513. /* Function recursion bits */
  514. TRACE_FTRACE_BIT,
  515. TRACE_FTRACE_NMI_BIT,
  516. TRACE_FTRACE_IRQ_BIT,
  517. TRACE_FTRACE_SIRQ_BIT,
  518. TRACE_FTRACE_TRANSITION_BIT,
  519. /* Internal use recursion bits */
  520. TRACE_INTERNAL_BIT,
  521. TRACE_INTERNAL_NMI_BIT,
  522. TRACE_INTERNAL_IRQ_BIT,
  523. TRACE_INTERNAL_SIRQ_BIT,
  524. TRACE_INTERNAL_TRANSITION_BIT,
  525. TRACE_BRANCH_BIT,
  526. /*
  527. * Abuse of the trace_recursion.
  528. * As we need a way to maintain state if we are tracing the function
  529. * graph in irq because we want to trace a particular function that
  530. * was called in irq context but we have irq tracing off. Since this
  531. * can only be modified by current, we can reuse trace_recursion.
  532. */
  533. TRACE_IRQ_BIT,
  534. /* Set if the function is in the set_graph_function file */
  535. TRACE_GRAPH_BIT,
  536. /*
  537. * In the very unlikely case that an interrupt came in
  538. * at a start of graph tracing, and we want to trace
  539. * the function in that interrupt, the depth can be greater
  540. * than zero, because of the preempted start of a previous
  541. * trace. In an even more unlikely case, depth could be 2
  542. * if a softirq interrupted the start of graph tracing,
  543. * followed by an interrupt preempting a start of graph
  544. * tracing in the softirq, and depth can even be 3
  545. * if an NMI came in at the start of an interrupt function
  546. * that preempted a softirq start of a function that
  547. * preempted normal context!!!! Luckily, it can't be
  548. * greater than 3, so the next two bits are a mask
  549. * of what the depth is when we set TRACE_GRAPH_BIT
  550. */
  551. TRACE_GRAPH_DEPTH_START_BIT,
  552. TRACE_GRAPH_DEPTH_END_BIT,
  553. /*
  554. * To implement set_graph_notrace, if this bit is set, we ignore
  555. * function graph tracing of called functions, until the return
  556. * function is called to clear it.
  557. */
  558. TRACE_GRAPH_NOTRACE_BIT,
  559. };
  560. #define trace_recursion_set(bit) do { (current)->trace_recursion |= (1<<(bit)); } while (0)
  561. #define trace_recursion_clear(bit) do { (current)->trace_recursion &= ~(1<<(bit)); } while (0)
  562. #define trace_recursion_test(bit) ((current)->trace_recursion & (1<<(bit)))
  563. #define trace_recursion_depth() \
  564. (((current)->trace_recursion >> TRACE_GRAPH_DEPTH_START_BIT) & 3)
  565. #define trace_recursion_set_depth(depth) \
  566. do { \
  567. current->trace_recursion &= \
  568. ~(3 << TRACE_GRAPH_DEPTH_START_BIT); \
  569. current->trace_recursion |= \
  570. ((depth) & 3) << TRACE_GRAPH_DEPTH_START_BIT; \
  571. } while (0)
  572. #define TRACE_CONTEXT_BITS 4
  573. #define TRACE_FTRACE_START TRACE_FTRACE_BIT
  574. #define TRACE_LIST_START TRACE_INTERNAL_BIT
  575. #define TRACE_CONTEXT_MASK ((1 << (TRACE_LIST_START + TRACE_CONTEXT_BITS)) - 1)
  576. enum {
  577. TRACE_CTX_NMI,
  578. TRACE_CTX_IRQ,
  579. TRACE_CTX_SOFTIRQ,
  580. TRACE_CTX_NORMAL,
  581. TRACE_CTX_TRANSITION,
  582. };
  583. static __always_inline int trace_get_context_bit(void)
  584. {
  585. int bit;
  586. if (in_interrupt()) {
  587. if (in_nmi())
  588. bit = TRACE_CTX_NMI;
  589. else if (in_irq())
  590. bit = TRACE_CTX_IRQ;
  591. else
  592. bit = TRACE_CTX_SOFTIRQ;
  593. } else
  594. bit = TRACE_CTX_NORMAL;
  595. return bit;
  596. }
  597. static __always_inline int trace_test_and_set_recursion(int start)
  598. {
  599. unsigned int val = current->trace_recursion;
  600. int bit;
  601. bit = trace_get_context_bit() + start;
  602. if (unlikely(val & (1 << bit))) {
  603. /*
  604. * It could be that preempt_count has not been updated during
  605. * a switch between contexts. Allow for a single recursion.
  606. */
  607. bit = start + TRACE_CTX_TRANSITION;
  608. if (trace_recursion_test(bit))
  609. return -1;
  610. trace_recursion_set(bit);
  611. barrier();
  612. return bit;
  613. }
  614. val |= 1 << bit;
  615. current->trace_recursion = val;
  616. barrier();
  617. return bit;
  618. }
  619. static __always_inline void trace_clear_recursion(int bit)
  620. {
  621. unsigned int val = current->trace_recursion;
  622. bit = 1 << bit;
  623. val &= ~bit;
  624. barrier();
  625. current->trace_recursion = val;
  626. }
  627. static inline struct ring_buffer_iter *
  628. trace_buffer_iter(struct trace_iterator *iter, int cpu)
  629. {
  630. return iter->buffer_iter ? iter->buffer_iter[cpu] : NULL;
  631. }
  632. int tracer_init(struct tracer *t, struct trace_array *tr);
  633. int tracing_is_enabled(void);
  634. void tracing_reset_online_cpus(struct array_buffer *buf);
  635. void tracing_reset_current(int cpu);
  636. void tracing_reset_all_online_cpus(void);
  637. int tracing_open_generic(struct inode *inode, struct file *filp);
  638. int tracing_open_generic_tr(struct inode *inode, struct file *filp);
  639. bool tracing_is_disabled(void);
  640. bool tracer_tracing_is_on(struct trace_array *tr);
  641. void tracer_tracing_on(struct trace_array *tr);
  642. void tracer_tracing_off(struct trace_array *tr);
  643. struct dentry *trace_create_file(const char *name,
  644. umode_t mode,
  645. struct dentry *parent,
  646. void *data,
  647. const struct file_operations *fops);
  648. int tracing_init_dentry(void);
  649. struct ring_buffer_event;
  650. struct ring_buffer_event *
  651. trace_buffer_lock_reserve(struct trace_buffer *buffer,
  652. int type,
  653. unsigned long len,
  654. unsigned long flags,
  655. int pc);
  656. struct trace_entry *tracing_get_trace_entry(struct trace_array *tr,
  657. struct trace_array_cpu *data);
  658. struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
  659. int *ent_cpu, u64 *ent_ts);
  660. void trace_buffer_unlock_commit_nostack(struct trace_buffer *buffer,
  661. struct ring_buffer_event *event);
  662. int trace_empty(struct trace_iterator *iter);
  663. void *trace_find_next_entry_inc(struct trace_iterator *iter);
  664. void trace_init_global_iter(struct trace_iterator *iter);
  665. void tracing_iter_reset(struct trace_iterator *iter, int cpu);
  666. unsigned long trace_total_entries_cpu(struct trace_array *tr, int cpu);
  667. unsigned long trace_total_entries(struct trace_array *tr);
  668. void trace_function(struct trace_array *tr,
  669. unsigned long ip,
  670. unsigned long parent_ip,
  671. unsigned long flags, int pc);
  672. void trace_graph_function(struct trace_array *tr,
  673. unsigned long ip,
  674. unsigned long parent_ip,
  675. unsigned long flags, int pc);
  676. void trace_latency_header(struct seq_file *m);
  677. void trace_default_header(struct seq_file *m);
  678. void print_trace_header(struct seq_file *m, struct trace_iterator *iter);
  679. int trace_empty(struct trace_iterator *iter);
  680. void trace_graph_return(struct ftrace_graph_ret *trace);
  681. int trace_graph_entry(struct ftrace_graph_ent *trace);
  682. void set_graph_array(struct trace_array *tr);
  683. void tracing_start_cmdline_record(void);
  684. void tracing_stop_cmdline_record(void);
  685. void tracing_start_tgid_record(void);
  686. void tracing_stop_tgid_record(void);
  687. int register_tracer(struct tracer *type);
  688. int is_tracing_stopped(void);
  689. loff_t tracing_lseek(struct file *file, loff_t offset, int whence);
  690. extern cpumask_var_t __read_mostly tracing_buffer_mask;
  691. #define for_each_tracing_cpu(cpu) \
  692. for_each_cpu(cpu, tracing_buffer_mask)
  693. extern unsigned long nsecs_to_usecs(unsigned long nsecs);
  694. extern unsigned long tracing_thresh;
  695. /* PID filtering */
  696. extern int pid_max;
  697. bool trace_find_filtered_pid(struct trace_pid_list *filtered_pids,
  698. pid_t search_pid);
  699. bool trace_ignore_this_task(struct trace_pid_list *filtered_pids,
  700. struct trace_pid_list *filtered_no_pids,
  701. struct task_struct *task);
  702. void trace_filter_add_remove_task(struct trace_pid_list *pid_list,
  703. struct task_struct *self,
  704. struct task_struct *task);
  705. void *trace_pid_next(struct trace_pid_list *pid_list, void *v, loff_t *pos);
  706. void *trace_pid_start(struct trace_pid_list *pid_list, loff_t *pos);
  707. int trace_pid_show(struct seq_file *m, void *v);
  708. void trace_free_pid_list(struct trace_pid_list *pid_list);
  709. int trace_pid_write(struct trace_pid_list *filtered_pids,
  710. struct trace_pid_list **new_pid_list,
  711. const char __user *ubuf, size_t cnt);
  712. #ifdef CONFIG_TRACER_MAX_TRACE
  713. void update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu,
  714. void *cond_data);
  715. void update_max_tr_single(struct trace_array *tr,
  716. struct task_struct *tsk, int cpu);
  717. #endif /* CONFIG_TRACER_MAX_TRACE */
  718. #if (defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)) && \
  719. defined(CONFIG_FSNOTIFY)
  720. void latency_fsnotify(struct trace_array *tr);
  721. #else
  722. static inline void latency_fsnotify(struct trace_array *tr) { }
  723. #endif
  724. #ifdef CONFIG_STACKTRACE
  725. void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
  726. int pc);
  727. #else
  728. static inline void __trace_stack(struct trace_array *tr, unsigned long flags,
  729. int skip, int pc)
  730. {
  731. }
  732. #endif /* CONFIG_STACKTRACE */
  733. extern u64 ftrace_now(int cpu);
  734. extern void trace_find_cmdline(int pid, char comm[]);
  735. extern int trace_find_tgid(int pid);
  736. extern void trace_event_follow_fork(struct trace_array *tr, bool enable);
  737. #ifdef CONFIG_DYNAMIC_FTRACE
  738. extern unsigned long ftrace_update_tot_cnt;
  739. extern unsigned long ftrace_number_of_pages;
  740. extern unsigned long ftrace_number_of_groups;
  741. void ftrace_init_trace_array(struct trace_array *tr);
  742. #else
  743. static inline void ftrace_init_trace_array(struct trace_array *tr) { }
  744. #endif
  745. #define DYN_FTRACE_TEST_NAME trace_selftest_dynamic_test_func
  746. extern int DYN_FTRACE_TEST_NAME(void);
  747. #define DYN_FTRACE_TEST_NAME2 trace_selftest_dynamic_test_func2
  748. extern int DYN_FTRACE_TEST_NAME2(void);
  749. extern bool ring_buffer_expanded;
  750. extern bool tracing_selftest_disabled;
  751. #ifdef CONFIG_FTRACE_STARTUP_TEST
  752. extern void __init disable_tracing_selftest(const char *reason);
  753. extern int trace_selftest_startup_function(struct tracer *trace,
  754. struct trace_array *tr);
  755. extern int trace_selftest_startup_function_graph(struct tracer *trace,
  756. struct trace_array *tr);
  757. extern int trace_selftest_startup_irqsoff(struct tracer *trace,
  758. struct trace_array *tr);
  759. extern int trace_selftest_startup_preemptoff(struct tracer *trace,
  760. struct trace_array *tr);
  761. extern int trace_selftest_startup_preemptirqsoff(struct tracer *trace,
  762. struct trace_array *tr);
  763. extern int trace_selftest_startup_wakeup(struct tracer *trace,
  764. struct trace_array *tr);
  765. extern int trace_selftest_startup_nop(struct tracer *trace,
  766. struct trace_array *tr);
  767. extern int trace_selftest_startup_branch(struct tracer *trace,
  768. struct trace_array *tr);
  769. /*
  770. * Tracer data references selftest functions that only occur
  771. * on boot up. These can be __init functions. Thus, when selftests
  772. * are enabled, then the tracers need to reference __init functions.
  773. */
  774. #define __tracer_data __refdata
  775. #else
  776. static inline void __init disable_tracing_selftest(const char *reason)
  777. {
  778. }
  779. /* Tracers are seldom changed. Optimize when selftests are disabled. */
  780. #define __tracer_data __read_mostly
  781. #endif /* CONFIG_FTRACE_STARTUP_TEST */
  782. extern void *head_page(struct trace_array_cpu *data);
  783. extern unsigned long long ns2usecs(u64 nsec);
  784. extern int
  785. trace_vbprintk(unsigned long ip, const char *fmt, va_list args);
  786. extern int
  787. trace_vprintk(unsigned long ip, const char *fmt, va_list args);
  788. extern int
  789. trace_array_vprintk(struct trace_array *tr,
  790. unsigned long ip, const char *fmt, va_list args);
  791. int trace_array_printk_buf(struct trace_buffer *buffer,
  792. unsigned long ip, const char *fmt, ...);
  793. void trace_printk_seq(struct trace_seq *s);
  794. enum print_line_t print_trace_line(struct trace_iterator *iter);
  795. extern char trace_find_mark(unsigned long long duration);
  796. struct ftrace_hash;
  797. struct ftrace_mod_load {
  798. struct list_head list;
  799. char *func;
  800. char *module;
  801. int enable;
  802. };
  803. enum {
  804. FTRACE_HASH_FL_MOD = (1 << 0),
  805. };
  806. struct ftrace_hash {
  807. unsigned long size_bits;
  808. struct hlist_head *buckets;
  809. unsigned long count;
  810. unsigned long flags;
  811. struct rcu_head rcu;
  812. };
  813. struct ftrace_func_entry *
  814. ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip);
  815. static __always_inline bool ftrace_hash_empty(struct ftrace_hash *hash)
  816. {
  817. return !hash || !(hash->count || (hash->flags & FTRACE_HASH_FL_MOD));
  818. }
  819. /* Standard output formatting function used for function return traces */
  820. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  821. /* Flag options */
  822. #define TRACE_GRAPH_PRINT_OVERRUN 0x1
  823. #define TRACE_GRAPH_PRINT_CPU 0x2
  824. #define TRACE_GRAPH_PRINT_OVERHEAD 0x4
  825. #define TRACE_GRAPH_PRINT_PROC 0x8
  826. #define TRACE_GRAPH_PRINT_DURATION 0x10
  827. #define TRACE_GRAPH_PRINT_ABS_TIME 0x20
  828. #define TRACE_GRAPH_PRINT_REL_TIME 0x40
  829. #define TRACE_GRAPH_PRINT_IRQS 0x80
  830. #define TRACE_GRAPH_PRINT_TAIL 0x100
  831. #define TRACE_GRAPH_SLEEP_TIME 0x200
  832. #define TRACE_GRAPH_GRAPH_TIME 0x400
  833. #define TRACE_GRAPH_PRINT_FILL_SHIFT 28
  834. #define TRACE_GRAPH_PRINT_FILL_MASK (0x3 << TRACE_GRAPH_PRINT_FILL_SHIFT)
  835. extern void ftrace_graph_sleep_time_control(bool enable);
  836. #ifdef CONFIG_FUNCTION_PROFILER
  837. extern void ftrace_graph_graph_time_control(bool enable);
  838. #else
  839. static inline void ftrace_graph_graph_time_control(bool enable) { }
  840. #endif
  841. extern enum print_line_t
  842. print_graph_function_flags(struct trace_iterator *iter, u32 flags);
  843. extern void print_graph_headers_flags(struct seq_file *s, u32 flags);
  844. extern void
  845. trace_print_graph_duration(unsigned long long duration, struct trace_seq *s);
  846. extern void graph_trace_open(struct trace_iterator *iter);
  847. extern void graph_trace_close(struct trace_iterator *iter);
  848. extern int __trace_graph_entry(struct trace_array *tr,
  849. struct ftrace_graph_ent *trace,
  850. unsigned long flags, int pc);
  851. extern void __trace_graph_return(struct trace_array *tr,
  852. struct ftrace_graph_ret *trace,
  853. unsigned long flags, int pc);
  854. #ifdef CONFIG_DYNAMIC_FTRACE
  855. extern struct ftrace_hash __rcu *ftrace_graph_hash;
  856. extern struct ftrace_hash __rcu *ftrace_graph_notrace_hash;
  857. static inline int ftrace_graph_addr(struct ftrace_graph_ent *trace)
  858. {
  859. unsigned long addr = trace->func;
  860. int ret = 0;
  861. struct ftrace_hash *hash;
  862. preempt_disable_notrace();
  863. /*
  864. * Have to open code "rcu_dereference_sched()" because the
  865. * function graph tracer can be called when RCU is not
  866. * "watching".
  867. * Protected with schedule_on_each_cpu(ftrace_sync)
  868. */
  869. hash = rcu_dereference_protected(ftrace_graph_hash, !preemptible());
  870. if (ftrace_hash_empty(hash)) {
  871. ret = 1;
  872. goto out;
  873. }
  874. if (ftrace_lookup_ip(hash, addr)) {
  875. /*
  876. * This needs to be cleared on the return functions
  877. * when the depth is zero.
  878. */
  879. trace_recursion_set(TRACE_GRAPH_BIT);
  880. trace_recursion_set_depth(trace->depth);
  881. /*
  882. * If no irqs are to be traced, but a set_graph_function
  883. * is set, and called by an interrupt handler, we still
  884. * want to trace it.
  885. */
  886. if (in_irq())
  887. trace_recursion_set(TRACE_IRQ_BIT);
  888. else
  889. trace_recursion_clear(TRACE_IRQ_BIT);
  890. ret = 1;
  891. }
  892. out:
  893. preempt_enable_notrace();
  894. return ret;
  895. }
  896. static inline void ftrace_graph_addr_finish(struct ftrace_graph_ret *trace)
  897. {
  898. if (trace_recursion_test(TRACE_GRAPH_BIT) &&
  899. trace->depth == trace_recursion_depth())
  900. trace_recursion_clear(TRACE_GRAPH_BIT);
  901. }
  902. static inline int ftrace_graph_notrace_addr(unsigned long addr)
  903. {
  904. int ret = 0;
  905. struct ftrace_hash *notrace_hash;
  906. preempt_disable_notrace();
  907. /*
  908. * Have to open code "rcu_dereference_sched()" because the
  909. * function graph tracer can be called when RCU is not
  910. * "watching".
  911. * Protected with schedule_on_each_cpu(ftrace_sync)
  912. */
  913. notrace_hash = rcu_dereference_protected(ftrace_graph_notrace_hash,
  914. !preemptible());
  915. if (ftrace_lookup_ip(notrace_hash, addr))
  916. ret = 1;
  917. preempt_enable_notrace();
  918. return ret;
  919. }
  920. #else
  921. static inline int ftrace_graph_addr(struct ftrace_graph_ent *trace)
  922. {
  923. return 1;
  924. }
  925. static inline int ftrace_graph_notrace_addr(unsigned long addr)
  926. {
  927. return 0;
  928. }
  929. static inline void ftrace_graph_addr_finish(struct ftrace_graph_ret *trace)
  930. { }
  931. #endif /* CONFIG_DYNAMIC_FTRACE */
  932. extern unsigned int fgraph_max_depth;
  933. static inline bool ftrace_graph_ignore_func(struct ftrace_graph_ent *trace)
  934. {
  935. /* trace it when it is-nested-in or is a function enabled. */
  936. return !(trace_recursion_test(TRACE_GRAPH_BIT) ||
  937. ftrace_graph_addr(trace)) ||
  938. (trace->depth < 0) ||
  939. (fgraph_max_depth && trace->depth >= fgraph_max_depth);
  940. }
  941. #else /* CONFIG_FUNCTION_GRAPH_TRACER */
  942. static inline enum print_line_t
  943. print_graph_function_flags(struct trace_iterator *iter, u32 flags)
  944. {
  945. return TRACE_TYPE_UNHANDLED;
  946. }
  947. #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
  948. extern struct list_head ftrace_pids;
  949. #ifdef CONFIG_FUNCTION_TRACER
  950. #define FTRACE_PID_IGNORE -1
  951. #define FTRACE_PID_TRACE -2
  952. struct ftrace_func_command {
  953. struct list_head list;
  954. char *name;
  955. int (*func)(struct trace_array *tr,
  956. struct ftrace_hash *hash,
  957. char *func, char *cmd,
  958. char *params, int enable);
  959. };
  960. extern bool ftrace_filter_param __initdata;
  961. static inline int ftrace_trace_task(struct trace_array *tr)
  962. {
  963. return this_cpu_read(tr->array_buffer.data->ftrace_ignore_pid) !=
  964. FTRACE_PID_IGNORE;
  965. }
  966. extern int ftrace_is_dead(void);
  967. int ftrace_create_function_files(struct trace_array *tr,
  968. struct dentry *parent);
  969. void ftrace_destroy_function_files(struct trace_array *tr);
  970. int ftrace_allocate_ftrace_ops(struct trace_array *tr);
  971. void ftrace_free_ftrace_ops(struct trace_array *tr);
  972. void ftrace_init_global_array_ops(struct trace_array *tr);
  973. void ftrace_init_array_ops(struct trace_array *tr, ftrace_func_t func);
  974. void ftrace_reset_array_ops(struct trace_array *tr);
  975. void ftrace_init_tracefs(struct trace_array *tr, struct dentry *d_tracer);
  976. void ftrace_init_tracefs_toplevel(struct trace_array *tr,
  977. struct dentry *d_tracer);
  978. void ftrace_clear_pids(struct trace_array *tr);
  979. int init_function_trace(void);
  980. void ftrace_pid_follow_fork(struct trace_array *tr, bool enable);
  981. #else
  982. static inline int ftrace_trace_task(struct trace_array *tr)
  983. {
  984. return 1;
  985. }
  986. static inline int ftrace_is_dead(void) { return 0; }
  987. static inline int
  988. ftrace_create_function_files(struct trace_array *tr,
  989. struct dentry *parent)
  990. {
  991. return 0;
  992. }
  993. static inline int ftrace_allocate_ftrace_ops(struct trace_array *tr)
  994. {
  995. return 0;
  996. }
  997. static inline void ftrace_free_ftrace_ops(struct trace_array *tr) { }
  998. static inline void ftrace_destroy_function_files(struct trace_array *tr) { }
  999. static inline __init void
  1000. ftrace_init_global_array_ops(struct trace_array *tr) { }
  1001. static inline void ftrace_reset_array_ops(struct trace_array *tr) { }
  1002. static inline void ftrace_init_tracefs(struct trace_array *tr, struct dentry *d) { }
  1003. static inline void ftrace_init_tracefs_toplevel(struct trace_array *tr, struct dentry *d) { }
  1004. static inline void ftrace_clear_pids(struct trace_array *tr) { }
  1005. static inline int init_function_trace(void) { return 0; }
  1006. static inline void ftrace_pid_follow_fork(struct trace_array *tr, bool enable) { }
  1007. /* ftace_func_t type is not defined, use macro instead of static inline */
  1008. #define ftrace_init_array_ops(tr, func) do { } while (0)
  1009. #endif /* CONFIG_FUNCTION_TRACER */
  1010. #if defined(CONFIG_FUNCTION_TRACER) && defined(CONFIG_DYNAMIC_FTRACE)
  1011. struct ftrace_probe_ops {
  1012. void (*func)(unsigned long ip,
  1013. unsigned long parent_ip,
  1014. struct trace_array *tr,
  1015. struct ftrace_probe_ops *ops,
  1016. void *data);
  1017. int (*init)(struct ftrace_probe_ops *ops,
  1018. struct trace_array *tr,
  1019. unsigned long ip, void *init_data,
  1020. void **data);
  1021. void (*free)(struct ftrace_probe_ops *ops,
  1022. struct trace_array *tr,
  1023. unsigned long ip, void *data);
  1024. int (*print)(struct seq_file *m,
  1025. unsigned long ip,
  1026. struct ftrace_probe_ops *ops,
  1027. void *data);
  1028. };
  1029. struct ftrace_func_mapper;
  1030. typedef int (*ftrace_mapper_func)(void *data);
  1031. struct ftrace_func_mapper *allocate_ftrace_func_mapper(void);
  1032. void **ftrace_func_mapper_find_ip(struct ftrace_func_mapper *mapper,
  1033. unsigned long ip);
  1034. int ftrace_func_mapper_add_ip(struct ftrace_func_mapper *mapper,
  1035. unsigned long ip, void *data);
  1036. void *ftrace_func_mapper_remove_ip(struct ftrace_func_mapper *mapper,
  1037. unsigned long ip);
  1038. void free_ftrace_func_mapper(struct ftrace_func_mapper *mapper,
  1039. ftrace_mapper_func free_func);
  1040. extern int
  1041. register_ftrace_function_probe(char *glob, struct trace_array *tr,
  1042. struct ftrace_probe_ops *ops, void *data);
  1043. extern int
  1044. unregister_ftrace_function_probe_func(char *glob, struct trace_array *tr,
  1045. struct ftrace_probe_ops *ops);
  1046. extern void clear_ftrace_function_probes(struct trace_array *tr);
  1047. int register_ftrace_command(struct ftrace_func_command *cmd);
  1048. int unregister_ftrace_command(struct ftrace_func_command *cmd);
  1049. void ftrace_create_filter_files(struct ftrace_ops *ops,
  1050. struct dentry *parent);
  1051. void ftrace_destroy_filter_files(struct ftrace_ops *ops);
  1052. extern int ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf,
  1053. int len, int reset);
  1054. extern int ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf,
  1055. int len, int reset);
  1056. #else
  1057. struct ftrace_func_command;
  1058. static inline __init int register_ftrace_command(struct ftrace_func_command *cmd)
  1059. {
  1060. return -EINVAL;
  1061. }
  1062. static inline __init int unregister_ftrace_command(char *cmd_name)
  1063. {
  1064. return -EINVAL;
  1065. }
  1066. static inline void clear_ftrace_function_probes(struct trace_array *tr)
  1067. {
  1068. }
  1069. /*
  1070. * The ops parameter passed in is usually undefined.
  1071. * This must be a macro.
  1072. */
  1073. #define ftrace_create_filter_files(ops, parent) do { } while (0)
  1074. #define ftrace_destroy_filter_files(ops) do { } while (0)
  1075. #endif /* CONFIG_FUNCTION_TRACER && CONFIG_DYNAMIC_FTRACE */
  1076. bool ftrace_event_is_function(struct trace_event_call *call);
  1077. /*
  1078. * struct trace_parser - servers for reading the user input separated by spaces
  1079. * @cont: set if the input is not complete - no final space char was found
  1080. * @buffer: holds the parsed user input
  1081. * @idx: user input length
  1082. * @size: buffer size
  1083. */
  1084. struct trace_parser {
  1085. bool cont;
  1086. char *buffer;
  1087. unsigned idx;
  1088. unsigned size;
  1089. };
  1090. static inline bool trace_parser_loaded(struct trace_parser *parser)
  1091. {
  1092. return (parser->idx != 0);
  1093. }
  1094. static inline bool trace_parser_cont(struct trace_parser *parser)
  1095. {
  1096. return parser->cont;
  1097. }
  1098. static inline void trace_parser_clear(struct trace_parser *parser)
  1099. {
  1100. parser->cont = false;
  1101. parser->idx = 0;
  1102. }
  1103. extern int trace_parser_get_init(struct trace_parser *parser, int size);
  1104. extern void trace_parser_put(struct trace_parser *parser);
  1105. extern int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
  1106. size_t cnt, loff_t *ppos);
  1107. /*
  1108. * Only create function graph options if function graph is configured.
  1109. */
  1110. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  1111. # define FGRAPH_FLAGS \
  1112. C(DISPLAY_GRAPH, "display-graph"),
  1113. #else
  1114. # define FGRAPH_FLAGS
  1115. #endif
  1116. #ifdef CONFIG_BRANCH_TRACER
  1117. # define BRANCH_FLAGS \
  1118. C(BRANCH, "branch"),
  1119. #else
  1120. # define BRANCH_FLAGS
  1121. #endif
  1122. #ifdef CONFIG_FUNCTION_TRACER
  1123. # define FUNCTION_FLAGS \
  1124. C(FUNCTION, "function-trace"), \
  1125. C(FUNC_FORK, "function-fork"),
  1126. # define FUNCTION_DEFAULT_FLAGS TRACE_ITER_FUNCTION
  1127. #else
  1128. # define FUNCTION_FLAGS
  1129. # define FUNCTION_DEFAULT_FLAGS 0UL
  1130. # define TRACE_ITER_FUNC_FORK 0UL
  1131. #endif
  1132. #ifdef CONFIG_STACKTRACE
  1133. # define STACK_FLAGS \
  1134. C(STACKTRACE, "stacktrace"),
  1135. #else
  1136. # define STACK_FLAGS
  1137. #endif
  1138. /*
  1139. * trace_iterator_flags is an enumeration that defines bit
  1140. * positions into trace_flags that controls the output.
  1141. *
  1142. * NOTE: These bits must match the trace_options array in
  1143. * trace.c (this macro guarantees it).
  1144. */
  1145. #define TRACE_FLAGS \
  1146. C(PRINT_PARENT, "print-parent"), \
  1147. C(SYM_OFFSET, "sym-offset"), \
  1148. C(SYM_ADDR, "sym-addr"), \
  1149. C(VERBOSE, "verbose"), \
  1150. C(RAW, "raw"), \
  1151. C(HEX, "hex"), \
  1152. C(BIN, "bin"), \
  1153. C(BLOCK, "block"), \
  1154. C(PRINTK, "trace_printk"), \
  1155. C(ANNOTATE, "annotate"), \
  1156. C(USERSTACKTRACE, "userstacktrace"), \
  1157. C(SYM_USEROBJ, "sym-userobj"), \
  1158. C(PRINTK_MSGONLY, "printk-msg-only"), \
  1159. C(CONTEXT_INFO, "context-info"), /* Print pid/cpu/time */ \
  1160. C(LATENCY_FMT, "latency-format"), \
  1161. C(RECORD_CMD, "record-cmd"), \
  1162. C(RECORD_TGID, "record-tgid"), \
  1163. C(OVERWRITE, "overwrite"), \
  1164. C(STOP_ON_FREE, "disable_on_free"), \
  1165. C(IRQ_INFO, "irq-info"), \
  1166. C(MARKERS, "markers"), \
  1167. C(EVENT_FORK, "event-fork"), \
  1168. C(PAUSE_ON_TRACE, "pause-on-trace"), \
  1169. FUNCTION_FLAGS \
  1170. FGRAPH_FLAGS \
  1171. STACK_FLAGS \
  1172. BRANCH_FLAGS
  1173. /*
  1174. * By defining C, we can make TRACE_FLAGS a list of bit names
  1175. * that will define the bits for the flag masks.
  1176. */
  1177. #undef C
  1178. #define C(a, b) TRACE_ITER_##a##_BIT
  1179. enum trace_iterator_bits {
  1180. TRACE_FLAGS
  1181. /* Make sure we don't go more than we have bits for */
  1182. TRACE_ITER_LAST_BIT
  1183. };
  1184. /*
  1185. * By redefining C, we can make TRACE_FLAGS a list of masks that
  1186. * use the bits as defined above.
  1187. */
  1188. #undef C
  1189. #define C(a, b) TRACE_ITER_##a = (1 << TRACE_ITER_##a##_BIT)
  1190. enum trace_iterator_flags { TRACE_FLAGS };
  1191. /*
  1192. * TRACE_ITER_SYM_MASK masks the options in trace_flags that
  1193. * control the output of kernel symbols.
  1194. */
  1195. #define TRACE_ITER_SYM_MASK \
  1196. (TRACE_ITER_PRINT_PARENT|TRACE_ITER_SYM_OFFSET|TRACE_ITER_SYM_ADDR)
  1197. extern struct tracer nop_trace;
  1198. #ifdef CONFIG_BRANCH_TRACER
  1199. extern int enable_branch_tracing(struct trace_array *tr);
  1200. extern void disable_branch_tracing(void);
  1201. static inline int trace_branch_enable(struct trace_array *tr)
  1202. {
  1203. if (tr->trace_flags & TRACE_ITER_BRANCH)
  1204. return enable_branch_tracing(tr);
  1205. return 0;
  1206. }
  1207. static inline void trace_branch_disable(void)
  1208. {
  1209. /* due to races, always disable */
  1210. disable_branch_tracing();
  1211. }
  1212. #else
  1213. static inline int trace_branch_enable(struct trace_array *tr)
  1214. {
  1215. return 0;
  1216. }
  1217. static inline void trace_branch_disable(void)
  1218. {
  1219. }
  1220. #endif /* CONFIG_BRANCH_TRACER */
  1221. /* set ring buffers to default size if not already done so */
  1222. int tracing_update_buffers(void);
  1223. struct ftrace_event_field {
  1224. struct list_head link;
  1225. const char *name;
  1226. const char *type;
  1227. int filter_type;
  1228. int offset;
  1229. int size;
  1230. int is_signed;
  1231. };
  1232. struct prog_entry;
  1233. struct event_filter {
  1234. struct prog_entry __rcu *prog;
  1235. char *filter_string;
  1236. };
  1237. struct event_subsystem {
  1238. struct list_head list;
  1239. const char *name;
  1240. struct event_filter *filter;
  1241. int ref_count;
  1242. };
  1243. struct trace_subsystem_dir {
  1244. struct list_head list;
  1245. struct event_subsystem *subsystem;
  1246. struct trace_array *tr;
  1247. struct dentry *entry;
  1248. int ref_count;
  1249. int nr_events;
  1250. };
  1251. extern int call_filter_check_discard(struct trace_event_call *call, void *rec,
  1252. struct trace_buffer *buffer,
  1253. struct ring_buffer_event *event);
  1254. void trace_buffer_unlock_commit_regs(struct trace_array *tr,
  1255. struct trace_buffer *buffer,
  1256. struct ring_buffer_event *event,
  1257. unsigned long flags, int pc,
  1258. struct pt_regs *regs);
  1259. static inline void trace_buffer_unlock_commit(struct trace_array *tr,
  1260. struct trace_buffer *buffer,
  1261. struct ring_buffer_event *event,
  1262. unsigned long flags, int pc)
  1263. {
  1264. trace_buffer_unlock_commit_regs(tr, buffer, event, flags, pc, NULL);
  1265. }
  1266. DECLARE_PER_CPU(struct ring_buffer_event *, trace_buffered_event);
  1267. DECLARE_PER_CPU(int, trace_buffered_event_cnt);
  1268. void trace_buffered_event_disable(void);
  1269. void trace_buffered_event_enable(void);
  1270. static inline void
  1271. __trace_event_discard_commit(struct trace_buffer *buffer,
  1272. struct ring_buffer_event *event)
  1273. {
  1274. if (this_cpu_read(trace_buffered_event) == event) {
  1275. /* Simply release the temp buffer */
  1276. this_cpu_dec(trace_buffered_event_cnt);
  1277. return;
  1278. }
  1279. ring_buffer_discard_commit(buffer, event);
  1280. }
  1281. /*
  1282. * Helper function for event_trigger_unlock_commit{_regs}().
  1283. * If there are event triggers attached to this event that requires
  1284. * filtering against its fields, then they will be called as the
  1285. * entry already holds the field information of the current event.
  1286. *
  1287. * It also checks if the event should be discarded or not.
  1288. * It is to be discarded if the event is soft disabled and the
  1289. * event was only recorded to process triggers, or if the event
  1290. * filter is active and this event did not match the filters.
  1291. *
  1292. * Returns true if the event is discarded, false otherwise.
  1293. */
  1294. static inline bool
  1295. __event_trigger_test_discard(struct trace_event_file *file,
  1296. struct trace_buffer *buffer,
  1297. struct ring_buffer_event *event,
  1298. void *entry,
  1299. enum event_trigger_type *tt)
  1300. {
  1301. unsigned long eflags = file->flags;
  1302. if (eflags & EVENT_FILE_FL_TRIGGER_COND)
  1303. *tt = event_triggers_call(file, entry, event);
  1304. if (likely(!(file->flags & (EVENT_FILE_FL_SOFT_DISABLED |
  1305. EVENT_FILE_FL_FILTERED |
  1306. EVENT_FILE_FL_PID_FILTER))))
  1307. return false;
  1308. if (file->flags & EVENT_FILE_FL_SOFT_DISABLED)
  1309. goto discard;
  1310. if (file->flags & EVENT_FILE_FL_FILTERED &&
  1311. !filter_match_preds(file->filter, entry))
  1312. goto discard;
  1313. if ((file->flags & EVENT_FILE_FL_PID_FILTER) &&
  1314. trace_event_ignore_this_pid(file))
  1315. goto discard;
  1316. return false;
  1317. discard:
  1318. __trace_event_discard_commit(buffer, event);
  1319. return true;
  1320. }
  1321. /**
  1322. * event_trigger_unlock_commit - handle triggers and finish event commit
  1323. * @file: The file pointer assoctiated to the event
  1324. * @buffer: The ring buffer that the event is being written to
  1325. * @event: The event meta data in the ring buffer
  1326. * @entry: The event itself
  1327. * @irq_flags: The state of the interrupts at the start of the event
  1328. * @pc: The state of the preempt count at the start of the event.
  1329. *
  1330. * This is a helper function to handle triggers that require data
  1331. * from the event itself. It also tests the event against filters and
  1332. * if the event is soft disabled and should be discarded.
  1333. */
  1334. static inline void
  1335. event_trigger_unlock_commit(struct trace_event_file *file,
  1336. struct trace_buffer *buffer,
  1337. struct ring_buffer_event *event,
  1338. void *entry, unsigned long irq_flags, int pc)
  1339. {
  1340. enum event_trigger_type tt = ETT_NONE;
  1341. if (!__event_trigger_test_discard(file, buffer, event, entry, &tt))
  1342. trace_buffer_unlock_commit(file->tr, buffer, event, irq_flags, pc);
  1343. if (tt)
  1344. event_triggers_post_call(file, tt);
  1345. }
  1346. /**
  1347. * event_trigger_unlock_commit_regs - handle triggers and finish event commit
  1348. * @file: The file pointer assoctiated to the event
  1349. * @buffer: The ring buffer that the event is being written to
  1350. * @event: The event meta data in the ring buffer
  1351. * @entry: The event itself
  1352. * @irq_flags: The state of the interrupts at the start of the event
  1353. * @pc: The state of the preempt count at the start of the event.
  1354. *
  1355. * This is a helper function to handle triggers that require data
  1356. * from the event itself. It also tests the event against filters and
  1357. * if the event is soft disabled and should be discarded.
  1358. *
  1359. * Same as event_trigger_unlock_commit() but calls
  1360. * trace_buffer_unlock_commit_regs() instead of trace_buffer_unlock_commit().
  1361. */
  1362. static inline void
  1363. event_trigger_unlock_commit_regs(struct trace_event_file *file,
  1364. struct trace_buffer *buffer,
  1365. struct ring_buffer_event *event,
  1366. void *entry, unsigned long irq_flags, int pc,
  1367. struct pt_regs *regs)
  1368. {
  1369. enum event_trigger_type tt = ETT_NONE;
  1370. if (!__event_trigger_test_discard(file, buffer, event, entry, &tt))
  1371. trace_buffer_unlock_commit_regs(file->tr, buffer, event,
  1372. irq_flags, pc, regs);
  1373. if (tt)
  1374. event_triggers_post_call(file, tt);
  1375. }
  1376. #define FILTER_PRED_INVALID ((unsigned short)-1)
  1377. #define FILTER_PRED_IS_RIGHT (1 << 15)
  1378. #define FILTER_PRED_FOLD (1 << 15)
  1379. /*
  1380. * The max preds is the size of unsigned short with
  1381. * two flags at the MSBs. One bit is used for both the IS_RIGHT
  1382. * and FOLD flags. The other is reserved.
  1383. *
  1384. * 2^14 preds is way more than enough.
  1385. */
  1386. #define MAX_FILTER_PRED 16384
  1387. struct filter_pred;
  1388. struct regex;
  1389. typedef int (*filter_pred_fn_t) (struct filter_pred *pred, void *event);
  1390. typedef int (*regex_match_func)(char *str, struct regex *r, int len);
  1391. enum regex_type {
  1392. MATCH_FULL = 0,
  1393. MATCH_FRONT_ONLY,
  1394. MATCH_MIDDLE_ONLY,
  1395. MATCH_END_ONLY,
  1396. MATCH_GLOB,
  1397. MATCH_INDEX,
  1398. };
  1399. struct regex {
  1400. char pattern[MAX_FILTER_STR_VAL];
  1401. int len;
  1402. int field_len;
  1403. regex_match_func match;
  1404. };
  1405. struct filter_pred {
  1406. filter_pred_fn_t fn;
  1407. u64 val;
  1408. struct regex regex;
  1409. unsigned short *ops;
  1410. struct ftrace_event_field *field;
  1411. int offset;
  1412. int not;
  1413. int op;
  1414. };
  1415. static inline bool is_string_field(struct ftrace_event_field *field)
  1416. {
  1417. return field->filter_type == FILTER_DYN_STRING ||
  1418. field->filter_type == FILTER_STATIC_STRING ||
  1419. field->filter_type == FILTER_PTR_STRING ||
  1420. field->filter_type == FILTER_COMM;
  1421. }
  1422. static inline bool is_function_field(struct ftrace_event_field *field)
  1423. {
  1424. return field->filter_type == FILTER_TRACE_FN;
  1425. }
  1426. extern enum regex_type
  1427. filter_parse_regex(char *buff, int len, char **search, int *not);
  1428. extern void print_event_filter(struct trace_event_file *file,
  1429. struct trace_seq *s);
  1430. extern int apply_event_filter(struct trace_event_file *file,
  1431. char *filter_string);
  1432. extern int apply_subsystem_event_filter(struct trace_subsystem_dir *dir,
  1433. char *filter_string);
  1434. extern void print_subsystem_event_filter(struct event_subsystem *system,
  1435. struct trace_seq *s);
  1436. extern int filter_assign_type(const char *type);
  1437. extern int create_event_filter(struct trace_array *tr,
  1438. struct trace_event_call *call,
  1439. char *filter_str, bool set_str,
  1440. struct event_filter **filterp);
  1441. extern void free_event_filter(struct event_filter *filter);
  1442. struct ftrace_event_field *
  1443. trace_find_event_field(struct trace_event_call *call, char *name);
  1444. extern void trace_event_enable_cmd_record(bool enable);
  1445. extern void trace_event_enable_tgid_record(bool enable);
  1446. extern int event_trace_init(void);
  1447. extern int event_trace_add_tracer(struct dentry *parent, struct trace_array *tr);
  1448. extern int event_trace_del_tracer(struct trace_array *tr);
  1449. extern void __trace_early_add_events(struct trace_array *tr);
  1450. extern struct trace_event_file *__find_event_file(struct trace_array *tr,
  1451. const char *system,
  1452. const char *event);
  1453. extern struct trace_event_file *find_event_file(struct trace_array *tr,
  1454. const char *system,
  1455. const char *event);
  1456. static inline void *event_file_data(struct file *filp)
  1457. {
  1458. return READ_ONCE(file_inode(filp)->i_private);
  1459. }
  1460. extern struct mutex event_mutex;
  1461. extern struct list_head ftrace_events;
  1462. extern const struct file_operations event_trigger_fops;
  1463. extern const struct file_operations event_hist_fops;
  1464. extern const struct file_operations event_hist_debug_fops;
  1465. extern const struct file_operations event_inject_fops;
  1466. #ifdef CONFIG_HIST_TRIGGERS
  1467. extern int register_trigger_hist_cmd(void);
  1468. extern int register_trigger_hist_enable_disable_cmds(void);
  1469. #else
  1470. static inline int register_trigger_hist_cmd(void) { return 0; }
  1471. static inline int register_trigger_hist_enable_disable_cmds(void) { return 0; }
  1472. #endif
  1473. extern int register_trigger_cmds(void);
  1474. extern void clear_event_triggers(struct trace_array *tr);
  1475. struct event_trigger_data {
  1476. unsigned long count;
  1477. int ref;
  1478. struct event_trigger_ops *ops;
  1479. struct event_command *cmd_ops;
  1480. struct event_filter __rcu *filter;
  1481. char *filter_str;
  1482. void *private_data;
  1483. bool paused;
  1484. bool paused_tmp;
  1485. struct list_head list;
  1486. char *name;
  1487. struct list_head named_list;
  1488. struct event_trigger_data *named_data;
  1489. };
  1490. /* Avoid typos */
  1491. #define ENABLE_EVENT_STR "enable_event"
  1492. #define DISABLE_EVENT_STR "disable_event"
  1493. #define ENABLE_HIST_STR "enable_hist"
  1494. #define DISABLE_HIST_STR "disable_hist"
  1495. struct enable_trigger_data {
  1496. struct trace_event_file *file;
  1497. bool enable;
  1498. bool hist;
  1499. };
  1500. extern int event_enable_trigger_print(struct seq_file *m,
  1501. struct event_trigger_ops *ops,
  1502. struct event_trigger_data *data);
  1503. extern void event_enable_trigger_free(struct event_trigger_ops *ops,
  1504. struct event_trigger_data *data);
  1505. extern int event_enable_trigger_func(struct event_command *cmd_ops,
  1506. struct trace_event_file *file,
  1507. char *glob, char *cmd, char *param);
  1508. extern int event_enable_register_trigger(char *glob,
  1509. struct event_trigger_ops *ops,
  1510. struct event_trigger_data *data,
  1511. struct trace_event_file *file);
  1512. extern void event_enable_unregister_trigger(char *glob,
  1513. struct event_trigger_ops *ops,
  1514. struct event_trigger_data *test,
  1515. struct trace_event_file *file);
  1516. extern void trigger_data_free(struct event_trigger_data *data);
  1517. extern int event_trigger_init(struct event_trigger_ops *ops,
  1518. struct event_trigger_data *data);
  1519. extern int trace_event_trigger_enable_disable(struct trace_event_file *file,
  1520. int trigger_enable);
  1521. extern void update_cond_flag(struct trace_event_file *file);
  1522. extern int set_trigger_filter(char *filter_str,
  1523. struct event_trigger_data *trigger_data,
  1524. struct trace_event_file *file);
  1525. extern struct event_trigger_data *find_named_trigger(const char *name);
  1526. extern bool is_named_trigger(struct event_trigger_data *test);
  1527. extern int save_named_trigger(const char *name,
  1528. struct event_trigger_data *data);
  1529. extern void del_named_trigger(struct event_trigger_data *data);
  1530. extern void pause_named_trigger(struct event_trigger_data *data);
  1531. extern void unpause_named_trigger(struct event_trigger_data *data);
  1532. extern void set_named_trigger_data(struct event_trigger_data *data,
  1533. struct event_trigger_data *named_data);
  1534. extern struct event_trigger_data *
  1535. get_named_trigger_data(struct event_trigger_data *data);
  1536. extern int register_event_command(struct event_command *cmd);
  1537. extern int unregister_event_command(struct event_command *cmd);
  1538. extern int register_trigger_hist_enable_disable_cmds(void);
  1539. /**
  1540. * struct event_trigger_ops - callbacks for trace event triggers
  1541. *
  1542. * The methods in this structure provide per-event trigger hooks for
  1543. * various trigger operations.
  1544. *
  1545. * All the methods below, except for @init() and @free(), must be
  1546. * implemented.
  1547. *
  1548. * @func: The trigger 'probe' function called when the triggering
  1549. * event occurs. The data passed into this callback is the data
  1550. * that was supplied to the event_command @reg() function that
  1551. * registered the trigger (see struct event_command) along with
  1552. * the trace record, rec.
  1553. *
  1554. * @init: An optional initialization function called for the trigger
  1555. * when the trigger is registered (via the event_command reg()
  1556. * function). This can be used to perform per-trigger
  1557. * initialization such as incrementing a per-trigger reference
  1558. * count, for instance. This is usually implemented by the
  1559. * generic utility function @event_trigger_init() (see
  1560. * trace_event_triggers.c).
  1561. *
  1562. * @free: An optional de-initialization function called for the
  1563. * trigger when the trigger is unregistered (via the
  1564. * event_command @reg() function). This can be used to perform
  1565. * per-trigger de-initialization such as decrementing a
  1566. * per-trigger reference count and freeing corresponding trigger
  1567. * data, for instance. This is usually implemented by the
  1568. * generic utility function @event_trigger_free() (see
  1569. * trace_event_triggers.c).
  1570. *
  1571. * @print: The callback function invoked to have the trigger print
  1572. * itself. This is usually implemented by a wrapper function
  1573. * that calls the generic utility function @event_trigger_print()
  1574. * (see trace_event_triggers.c).
  1575. */
  1576. struct event_trigger_ops {
  1577. void (*func)(struct event_trigger_data *data,
  1578. void *rec,
  1579. struct ring_buffer_event *rbe);
  1580. int (*init)(struct event_trigger_ops *ops,
  1581. struct event_trigger_data *data);
  1582. void (*free)(struct event_trigger_ops *ops,
  1583. struct event_trigger_data *data);
  1584. int (*print)(struct seq_file *m,
  1585. struct event_trigger_ops *ops,
  1586. struct event_trigger_data *data);
  1587. };
  1588. /**
  1589. * struct event_command - callbacks and data members for event commands
  1590. *
  1591. * Event commands are invoked by users by writing the command name
  1592. * into the 'trigger' file associated with a trace event. The
  1593. * parameters associated with a specific invocation of an event
  1594. * command are used to create an event trigger instance, which is
  1595. * added to the list of trigger instances associated with that trace
  1596. * event. When the event is hit, the set of triggers associated with
  1597. * that event is invoked.
  1598. *
  1599. * The data members in this structure provide per-event command data
  1600. * for various event commands.
  1601. *
  1602. * All the data members below, except for @post_trigger, must be set
  1603. * for each event command.
  1604. *
  1605. * @name: The unique name that identifies the event command. This is
  1606. * the name used when setting triggers via trigger files.
  1607. *
  1608. * @trigger_type: A unique id that identifies the event command
  1609. * 'type'. This value has two purposes, the first to ensure that
  1610. * only one trigger of the same type can be set at a given time
  1611. * for a particular event e.g. it doesn't make sense to have both
  1612. * a traceon and traceoff trigger attached to a single event at
  1613. * the same time, so traceon and traceoff have the same type
  1614. * though they have different names. The @trigger_type value is
  1615. * also used as a bit value for deferring the actual trigger
  1616. * action until after the current event is finished. Some
  1617. * commands need to do this if they themselves log to the trace
  1618. * buffer (see the @post_trigger() member below). @trigger_type
  1619. * values are defined by adding new values to the trigger_type
  1620. * enum in include/linux/trace_events.h.
  1621. *
  1622. * @flags: See the enum event_command_flags below.
  1623. *
  1624. * All the methods below, except for @set_filter() and @unreg_all(),
  1625. * must be implemented.
  1626. *
  1627. * @func: The callback function responsible for parsing and
  1628. * registering the trigger written to the 'trigger' file by the
  1629. * user. It allocates the trigger instance and registers it with
  1630. * the appropriate trace event. It makes use of the other
  1631. * event_command callback functions to orchestrate this, and is
  1632. * usually implemented by the generic utility function
  1633. * @event_trigger_callback() (see trace_event_triggers.c).
  1634. *
  1635. * @reg: Adds the trigger to the list of triggers associated with the
  1636. * event, and enables the event trigger itself, after
  1637. * initializing it (via the event_trigger_ops @init() function).
  1638. * This is also where commands can use the @trigger_type value to
  1639. * make the decision as to whether or not multiple instances of
  1640. * the trigger should be allowed. This is usually implemented by
  1641. * the generic utility function @register_trigger() (see
  1642. * trace_event_triggers.c).
  1643. *
  1644. * @unreg: Removes the trigger from the list of triggers associated
  1645. * with the event, and disables the event trigger itself, after
  1646. * initializing it (via the event_trigger_ops @free() function).
  1647. * This is usually implemented by the generic utility function
  1648. * @unregister_trigger() (see trace_event_triggers.c).
  1649. *
  1650. * @unreg_all: An optional function called to remove all the triggers
  1651. * from the list of triggers associated with the event. Called
  1652. * when a trigger file is opened in truncate mode.
  1653. *
  1654. * @set_filter: An optional function called to parse and set a filter
  1655. * for the trigger. If no @set_filter() method is set for the
  1656. * event command, filters set by the user for the command will be
  1657. * ignored. This is usually implemented by the generic utility
  1658. * function @set_trigger_filter() (see trace_event_triggers.c).
  1659. *
  1660. * @get_trigger_ops: The callback function invoked to retrieve the
  1661. * event_trigger_ops implementation associated with the command.
  1662. */
  1663. struct event_command {
  1664. struct list_head list;
  1665. char *name;
  1666. enum event_trigger_type trigger_type;
  1667. int flags;
  1668. int (*func)(struct event_command *cmd_ops,
  1669. struct trace_event_file *file,
  1670. char *glob, char *cmd, char *params);
  1671. int (*reg)(char *glob,
  1672. struct event_trigger_ops *ops,
  1673. struct event_trigger_data *data,
  1674. struct trace_event_file *file);
  1675. void (*unreg)(char *glob,
  1676. struct event_trigger_ops *ops,
  1677. struct event_trigger_data *data,
  1678. struct trace_event_file *file);
  1679. void (*unreg_all)(struct trace_event_file *file);
  1680. int (*set_filter)(char *filter_str,
  1681. struct event_trigger_data *data,
  1682. struct trace_event_file *file);
  1683. struct event_trigger_ops *(*get_trigger_ops)(char *cmd, char *param);
  1684. };
  1685. /**
  1686. * enum event_command_flags - flags for struct event_command
  1687. *
  1688. * @POST_TRIGGER: A flag that says whether or not this command needs
  1689. * to have its action delayed until after the current event has
  1690. * been closed. Some triggers need to avoid being invoked while
  1691. * an event is currently in the process of being logged, since
  1692. * the trigger may itself log data into the trace buffer. Thus
  1693. * we make sure the current event is committed before invoking
  1694. * those triggers. To do that, the trigger invocation is split
  1695. * in two - the first part checks the filter using the current
  1696. * trace record; if a command has the @post_trigger flag set, it
  1697. * sets a bit for itself in the return value, otherwise it
  1698. * directly invokes the trigger. Once all commands have been
  1699. * either invoked or set their return flag, the current record is
  1700. * either committed or discarded. At that point, if any commands
  1701. * have deferred their triggers, those commands are finally
  1702. * invoked following the close of the current event. In other
  1703. * words, if the event_trigger_ops @func() probe implementation
  1704. * itself logs to the trace buffer, this flag should be set,
  1705. * otherwise it can be left unspecified.
  1706. *
  1707. * @NEEDS_REC: A flag that says whether or not this command needs
  1708. * access to the trace record in order to perform its function,
  1709. * regardless of whether or not it has a filter associated with
  1710. * it (filters make a trigger require access to the trace record
  1711. * but are not always present).
  1712. */
  1713. enum event_command_flags {
  1714. EVENT_CMD_FL_POST_TRIGGER = 1,
  1715. EVENT_CMD_FL_NEEDS_REC = 2,
  1716. };
  1717. static inline bool event_command_post_trigger(struct event_command *cmd_ops)
  1718. {
  1719. return cmd_ops->flags & EVENT_CMD_FL_POST_TRIGGER;
  1720. }
  1721. static inline bool event_command_needs_rec(struct event_command *cmd_ops)
  1722. {
  1723. return cmd_ops->flags & EVENT_CMD_FL_NEEDS_REC;
  1724. }
  1725. extern int trace_event_enable_disable(struct trace_event_file *file,
  1726. int enable, int soft_disable);
  1727. extern int tracing_alloc_snapshot(void);
  1728. extern void tracing_snapshot_cond(struct trace_array *tr, void *cond_data);
  1729. extern int tracing_snapshot_cond_enable(struct trace_array *tr, void *cond_data, cond_update_fn_t update);
  1730. extern int tracing_snapshot_cond_disable(struct trace_array *tr);
  1731. extern void *tracing_cond_snapshot_data(struct trace_array *tr);
  1732. extern const char *__start___trace_bprintk_fmt[];
  1733. extern const char *__stop___trace_bprintk_fmt[];
  1734. extern const char *__start___tracepoint_str[];
  1735. extern const char *__stop___tracepoint_str[];
  1736. void trace_printk_control(bool enabled);
  1737. void trace_printk_start_comm(void);
  1738. int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set);
  1739. int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled);
  1740. /* Used from boot time tracer */
  1741. extern int trace_set_options(struct trace_array *tr, char *option);
  1742. extern int tracing_set_tracer(struct trace_array *tr, const char *buf);
  1743. extern ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
  1744. unsigned long size, int cpu_id);
  1745. extern int tracing_set_cpumask(struct trace_array *tr,
  1746. cpumask_var_t tracing_cpumask_new);
  1747. #define MAX_EVENT_NAME_LEN 64
  1748. extern int trace_run_command(const char *buf, int (*createfn)(int, char**));
  1749. extern ssize_t trace_parse_run_command(struct file *file,
  1750. const char __user *buffer, size_t count, loff_t *ppos,
  1751. int (*createfn)(int, char**));
  1752. extern unsigned int err_pos(char *cmd, const char *str);
  1753. extern void tracing_log_err(struct trace_array *tr,
  1754. const char *loc, const char *cmd,
  1755. const char **errs, u8 type, u8 pos);
  1756. /*
  1757. * Normal trace_printk() and friends allocates special buffers
  1758. * to do the manipulation, as well as saves the print formats
  1759. * into sections to display. But the trace infrastructure wants
  1760. * to use these without the added overhead at the price of being
  1761. * a bit slower (used mainly for warnings, where we don't care
  1762. * about performance). The internal_trace_puts() is for such
  1763. * a purpose.
  1764. */
  1765. #define internal_trace_puts(str) __trace_puts(_THIS_IP_, str, strlen(str))
  1766. #undef FTRACE_ENTRY
  1767. #define FTRACE_ENTRY(call, struct_name, id, tstruct, print) \
  1768. extern struct trace_event_call \
  1769. __aligned(4) event_##call;
  1770. #undef FTRACE_ENTRY_DUP
  1771. #define FTRACE_ENTRY_DUP(call, struct_name, id, tstruct, print) \
  1772. FTRACE_ENTRY(call, struct_name, id, PARAMS(tstruct), PARAMS(print))
  1773. #undef FTRACE_ENTRY_PACKED
  1774. #define FTRACE_ENTRY_PACKED(call, struct_name, id, tstruct, print) \
  1775. FTRACE_ENTRY(call, struct_name, id, PARAMS(tstruct), PARAMS(print))
  1776. #include "trace_entries.h"
  1777. #if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_FUNCTION_TRACER)
  1778. int perf_ftrace_event_register(struct trace_event_call *call,
  1779. enum trace_reg type, void *data);
  1780. #else
  1781. #define perf_ftrace_event_register NULL
  1782. #endif
  1783. #ifdef CONFIG_FTRACE_SYSCALLS
  1784. void init_ftrace_syscalls(void);
  1785. const char *get_syscall_name(int syscall);
  1786. #else
  1787. static inline void init_ftrace_syscalls(void) { }
  1788. static inline const char *get_syscall_name(int syscall)
  1789. {
  1790. return NULL;
  1791. }
  1792. #endif
  1793. #ifdef CONFIG_EVENT_TRACING
  1794. void trace_event_init(void);
  1795. void trace_event_eval_update(struct trace_eval_map **map, int len);
  1796. /* Used from boot time tracer */
  1797. extern int ftrace_set_clr_event(struct trace_array *tr, char *buf, int set);
  1798. extern int trigger_process_regex(struct trace_event_file *file, char *buff);
  1799. #else
  1800. static inline void __init trace_event_init(void) { }
  1801. static inline void trace_event_eval_update(struct trace_eval_map **map, int len) { }
  1802. #endif
  1803. #ifdef CONFIG_TRACER_SNAPSHOT
  1804. void tracing_snapshot_instance(struct trace_array *tr);
  1805. int tracing_alloc_snapshot_instance(struct trace_array *tr);
  1806. #else
  1807. static inline void tracing_snapshot_instance(struct trace_array *tr) { }
  1808. static inline int tracing_alloc_snapshot_instance(struct trace_array *tr)
  1809. {
  1810. return 0;
  1811. }
  1812. #endif
  1813. #ifdef CONFIG_PREEMPT_TRACER
  1814. void tracer_preempt_on(unsigned long a0, unsigned long a1);
  1815. void tracer_preempt_off(unsigned long a0, unsigned long a1);
  1816. #else
  1817. static inline void tracer_preempt_on(unsigned long a0, unsigned long a1) { }
  1818. static inline void tracer_preempt_off(unsigned long a0, unsigned long a1) { }
  1819. #endif
  1820. #ifdef CONFIG_IRQSOFF_TRACER
  1821. void tracer_hardirqs_on(unsigned long a0, unsigned long a1);
  1822. void tracer_hardirqs_off(unsigned long a0, unsigned long a1);
  1823. #else
  1824. static inline void tracer_hardirqs_on(unsigned long a0, unsigned long a1) { }
  1825. static inline void tracer_hardirqs_off(unsigned long a0, unsigned long a1) { }
  1826. #endif
  1827. extern struct trace_iterator *tracepoint_print_iter;
  1828. /*
  1829. * Reset the state of the trace_iterator so that it can read consumed data.
  1830. * Normally, the trace_iterator is used for reading the data when it is not
  1831. * consumed, and must retain state.
  1832. */
  1833. static __always_inline void trace_iterator_reset(struct trace_iterator *iter)
  1834. {
  1835. const size_t offset = offsetof(struct trace_iterator, seq);
  1836. /*
  1837. * Keep gcc from complaining about overwriting more than just one
  1838. * member in the structure.
  1839. */
  1840. memset((char *)iter + offset, 0, sizeof(struct trace_iterator) - offset);
  1841. iter->pos = -1;
  1842. }
  1843. /* Check the name is good for event/group/fields */
  1844. static inline bool is_good_name(const char *name)
  1845. {
  1846. if (!isalpha(*name) && *name != '_')
  1847. return false;
  1848. while (*++name != '\0') {
  1849. if (!isalpha(*name) && !isdigit(*name) && *name != '_')
  1850. return false;
  1851. }
  1852. return true;
  1853. }
  1854. #endif /* _LINUX_KERNEL_TRACE_H */