printk_ringbuffer.h 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. #ifndef _KERNEL_PRINTK_RINGBUFFER_H
  3. #define _KERNEL_PRINTK_RINGBUFFER_H
  4. #include <linux/atomic.h>
  5. #include <linux/dev_printk.h>
  6. /*
  7. * Meta information about each stored message.
  8. *
  9. * All fields are set by the printk code except for @seq, which is
  10. * set by the ringbuffer code.
  11. */
  12. struct printk_info {
  13. u64 seq; /* sequence number */
  14. u64 ts_nsec; /* timestamp in nanoseconds */
  15. u16 text_len; /* length of text message */
  16. u8 facility; /* syslog facility */
  17. u8 flags:5; /* internal record flags */
  18. u8 level:3; /* syslog level */
  19. u32 caller_id; /* thread id or processor id */
  20. struct dev_printk_info dev_info;
  21. };
  22. /*
  23. * A structure providing the buffers, used by writers and readers.
  24. *
  25. * Writers:
  26. * Using prb_rec_init_wr(), a writer sets @text_buf_size before calling
  27. * prb_reserve(). On success, prb_reserve() sets @info and @text_buf to
  28. * buffers reserved for that writer.
  29. *
  30. * Readers:
  31. * Using prb_rec_init_rd(), a reader sets all fields before calling
  32. * prb_read_valid(). Note that the reader provides the @info and @text_buf,
  33. * buffers. On success, the struct pointed to by @info will be filled and
  34. * the char array pointed to by @text_buf will be filled with text data.
  35. */
  36. struct printk_record {
  37. struct printk_info *info;
  38. char *text_buf;
  39. unsigned int text_buf_size;
  40. };
  41. /* Specifies the logical position and span of a data block. */
  42. struct prb_data_blk_lpos {
  43. unsigned long begin;
  44. unsigned long next;
  45. };
  46. /*
  47. * A descriptor: the complete meta-data for a record.
  48. *
  49. * @state_var: A bitwise combination of descriptor ID and descriptor state.
  50. */
  51. struct prb_desc {
  52. atomic_long_t state_var;
  53. struct prb_data_blk_lpos text_blk_lpos;
  54. };
  55. /* A ringbuffer of "ID + data" elements. */
  56. struct prb_data_ring {
  57. unsigned int size_bits;
  58. char *data;
  59. atomic_long_t head_lpos;
  60. atomic_long_t tail_lpos;
  61. };
  62. /* A ringbuffer of "struct prb_desc" elements. */
  63. struct prb_desc_ring {
  64. unsigned int count_bits;
  65. struct prb_desc *descs;
  66. struct printk_info *infos;
  67. atomic_long_t head_id;
  68. atomic_long_t tail_id;
  69. atomic_long_t last_finalized_id;
  70. };
  71. /*
  72. * The high level structure representing the printk ringbuffer.
  73. *
  74. * @fail: Count of failed prb_reserve() calls where not even a data-less
  75. * record was created.
  76. */
  77. struct printk_ringbuffer {
  78. struct prb_desc_ring desc_ring;
  79. struct prb_data_ring text_data_ring;
  80. atomic_long_t fail;
  81. };
  82. /*
  83. * Used by writers as a reserve/commit handle.
  84. *
  85. * @rb: Ringbuffer where the entry is reserved.
  86. * @irqflags: Saved irq flags to restore on entry commit.
  87. * @id: ID of the reserved descriptor.
  88. * @text_space: Total occupied buffer space in the text data ring, including
  89. * ID, alignment padding, and wrapping data blocks.
  90. *
  91. * This structure is an opaque handle for writers. Its contents are only
  92. * to be used by the ringbuffer implementation.
  93. */
  94. struct prb_reserved_entry {
  95. struct printk_ringbuffer *rb;
  96. unsigned long irqflags;
  97. unsigned long id;
  98. unsigned int text_space;
  99. };
  100. /* The possible responses of a descriptor state-query. */
  101. enum desc_state {
  102. desc_miss = -1, /* ID mismatch (pseudo state) */
  103. desc_reserved = 0x0, /* reserved, in use by writer */
  104. desc_committed = 0x1, /* committed by writer, could get reopened */
  105. desc_finalized = 0x2, /* committed, no further modification allowed */
  106. desc_reusable = 0x3, /* free, not yet used by any writer */
  107. };
  108. #define _DATA_SIZE(sz_bits) (1UL << (sz_bits))
  109. #define _DESCS_COUNT(ct_bits) (1U << (ct_bits))
  110. #define DESC_SV_BITS (sizeof(unsigned long) * 8)
  111. #define DESC_FLAGS_SHIFT (DESC_SV_BITS - 2)
  112. #define DESC_FLAGS_MASK (3UL << DESC_FLAGS_SHIFT)
  113. #define DESC_STATE(sv) (3UL & (sv >> DESC_FLAGS_SHIFT))
  114. #define DESC_SV(id, state) (((unsigned long)state << DESC_FLAGS_SHIFT) | id)
  115. #define DESC_ID_MASK (~DESC_FLAGS_MASK)
  116. #define DESC_ID(sv) ((sv) & DESC_ID_MASK)
  117. #define FAILED_LPOS 0x1
  118. #define NO_LPOS 0x3
  119. #define FAILED_BLK_LPOS \
  120. { \
  121. .begin = FAILED_LPOS, \
  122. .next = FAILED_LPOS, \
  123. }
  124. /*
  125. * Descriptor Bootstrap
  126. *
  127. * The descriptor array is minimally initialized to allow immediate usage
  128. * by readers and writers. The requirements that the descriptor array
  129. * initialization must satisfy:
  130. *
  131. * Req1
  132. * The tail must point to an existing (committed or reusable) descriptor.
  133. * This is required by the implementation of prb_first_seq().
  134. *
  135. * Req2
  136. * Readers must see that the ringbuffer is initially empty.
  137. *
  138. * Req3
  139. * The first record reserved by a writer is assigned sequence number 0.
  140. *
  141. * To satisfy Req1, the tail initially points to a descriptor that is
  142. * minimally initialized (having no data block, i.e. data-less with the
  143. * data block's lpos @begin and @next values set to FAILED_LPOS).
  144. *
  145. * To satisfy Req2, the initial tail descriptor is initialized to the
  146. * reusable state. Readers recognize reusable descriptors as existing
  147. * records, but skip over them.
  148. *
  149. * To satisfy Req3, the last descriptor in the array is used as the initial
  150. * head (and tail) descriptor. This allows the first record reserved by a
  151. * writer (head + 1) to be the first descriptor in the array. (Only the first
  152. * descriptor in the array could have a valid sequence number of 0.)
  153. *
  154. * The first time a descriptor is reserved, it is assigned a sequence number
  155. * with the value of the array index. A "first time reserved" descriptor can
  156. * be recognized because it has a sequence number of 0 but does not have an
  157. * index of 0. (Only the first descriptor in the array could have a valid
  158. * sequence number of 0.) After the first reservation, all future reservations
  159. * (recycling) simply involve incrementing the sequence number by the array
  160. * count.
  161. *
  162. * Hack #1
  163. * Only the first descriptor in the array is allowed to have the sequence
  164. * number 0. In this case it is not possible to recognize if it is being
  165. * reserved the first time (set to index value) or has been reserved
  166. * previously (increment by the array count). This is handled by _always_
  167. * incrementing the sequence number by the array count when reserving the
  168. * first descriptor in the array. In order to satisfy Req3, the sequence
  169. * number of the first descriptor in the array is initialized to minus
  170. * the array count. Then, upon the first reservation, it is incremented
  171. * to 0, thus satisfying Req3.
  172. *
  173. * Hack #2
  174. * prb_first_seq() can be called at any time by readers to retrieve the
  175. * sequence number of the tail descriptor. However, due to Req2 and Req3,
  176. * initially there are no records to report the sequence number of
  177. * (sequence numbers are u64 and there is nothing less than 0). To handle
  178. * this, the sequence number of the initial tail descriptor is initialized
  179. * to 0. Technically this is incorrect, because there is no record with
  180. * sequence number 0 (yet) and the tail descriptor is not the first
  181. * descriptor in the array. But it allows prb_read_valid() to correctly
  182. * report the existence of a record for _any_ given sequence number at all
  183. * times. Bootstrapping is complete when the tail is pushed the first
  184. * time, thus finally pointing to the first descriptor reserved by a
  185. * writer, which has the assigned sequence number 0.
  186. */
  187. /*
  188. * Initiating Logical Value Overflows
  189. *
  190. * Both logical position (lpos) and ID values can be mapped to array indexes
  191. * but may experience overflows during the lifetime of the system. To ensure
  192. * that printk_ringbuffer can handle the overflows for these types, initial
  193. * values are chosen that map to the correct initial array indexes, but will
  194. * result in overflows soon.
  195. *
  196. * BLK0_LPOS
  197. * The initial @head_lpos and @tail_lpos for data rings. It is at index
  198. * 0 and the lpos value is such that it will overflow on the first wrap.
  199. *
  200. * DESC0_ID
  201. * The initial @head_id and @tail_id for the desc ring. It is at the last
  202. * index of the descriptor array (see Req3 above) and the ID value is such
  203. * that it will overflow on the second wrap.
  204. */
  205. #define BLK0_LPOS(sz_bits) (-(_DATA_SIZE(sz_bits)))
  206. #define DESC0_ID(ct_bits) DESC_ID(-(_DESCS_COUNT(ct_bits) + 1))
  207. #define DESC0_SV(ct_bits) DESC_SV(DESC0_ID(ct_bits), desc_reusable)
  208. /*
  209. * Define a ringbuffer with an external text data buffer. The same as
  210. * DEFINE_PRINTKRB() but requires specifying an external buffer for the
  211. * text data.
  212. *
  213. * Note: The specified external buffer must be of the size:
  214. * 2 ^ (descbits + avgtextbits)
  215. */
  216. #define _DEFINE_PRINTKRB(name, descbits, avgtextbits, text_buf) \
  217. static struct prb_desc _##name##_descs[_DESCS_COUNT(descbits)] = { \
  218. /* the initial head and tail */ \
  219. [_DESCS_COUNT(descbits) - 1] = { \
  220. /* reusable */ \
  221. .state_var = ATOMIC_INIT(DESC0_SV(descbits)), \
  222. /* no associated data block */ \
  223. .text_blk_lpos = FAILED_BLK_LPOS, \
  224. }, \
  225. }; \
  226. static struct printk_info _##name##_infos[_DESCS_COUNT(descbits)] = { \
  227. /* this will be the first record reserved by a writer */ \
  228. [0] = { \
  229. /* will be incremented to 0 on the first reservation */ \
  230. .seq = -(u64)_DESCS_COUNT(descbits), \
  231. }, \
  232. /* the initial head and tail */ \
  233. [_DESCS_COUNT(descbits) - 1] = { \
  234. /* reports the first seq value during the bootstrap phase */ \
  235. .seq = 0, \
  236. }, \
  237. }; \
  238. static struct printk_ringbuffer name = { \
  239. .desc_ring = { \
  240. .count_bits = descbits, \
  241. .descs = &_##name##_descs[0], \
  242. .infos = &_##name##_infos[0], \
  243. .head_id = ATOMIC_INIT(DESC0_ID(descbits)), \
  244. .tail_id = ATOMIC_INIT(DESC0_ID(descbits)), \
  245. .last_finalized_id = ATOMIC_INIT(DESC0_ID(descbits)), \
  246. }, \
  247. .text_data_ring = { \
  248. .size_bits = (avgtextbits) + (descbits), \
  249. .data = text_buf, \
  250. .head_lpos = ATOMIC_LONG_INIT(BLK0_LPOS((avgtextbits) + (descbits))), \
  251. .tail_lpos = ATOMIC_LONG_INIT(BLK0_LPOS((avgtextbits) + (descbits))), \
  252. }, \
  253. .fail = ATOMIC_LONG_INIT(0), \
  254. }
  255. /**
  256. * DEFINE_PRINTKRB() - Define a ringbuffer.
  257. *
  258. * @name: The name of the ringbuffer variable.
  259. * @descbits: The number of descriptors as a power-of-2 value.
  260. * @avgtextbits: The average text data size per record as a power-of-2 value.
  261. *
  262. * This is a macro for defining a ringbuffer and all internal structures
  263. * such that it is ready for immediate use. See _DEFINE_PRINTKRB() for a
  264. * variant where the text data buffer can be specified externally.
  265. */
  266. #define DEFINE_PRINTKRB(name, descbits, avgtextbits) \
  267. static char _##name##_text[1U << ((avgtextbits) + (descbits))] \
  268. __aligned(__alignof__(unsigned long)); \
  269. _DEFINE_PRINTKRB(name, descbits, avgtextbits, &_##name##_text[0])
  270. /* Writer Interface */
  271. /**
  272. * prb_rec_init_wd() - Initialize a buffer for writing records.
  273. *
  274. * @r: The record to initialize.
  275. * @text_buf_size: The needed text buffer size.
  276. */
  277. static inline void prb_rec_init_wr(struct printk_record *r,
  278. unsigned int text_buf_size)
  279. {
  280. r->info = NULL;
  281. r->text_buf = NULL;
  282. r->text_buf_size = text_buf_size;
  283. }
  284. bool prb_reserve(struct prb_reserved_entry *e, struct printk_ringbuffer *rb,
  285. struct printk_record *r);
  286. bool prb_reserve_in_last(struct prb_reserved_entry *e, struct printk_ringbuffer *rb,
  287. struct printk_record *r, u32 caller_id, unsigned int max_size);
  288. void prb_commit(struct prb_reserved_entry *e);
  289. void prb_final_commit(struct prb_reserved_entry *e);
  290. void prb_init(struct printk_ringbuffer *rb,
  291. char *text_buf, unsigned int text_buf_size,
  292. struct prb_desc *descs, unsigned int descs_count_bits,
  293. struct printk_info *infos);
  294. unsigned int prb_record_text_space(struct prb_reserved_entry *e);
  295. /* Reader Interface */
  296. /**
  297. * prb_rec_init_rd() - Initialize a buffer for reading records.
  298. *
  299. * @r: The record to initialize.
  300. * @info: A buffer to store record meta-data.
  301. * @text_buf: A buffer to store text data.
  302. * @text_buf_size: The size of @text_buf.
  303. *
  304. * Initialize all the fields that a reader is interested in. All arguments
  305. * (except @r) are optional. Only record data for arguments that are
  306. * non-NULL or non-zero will be read.
  307. */
  308. static inline void prb_rec_init_rd(struct printk_record *r,
  309. struct printk_info *info,
  310. char *text_buf, unsigned int text_buf_size)
  311. {
  312. r->info = info;
  313. r->text_buf = text_buf;
  314. r->text_buf_size = text_buf_size;
  315. }
  316. /**
  317. * prb_for_each_record() - Iterate over the records of a ringbuffer.
  318. *
  319. * @from: The sequence number to begin with.
  320. * @rb: The ringbuffer to iterate over.
  321. * @s: A u64 to store the sequence number on each iteration.
  322. * @r: A printk_record to store the record on each iteration.
  323. *
  324. * This is a macro for conveniently iterating over a ringbuffer.
  325. * Note that @s may not be the sequence number of the record on each
  326. * iteration. For the sequence number, @r->info->seq should be checked.
  327. *
  328. * Context: Any context.
  329. */
  330. #define prb_for_each_record(from, rb, s, r) \
  331. for ((s) = from; prb_read_valid(rb, s, r); (s) = (r)->info->seq + 1)
  332. /**
  333. * prb_for_each_info() - Iterate over the meta data of a ringbuffer.
  334. *
  335. * @from: The sequence number to begin with.
  336. * @rb: The ringbuffer to iterate over.
  337. * @s: A u64 to store the sequence number on each iteration.
  338. * @i: A printk_info to store the record meta data on each iteration.
  339. * @lc: An unsigned int to store the text line count of each record.
  340. *
  341. * This is a macro for conveniently iterating over a ringbuffer.
  342. * Note that @s may not be the sequence number of the record on each
  343. * iteration. For the sequence number, @r->info->seq should be checked.
  344. *
  345. * Context: Any context.
  346. */
  347. #define prb_for_each_info(from, rb, s, i, lc) \
  348. for ((s) = from; prb_read_valid_info(rb, s, i, lc); (s) = (i)->seq + 1)
  349. bool prb_read_valid(struct printk_ringbuffer *rb, u64 seq,
  350. struct printk_record *r);
  351. bool prb_read_valid_info(struct printk_ringbuffer *rb, u64 seq,
  352. struct printk_info *info, unsigned int *line_count);
  353. u64 prb_first_valid_seq(struct printk_ringbuffer *rb);
  354. u64 prb_next_seq(struct printk_ringbuffer *rb);
  355. #endif /* _KERNEL_PRINTK_RINGBUFFER_H */