sorttable.h 9.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380
  1. /* SPDX-License-Identifier: GPL-2.0-only */
  2. /*
  3. * sorttable.h
  4. *
  5. * Added ORC unwind tables sort support and other updates:
  6. * Copyright (C) 1999-2019 Alibaba Group Holding Limited. by:
  7. * Shile Zhang <shile.zhang@linux.alibaba.com>
  8. *
  9. * Copyright 2011 - 2012 Cavium, Inc.
  10. *
  11. * Some of code was taken out of arch/x86/kernel/unwind_orc.c, written by:
  12. * Copyright (C) 2017 Josh Poimboeuf <jpoimboe@redhat.com>
  13. *
  14. * Some of this code was taken out of recordmcount.h written by:
  15. *
  16. * Copyright 2009 John F. Reiser <jreiser@BitWagon.com>. All rights reserved.
  17. * Copyright 2010 Steven Rostedt <srostedt@redhat.com>, Red Hat Inc.
  18. */
  19. #undef extable_ent_size
  20. #undef compare_extable
  21. #undef do_sort
  22. #undef Elf_Addr
  23. #undef Elf_Ehdr
  24. #undef Elf_Shdr
  25. #undef Elf_Rel
  26. #undef Elf_Rela
  27. #undef Elf_Sym
  28. #undef ELF_R_SYM
  29. #undef Elf_r_sym
  30. #undef ELF_R_INFO
  31. #undef Elf_r_info
  32. #undef ELF_ST_BIND
  33. #undef ELF_ST_TYPE
  34. #undef fn_ELF_R_SYM
  35. #undef fn_ELF_R_INFO
  36. #undef uint_t
  37. #undef _r
  38. #undef _w
  39. #ifdef SORTTABLE_64
  40. # define extable_ent_size 16
  41. # define compare_extable compare_extable_64
  42. # define do_sort do_sort_64
  43. # define Elf_Addr Elf64_Addr
  44. # define Elf_Ehdr Elf64_Ehdr
  45. # define Elf_Shdr Elf64_Shdr
  46. # define Elf_Rel Elf64_Rel
  47. # define Elf_Rela Elf64_Rela
  48. # define Elf_Sym Elf64_Sym
  49. # define ELF_R_SYM ELF64_R_SYM
  50. # define Elf_r_sym Elf64_r_sym
  51. # define ELF_R_INFO ELF64_R_INFO
  52. # define Elf_r_info Elf64_r_info
  53. # define ELF_ST_BIND ELF64_ST_BIND
  54. # define ELF_ST_TYPE ELF64_ST_TYPE
  55. # define fn_ELF_R_SYM fn_ELF64_R_SYM
  56. # define fn_ELF_R_INFO fn_ELF64_R_INFO
  57. # define uint_t uint64_t
  58. # define _r r8
  59. # define _w w8
  60. #else
  61. # define extable_ent_size 8
  62. # define compare_extable compare_extable_32
  63. # define do_sort do_sort_32
  64. # define Elf_Addr Elf32_Addr
  65. # define Elf_Ehdr Elf32_Ehdr
  66. # define Elf_Shdr Elf32_Shdr
  67. # define Elf_Rel Elf32_Rel
  68. # define Elf_Rela Elf32_Rela
  69. # define Elf_Sym Elf32_Sym
  70. # define ELF_R_SYM ELF32_R_SYM
  71. # define Elf_r_sym Elf32_r_sym
  72. # define ELF_R_INFO ELF32_R_INFO
  73. # define Elf_r_info Elf32_r_info
  74. # define ELF_ST_BIND ELF32_ST_BIND
  75. # define ELF_ST_TYPE ELF32_ST_TYPE
  76. # define fn_ELF_R_SYM fn_ELF32_R_SYM
  77. # define fn_ELF_R_INFO fn_ELF32_R_INFO
  78. # define uint_t uint32_t
  79. # define _r r
  80. # define _w w
  81. #endif
  82. #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED)
  83. /* ORC unwinder only support X86_64 */
  84. #include <errno.h>
  85. #include <pthread.h>
  86. #include <asm/orc_types.h>
  87. #define ERRSTR_MAXSZ 256
  88. char g_err[ERRSTR_MAXSZ];
  89. int *g_orc_ip_table;
  90. struct orc_entry *g_orc_table;
  91. pthread_t orc_sort_thread;
  92. static inline unsigned long orc_ip(const int *ip)
  93. {
  94. return (unsigned long)ip + *ip;
  95. }
  96. static int orc_sort_cmp(const void *_a, const void *_b)
  97. {
  98. struct orc_entry *orc_a;
  99. const int *a = g_orc_ip_table + *(int *)_a;
  100. const int *b = g_orc_ip_table + *(int *)_b;
  101. unsigned long a_val = orc_ip(a);
  102. unsigned long b_val = orc_ip(b);
  103. if (a_val > b_val)
  104. return 1;
  105. if (a_val < b_val)
  106. return -1;
  107. /*
  108. * The "weak" section terminator entries need to always be on the left
  109. * to ensure the lookup code skips them in favor of real entries.
  110. * These terminator entries exist to handle any gaps created by
  111. * whitelisted .o files which didn't get objtool generation.
  112. */
  113. orc_a = g_orc_table + (a - g_orc_ip_table);
  114. return orc_a->sp_reg == ORC_REG_UNDEFINED && !orc_a->end ? -1 : 1;
  115. }
  116. static void *sort_orctable(void *arg)
  117. {
  118. int i;
  119. int *idxs = NULL;
  120. int *tmp_orc_ip_table = NULL;
  121. struct orc_entry *tmp_orc_table = NULL;
  122. unsigned int *orc_ip_size = (unsigned int *)arg;
  123. unsigned int num_entries = *orc_ip_size / sizeof(int);
  124. unsigned int orc_size = num_entries * sizeof(struct orc_entry);
  125. idxs = (int *)malloc(*orc_ip_size);
  126. if (!idxs) {
  127. snprintf(g_err, ERRSTR_MAXSZ, "malloc idxs: %s",
  128. strerror(errno));
  129. pthread_exit(g_err);
  130. }
  131. tmp_orc_ip_table = (int *)malloc(*orc_ip_size);
  132. if (!tmp_orc_ip_table) {
  133. snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_ip_table: %s",
  134. strerror(errno));
  135. pthread_exit(g_err);
  136. }
  137. tmp_orc_table = (struct orc_entry *)malloc(orc_size);
  138. if (!tmp_orc_table) {
  139. snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_table: %s",
  140. strerror(errno));
  141. pthread_exit(g_err);
  142. }
  143. /* initialize indices array, convert ip_table to absolute address */
  144. for (i = 0; i < num_entries; i++) {
  145. idxs[i] = i;
  146. tmp_orc_ip_table[i] = g_orc_ip_table[i] + i * sizeof(int);
  147. }
  148. memcpy(tmp_orc_table, g_orc_table, orc_size);
  149. qsort(idxs, num_entries, sizeof(int), orc_sort_cmp);
  150. for (i = 0; i < num_entries; i++) {
  151. if (idxs[i] == i)
  152. continue;
  153. /* convert back to relative address */
  154. g_orc_ip_table[i] = tmp_orc_ip_table[idxs[i]] - i * sizeof(int);
  155. g_orc_table[i] = tmp_orc_table[idxs[i]];
  156. }
  157. free(idxs);
  158. free(tmp_orc_ip_table);
  159. free(tmp_orc_table);
  160. pthread_exit(NULL);
  161. }
  162. #endif
  163. static int compare_extable(const void *a, const void *b)
  164. {
  165. Elf_Addr av = _r(a);
  166. Elf_Addr bv = _r(b);
  167. if (av < bv)
  168. return -1;
  169. if (av > bv)
  170. return 1;
  171. return 0;
  172. }
  173. static int do_sort(Elf_Ehdr *ehdr,
  174. char const *const fname,
  175. table_sort_t custom_sort)
  176. {
  177. int rc = -1;
  178. Elf_Shdr *s, *shdr = (Elf_Shdr *)((char *)ehdr + _r(&ehdr->e_shoff));
  179. Elf_Shdr *strtab_sec = NULL;
  180. Elf_Shdr *symtab_sec = NULL;
  181. Elf_Shdr *extab_sec = NULL;
  182. Elf_Sym *sym;
  183. const Elf_Sym *symtab;
  184. Elf32_Word *symtab_shndx = NULL;
  185. Elf_Sym *sort_needed_sym = NULL;
  186. Elf_Shdr *sort_needed_sec;
  187. Elf_Rel *relocs = NULL;
  188. int relocs_size = 0;
  189. uint32_t *sort_needed_loc;
  190. const char *secstrings;
  191. const char *strtab;
  192. char *extab_image;
  193. int extab_index = 0;
  194. int i;
  195. int idx;
  196. unsigned int shnum;
  197. unsigned int shstrndx;
  198. #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED)
  199. unsigned int orc_ip_size = 0;
  200. unsigned int orc_size = 0;
  201. unsigned int orc_num_entries = 0;
  202. #endif
  203. shstrndx = r2(&ehdr->e_shstrndx);
  204. if (shstrndx == SHN_XINDEX)
  205. shstrndx = r(&shdr[0].sh_link);
  206. secstrings = (const char *)ehdr + _r(&shdr[shstrndx].sh_offset);
  207. shnum = r2(&ehdr->e_shnum);
  208. if (shnum == SHN_UNDEF)
  209. shnum = _r(&shdr[0].sh_size);
  210. for (i = 0, s = shdr; s < shdr + shnum; i++, s++) {
  211. idx = r(&s->sh_name);
  212. if (!strcmp(secstrings + idx, "__ex_table")) {
  213. extab_sec = s;
  214. extab_index = i;
  215. }
  216. if (!strcmp(secstrings + idx, ".symtab"))
  217. symtab_sec = s;
  218. if (!strcmp(secstrings + idx, ".strtab"))
  219. strtab_sec = s;
  220. if ((r(&s->sh_type) == SHT_REL ||
  221. r(&s->sh_type) == SHT_RELA) &&
  222. r(&s->sh_info) == extab_index) {
  223. relocs = (void *)ehdr + _r(&s->sh_offset);
  224. relocs_size = _r(&s->sh_size);
  225. }
  226. if (r(&s->sh_type) == SHT_SYMTAB_SHNDX)
  227. symtab_shndx = (Elf32_Word *)((const char *)ehdr +
  228. _r(&s->sh_offset));
  229. #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED)
  230. /* locate the ORC unwind tables */
  231. if (!strcmp(secstrings + idx, ".orc_unwind_ip")) {
  232. orc_ip_size = s->sh_size;
  233. g_orc_ip_table = (int *)((void *)ehdr +
  234. s->sh_offset);
  235. }
  236. if (!strcmp(secstrings + idx, ".orc_unwind")) {
  237. orc_size = s->sh_size;
  238. g_orc_table = (struct orc_entry *)((void *)ehdr +
  239. s->sh_offset);
  240. }
  241. #endif
  242. } /* for loop */
  243. #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED)
  244. if (!g_orc_ip_table || !g_orc_table) {
  245. fprintf(stderr,
  246. "incomplete ORC unwind tables in file: %s\n", fname);
  247. goto out;
  248. }
  249. orc_num_entries = orc_ip_size / sizeof(int);
  250. if (orc_ip_size % sizeof(int) != 0 ||
  251. orc_size % sizeof(struct orc_entry) != 0 ||
  252. orc_num_entries != orc_size / sizeof(struct orc_entry)) {
  253. fprintf(stderr,
  254. "inconsistent ORC unwind table entries in file: %s\n",
  255. fname);
  256. goto out;
  257. }
  258. /* create thread to sort ORC unwind tables concurrently */
  259. if (pthread_create(&orc_sort_thread, NULL,
  260. sort_orctable, &orc_ip_size)) {
  261. fprintf(stderr,
  262. "pthread_create orc_sort_thread failed '%s': %s\n",
  263. strerror(errno), fname);
  264. goto out;
  265. }
  266. #endif
  267. if (!extab_sec) {
  268. fprintf(stderr, "no __ex_table in file: %s\n", fname);
  269. goto out;
  270. }
  271. if (!symtab_sec) {
  272. fprintf(stderr, "no .symtab in file: %s\n", fname);
  273. goto out;
  274. }
  275. if (!strtab_sec) {
  276. fprintf(stderr, "no .strtab in file: %s\n", fname);
  277. goto out;
  278. }
  279. extab_image = (void *)ehdr + _r(&extab_sec->sh_offset);
  280. strtab = (const char *)ehdr + _r(&strtab_sec->sh_offset);
  281. symtab = (const Elf_Sym *)((const char *)ehdr +
  282. _r(&symtab_sec->sh_offset));
  283. if (custom_sort) {
  284. custom_sort(extab_image, _r(&extab_sec->sh_size));
  285. } else {
  286. int num_entries = _r(&extab_sec->sh_size) / extable_ent_size;
  287. qsort(extab_image, num_entries,
  288. extable_ent_size, compare_extable);
  289. }
  290. /* If there were relocations, we no longer need them. */
  291. if (relocs)
  292. memset(relocs, 0, relocs_size);
  293. /* find the flag main_extable_sort_needed */
  294. for (sym = (void *)ehdr + _r(&symtab_sec->sh_offset);
  295. sym < sym + _r(&symtab_sec->sh_size) / sizeof(Elf_Sym);
  296. sym++) {
  297. if (ELF_ST_TYPE(sym->st_info) != STT_OBJECT)
  298. continue;
  299. if (!strcmp(strtab + r(&sym->st_name),
  300. "main_extable_sort_needed")) {
  301. sort_needed_sym = sym;
  302. break;
  303. }
  304. }
  305. if (!sort_needed_sym) {
  306. fprintf(stderr,
  307. "no main_extable_sort_needed symbol in file: %s\n",
  308. fname);
  309. goto out;
  310. }
  311. sort_needed_sec = &shdr[get_secindex(r2(&sym->st_shndx),
  312. sort_needed_sym - symtab,
  313. symtab_shndx)];
  314. sort_needed_loc = (void *)ehdr +
  315. _r(&sort_needed_sec->sh_offset) +
  316. _r(&sort_needed_sym->st_value) -
  317. _r(&sort_needed_sec->sh_addr);
  318. /* extable has been sorted, clear the flag */
  319. w(0, sort_needed_loc);
  320. rc = 0;
  321. out:
  322. #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED)
  323. if (orc_sort_thread) {
  324. void *retval = NULL;
  325. /* wait for ORC tables sort done */
  326. rc = pthread_join(orc_sort_thread, &retval);
  327. if (rc)
  328. fprintf(stderr,
  329. "pthread_join failed '%s': %s\n",
  330. strerror(errno), fname);
  331. else if (retval) {
  332. rc = -1;
  333. fprintf(stderr,
  334. "failed to sort ORC tables '%s': %s\n",
  335. (char *)retval, fname);
  336. }
  337. }
  338. #endif
  339. return rc;
  340. }