package.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318
  1. #
  2. # SPDX-License-Identifier: GPL-2.0-only
  3. #
  4. import stat
  5. import mmap
  6. import subprocess
  7. def runstrip(arg):
  8. # Function to strip a single file, called from split_and_strip_files below
  9. # A working 'file' (one which works on the target architecture)
  10. #
  11. # The elftype is a bit pattern (explained in is_elf below) to tell
  12. # us what type of file we're processing...
  13. # 4 - executable
  14. # 8 - shared library
  15. # 16 - kernel module
  16. (file, elftype, strip) = arg
  17. newmode = None
  18. if not os.access(file, os.W_OK) or os.access(file, os.R_OK):
  19. origmode = os.stat(file)[stat.ST_MODE]
  20. newmode = origmode | stat.S_IWRITE | stat.S_IREAD
  21. os.chmod(file, newmode)
  22. stripcmd = [strip]
  23. skip_strip = False
  24. # kernel module
  25. if elftype & 16:
  26. if is_kernel_module_signed(file):
  27. bb.debug(1, "Skip strip on signed module %s" % file)
  28. skip_strip = True
  29. else:
  30. stripcmd.extend(["--strip-debug", "--remove-section=.comment",
  31. "--remove-section=.note", "--preserve-dates"])
  32. # .so and shared library
  33. elif ".so" in file and elftype & 8:
  34. stripcmd.extend(["--remove-section=.comment", "--remove-section=.note", "--strip-unneeded"])
  35. # shared or executable:
  36. elif elftype & 8 or elftype & 4:
  37. stripcmd.extend(["--remove-section=.comment", "--remove-section=.note"])
  38. stripcmd.append(file)
  39. bb.debug(1, "runstrip: %s" % stripcmd)
  40. if not skip_strip:
  41. output = subprocess.check_output(stripcmd, stderr=subprocess.STDOUT)
  42. if newmode:
  43. os.chmod(file, origmode)
  44. # Detect .ko module by searching for "vermagic=" string
  45. def is_kernel_module(path):
  46. with open(path) as f:
  47. return mmap.mmap(f.fileno(), 0, prot=mmap.PROT_READ).find(b"vermagic=") >= 0
  48. # Detect if .ko module is signed
  49. def is_kernel_module_signed(path):
  50. with open(path, "rb") as f:
  51. f.seek(-28, 2)
  52. module_tail = f.read()
  53. return "Module signature appended" in "".join(chr(c) for c in bytearray(module_tail))
  54. # Return type (bits):
  55. # 0 - not elf
  56. # 1 - ELF
  57. # 2 - stripped
  58. # 4 - executable
  59. # 8 - shared library
  60. # 16 - kernel module
  61. def is_elf(path):
  62. exec_type = 0
  63. result = subprocess.check_output(["file", "-b", path], stderr=subprocess.STDOUT).decode("utf-8")
  64. if "ELF" in result:
  65. exec_type |= 1
  66. if "not stripped" not in result:
  67. exec_type |= 2
  68. if "executable" in result:
  69. exec_type |= 4
  70. if "shared" in result:
  71. exec_type |= 8
  72. if "relocatable" in result:
  73. if path.endswith(".ko") and path.find("/lib/modules/") != -1 and is_kernel_module(path):
  74. exec_type |= 16
  75. return (path, exec_type)
  76. def is_static_lib(path):
  77. if path.endswith('.a') and not os.path.islink(path):
  78. with open(path, 'rb') as fh:
  79. # The magic must include the first slash to avoid
  80. # matching golang static libraries
  81. magic = b'!<arch>\x0a/'
  82. start = fh.read(len(magic))
  83. return start == magic
  84. return False
  85. def strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, d, qa_already_stripped=False):
  86. """
  87. Strip executable code (like executables, shared libraries) _in_place_
  88. - Based on sysroot_strip in staging.bbclass
  89. :param dstdir: directory in which to strip files
  90. :param strip_cmd: Strip command (usually ${STRIP})
  91. :param libdir: ${libdir} - strip .so files in this directory
  92. :param base_libdir: ${base_libdir} - strip .so files in this directory
  93. :param qa_already_stripped: Set to True if already-stripped' in ${INSANE_SKIP}
  94. This is for proper logging and messages only.
  95. """
  96. import stat, errno, oe.path, oe.utils
  97. elffiles = {}
  98. inodes = {}
  99. libdir = os.path.abspath(dstdir + os.sep + libdir)
  100. base_libdir = os.path.abspath(dstdir + os.sep + base_libdir)
  101. exec_mask = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
  102. #
  103. # First lets figure out all of the files we may have to process
  104. #
  105. checkelf = []
  106. inodecache = {}
  107. for root, dirs, files in os.walk(dstdir):
  108. for f in files:
  109. file = os.path.join(root, f)
  110. try:
  111. ltarget = oe.path.realpath(file, dstdir, False)
  112. s = os.lstat(ltarget)
  113. except OSError as e:
  114. (err, strerror) = e.args
  115. if err != errno.ENOENT:
  116. raise
  117. # Skip broken symlinks
  118. continue
  119. if not s:
  120. continue
  121. # Check its an excutable
  122. if s[stat.ST_MODE] & exec_mask \
  123. or ((file.startswith(libdir) or file.startswith(base_libdir)) and ".so" in f) \
  124. or file.endswith('.ko'):
  125. # If it's a symlink, and points to an ELF file, we capture the readlink target
  126. if os.path.islink(file):
  127. continue
  128. # It's a file (or hardlink), not a link
  129. # ...but is it ELF, and is it already stripped?
  130. checkelf.append(file)
  131. inodecache[file] = s.st_ino
  132. results = oe.utils.multiprocess_launch(is_elf, checkelf, d)
  133. for (file, elf_file) in results:
  134. #elf_file = is_elf(file)
  135. if elf_file & 1:
  136. if elf_file & 2:
  137. if qa_already_stripped:
  138. bb.note("Skipping file %s from %s for already-stripped QA test" % (file[len(dstdir):], pn))
  139. else:
  140. bb.warn("File '%s' from %s was already stripped, this will prevent future debugging!" % (file[len(dstdir):], pn))
  141. continue
  142. if inodecache[file] in inodes:
  143. os.unlink(file)
  144. os.link(inodes[inodecache[file]], file)
  145. else:
  146. # break hardlinks so that we do not strip the original.
  147. inodes[inodecache[file]] = file
  148. bb.utils.break_hardlinks(file)
  149. elffiles[file] = elf_file
  150. #
  151. # Now strip them (in parallel)
  152. #
  153. sfiles = []
  154. for file in elffiles:
  155. elf_file = int(elffiles[file])
  156. sfiles.append((file, elf_file, strip_cmd))
  157. oe.utils.multiprocess_launch(runstrip, sfiles, d)
  158. def file_translate(file):
  159. ft = file.replace("@", "@at@")
  160. ft = ft.replace(" ", "@space@")
  161. ft = ft.replace("\t", "@tab@")
  162. ft = ft.replace("[", "@openbrace@")
  163. ft = ft.replace("]", "@closebrace@")
  164. ft = ft.replace("_", "@underscore@")
  165. return ft
  166. def filedeprunner(arg):
  167. import re, subprocess, shlex
  168. (pkg, pkgfiles, rpmdeps, pkgdest) = arg
  169. provides = {}
  170. requires = {}
  171. file_re = re.compile(r'\s+\d+\s(.*)')
  172. dep_re = re.compile(r'\s+(\S)\s+(.*)')
  173. r = re.compile(r'[<>=]+\s+\S*')
  174. def process_deps(pipe, pkg, pkgdest, provides, requires):
  175. file = None
  176. for line in pipe.split("\n"):
  177. m = file_re.match(line)
  178. if m:
  179. file = m.group(1)
  180. file = file.replace(pkgdest + "/" + pkg, "")
  181. file = file_translate(file)
  182. continue
  183. m = dep_re.match(line)
  184. if not m or not file:
  185. continue
  186. type, dep = m.groups()
  187. if type == 'R':
  188. i = requires
  189. elif type == 'P':
  190. i = provides
  191. else:
  192. continue
  193. if dep.startswith("python("):
  194. continue
  195. # Ignore all perl(VMS::...) and perl(Mac::...) dependencies. These
  196. # are typically used conditionally from the Perl code, but are
  197. # generated as unconditional dependencies.
  198. if dep.startswith('perl(VMS::') or dep.startswith('perl(Mac::'):
  199. continue
  200. # Ignore perl dependencies on .pl files.
  201. if dep.startswith('perl(') and dep.endswith('.pl)'):
  202. continue
  203. # Remove perl versions and perl module versions since they typically
  204. # do not make sense when used as package versions.
  205. if dep.startswith('perl') and r.search(dep):
  206. dep = dep.split()[0]
  207. # Put parentheses around any version specifications.
  208. dep = r.sub(r'(\g<0>)',dep)
  209. if file not in i:
  210. i[file] = []
  211. i[file].append(dep)
  212. return provides, requires
  213. output = subprocess.check_output(shlex.split(rpmdeps) + pkgfiles, stderr=subprocess.STDOUT).decode("utf-8")
  214. provides, requires = process_deps(output, pkg, pkgdest, provides, requires)
  215. return (pkg, provides, requires)
  216. def read_shlib_providers(d):
  217. import re
  218. shlib_provider = {}
  219. shlibs_dirs = d.getVar('SHLIBSDIRS').split()
  220. list_re = re.compile(r'^(.*)\.list$')
  221. # Go from least to most specific since the last one found wins
  222. for dir in reversed(shlibs_dirs):
  223. bb.debug(2, "Reading shlib providers in %s" % (dir))
  224. if not os.path.exists(dir):
  225. continue
  226. for file in os.listdir(dir):
  227. m = list_re.match(file)
  228. if m:
  229. dep_pkg = m.group(1)
  230. try:
  231. fd = open(os.path.join(dir, file))
  232. except IOError:
  233. # During a build unrelated shlib files may be deleted, so
  234. # handle files disappearing between the listdirs and open.
  235. continue
  236. lines = fd.readlines()
  237. fd.close()
  238. for l in lines:
  239. s = l.strip().split(":")
  240. if s[0] not in shlib_provider:
  241. shlib_provider[s[0]] = {}
  242. shlib_provider[s[0]][s[1]] = (dep_pkg, s[2])
  243. return shlib_provider
  244. def npm_split_package_dirs(pkgdir):
  245. """
  246. Work out the packages fetched and unpacked by BitBake's npm fetcher
  247. Returns a dict of packagename -> (relpath, package.json) ordered
  248. such that it is suitable for use in PACKAGES and FILES
  249. """
  250. from collections import OrderedDict
  251. import json
  252. packages = {}
  253. for root, dirs, files in os.walk(pkgdir):
  254. if os.path.basename(root) == 'node_modules':
  255. for dn in dirs:
  256. relpth = os.path.relpath(os.path.join(root, dn), pkgdir)
  257. pkgitems = ['${PN}']
  258. for pathitem in relpth.split('/'):
  259. if pathitem == 'node_modules':
  260. continue
  261. pkgitems.append(pathitem)
  262. pkgname = '-'.join(pkgitems).replace('_', '-')
  263. pkgname = pkgname.replace('@', '')
  264. pkgfile = os.path.join(root, dn, 'package.json')
  265. data = None
  266. if os.path.exists(pkgfile):
  267. with open(pkgfile, 'r') as f:
  268. data = json.loads(f.read())
  269. packages[pkgname] = (relpth, data)
  270. # We want the main package for a module sorted *after* its subpackages
  271. # (so that it doesn't otherwise steal the files for the subpackage), so
  272. # this is a cheap way to do that whilst still having an otherwise
  273. # alphabetical sort
  274. return OrderedDict((key, packages[key]) for key in sorted(packages, key=lambda pkg: pkg + '~'))