direct.py 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602
  1. #
  2. # Copyright (c) 2013, Intel Corporation.
  3. #
  4. # SPDX-License-Identifier: GPL-2.0-only
  5. #
  6. # DESCRIPTION
  7. # This implements the 'direct' imager plugin class for 'wic'
  8. #
  9. # AUTHORS
  10. # Tom Zanussi <tom.zanussi (at] linux.intel.com>
  11. #
  12. import logging
  13. import os
  14. import random
  15. import shutil
  16. import tempfile
  17. import uuid
  18. from time import strftime
  19. from oe.path import copyhardlinktree
  20. from wic import WicError
  21. from wic.filemap import sparse_copy
  22. from wic.ksparser import KickStart, KickStartError
  23. from wic.pluginbase import PluginMgr, ImagerPlugin
  24. from wic.misc import get_bitbake_var, exec_cmd, exec_native_cmd
  25. logger = logging.getLogger('wic')
  26. class DirectPlugin(ImagerPlugin):
  27. """
  28. Install a system into a file containing a partitioned disk image.
  29. An image file is formatted with a partition table, each partition
  30. created from a rootfs or other OpenEmbedded build artifact and dd'ed
  31. into the virtual disk. The disk image can subsequently be dd'ed onto
  32. media and used on actual hardware.
  33. """
  34. name = 'direct'
  35. def __init__(self, wks_file, rootfs_dir, bootimg_dir, kernel_dir,
  36. native_sysroot, oe_builddir, options):
  37. try:
  38. self.ks = KickStart(wks_file)
  39. except KickStartError as err:
  40. raise WicError(str(err))
  41. # parse possible 'rootfs=name' items
  42. self.rootfs_dir = dict(rdir.split('=') for rdir in rootfs_dir.split(' '))
  43. self.bootimg_dir = bootimg_dir
  44. self.kernel_dir = kernel_dir
  45. self.native_sysroot = native_sysroot
  46. self.oe_builddir = oe_builddir
  47. self.outdir = options.outdir
  48. self.compressor = options.compressor
  49. self.bmap = options.bmap
  50. self.no_fstab_update = options.no_fstab_update
  51. self.original_fstab = None
  52. self.name = "%s-%s" % (os.path.splitext(os.path.basename(wks_file))[0],
  53. strftime("%Y%m%d%H%M"))
  54. self.workdir = tempfile.mkdtemp(dir=self.outdir, prefix='tmp.wic.')
  55. self._image = None
  56. self.ptable_format = self.ks.bootloader.ptable
  57. self.parts = self.ks.partitions
  58. # as a convenience, set source to the boot partition source
  59. # instead of forcing it to be set via bootloader --source
  60. for part in self.parts:
  61. if not self.ks.bootloader.source and part.mountpoint == "/boot":
  62. self.ks.bootloader.source = part.source
  63. break
  64. image_path = self._full_path(self.workdir, self.parts[0].disk, "direct")
  65. self._image = PartitionedImage(image_path, self.ptable_format,
  66. self.parts, self.native_sysroot)
  67. def do_create(self):
  68. """
  69. Plugin entry point.
  70. """
  71. try:
  72. self.create()
  73. self.assemble()
  74. self.finalize()
  75. self.print_info()
  76. finally:
  77. self.cleanup()
  78. def _write_fstab(self, image_rootfs):
  79. """overriden to generate fstab (temporarily) in rootfs. This is called
  80. from _create, make sure it doesn't get called from
  81. BaseImage.create()
  82. """
  83. if not image_rootfs:
  84. return
  85. fstab_path = image_rootfs + "/etc/fstab"
  86. if not os.path.isfile(fstab_path):
  87. return
  88. with open(fstab_path) as fstab:
  89. fstab_lines = fstab.readlines()
  90. self.original_fstab = fstab_lines.copy()
  91. if self._update_fstab(fstab_lines, self.parts):
  92. with open(fstab_path, "w") as fstab:
  93. fstab.writelines(fstab_lines)
  94. else:
  95. self.original_fstab = None
  96. def _update_fstab(self, fstab_lines, parts):
  97. """Assume partition order same as in wks"""
  98. updated = False
  99. for part in parts:
  100. if not part.realnum or not part.mountpoint \
  101. or part.mountpoint == "/":
  102. continue
  103. if part.use_uuid:
  104. if part.fsuuid:
  105. # FAT UUID is different from others
  106. if len(part.fsuuid) == 10:
  107. device_name = "UUID=%s-%s" % \
  108. (part.fsuuid[2:6], part.fsuuid[6:])
  109. else:
  110. device_name = "UUID=%s" % part.fsuuid
  111. else:
  112. device_name = "PARTUUID=%s" % part.uuid
  113. elif part.use_label:
  114. device_name = "LABEL=%s" % part.label
  115. else:
  116. # mmc device partitions are named mmcblk0p1, mmcblk0p2..
  117. prefix = 'p' if part.disk.startswith('mmcblk') else ''
  118. device_name = "/dev/%s%s%d" % (part.disk, prefix, part.realnum)
  119. opts = part.fsopts if part.fsopts else "defaults"
  120. line = "\t".join([device_name, part.mountpoint, part.fstype,
  121. opts, "0", "0"]) + "\n"
  122. fstab_lines.append(line)
  123. updated = True
  124. return updated
  125. def _full_path(self, path, name, extention):
  126. """ Construct full file path to a file we generate. """
  127. return os.path.join(path, "%s-%s.%s" % (self.name, name, extention))
  128. #
  129. # Actual implemention
  130. #
  131. def create(self):
  132. """
  133. For 'wic', we already have our build artifacts - we just create
  134. filesystems from the artifacts directly and combine them into
  135. a partitioned image.
  136. """
  137. if not self.no_fstab_update:
  138. self._write_fstab(self.rootfs_dir.get("ROOTFS_DIR"))
  139. for part in self.parts:
  140. # get rootfs size from bitbake variable if it's not set in .ks file
  141. if not part.size:
  142. # and if rootfs name is specified for the partition
  143. image_name = self.rootfs_dir.get(part.rootfs_dir)
  144. if image_name and os.path.sep not in image_name:
  145. # Bitbake variable ROOTFS_SIZE is calculated in
  146. # Image._get_rootfs_size method from meta/lib/oe/image.py
  147. # using IMAGE_ROOTFS_SIZE, IMAGE_ROOTFS_ALIGNMENT,
  148. # IMAGE_OVERHEAD_FACTOR and IMAGE_ROOTFS_EXTRA_SPACE
  149. rsize_bb = get_bitbake_var('ROOTFS_SIZE', image_name)
  150. if rsize_bb:
  151. part.size = int(round(float(rsize_bb)))
  152. self._image.prepare(self)
  153. self._image.layout_partitions()
  154. self._image.create()
  155. def assemble(self):
  156. """
  157. Assemble partitions into disk image
  158. """
  159. self._image.assemble()
  160. def finalize(self):
  161. """
  162. Finalize the disk image.
  163. For example, prepare the image to be bootable by e.g.
  164. creating and installing a bootloader configuration.
  165. """
  166. source_plugin = self.ks.bootloader.source
  167. disk_name = self.parts[0].disk
  168. if source_plugin:
  169. plugin = PluginMgr.get_plugins('source')[source_plugin]
  170. plugin.do_install_disk(self._image, disk_name, self, self.workdir,
  171. self.oe_builddir, self.bootimg_dir,
  172. self.kernel_dir, self.native_sysroot)
  173. full_path = self._image.path
  174. # Generate .bmap
  175. if self.bmap:
  176. logger.debug("Generating bmap file for %s", disk_name)
  177. python = os.path.join(self.native_sysroot, 'usr/bin/python3-native/python3')
  178. bmaptool = os.path.join(self.native_sysroot, 'usr/bin/bmaptool')
  179. exec_native_cmd("%s %s create %s -o %s.bmap" % \
  180. (python, bmaptool, full_path, full_path), self.native_sysroot)
  181. # Compress the image
  182. if self.compressor:
  183. logger.debug("Compressing disk %s with %s", disk_name, self.compressor)
  184. exec_cmd("%s %s" % (self.compressor, full_path))
  185. def print_info(self):
  186. """
  187. Print the image(s) and artifacts used, for the user.
  188. """
  189. msg = "The new image(s) can be found here:\n"
  190. extension = "direct" + {"gzip": ".gz",
  191. "bzip2": ".bz2",
  192. "xz": ".xz",
  193. None: ""}.get(self.compressor)
  194. full_path = self._full_path(self.outdir, self.parts[0].disk, extension)
  195. msg += ' %s\n\n' % full_path
  196. msg += 'The following build artifacts were used to create the image(s):\n'
  197. for part in self.parts:
  198. if part.rootfs_dir is None:
  199. continue
  200. if part.mountpoint == '/':
  201. suffix = ':'
  202. else:
  203. suffix = '["%s"]:' % (part.mountpoint or part.label)
  204. rootdir = part.rootfs_dir
  205. msg += ' ROOTFS_DIR%s%s\n' % (suffix.ljust(20), rootdir)
  206. msg += ' BOOTIMG_DIR: %s\n' % self.bootimg_dir
  207. msg += ' KERNEL_DIR: %s\n' % self.kernel_dir
  208. msg += ' NATIVE_SYSROOT: %s\n' % self.native_sysroot
  209. logger.info(msg)
  210. @property
  211. def rootdev(self):
  212. """
  213. Get root device name to use as a 'root' parameter
  214. in kernel command line.
  215. Assume partition order same as in wks
  216. """
  217. for part in self.parts:
  218. if part.mountpoint == "/":
  219. if part.uuid:
  220. return "PARTUUID=%s" % part.uuid
  221. else:
  222. suffix = 'p' if part.disk.startswith('mmcblk') else ''
  223. return "/dev/%s%s%-d" % (part.disk, suffix, part.realnum)
  224. def cleanup(self):
  225. if self._image:
  226. self._image.cleanup()
  227. # Move results to the output dir
  228. if not os.path.exists(self.outdir):
  229. os.makedirs(self.outdir)
  230. for fname in os.listdir(self.workdir):
  231. path = os.path.join(self.workdir, fname)
  232. if os.path.isfile(path):
  233. shutil.move(path, os.path.join(self.outdir, fname))
  234. #Restore original fstab
  235. if self.original_fstab:
  236. fstab_path = self.rootfs_dir.get("ROOTFS_DIR") + "/etc/fstab"
  237. with open(fstab_path, "w") as fstab:
  238. fstab.writelines(self.original_fstab)
  239. # remove work directory
  240. shutil.rmtree(self.workdir, ignore_errors=True)
  241. # Overhead of the MBR partitioning scheme (just one sector)
  242. MBR_OVERHEAD = 1
  243. # Overhead of the GPT partitioning scheme
  244. GPT_OVERHEAD = 34
  245. # Size of a sector in bytes
  246. SECTOR_SIZE = 512
  247. class PartitionedImage():
  248. """
  249. Partitioned image in a file.
  250. """
  251. def __init__(self, path, ptable_format, partitions, native_sysroot=None):
  252. self.path = path # Path to the image file
  253. self.numpart = 0 # Number of allocated partitions
  254. self.realpart = 0 # Number of partitions in the partition table
  255. self.primary_part_num = 0 # Number of primary partitions (msdos)
  256. self.extendedpart = 0 # Create extended partition before this logical partition (msdos)
  257. self.extended_size_sec = 0 # Size of exteded partition (msdos)
  258. self.logical_part_cnt = 0 # Number of total logical paritions (msdos)
  259. self.offset = 0 # Offset of next partition (in sectors)
  260. self.min_size = 0 # Minimum required disk size to fit
  261. # all partitions (in bytes)
  262. self.ptable_format = ptable_format # Partition table format
  263. # Disk system identifier
  264. self.identifier = random.SystemRandom().randint(1, 0xffffffff)
  265. self.partitions = partitions
  266. self.partimages = []
  267. # Size of a sector used in calculations
  268. self.sector_size = SECTOR_SIZE
  269. self.native_sysroot = native_sysroot
  270. num_real_partitions = len([p for p in self.partitions if not p.no_table])
  271. # calculate the real partition number, accounting for partitions not
  272. # in the partition table and logical partitions
  273. realnum = 0
  274. for part in self.partitions:
  275. if part.no_table:
  276. part.realnum = 0
  277. else:
  278. realnum += 1
  279. if self.ptable_format == 'msdos' and realnum > 3 and num_real_partitions > 4:
  280. part.realnum = realnum + 1
  281. continue
  282. part.realnum = realnum
  283. # generate parition and filesystem UUIDs
  284. for part in self.partitions:
  285. if not part.uuid and part.use_uuid:
  286. if self.ptable_format == 'gpt':
  287. part.uuid = str(uuid.uuid4())
  288. else: # msdos partition table
  289. part.uuid = '%08x-%02d' % (self.identifier, part.realnum)
  290. if not part.fsuuid:
  291. if part.fstype == 'vfat' or part.fstype == 'msdos':
  292. part.fsuuid = '0x' + str(uuid.uuid4())[:8].upper()
  293. else:
  294. part.fsuuid = str(uuid.uuid4())
  295. def prepare(self, imager):
  296. """Prepare an image. Call prepare method of all image partitions."""
  297. for part in self.partitions:
  298. # need to create the filesystems in order to get their
  299. # sizes before we can add them and do the layout.
  300. part.prepare(imager, imager.workdir, imager.oe_builddir,
  301. imager.rootfs_dir, imager.bootimg_dir,
  302. imager.kernel_dir, imager.native_sysroot)
  303. # Converting kB to sectors for parted
  304. part.size_sec = part.disk_size * 1024 // self.sector_size
  305. def layout_partitions(self):
  306. """ Layout the partitions, meaning calculate the position of every
  307. partition on the disk. The 'ptable_format' parameter defines the
  308. partition table format and may be "msdos". """
  309. logger.debug("Assigning %s partitions to disks", self.ptable_format)
  310. # The number of primary and logical partitions. Extended partition and
  311. # partitions not listed in the table are not included.
  312. num_real_partitions = len([p for p in self.partitions if not p.no_table])
  313. # Go through partitions in the order they are added in .ks file
  314. for num in range(len(self.partitions)):
  315. part = self.partitions[num]
  316. if self.ptable_format == 'msdos' and part.part_name:
  317. raise WicError("setting custom partition name is not " \
  318. "implemented for msdos partitions")
  319. if self.ptable_format == 'msdos' and part.part_type:
  320. # The --part-type can also be implemented for MBR partitions,
  321. # in which case it would map to the 1-byte "partition type"
  322. # filed at offset 3 of the partition entry.
  323. raise WicError("setting custom partition type is not " \
  324. "implemented for msdos partitions")
  325. # Get the disk where the partition is located
  326. self.numpart += 1
  327. if not part.no_table:
  328. self.realpart += 1
  329. if self.numpart == 1:
  330. if self.ptable_format == "msdos":
  331. overhead = MBR_OVERHEAD
  332. elif self.ptable_format == "gpt":
  333. overhead = GPT_OVERHEAD
  334. # Skip one sector required for the partitioning scheme overhead
  335. self.offset += overhead
  336. if self.ptable_format == "msdos":
  337. if self.primary_part_num > 3 or \
  338. (self.extendedpart == 0 and self.primary_part_num >= 3 and num_real_partitions > 4):
  339. part.type = 'logical'
  340. # Reserve a sector for EBR for every logical partition
  341. # before alignment is performed.
  342. if part.type == 'logical':
  343. self.offset += 1
  344. align_sectors = 0
  345. if part.align:
  346. # If not first partition and we do have alignment set we need
  347. # to align the partition.
  348. # FIXME: This leaves a empty spaces to the disk. To fill the
  349. # gaps we could enlargea the previous partition?
  350. # Calc how much the alignment is off.
  351. align_sectors = self.offset % (part.align * 1024 // self.sector_size)
  352. if align_sectors:
  353. # If partition is not aligned as required, we need
  354. # to move forward to the next alignment point
  355. align_sectors = (part.align * 1024 // self.sector_size) - align_sectors
  356. logger.debug("Realignment for %s%s with %s sectors, original"
  357. " offset %s, target alignment is %sK.",
  358. part.disk, self.numpart, align_sectors,
  359. self.offset, part.align)
  360. # increase the offset so we actually start the partition on right alignment
  361. self.offset += align_sectors
  362. part.start = self.offset
  363. self.offset += part.size_sec
  364. if not part.no_table:
  365. part.num = self.realpart
  366. else:
  367. part.num = 0
  368. if self.ptable_format == "msdos" and not part.no_table:
  369. if part.type == 'logical':
  370. self.logical_part_cnt += 1
  371. part.num = self.logical_part_cnt + 4
  372. if self.extendedpart == 0:
  373. # Create extended partition as a primary partition
  374. self.primary_part_num += 1
  375. self.extendedpart = part.num
  376. else:
  377. self.extended_size_sec += align_sectors
  378. self.extended_size_sec += part.size_sec + 1
  379. else:
  380. self.primary_part_num += 1
  381. part.num = self.primary_part_num
  382. logger.debug("Assigned %s to %s%d, sectors range %d-%d size %d "
  383. "sectors (%d bytes).", part.mountpoint, part.disk,
  384. part.num, part.start, self.offset - 1, part.size_sec,
  385. part.size_sec * self.sector_size)
  386. # Once all the partitions have been layed out, we can calculate the
  387. # minumim disk size
  388. self.min_size = self.offset
  389. if self.ptable_format == "gpt":
  390. self.min_size += GPT_OVERHEAD
  391. self.min_size *= self.sector_size
  392. def _create_partition(self, device, parttype, fstype, start, size):
  393. """ Create a partition on an image described by the 'device' object. """
  394. # Start is included to the size so we need to substract one from the end.
  395. end = start + size - 1
  396. logger.debug("Added '%s' partition, sectors %d-%d, size %d sectors",
  397. parttype, start, end, size)
  398. cmd = "parted -s %s unit s mkpart %s" % (device, parttype)
  399. if fstype:
  400. cmd += " %s" % fstype
  401. cmd += " %d %d" % (start, end)
  402. return exec_native_cmd(cmd, self.native_sysroot)
  403. def create(self):
  404. logger.debug("Creating sparse file %s", self.path)
  405. with open(self.path, 'w') as sparse:
  406. os.ftruncate(sparse.fileno(), self.min_size)
  407. logger.debug("Initializing partition table for %s", self.path)
  408. exec_native_cmd("parted -s %s mklabel %s" %
  409. (self.path, self.ptable_format), self.native_sysroot)
  410. logger.debug("Set disk identifier %x", self.identifier)
  411. with open(self.path, 'r+b') as img:
  412. img.seek(0x1B8)
  413. img.write(self.identifier.to_bytes(4, 'little'))
  414. logger.debug("Creating partitions")
  415. for part in self.partitions:
  416. if part.num == 0:
  417. continue
  418. if self.ptable_format == "msdos" and part.num == self.extendedpart:
  419. # Create an extended partition (note: extended
  420. # partition is described in MBR and contains all
  421. # logical partitions). The logical partitions save a
  422. # sector for an EBR just before the start of a
  423. # partition. The extended partition must start one
  424. # sector before the start of the first logical
  425. # partition. This way the first EBR is inside of the
  426. # extended partition. Since the extended partitions
  427. # starts a sector before the first logical partition,
  428. # add a sector at the back, so that there is enough
  429. # room for all logical partitions.
  430. self._create_partition(self.path, "extended",
  431. None, part.start - 1,
  432. self.extended_size_sec)
  433. if part.fstype == "swap":
  434. parted_fs_type = "linux-swap"
  435. elif part.fstype == "vfat":
  436. parted_fs_type = "fat32"
  437. elif part.fstype == "msdos":
  438. parted_fs_type = "fat16"
  439. if not part.system_id:
  440. part.system_id = '0x6' # FAT16
  441. else:
  442. # Type for ext2/ext3/ext4/btrfs
  443. parted_fs_type = "ext2"
  444. # Boot ROM of OMAP boards require vfat boot partition to have an
  445. # even number of sectors.
  446. if part.mountpoint == "/boot" and part.fstype in ["vfat", "msdos"] \
  447. and part.size_sec % 2:
  448. logger.debug("Subtracting one sector from '%s' partition to "
  449. "get even number of sectors for the partition",
  450. part.mountpoint)
  451. part.size_sec -= 1
  452. self._create_partition(self.path, part.type,
  453. parted_fs_type, part.start, part.size_sec)
  454. if part.part_name:
  455. logger.debug("partition %d: set name to %s",
  456. part.num, part.part_name)
  457. exec_native_cmd("sgdisk --change-name=%d:%s %s" % \
  458. (part.num, part.part_name,
  459. self.path), self.native_sysroot)
  460. if part.part_type:
  461. logger.debug("partition %d: set type UID to %s",
  462. part.num, part.part_type)
  463. exec_native_cmd("sgdisk --typecode=%d:%s %s" % \
  464. (part.num, part.part_type,
  465. self.path), self.native_sysroot)
  466. if part.uuid and self.ptable_format == "gpt":
  467. logger.debug("partition %d: set UUID to %s",
  468. part.num, part.uuid)
  469. exec_native_cmd("sgdisk --partition-guid=%d:%s %s" % \
  470. (part.num, part.uuid, self.path),
  471. self.native_sysroot)
  472. if part.label and self.ptable_format == "gpt":
  473. logger.debug("partition %d: set name to %s",
  474. part.num, part.label)
  475. exec_native_cmd("parted -s %s name %d %s" % \
  476. (self.path, part.num, part.label),
  477. self.native_sysroot)
  478. if part.active:
  479. flag_name = "legacy_boot" if self.ptable_format == 'gpt' else "boot"
  480. logger.debug("Set '%s' flag for partition '%s' on disk '%s'",
  481. flag_name, part.num, self.path)
  482. exec_native_cmd("parted -s %s set %d %s on" % \
  483. (self.path, part.num, flag_name),
  484. self.native_sysroot)
  485. if part.system_id:
  486. exec_native_cmd("sfdisk --part-type %s %s %s" % \
  487. (self.path, part.num, part.system_id),
  488. self.native_sysroot)
  489. def cleanup(self):
  490. # remove partition images
  491. for image in set(self.partimages):
  492. os.remove(image)
  493. def assemble(self):
  494. logger.debug("Installing partitions")
  495. for part in self.partitions:
  496. source = part.source_file
  497. if source:
  498. # install source_file contents into a partition
  499. sparse_copy(source, self.path, seek=part.start * self.sector_size)
  500. logger.debug("Installed %s in partition %d, sectors %d-%d, "
  501. "size %d sectors", source, part.num, part.start,
  502. part.start + part.size_sec - 1, part.size_sec)
  503. partimage = self.path + '.p%d' % part.num
  504. os.rename(source, partimage)
  505. self.partimages.append(partimage)