filesystem.go 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509
  1. // Copyright (C) 2020 The Android Open Source Project
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package filesystem
  15. import (
  16. "crypto/sha256"
  17. "fmt"
  18. "io"
  19. "path/filepath"
  20. "strings"
  21. "android/soong/android"
  22. "android/soong/cc"
  23. "github.com/google/blueprint"
  24. "github.com/google/blueprint/proptools"
  25. )
  26. func init() {
  27. registerBuildComponents(android.InitRegistrationContext)
  28. }
  29. func registerBuildComponents(ctx android.RegistrationContext) {
  30. ctx.RegisterModuleType("android_filesystem", filesystemFactory)
  31. ctx.RegisterModuleType("android_system_image", systemImageFactory)
  32. ctx.RegisterModuleType("avb_add_hash_footer", avbAddHashFooterFactory)
  33. ctx.RegisterModuleType("avb_gen_vbmeta_image", avbGenVbmetaImageFactory)
  34. }
  35. type filesystem struct {
  36. android.ModuleBase
  37. android.PackagingBase
  38. properties filesystemProperties
  39. // Function that builds extra files under the root directory and returns the files
  40. buildExtraFiles func(ctx android.ModuleContext, root android.OutputPath) android.OutputPaths
  41. // Function that filters PackagingSpecs returned by PackagingBase.GatherPackagingSpecs()
  42. filterPackagingSpecs func(specs map[string]android.PackagingSpec)
  43. output android.OutputPath
  44. installDir android.InstallPath
  45. // For testing. Keeps the result of CopyDepsToZip()
  46. entries []string
  47. }
  48. type symlinkDefinition struct {
  49. Target *string
  50. Name *string
  51. }
  52. type filesystemProperties struct {
  53. // When set to true, sign the image with avbtool. Default is false.
  54. Use_avb *bool
  55. // Path to the private key that avbtool will use to sign this filesystem image.
  56. // TODO(jiyong): allow apex_key to be specified here
  57. Avb_private_key *string `android:"path"`
  58. // Signing algorithm for avbtool. Default is SHA256_RSA4096.
  59. Avb_algorithm *string
  60. // Hash algorithm used for avbtool (for descriptors). This is passed as hash_algorithm to
  61. // avbtool. Default used by avbtool is sha1.
  62. Avb_hash_algorithm *string
  63. // Name of the partition stored in vbmeta desc. Defaults to the name of this module.
  64. Partition_name *string
  65. // Type of the filesystem. Currently, ext4, cpio, and compressed_cpio are supported. Default
  66. // is ext4.
  67. Type *string
  68. // file_contexts file to make image. Currently, only ext4 is supported.
  69. File_contexts *string `android:"path"`
  70. // Base directory relative to root, to which deps are installed, e.g. "system". Default is "."
  71. // (root).
  72. Base_dir *string
  73. // Directories to be created under root. e.g. /dev, /proc, etc.
  74. Dirs []string
  75. // Symbolic links to be created under root with "ln -sf <target> <name>".
  76. Symlinks []symlinkDefinition
  77. // Seconds since unix epoch to override timestamps of file entries
  78. Fake_timestamp *string
  79. // When set, passed to mkuserimg_mke2fs --mke2fs_uuid & --mke2fs_hash_seed.
  80. // Otherwise, they'll be set as random which might cause indeterministic build output.
  81. Uuid *string
  82. }
  83. // android_filesystem packages a set of modules and their transitive dependencies into a filesystem
  84. // image. The filesystem images are expected to be mounted in the target device, which means the
  85. // modules in the filesystem image are built for the target device (i.e. Android, not Linux host).
  86. // The modules are placed in the filesystem image just like they are installed to the ordinary
  87. // partitions like system.img. For example, cc_library modules are placed under ./lib[64] directory.
  88. func filesystemFactory() android.Module {
  89. module := &filesystem{}
  90. initFilesystemModule(module)
  91. return module
  92. }
  93. func initFilesystemModule(module *filesystem) {
  94. module.AddProperties(&module.properties)
  95. android.InitPackageModule(module)
  96. android.InitAndroidMultiTargetsArchModule(module, android.DeviceSupported, android.MultilibCommon)
  97. }
  98. var dependencyTag = struct {
  99. blueprint.BaseDependencyTag
  100. android.PackagingItemAlwaysDepTag
  101. }{}
  102. func (f *filesystem) DepsMutator(ctx android.BottomUpMutatorContext) {
  103. f.AddDeps(ctx, dependencyTag)
  104. }
  105. type fsType int
  106. const (
  107. ext4Type fsType = iota
  108. compressedCpioType
  109. cpioType // uncompressed
  110. unknown
  111. )
  112. func (f *filesystem) fsType(ctx android.ModuleContext) fsType {
  113. typeStr := proptools.StringDefault(f.properties.Type, "ext4")
  114. switch typeStr {
  115. case "ext4":
  116. return ext4Type
  117. case "compressed_cpio":
  118. return compressedCpioType
  119. case "cpio":
  120. return cpioType
  121. default:
  122. ctx.PropertyErrorf("type", "%q not supported", typeStr)
  123. return unknown
  124. }
  125. }
  126. func (f *filesystem) installFileName() string {
  127. return f.BaseModuleName() + ".img"
  128. }
  129. var pctx = android.NewPackageContext("android/soong/filesystem")
  130. func (f *filesystem) GenerateAndroidBuildActions(ctx android.ModuleContext) {
  131. switch f.fsType(ctx) {
  132. case ext4Type:
  133. f.output = f.buildImageUsingBuildImage(ctx)
  134. case compressedCpioType:
  135. f.output = f.buildCpioImage(ctx, true)
  136. case cpioType:
  137. f.output = f.buildCpioImage(ctx, false)
  138. default:
  139. return
  140. }
  141. f.installDir = android.PathForModuleInstall(ctx, "etc")
  142. ctx.InstallFile(f.installDir, f.installFileName(), f.output)
  143. }
  144. // root zip will contain extra files/dirs that are not from the `deps` property.
  145. func (f *filesystem) buildRootZip(ctx android.ModuleContext) android.OutputPath {
  146. rootDir := android.PathForModuleGen(ctx, "root").OutputPath
  147. builder := android.NewRuleBuilder(pctx, ctx)
  148. builder.Command().Text("rm -rf").Text(rootDir.String())
  149. builder.Command().Text("mkdir -p").Text(rootDir.String())
  150. // create dirs and symlinks
  151. for _, dir := range f.properties.Dirs {
  152. // OutputPath.Join verifies dir
  153. builder.Command().Text("mkdir -p").Text(rootDir.Join(ctx, dir).String())
  154. }
  155. for _, symlink := range f.properties.Symlinks {
  156. name := strings.TrimSpace(proptools.String(symlink.Name))
  157. target := strings.TrimSpace(proptools.String(symlink.Target))
  158. if name == "" {
  159. ctx.PropertyErrorf("symlinks", "Name can't be empty")
  160. continue
  161. }
  162. if target == "" {
  163. ctx.PropertyErrorf("symlinks", "Target can't be empty")
  164. continue
  165. }
  166. // OutputPath.Join verifies name. don't need to verify target.
  167. dst := rootDir.Join(ctx, name)
  168. builder.Command().Text("mkdir -p").Text(filepath.Dir(dst.String()))
  169. builder.Command().Text("ln -sf").Text(proptools.ShellEscape(target)).Text(dst.String())
  170. }
  171. // create extra files if there's any
  172. rootForExtraFiles := android.PathForModuleGen(ctx, "root-extra").OutputPath
  173. var extraFiles android.OutputPaths
  174. if f.buildExtraFiles != nil {
  175. extraFiles = f.buildExtraFiles(ctx, rootForExtraFiles)
  176. for _, f := range extraFiles {
  177. rel, _ := filepath.Rel(rootForExtraFiles.String(), f.String())
  178. if strings.HasPrefix(rel, "..") {
  179. panic(fmt.Errorf("%q is not under %q\n", f, rootForExtraFiles))
  180. }
  181. }
  182. }
  183. // Zip them all
  184. zipOut := android.PathForModuleGen(ctx, "root.zip").OutputPath
  185. zipCommand := builder.Command().BuiltTool("soong_zip")
  186. zipCommand.FlagWithOutput("-o ", zipOut).
  187. FlagWithArg("-C ", rootDir.String()).
  188. Flag("-L 0"). // no compression because this will be unzipped soon
  189. FlagWithArg("-D ", rootDir.String()).
  190. Flag("-d") // include empty directories
  191. if len(extraFiles) > 0 {
  192. zipCommand.FlagWithArg("-C ", rootForExtraFiles.String())
  193. for _, f := range extraFiles {
  194. zipCommand.FlagWithInput("-f ", f)
  195. }
  196. }
  197. builder.Command().Text("rm -rf").Text(rootDir.String())
  198. builder.Build("zip_root", fmt.Sprintf("zipping root contents for %s", ctx.ModuleName()))
  199. return zipOut
  200. }
  201. func (f *filesystem) buildImageUsingBuildImage(ctx android.ModuleContext) android.OutputPath {
  202. depsZipFile := android.PathForModuleOut(ctx, "deps.zip").OutputPath
  203. f.entries = f.CopyDepsToZip(ctx, f.gatherFilteredPackagingSpecs(ctx), depsZipFile)
  204. builder := android.NewRuleBuilder(pctx, ctx)
  205. depsBase := proptools.StringDefault(f.properties.Base_dir, ".")
  206. rebasedDepsZip := android.PathForModuleOut(ctx, "rebased_deps.zip").OutputPath
  207. builder.Command().
  208. BuiltTool("zip2zip").
  209. FlagWithInput("-i ", depsZipFile).
  210. FlagWithOutput("-o ", rebasedDepsZip).
  211. Text("**/*:" + proptools.ShellEscape(depsBase)) // zip2zip verifies depsBase
  212. rootDir := android.PathForModuleOut(ctx, "root").OutputPath
  213. rootZip := f.buildRootZip(ctx)
  214. builder.Command().
  215. BuiltTool("zipsync").
  216. FlagWithArg("-d ", rootDir.String()). // zipsync wipes this. No need to clear.
  217. Input(rootZip).
  218. Input(rebasedDepsZip)
  219. // run host_init_verifier
  220. // Ideally we should have a concept of pluggable linters that verify the generated image.
  221. // While such concept is not implement this will do.
  222. // TODO(b/263574231): substitute with pluggable linter.
  223. builder.Command().
  224. BuiltTool("host_init_verifier").
  225. FlagWithArg("--out_system=", rootDir.String()+"/system")
  226. propFile, toolDeps := f.buildPropFile(ctx)
  227. output := android.PathForModuleOut(ctx, f.installFileName()).OutputPath
  228. builder.Command().BuiltTool("build_image").
  229. Text(rootDir.String()). // input directory
  230. Input(propFile).
  231. Implicits(toolDeps).
  232. Output(output).
  233. Text(rootDir.String()) // directory where to find fs_config_files|dirs
  234. // rootDir is not deleted. Might be useful for quick inspection.
  235. builder.Build("build_filesystem_image", fmt.Sprintf("Creating filesystem %s", f.BaseModuleName()))
  236. return output
  237. }
  238. func (f *filesystem) buildFileContexts(ctx android.ModuleContext) android.OutputPath {
  239. builder := android.NewRuleBuilder(pctx, ctx)
  240. fcBin := android.PathForModuleOut(ctx, "file_contexts.bin")
  241. builder.Command().BuiltTool("sefcontext_compile").
  242. FlagWithOutput("-o ", fcBin).
  243. Input(android.PathForModuleSrc(ctx, proptools.String(f.properties.File_contexts)))
  244. builder.Build("build_filesystem_file_contexts", fmt.Sprintf("Creating filesystem file contexts for %s", f.BaseModuleName()))
  245. return fcBin.OutputPath
  246. }
  247. // Calculates avb_salt from entry list (sorted) for deterministic output.
  248. func (f *filesystem) salt() string {
  249. return sha1sum(f.entries)
  250. }
  251. func (f *filesystem) buildPropFile(ctx android.ModuleContext) (propFile android.OutputPath, toolDeps android.Paths) {
  252. type prop struct {
  253. name string
  254. value string
  255. }
  256. var props []prop
  257. var deps android.Paths
  258. addStr := func(name string, value string) {
  259. props = append(props, prop{name, value})
  260. }
  261. addPath := func(name string, path android.Path) {
  262. props = append(props, prop{name, path.String()})
  263. deps = append(deps, path)
  264. }
  265. // Type string that build_image.py accepts.
  266. fsTypeStr := func(t fsType) string {
  267. switch t {
  268. // TODO(jiyong): add more types like f2fs, erofs, etc.
  269. case ext4Type:
  270. return "ext4"
  271. }
  272. panic(fmt.Errorf("unsupported fs type %v", t))
  273. }
  274. addStr("fs_type", fsTypeStr(f.fsType(ctx)))
  275. addStr("mount_point", "/")
  276. addStr("use_dynamic_partition_size", "true")
  277. addPath("ext_mkuserimg", ctx.Config().HostToolPath(ctx, "mkuserimg_mke2fs"))
  278. // b/177813163 deps of the host tools have to be added. Remove this.
  279. for _, t := range []string{"mke2fs", "e2fsdroid", "tune2fs"} {
  280. deps = append(deps, ctx.Config().HostToolPath(ctx, t))
  281. }
  282. if proptools.Bool(f.properties.Use_avb) {
  283. addStr("avb_hashtree_enable", "true")
  284. addPath("avb_avbtool", ctx.Config().HostToolPath(ctx, "avbtool"))
  285. algorithm := proptools.StringDefault(f.properties.Avb_algorithm, "SHA256_RSA4096")
  286. addStr("avb_algorithm", algorithm)
  287. key := android.PathForModuleSrc(ctx, proptools.String(f.properties.Avb_private_key))
  288. addPath("avb_key_path", key)
  289. avb_add_hashtree_footer_args := "--do_not_generate_fec"
  290. if hashAlgorithm := proptools.String(f.properties.Avb_hash_algorithm); hashAlgorithm != "" {
  291. avb_add_hashtree_footer_args += " --hash_algorithm " + hashAlgorithm
  292. }
  293. addStr("avb_add_hashtree_footer_args", avb_add_hashtree_footer_args)
  294. partitionName := proptools.StringDefault(f.properties.Partition_name, f.Name())
  295. addStr("partition_name", partitionName)
  296. addStr("avb_salt", f.salt())
  297. }
  298. if proptools.String(f.properties.File_contexts) != "" {
  299. addPath("selinux_fc", f.buildFileContexts(ctx))
  300. }
  301. if timestamp := proptools.String(f.properties.Fake_timestamp); timestamp != "" {
  302. addStr("timestamp", timestamp)
  303. }
  304. if uuid := proptools.String(f.properties.Uuid); uuid != "" {
  305. addStr("uuid", uuid)
  306. addStr("hash_seed", uuid)
  307. }
  308. propFile = android.PathForModuleOut(ctx, "prop").OutputPath
  309. builder := android.NewRuleBuilder(pctx, ctx)
  310. builder.Command().Text("rm").Flag("-rf").Output(propFile)
  311. for _, p := range props {
  312. builder.Command().
  313. Text("echo").
  314. Flag(`"` + p.name + "=" + p.value + `"`).
  315. Text(">>").Output(propFile)
  316. }
  317. builder.Build("build_filesystem_prop", fmt.Sprintf("Creating filesystem props for %s", f.BaseModuleName()))
  318. return propFile, deps
  319. }
  320. func (f *filesystem) buildCpioImage(ctx android.ModuleContext, compressed bool) android.OutputPath {
  321. if proptools.Bool(f.properties.Use_avb) {
  322. ctx.PropertyErrorf("use_avb", "signing compresed cpio image using avbtool is not supported."+
  323. "Consider adding this to bootimg module and signing the entire boot image.")
  324. }
  325. if proptools.String(f.properties.File_contexts) != "" {
  326. ctx.PropertyErrorf("file_contexts", "file_contexts is not supported for compressed cpio image.")
  327. }
  328. depsZipFile := android.PathForModuleOut(ctx, "deps.zip").OutputPath
  329. f.entries = f.CopyDepsToZip(ctx, f.gatherFilteredPackagingSpecs(ctx), depsZipFile)
  330. builder := android.NewRuleBuilder(pctx, ctx)
  331. depsBase := proptools.StringDefault(f.properties.Base_dir, ".")
  332. rebasedDepsZip := android.PathForModuleOut(ctx, "rebased_deps.zip").OutputPath
  333. builder.Command().
  334. BuiltTool("zip2zip").
  335. FlagWithInput("-i ", depsZipFile).
  336. FlagWithOutput("-o ", rebasedDepsZip).
  337. Text("**/*:" + proptools.ShellEscape(depsBase)) // zip2zip verifies depsBase
  338. rootDir := android.PathForModuleOut(ctx, "root").OutputPath
  339. rootZip := f.buildRootZip(ctx)
  340. builder.Command().
  341. BuiltTool("zipsync").
  342. FlagWithArg("-d ", rootDir.String()). // zipsync wipes this. No need to clear.
  343. Input(rootZip).
  344. Input(rebasedDepsZip)
  345. output := android.PathForModuleOut(ctx, f.installFileName()).OutputPath
  346. cmd := builder.Command().
  347. BuiltTool("mkbootfs").
  348. Text(rootDir.String()) // input directory
  349. if compressed {
  350. cmd.Text("|").
  351. BuiltTool("lz4").
  352. Flag("--favor-decSpeed"). // for faster boot
  353. Flag("-12"). // maximum compression level
  354. Flag("-l"). // legacy format for kernel
  355. Text(">").Output(output)
  356. } else {
  357. cmd.Text(">").Output(output)
  358. }
  359. // rootDir is not deleted. Might be useful for quick inspection.
  360. builder.Build("build_cpio_image", fmt.Sprintf("Creating filesystem %s", f.BaseModuleName()))
  361. return output
  362. }
  363. var _ android.AndroidMkEntriesProvider = (*filesystem)(nil)
  364. // Implements android.AndroidMkEntriesProvider
  365. func (f *filesystem) AndroidMkEntries() []android.AndroidMkEntries {
  366. return []android.AndroidMkEntries{android.AndroidMkEntries{
  367. Class: "ETC",
  368. OutputFile: android.OptionalPathForPath(f.output),
  369. ExtraEntries: []android.AndroidMkExtraEntriesFunc{
  370. func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) {
  371. entries.SetString("LOCAL_MODULE_PATH", f.installDir.String())
  372. entries.SetString("LOCAL_INSTALLED_MODULE_STEM", f.installFileName())
  373. },
  374. },
  375. }}
  376. }
  377. var _ android.OutputFileProducer = (*filesystem)(nil)
  378. // Implements android.OutputFileProducer
  379. func (f *filesystem) OutputFiles(tag string) (android.Paths, error) {
  380. if tag == "" {
  381. return []android.Path{f.output}, nil
  382. }
  383. return nil, fmt.Errorf("unsupported module reference tag %q", tag)
  384. }
  385. // Filesystem is the public interface for the filesystem struct. Currently, it's only for the apex
  386. // package to have access to the output file.
  387. type Filesystem interface {
  388. android.Module
  389. OutputPath() android.Path
  390. // Returns the output file that is signed by avbtool. If this module is not signed, returns
  391. // nil.
  392. SignedOutputPath() android.Path
  393. }
  394. var _ Filesystem = (*filesystem)(nil)
  395. func (f *filesystem) OutputPath() android.Path {
  396. return f.output
  397. }
  398. func (f *filesystem) SignedOutputPath() android.Path {
  399. if proptools.Bool(f.properties.Use_avb) {
  400. return f.OutputPath()
  401. }
  402. return nil
  403. }
  404. // Filter the result of GatherPackagingSpecs to discard items targeting outside "system" partition.
  405. // Note that "apex" module installs its contents to "apex"(fake partition) as well
  406. // for symbol lookup by imitating "activated" paths.
  407. func (f *filesystem) gatherFilteredPackagingSpecs(ctx android.ModuleContext) map[string]android.PackagingSpec {
  408. specs := f.PackagingBase.GatherPackagingSpecs(ctx)
  409. if f.filterPackagingSpecs != nil {
  410. f.filterPackagingSpecs(specs)
  411. }
  412. return specs
  413. }
  414. func sha1sum(values []string) string {
  415. h := sha256.New()
  416. for _, value := range values {
  417. io.WriteString(h, value)
  418. }
  419. return fmt.Sprintf("%x", h.Sum(nil))
  420. }
  421. // Base cc.UseCoverage
  422. var _ cc.UseCoverage = (*filesystem)(nil)
  423. func (*filesystem) IsNativeCoverageNeeded(ctx android.BaseModuleContext) bool {
  424. return ctx.Device() && ctx.DeviceConfig().NativeCoverageEnabled()
  425. }