cleanbuild.go 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330
  1. // Copyright 2017 Google Inc. All rights reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package build
  15. import (
  16. "bytes"
  17. "fmt"
  18. "io/ioutil"
  19. "os"
  20. "path/filepath"
  21. "sort"
  22. "strings"
  23. "android/soong/ui/metrics"
  24. )
  25. // Given a series of glob patterns, remove matching files and directories from the filesystem.
  26. // For example, "malware*" would remove all files and directories in the current directory that begin with "malware".
  27. func removeGlobs(ctx Context, globs ...string) {
  28. for _, glob := range globs {
  29. // Find files and directories that match this glob pattern.
  30. files, err := filepath.Glob(glob)
  31. if err != nil {
  32. // Only possible error is ErrBadPattern
  33. panic(fmt.Errorf("%q: %s", glob, err))
  34. }
  35. for _, file := range files {
  36. err = os.RemoveAll(file)
  37. if err != nil {
  38. ctx.Fatalf("Failed to remove file %q: %v", file, err)
  39. }
  40. }
  41. }
  42. }
  43. // Based on https://stackoverflow.com/questions/28969455/how-to-properly-instantiate-os-filemode
  44. // Because Go doesn't provide a nice way to set bits on a filemode
  45. const (
  46. FILEMODE_READ = 04
  47. FILEMODE_WRITE = 02
  48. FILEMODE_EXECUTE = 01
  49. FILEMODE_USER_SHIFT = 6
  50. FILEMODE_USER_READ = FILEMODE_READ << FILEMODE_USER_SHIFT
  51. FILEMODE_USER_WRITE = FILEMODE_WRITE << FILEMODE_USER_SHIFT
  52. FILEMODE_USER_EXECUTE = FILEMODE_EXECUTE << FILEMODE_USER_SHIFT
  53. )
  54. // Remove everything under the out directory. Don't remove the out directory
  55. // itself in case it's a symlink.
  56. func clean(ctx Context, config Config) {
  57. removeGlobs(ctx, filepath.Join(config.OutDir(), "*"))
  58. ctx.Println("Entire build directory removed.")
  59. }
  60. // Remove everything in the data directory.
  61. func dataClean(ctx Context, config Config) {
  62. removeGlobs(ctx, filepath.Join(config.ProductOut(), "data", "*"))
  63. ctx.Println("Entire data directory removed.")
  64. }
  65. // installClean deletes all of the installed files -- the intent is to remove
  66. // files that may no longer be installed, either because the user previously
  67. // installed them, or they were previously installed by default but no longer
  68. // are.
  69. //
  70. // This is faster than a full clean, since we're not deleting the
  71. // intermediates. Instead of recompiling, we can just copy the results.
  72. func installClean(ctx Context, config Config) {
  73. dataClean(ctx, config)
  74. if hostCrossOutPath := config.hostCrossOut(); hostCrossOutPath != "" {
  75. hostCrossOut := func(path string) string {
  76. return filepath.Join(hostCrossOutPath, path)
  77. }
  78. removeGlobs(ctx,
  79. hostCrossOut("bin"),
  80. hostCrossOut("coverage"),
  81. hostCrossOut("lib*"),
  82. hostCrossOut("nativetest*"))
  83. }
  84. hostOutPath := config.HostOut()
  85. hostOut := func(path string) string {
  86. return filepath.Join(hostOutPath, path)
  87. }
  88. hostCommonOut := func(path string) string {
  89. return filepath.Join(config.hostOutRoot(), "common", path)
  90. }
  91. productOutPath := config.ProductOut()
  92. productOut := func(path string) string {
  93. return filepath.Join(productOutPath, path)
  94. }
  95. // Host bin, frameworks, and lib* are intentionally omitted, since
  96. // otherwise we'd have to rebuild any generated files created with
  97. // those tools.
  98. removeGlobs(ctx,
  99. hostOut("apex"),
  100. hostOut("obj/NOTICE_FILES"),
  101. hostOut("obj/PACKAGING"),
  102. hostOut("coverage"),
  103. hostOut("cts"),
  104. hostOut("nativetest*"),
  105. hostOut("sdk"),
  106. hostOut("sdk_addon"),
  107. hostOut("testcases"),
  108. hostOut("vts"),
  109. hostOut("vts10"),
  110. hostOut("vts-core"),
  111. hostCommonOut("obj/PACKAGING"),
  112. productOut("*.img"),
  113. productOut("*.zip"),
  114. productOut("android-info.txt"),
  115. productOut("misc_info.txt"),
  116. productOut("apex"),
  117. productOut("kernel"),
  118. productOut("kernel-*"),
  119. productOut("data"),
  120. productOut("skin"),
  121. productOut("obj/NOTICE_FILES"),
  122. productOut("obj/PACKAGING"),
  123. productOut("ramdisk"),
  124. productOut("ramdisk_16k"),
  125. productOut("debug_ramdisk"),
  126. productOut("vendor_ramdisk"),
  127. productOut("vendor_debug_ramdisk"),
  128. productOut("vendor_kernel_ramdisk"),
  129. productOut("test_harness_ramdisk"),
  130. productOut("recovery"),
  131. productOut("root"),
  132. productOut("system"),
  133. productOut("system_dlkm"),
  134. productOut("system_other"),
  135. productOut("vendor"),
  136. productOut("vendor_dlkm"),
  137. productOut("product"),
  138. productOut("system_ext"),
  139. productOut("oem"),
  140. productOut("obj/FAKE"),
  141. productOut("breakpad"),
  142. productOut("cache"),
  143. productOut("coverage"),
  144. productOut("installer"),
  145. productOut("odm"),
  146. productOut("odm_dlkm"),
  147. productOut("sysloader"),
  148. productOut("testcases"),
  149. productOut("symbols"))
  150. }
  151. // Since products and build variants (unfortunately) shared the same
  152. // PRODUCT_OUT staging directory, things can get out of sync if different
  153. // build configurations are built in the same tree. This function will
  154. // notice when the configuration has changed and call installClean to
  155. // remove the files necessary to keep things consistent.
  156. func installCleanIfNecessary(ctx Context, config Config) {
  157. configFile := config.DevicePreviousProductConfig()
  158. prefix := "PREVIOUS_BUILD_CONFIG := "
  159. suffix := "\n"
  160. currentConfig := prefix + config.TargetProduct() + "-" + config.TargetBuildVariant() + suffix
  161. ensureDirectoriesExist(ctx, filepath.Dir(configFile))
  162. writeConfig := func() {
  163. err := ioutil.WriteFile(configFile, []byte(currentConfig), 0666) // a+rw
  164. if err != nil {
  165. ctx.Fatalln("Failed to write product config:", err)
  166. }
  167. }
  168. previousConfigBytes, err := ioutil.ReadFile(configFile)
  169. if err != nil {
  170. if os.IsNotExist(err) {
  171. // Just write the new config file, no old config file to worry about.
  172. writeConfig()
  173. return
  174. } else {
  175. ctx.Fatalln("Failed to read previous product config:", err)
  176. }
  177. }
  178. previousConfig := string(previousConfigBytes)
  179. if previousConfig == currentConfig {
  180. // Same config as before - nothing to clean.
  181. return
  182. }
  183. if config.Environment().IsEnvTrue("DISABLE_AUTO_INSTALLCLEAN") {
  184. ctx.Println("DISABLE_AUTO_INSTALLCLEAN is set and true; skipping auto-clean. Your tree may be in an inconsistent state.")
  185. return
  186. }
  187. ctx.BeginTrace(metrics.PrimaryNinja, "installclean")
  188. defer ctx.EndTrace()
  189. previousProductAndVariant := strings.TrimPrefix(strings.TrimSuffix(previousConfig, suffix), prefix)
  190. currentProductAndVariant := strings.TrimPrefix(strings.TrimSuffix(currentConfig, suffix), prefix)
  191. ctx.Printf("Build configuration changed: %q -> %q, forcing installclean\n", previousProductAndVariant, currentProductAndVariant)
  192. installClean(ctx, config)
  193. writeConfig()
  194. }
  195. // cleanOldFiles takes an input file (with all paths relative to basePath), and removes files from
  196. // the filesystem if they were removed from the input file since the last execution.
  197. func cleanOldFiles(ctx Context, basePath, newFile string) {
  198. newFile = filepath.Join(basePath, newFile)
  199. oldFile := newFile + ".previous"
  200. if _, err := os.Stat(newFile); os.IsNotExist(err) {
  201. // If the file doesn't exist, assume no installed files exist either
  202. return
  203. } else if err != nil {
  204. ctx.Fatalf("Expected %q to be readable", newFile)
  205. }
  206. if _, err := os.Stat(oldFile); os.IsNotExist(err) {
  207. if err := os.Rename(newFile, oldFile); err != nil {
  208. ctx.Fatalf("Failed to rename file list (%q->%q): %v", newFile, oldFile, err)
  209. }
  210. return
  211. }
  212. var newData, oldData []byte
  213. if data, err := ioutil.ReadFile(newFile); err == nil {
  214. newData = data
  215. } else {
  216. ctx.Fatalf("Failed to read list of installable files (%q): %v", newFile, err)
  217. }
  218. if data, err := ioutil.ReadFile(oldFile); err == nil {
  219. oldData = data
  220. } else {
  221. ctx.Fatalf("Failed to read list of installable files (%q): %v", oldFile, err)
  222. }
  223. // Common case: nothing has changed
  224. if bytes.Equal(newData, oldData) {
  225. return
  226. }
  227. var newPaths, oldPaths []string
  228. newPaths = strings.Fields(string(newData))
  229. oldPaths = strings.Fields(string(oldData))
  230. // These should be mostly sorted by make already, but better make sure Go concurs
  231. sort.Strings(newPaths)
  232. sort.Strings(oldPaths)
  233. for len(oldPaths) > 0 {
  234. if len(newPaths) > 0 {
  235. if oldPaths[0] == newPaths[0] {
  236. // Same file; continue
  237. newPaths = newPaths[1:]
  238. oldPaths = oldPaths[1:]
  239. continue
  240. } else if oldPaths[0] > newPaths[0] {
  241. // New file; ignore
  242. newPaths = newPaths[1:]
  243. continue
  244. }
  245. }
  246. // File only exists in the old list; remove if it exists
  247. oldPath := filepath.Join(basePath, oldPaths[0])
  248. oldPaths = oldPaths[1:]
  249. if oldFile, err := os.Stat(oldPath); err == nil {
  250. if oldFile.IsDir() {
  251. if err := os.Remove(oldPath); err == nil {
  252. ctx.Println("Removed directory that is no longer installed: ", oldPath)
  253. cleanEmptyDirs(ctx, filepath.Dir(oldPath))
  254. } else {
  255. ctx.Println("Failed to remove directory that is no longer installed (%q): %v", oldPath, err)
  256. ctx.Println("It's recommended to run `m installclean`")
  257. }
  258. } else {
  259. // Removing a file, not a directory.
  260. if err := os.Remove(oldPath); err == nil {
  261. ctx.Println("Removed file that is no longer installed: ", oldPath)
  262. cleanEmptyDirs(ctx, filepath.Dir(oldPath))
  263. } else if !os.IsNotExist(err) {
  264. ctx.Fatalf("Failed to remove file that is no longer installed (%q): %v", oldPath, err)
  265. }
  266. }
  267. }
  268. }
  269. // Use the new list as the base for the next build
  270. os.Rename(newFile, oldFile)
  271. }
  272. // cleanEmptyDirs will delete a directory if it contains no files.
  273. // If a deletion occurs, then it also recurses upwards to try and delete empty parent directories.
  274. func cleanEmptyDirs(ctx Context, dir string) {
  275. files, err := ioutil.ReadDir(dir)
  276. if err != nil {
  277. ctx.Println("Could not read directory while trying to clean empty dirs: ", dir)
  278. return
  279. }
  280. if len(files) > 0 {
  281. // Directory is not empty.
  282. return
  283. }
  284. if err := os.Remove(dir); err == nil {
  285. ctx.Println("Removed empty directory (may no longer be installed?): ", dir)
  286. } else {
  287. ctx.Fatalf("Failed to remove empty directory (which may no longer be installed?) %q: (%v)", dir, err)
  288. }
  289. // Try and delete empty parent directories too.
  290. cleanEmptyDirs(ctx, filepath.Dir(dir))
  291. }