symlink_forest.go 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501
  1. // Copyright 2022 Google Inc. All rights reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package bp2build
  15. import (
  16. "fmt"
  17. "io/ioutil"
  18. "os"
  19. "path/filepath"
  20. "regexp"
  21. "sort"
  22. "strconv"
  23. "strings"
  24. "sync"
  25. "sync/atomic"
  26. "android/soong/shared"
  27. "github.com/google/blueprint/pathtools"
  28. )
  29. // A tree structure that describes what to do at each directory in the created
  30. // symlink tree. Currently it is used to enumerate which files/directories
  31. // should be excluded from symlinking. Each instance of "node" represents a file
  32. // or a directory. If excluded is true, then that file/directory should be
  33. // excluded from symlinking. Otherwise, the node is not excluded, but one of its
  34. // descendants is (otherwise the node in question would not exist)
  35. // This is a version int written to a file called symlink_forest_version at the root of the
  36. // symlink forest. If the version here does not match the version in the file, then we'll
  37. // clean the whole symlink forest and recreate it. This number can be bumped whenever there's
  38. // an incompatible change to the forest layout or a bug in incrementality that needs to be fixed
  39. // on machines that may still have the bug present in their forest.
  40. const symlinkForestVersion = 1
  41. type instructionsNode struct {
  42. name string
  43. excluded bool // If false, this is just an intermediate node
  44. children map[string]*instructionsNode
  45. }
  46. type symlinkForestContext struct {
  47. verbose bool
  48. topdir string // $TOPDIR
  49. // State
  50. wg sync.WaitGroup
  51. depCh chan string
  52. mkdirCount atomic.Uint64
  53. symlinkCount atomic.Uint64
  54. }
  55. // Ensures that the node for the given path exists in the tree and returns it.
  56. func ensureNodeExists(root *instructionsNode, path string) *instructionsNode {
  57. if path == "" {
  58. return root
  59. }
  60. if path[len(path)-1] == '/' {
  61. path = path[:len(path)-1] // filepath.Split() leaves a trailing slash
  62. }
  63. dir, base := filepath.Split(path)
  64. // First compute the parent node...
  65. dn := ensureNodeExists(root, dir)
  66. // then create the requested node as its direct child, if needed.
  67. if child, ok := dn.children[base]; ok {
  68. return child
  69. } else {
  70. dn.children[base] = &instructionsNode{base, false, make(map[string]*instructionsNode)}
  71. return dn.children[base]
  72. }
  73. }
  74. // Turns a list of paths to be excluded into a tree
  75. func instructionsFromExcludePathList(paths []string) *instructionsNode {
  76. result := &instructionsNode{"", false, make(map[string]*instructionsNode)}
  77. for _, p := range paths {
  78. ensureNodeExists(result, p).excluded = true
  79. }
  80. return result
  81. }
  82. func mergeBuildFiles(output string, srcBuildFile string, generatedBuildFile string, verbose bool) error {
  83. srcBuildFileContent, err := os.ReadFile(srcBuildFile)
  84. if err != nil {
  85. return err
  86. }
  87. generatedBuildFileContent, err := os.ReadFile(generatedBuildFile)
  88. if err != nil {
  89. return err
  90. }
  91. // There can't be a package() call in both the source and generated BUILD files.
  92. // bp2build will generate a package() call for licensing information, but if
  93. // there's no licensing information, it will still generate a package() call
  94. // that just sets default_visibility=public. If the handcrafted build file
  95. // also has a package() call, we'll allow it to override the bp2build
  96. // generated one if it doesn't have any licensing information. If the bp2build
  97. // one has licensing information and the handcrafted one exists, we'll leave
  98. // them both in for bazel to throw an error.
  99. packageRegex := regexp.MustCompile(`(?m)^package\s*\(`)
  100. packageDefaultVisibilityRegex := regexp.MustCompile(`(?m)^package\s*\(\s*default_visibility\s*=\s*\[\s*"//visibility:public",?\s*]\s*\)`)
  101. if packageRegex.Find(srcBuildFileContent) != nil {
  102. if verbose && packageDefaultVisibilityRegex.Find(generatedBuildFileContent) != nil {
  103. fmt.Fprintf(os.Stderr, "Both '%s' and '%s' have a package() target, removing the first one\n",
  104. generatedBuildFile, srcBuildFile)
  105. }
  106. generatedBuildFileContent = packageDefaultVisibilityRegex.ReplaceAll(generatedBuildFileContent, []byte{})
  107. }
  108. newContents := generatedBuildFileContent
  109. if newContents[len(newContents)-1] != '\n' {
  110. newContents = append(newContents, '\n')
  111. }
  112. newContents = append(newContents, srcBuildFileContent...)
  113. // Say you run bp2build 4 times:
  114. // - The first time there's only an Android.bp file. bp2build will convert it to a build file
  115. // under out/soong/bp2build, then symlink from the forest to that generated file
  116. // - Then you add a handcrafted BUILD file in the same directory. bp2build will merge this with
  117. // the generated one, and write the result to the output file in the forest. But the output
  118. // file was a symlink to out/soong/bp2build from the previous step! So we erroneously update
  119. // the file in out/soong/bp2build instead. So far this doesn't cause any problems...
  120. // - You run a 3rd bp2build with no relevant changes. Everything continues to work.
  121. // - You then add a comment to the handcrafted BUILD file. This causes a merge with the
  122. // generated file again. But since we wrote to the generated file in step 2, the generated
  123. // file has an old copy of the handcrafted file in it! This probably causes duplicate bazel
  124. // targets.
  125. // To solve this, if we see that the output file is a symlink from a previous build, remove it.
  126. stat, err := os.Lstat(output)
  127. if err != nil && !os.IsNotExist(err) {
  128. return err
  129. } else if err == nil {
  130. if stat.Mode()&os.ModeSymlink == os.ModeSymlink {
  131. if verbose {
  132. fmt.Fprintf(os.Stderr, "Removing symlink so that we can replace it with a merged file: %s\n", output)
  133. }
  134. err = os.Remove(output)
  135. if err != nil {
  136. return err
  137. }
  138. }
  139. }
  140. return pathtools.WriteFileIfChanged(output, newContents, 0666)
  141. }
  142. // Calls readdir() and returns it as a map from the basename of the files in dir
  143. // to os.FileInfo.
  144. func readdirToMap(dir string) map[string]os.FileInfo {
  145. entryList, err := ioutil.ReadDir(dir)
  146. result := make(map[string]os.FileInfo)
  147. if err != nil {
  148. if os.IsNotExist(err) {
  149. // It's okay if a directory doesn't exist; it just means that one of the
  150. // trees to be merged contains parts the other doesn't
  151. return result
  152. } else {
  153. fmt.Fprintf(os.Stderr, "Cannot readdir '%s': %s\n", dir, err)
  154. os.Exit(1)
  155. }
  156. }
  157. for _, fi := range entryList {
  158. result[fi.Name()] = fi
  159. }
  160. return result
  161. }
  162. // Creates a symbolic link at dst pointing to src
  163. func symlinkIntoForest(topdir, dst, src string) uint64 {
  164. srcPath := shared.JoinPath(topdir, src)
  165. dstPath := shared.JoinPath(topdir, dst)
  166. // Check if a symlink already exists.
  167. if dstInfo, err := os.Lstat(dstPath); err != nil {
  168. if !os.IsNotExist(err) {
  169. fmt.Fprintf(os.Stderr, "Failed to lstat '%s': %s", dst, err)
  170. os.Exit(1)
  171. }
  172. } else {
  173. if dstInfo.Mode()&os.ModeSymlink != 0 {
  174. // Assume that the link's target is correct, i.e. no manual tampering.
  175. // E.g. OUT_DIR could have been previously used with a different source tree check-out!
  176. return 0
  177. } else {
  178. if err := os.RemoveAll(dstPath); err != nil {
  179. fmt.Fprintf(os.Stderr, "Failed to remove '%s': %s", dst, err)
  180. os.Exit(1)
  181. }
  182. }
  183. }
  184. // Create symlink.
  185. if err := os.Symlink(srcPath, dstPath); err != nil {
  186. fmt.Fprintf(os.Stderr, "Cannot create symlink at '%s' pointing to '%s': %s", dst, src, err)
  187. os.Exit(1)
  188. }
  189. return 1
  190. }
  191. func isDir(path string, fi os.FileInfo) bool {
  192. if (fi.Mode() & os.ModeSymlink) != os.ModeSymlink {
  193. return fi.IsDir()
  194. }
  195. fi2, statErr := os.Stat(path)
  196. if statErr == nil {
  197. return fi2.IsDir()
  198. }
  199. // Check if this is a dangling symlink. If so, treat it like a file, not a dir.
  200. _, lstatErr := os.Lstat(path)
  201. if lstatErr != nil {
  202. fmt.Fprintf(os.Stderr, "Cannot stat or lstat '%s': %s\n%s\n", path, statErr, lstatErr)
  203. os.Exit(1)
  204. }
  205. return false
  206. }
  207. // maybeCleanSymlinkForest will remove the whole symlink forest directory if the version recorded
  208. // in the symlink_forest_version file is not equal to symlinkForestVersion.
  209. func maybeCleanSymlinkForest(topdir, forest string, verbose bool) error {
  210. versionFilePath := shared.JoinPath(topdir, forest, "symlink_forest_version")
  211. versionFileContents, err := os.ReadFile(versionFilePath)
  212. if err != nil && !os.IsNotExist(err) {
  213. return err
  214. }
  215. versionFileString := strings.TrimSpace(string(versionFileContents))
  216. symlinkForestVersionString := strconv.Itoa(symlinkForestVersion)
  217. if err != nil || versionFileString != symlinkForestVersionString {
  218. if verbose {
  219. fmt.Fprintf(os.Stderr, "Old symlink_forest_version was %q, current is %q. Cleaning symlink forest before recreating...\n", versionFileString, symlinkForestVersionString)
  220. }
  221. err = os.RemoveAll(shared.JoinPath(topdir, forest))
  222. if err != nil {
  223. return err
  224. }
  225. }
  226. return nil
  227. }
  228. // maybeWriteVersionFile will write the symlink_forest_version file containing symlinkForestVersion
  229. // if it doesn't exist already. If it exists we know it must contain symlinkForestVersion because
  230. // we checked for that already in maybeCleanSymlinkForest
  231. func maybeWriteVersionFile(topdir, forest string) error {
  232. versionFilePath := shared.JoinPath(topdir, forest, "symlink_forest_version")
  233. _, err := os.Stat(versionFilePath)
  234. if err != nil {
  235. if !os.IsNotExist(err) {
  236. return err
  237. }
  238. err = os.WriteFile(versionFilePath, []byte(strconv.Itoa(symlinkForestVersion)+"\n"), 0666)
  239. if err != nil {
  240. return err
  241. }
  242. }
  243. return nil
  244. }
  245. // Recursively plants a symlink forest at forestDir. The symlink tree will
  246. // contain every file in buildFilesDir and srcDir excluding the files in
  247. // instructions. Collects every directory encountered during the traversal of
  248. // srcDir .
  249. func plantSymlinkForestRecursive(context *symlinkForestContext, instructions *instructionsNode, forestDir string, buildFilesDir string, srcDir string) {
  250. defer context.wg.Done()
  251. if instructions != nil && instructions.excluded {
  252. // Excluded paths are skipped at the level of the non-excluded parent.
  253. fmt.Fprintf(os.Stderr, "may not specify a root-level exclude directory '%s'", srcDir)
  254. os.Exit(1)
  255. }
  256. // We don't add buildFilesDir here because the bp2build files marker files is
  257. // already a dependency which covers it. If we ever wanted to turn this into
  258. // a generic symlink forest creation tool, we'd need to add it, too.
  259. context.depCh <- srcDir
  260. srcDirMap := readdirToMap(shared.JoinPath(context.topdir, srcDir))
  261. buildFilesMap := readdirToMap(shared.JoinPath(context.topdir, buildFilesDir))
  262. renamingBuildFile := false
  263. if _, ok := srcDirMap["BUILD"]; ok {
  264. if _, ok := srcDirMap["BUILD.bazel"]; !ok {
  265. if _, ok := buildFilesMap["BUILD.bazel"]; ok {
  266. renamingBuildFile = true
  267. srcDirMap["BUILD.bazel"] = srcDirMap["BUILD"]
  268. delete(srcDirMap, "BUILD")
  269. if instructions != nil {
  270. if _, ok := instructions.children["BUILD"]; ok {
  271. instructions.children["BUILD.bazel"] = instructions.children["BUILD"]
  272. delete(instructions.children, "BUILD")
  273. }
  274. }
  275. }
  276. }
  277. }
  278. allEntries := make([]string, 0, len(srcDirMap)+len(buildFilesMap))
  279. for n := range srcDirMap {
  280. allEntries = append(allEntries, n)
  281. }
  282. for n := range buildFilesMap {
  283. if _, ok := srcDirMap[n]; !ok {
  284. allEntries = append(allEntries, n)
  285. }
  286. }
  287. // Tests read the error messages generated, so ensure their order is deterministic
  288. sort.Strings(allEntries)
  289. fullForestPath := shared.JoinPath(context.topdir, forestDir)
  290. createForestDir := false
  291. if fi, err := os.Lstat(fullForestPath); err != nil {
  292. if os.IsNotExist(err) {
  293. createForestDir = true
  294. } else {
  295. fmt.Fprintf(os.Stderr, "Could not read info for '%s': %s\n", forestDir, err)
  296. }
  297. } else if fi.Mode()&os.ModeDir == 0 {
  298. if err := os.RemoveAll(fullForestPath); err != nil {
  299. fmt.Fprintf(os.Stderr, "Failed to remove '%s': %s", forestDir, err)
  300. os.Exit(1)
  301. }
  302. createForestDir = true
  303. }
  304. if createForestDir {
  305. if err := os.MkdirAll(fullForestPath, 0777); err != nil {
  306. fmt.Fprintf(os.Stderr, "Could not mkdir '%s': %s\n", forestDir, err)
  307. os.Exit(1)
  308. }
  309. context.mkdirCount.Add(1)
  310. }
  311. // Start with a list of items that already exist in the forest, and remove
  312. // each element as it is processed in allEntries. Any remaining items in
  313. // forestMapForDeletion must be removed. (This handles files which were
  314. // removed since the previous forest generation).
  315. forestMapForDeletion := readdirToMap(shared.JoinPath(context.topdir, forestDir))
  316. for _, f := range allEntries {
  317. if f[0] == '.' {
  318. continue // Ignore dotfiles
  319. }
  320. delete(forestMapForDeletion, f)
  321. // todo add deletionCount metric
  322. // The full paths of children in the input trees and in the output tree
  323. forestChild := shared.JoinPath(forestDir, f)
  324. srcChild := shared.JoinPath(srcDir, f)
  325. if f == "BUILD.bazel" && renamingBuildFile {
  326. srcChild = shared.JoinPath(srcDir, "BUILD")
  327. }
  328. buildFilesChild := shared.JoinPath(buildFilesDir, f)
  329. // Descend in the instruction tree if it exists
  330. var instructionsChild *instructionsNode
  331. if instructions != nil {
  332. instructionsChild = instructions.children[f]
  333. }
  334. srcChildEntry, sExists := srcDirMap[f]
  335. buildFilesChildEntry, bExists := buildFilesMap[f]
  336. if instructionsChild != nil && instructionsChild.excluded {
  337. if bExists {
  338. context.symlinkCount.Add(symlinkIntoForest(context.topdir, forestChild, buildFilesChild))
  339. }
  340. continue
  341. }
  342. sDir := sExists && isDir(shared.JoinPath(context.topdir, srcChild), srcChildEntry)
  343. bDir := bExists && isDir(shared.JoinPath(context.topdir, buildFilesChild), buildFilesChildEntry)
  344. if !sExists {
  345. if bDir && instructionsChild != nil {
  346. // Not in the source tree, but we have to exclude something from under
  347. // this subtree, so descend
  348. context.wg.Add(1)
  349. go plantSymlinkForestRecursive(context, instructionsChild, forestChild, buildFilesChild, srcChild)
  350. } else {
  351. // Not in the source tree, symlink BUILD file
  352. context.symlinkCount.Add(symlinkIntoForest(context.topdir, forestChild, buildFilesChild))
  353. }
  354. } else if !bExists {
  355. if sDir && instructionsChild != nil {
  356. // Not in the build file tree, but we have to exclude something from
  357. // under this subtree, so descend
  358. context.wg.Add(1)
  359. go plantSymlinkForestRecursive(context, instructionsChild, forestChild, buildFilesChild, srcChild)
  360. } else {
  361. // Not in the build file tree, symlink source tree, carry on
  362. context.symlinkCount.Add(symlinkIntoForest(context.topdir, forestChild, srcChild))
  363. }
  364. } else if sDir && bDir {
  365. // Both are directories. Descend.
  366. context.wg.Add(1)
  367. go plantSymlinkForestRecursive(context, instructionsChild, forestChild, buildFilesChild, srcChild)
  368. } else if !sDir && !bDir {
  369. // Neither is a directory. Merge them.
  370. srcBuildFile := shared.JoinPath(context.topdir, srcChild)
  371. generatedBuildFile := shared.JoinPath(context.topdir, buildFilesChild)
  372. // The Android.bp file that codegen used to produce `buildFilesChild` is
  373. // already a dependency, we can ignore `buildFilesChild`.
  374. context.depCh <- srcChild
  375. if err := mergeBuildFiles(shared.JoinPath(context.topdir, forestChild), srcBuildFile, generatedBuildFile, context.verbose); err != nil {
  376. fmt.Fprintf(os.Stderr, "Error merging %s and %s: %s",
  377. srcBuildFile, generatedBuildFile, err)
  378. os.Exit(1)
  379. }
  380. } else {
  381. // Both exist and one is a file. This is an error.
  382. fmt.Fprintf(os.Stderr,
  383. "Conflict in workspace symlink tree creation: both '%s' and '%s' exist and exactly one is a directory\n",
  384. srcChild, buildFilesChild)
  385. os.Exit(1)
  386. }
  387. }
  388. // Remove all files in the forest that exist in neither the source
  389. // tree nor the build files tree. (This handles files which were removed
  390. // since the previous forest generation).
  391. for f := range forestMapForDeletion {
  392. var instructionsChild *instructionsNode
  393. if instructions != nil {
  394. instructionsChild = instructions.children[f]
  395. }
  396. if instructionsChild != nil && instructionsChild.excluded {
  397. // This directory may be excluded because bazel writes to it under the
  398. // forest root. Thus this path is intentionally left alone.
  399. continue
  400. }
  401. forestChild := shared.JoinPath(context.topdir, forestDir, f)
  402. if err := os.RemoveAll(forestChild); err != nil {
  403. fmt.Fprintf(os.Stderr, "Failed to remove '%s/%s': %s", forestDir, f, err)
  404. os.Exit(1)
  405. }
  406. }
  407. }
  408. // PlantSymlinkForest Creates a symlink forest by merging the directory tree at "buildFiles" and
  409. // "srcDir" while excluding paths listed in "exclude". Returns the set of paths
  410. // under srcDir on which readdir() had to be called to produce the symlink
  411. // forest.
  412. func PlantSymlinkForest(verbose bool, topdir string, forest string, buildFiles string, exclude []string) (deps []string, mkdirCount, symlinkCount uint64) {
  413. context := &symlinkForestContext{
  414. verbose: verbose,
  415. topdir: topdir,
  416. depCh: make(chan string),
  417. mkdirCount: atomic.Uint64{},
  418. symlinkCount: atomic.Uint64{},
  419. }
  420. err := maybeCleanSymlinkForest(topdir, forest, verbose)
  421. if err != nil {
  422. fmt.Fprintln(os.Stderr, err)
  423. os.Exit(1)
  424. }
  425. instructions := instructionsFromExcludePathList(exclude)
  426. go func() {
  427. context.wg.Add(1)
  428. plantSymlinkForestRecursive(context, instructions, forest, buildFiles, ".")
  429. context.wg.Wait()
  430. close(context.depCh)
  431. }()
  432. for dep := range context.depCh {
  433. deps = append(deps, dep)
  434. }
  435. err = maybeWriteVersionFile(topdir, forest)
  436. if err != nil {
  437. fmt.Fprintln(os.Stderr, err)
  438. os.Exit(1)
  439. }
  440. return deps, context.mkdirCount.Load(), context.symlinkCount.Load()
  441. }