finder.go 8.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247
  1. // Copyright 2017 Google Inc. All rights reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package build
  15. import (
  16. "bytes"
  17. "io/ioutil"
  18. "os"
  19. "path/filepath"
  20. "strings"
  21. "android/soong/finder"
  22. "android/soong/finder/fs"
  23. "android/soong/ui/logger"
  24. "android/soong/ui/metrics"
  25. )
  26. // This file provides an interface to the Finder type for soong_ui. Finder is
  27. // used to recursively traverse the source tree to gather paths of files, such
  28. // as Android.bp or Android.mk, and store the lists/database of paths in files
  29. // under `$OUT_DIR/.module_paths`. This directory can also be dist'd.
  30. // NewSourceFinder returns a new Finder configured to search for source files.
  31. // Callers of NewSourceFinder should call <f.Shutdown()> when done
  32. func NewSourceFinder(ctx Context, config Config) (f *finder.Finder) {
  33. ctx.BeginTrace(metrics.RunSetupTool, "find modules")
  34. defer ctx.EndTrace()
  35. // Set up the working directory for the Finder.
  36. dir, err := os.Getwd()
  37. if err != nil {
  38. ctx.Fatalf("No working directory for module-finder: %v", err.Error())
  39. }
  40. filesystem := fs.OsFs
  41. // .out-dir and .find-ignore are markers for Finder to ignore siblings and
  42. // subdirectories of the directory Finder finds them in, hence stopping the
  43. // search recursively down those branches. It's possible that these files
  44. // are in the root directory, and if they are, then the subsequent error
  45. // messages are very confusing, so check for that here.
  46. pruneFiles := []string{".out-dir", ".find-ignore"}
  47. for _, name := range pruneFiles {
  48. prunePath := filepath.Join(dir, name)
  49. _, statErr := filesystem.Lstat(prunePath)
  50. if statErr == nil {
  51. ctx.Fatalf("%v must not exist", prunePath)
  52. }
  53. }
  54. // Set up configuration parameters for the Finder cache.
  55. cacheParams := finder.CacheParams{
  56. WorkingDirectory: dir,
  57. RootDirs: androidBpSearchDirs(config),
  58. FollowSymlinks: config.environ.IsEnvTrue("ALLOW_BP_UNDER_SYMLINKS"),
  59. ExcludeDirs: []string{".git", ".repo"},
  60. PruneFiles: pruneFiles,
  61. IncludeFiles: []string{
  62. // Kati build definitions.
  63. "Android.mk",
  64. // Product configuration files.
  65. "AndroidProducts.mk",
  66. // General Soong build definitions, using the Blueprint syntax.
  67. "Android.bp",
  68. // Bazel build definitions.
  69. "BUILD.bazel",
  70. // Bazel build definitions.
  71. "BUILD",
  72. // Kati clean definitions.
  73. "CleanSpec.mk",
  74. // Ownership definition.
  75. "OWNERS",
  76. // Test configuration for modules in directories that contain this
  77. // file.
  78. "TEST_MAPPING",
  79. // Bazel top-level file to mark a directory as a Bazel workspace.
  80. "WORKSPACE",
  81. // METADATA file of packages
  82. "METADATA",
  83. },
  84. // Bazel Starlark configuration files and all .mk files for product/board configuration.
  85. IncludeSuffixes: []string{".bzl", ".mk"},
  86. }
  87. dumpDir := config.FileListDir()
  88. f, err = finder.New(cacheParams, filesystem, logger.New(ioutil.Discard),
  89. filepath.Join(dumpDir, "files.db"))
  90. if err != nil {
  91. ctx.Fatalf("Could not create module-finder: %v", err)
  92. }
  93. return f
  94. }
  95. func androidBpSearchDirs(config Config) []string {
  96. dirs := []string{"."} // always search from root of source tree.
  97. if config.searchApiDir {
  98. // Search in out/api_surfaces
  99. dirs = append(dirs, config.ApiSurfacesOutDir())
  100. }
  101. return dirs
  102. }
  103. // Finds the list of Bazel-related files (BUILD, WORKSPACE and Starlark) in the tree.
  104. func findBazelFiles(entries finder.DirEntries) (dirNames []string, fileNames []string) {
  105. matches := []string{}
  106. for _, foundName := range entries.FileNames {
  107. if foundName == "BUILD.bazel" || foundName == "BUILD" || foundName == "WORKSPACE" || strings.HasSuffix(foundName, ".bzl") {
  108. matches = append(matches, foundName)
  109. }
  110. }
  111. return entries.DirNames, matches
  112. }
  113. func findProductAndBoardConfigFiles(entries finder.DirEntries) (dirNames []string, fileNames []string) {
  114. matches := []string{}
  115. for _, foundName := range entries.FileNames {
  116. if foundName != "Android.mk" &&
  117. foundName != "AndroidProducts.mk" &&
  118. foundName != "CleanSpec.mk" &&
  119. strings.HasSuffix(foundName, ".mk") {
  120. matches = append(matches, foundName)
  121. }
  122. }
  123. return entries.DirNames, matches
  124. }
  125. // FindSources searches for source files known to <f> and writes them to the filesystem for
  126. // use later.
  127. func FindSources(ctx Context, config Config, f *finder.Finder) {
  128. // note that dumpDir in FindSources may be different than dumpDir in NewSourceFinder
  129. // if a caller such as multiproduct_kati wants to share one Finder among several builds
  130. dumpDir := config.FileListDir()
  131. os.MkdirAll(dumpDir, 0777)
  132. // Stop searching a subdirectory recursively after finding an Android.mk.
  133. androidMks := f.FindFirstNamedAt(".", "Android.mk")
  134. err := dumpListToFile(ctx, config, androidMks, filepath.Join(dumpDir, "Android.mk.list"))
  135. if err != nil {
  136. ctx.Fatalf("Could not export module list: %v", err)
  137. }
  138. // Gate collecting/reporting mk metrics on builds that specifically request
  139. // it, as identifying the total number of mk files adds 4-5ms onto null
  140. // builds.
  141. if config.reportMkMetrics {
  142. androidMksTotal := f.FindNamedAt(".", "Android.mk")
  143. ctx.Metrics.SetToplevelMakefiles(len(androidMks))
  144. ctx.Metrics.SetTotalMakefiles(len(androidMksTotal))
  145. ctx.Metrics.DumpMkMetrics(config.MkMetrics())
  146. }
  147. // Stop searching a subdirectory recursively after finding a CleanSpec.mk.
  148. cleanSpecs := f.FindFirstNamedAt(".", "CleanSpec.mk")
  149. err = dumpListToFile(ctx, config, cleanSpecs, filepath.Join(dumpDir, "CleanSpec.mk.list"))
  150. if err != nil {
  151. ctx.Fatalf("Could not export module list: %v", err)
  152. }
  153. // Only consider AndroidProducts.mk in device/, vendor/ and product/, recursively in these directories.
  154. androidProductsMks := f.FindNamedAt("device", "AndroidProducts.mk")
  155. androidProductsMks = append(androidProductsMks, f.FindNamedAt("vendor", "AndroidProducts.mk")...)
  156. androidProductsMks = append(androidProductsMks, f.FindNamedAt("product", "AndroidProducts.mk")...)
  157. err = dumpListToFile(ctx, config, androidProductsMks, filepath.Join(dumpDir, "AndroidProducts.mk.list"))
  158. if err != nil {
  159. ctx.Fatalf("Could not export product list: %v", err)
  160. }
  161. // Recursively look for all Bazel related files.
  162. bazelFiles := f.FindMatching(".", findBazelFiles)
  163. err = dumpListToFile(ctx, config, bazelFiles, filepath.Join(dumpDir, "bazel.list"))
  164. if err != nil {
  165. ctx.Fatalf("Could not export bazel BUILD list: %v", err)
  166. }
  167. // Recursively look for all OWNERS files.
  168. owners := f.FindNamedAt(".", "OWNERS")
  169. err = dumpListToFile(ctx, config, owners, filepath.Join(dumpDir, "OWNERS.list"))
  170. if err != nil {
  171. ctx.Fatalf("Could not find OWNERS: %v", err)
  172. }
  173. // Recursively look for all METADATA files.
  174. metadataFiles := f.FindNamedAt(".", "METADATA")
  175. err = dumpListToFile(ctx, config, metadataFiles, filepath.Join(dumpDir, "METADATA.list"))
  176. if err != nil {
  177. ctx.Fatalf("Could not find METADATA: %v", err)
  178. }
  179. // Recursively look for all TEST_MAPPING files.
  180. testMappings := f.FindNamedAt(".", "TEST_MAPPING")
  181. err = dumpListToFile(ctx, config, testMappings, filepath.Join(dumpDir, "TEST_MAPPING.list"))
  182. if err != nil {
  183. ctx.Fatalf("Could not find TEST_MAPPING: %v", err)
  184. }
  185. // Recursively look for all Android.bp files
  186. androidBps := f.FindNamedAt(".", "Android.bp")
  187. if len(androidBps) == 0 {
  188. ctx.Fatalf("No Android.bp found")
  189. }
  190. err = dumpListToFile(ctx, config, androidBps, filepath.Join(dumpDir, "Android.bp.list"))
  191. if err != nil {
  192. ctx.Fatalf("Could not find modules: %v", err)
  193. }
  194. // Recursively look for all product/board config files.
  195. configurationFiles := f.FindMatching(".", findProductAndBoardConfigFiles)
  196. err = dumpListToFile(ctx, config, configurationFiles, filepath.Join(dumpDir, "configuration.list"))
  197. if err != nil {
  198. ctx.Fatalf("Could not export product/board configuration list: %v", err)
  199. }
  200. if config.Dist() {
  201. f.WaitForDbDump()
  202. // Dist the files.db plain text database.
  203. distFile(ctx, config, f.DbPath, "module_paths")
  204. }
  205. }
  206. // Write the .list files to disk.
  207. func dumpListToFile(ctx Context, config Config, list []string, filePath string) (err error) {
  208. desiredText := strings.Join(list, "\n")
  209. desiredBytes := []byte(desiredText)
  210. actualBytes, readErr := ioutil.ReadFile(filePath)
  211. if readErr != nil || !bytes.Equal(desiredBytes, actualBytes) {
  212. err = ioutil.WriteFile(filePath, desiredBytes, 0777)
  213. if err != nil {
  214. return err
  215. }
  216. }
  217. distFile(ctx, config, filePath, "module_paths")
  218. return nil
  219. }