kati.go 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412
  1. // Copyright 2017 Google Inc. All rights reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package build
  15. import (
  16. "crypto/md5"
  17. "fmt"
  18. "io/ioutil"
  19. "os"
  20. "os/user"
  21. "path/filepath"
  22. "strings"
  23. "time"
  24. "android/soong/ui/metrics"
  25. "android/soong/ui/status"
  26. )
  27. var spaceSlashReplacer = strings.NewReplacer("/", "_", " ", "_")
  28. const katiBuildSuffix = ""
  29. const katiCleanspecSuffix = "-cleanspec"
  30. const katiPackageSuffix = "-package"
  31. // genKatiSuffix creates a filename suffix for kati-generated files so that we
  32. // can cache them based on their inputs. Such files include the generated Ninja
  33. // files and env.sh environment variable setup files.
  34. //
  35. // The filename suffix should encode all common changes to Kati inputs.
  36. // Currently that includes the TARGET_PRODUCT and kati-processed command line
  37. // arguments.
  38. func genKatiSuffix(ctx Context, config Config) {
  39. // Construct the base suffix.
  40. katiSuffix := "-" + config.TargetProduct()
  41. // Append kati arguments to the suffix.
  42. if args := config.KatiArgs(); len(args) > 0 {
  43. katiSuffix += "-" + spaceSlashReplacer.Replace(strings.Join(args, "_"))
  44. }
  45. // If the suffix is too long, replace it with a md5 hash and write a
  46. // file that contains the original suffix.
  47. if len(katiSuffix) > 64 {
  48. shortSuffix := "-" + fmt.Sprintf("%x", md5.Sum([]byte(katiSuffix)))
  49. config.SetKatiSuffix(shortSuffix)
  50. ctx.Verbosef("Kati ninja suffix too long: %q", katiSuffix)
  51. ctx.Verbosef("Replacing with: %q", shortSuffix)
  52. if err := ioutil.WriteFile(strings.TrimSuffix(config.KatiBuildNinjaFile(), "ninja")+"suf", []byte(katiSuffix), 0777); err != nil {
  53. ctx.Println("Error writing suffix file:", err)
  54. }
  55. } else {
  56. config.SetKatiSuffix(katiSuffix)
  57. }
  58. }
  59. func writeValueIfChanged(ctx Context, config Config, dir string, filename string, value string) {
  60. filePath := filepath.Join(dir, filename)
  61. previousValue := ""
  62. rawPreviousValue, err := ioutil.ReadFile(filePath)
  63. if err == nil {
  64. previousValue = string(rawPreviousValue)
  65. }
  66. if previousValue != value {
  67. if err = ioutil.WriteFile(filePath, []byte(value), 0666); err != nil {
  68. ctx.Fatalf("Failed to write: %v", err)
  69. }
  70. }
  71. }
  72. // Base function to construct and run the Kati command line with additional
  73. // arguments, and a custom function closure to mutate the environment Kati runs
  74. // in.
  75. func runKati(ctx Context, config Config, extraSuffix string, args []string, envFunc func(*Environment)) {
  76. executable := config.PrebuiltBuildTool("ckati")
  77. // cKati arguments.
  78. args = append([]string{
  79. // Instead of executing commands directly, generate a Ninja file.
  80. "--ninja",
  81. // Generate Ninja files in the output directory.
  82. "--ninja_dir=" + config.OutDir(),
  83. // Filename suffix of the generated Ninja file.
  84. "--ninja_suffix=" + config.KatiSuffix() + extraSuffix,
  85. // Remove common parts at the beginning of a Ninja file, like build_dir,
  86. // local_pool and _kati_always_build_. Allows Kati to be run multiple
  87. // times, with generated Ninja files combined in a single invocation
  88. // using 'include'.
  89. "--no_ninja_prelude",
  90. // Support declaring phony outputs in AOSP Ninja.
  91. "--use_ninja_phony_output",
  92. // Support declaring symlink outputs in AOSP Ninja.
  93. "--use_ninja_symlink_outputs",
  94. // Regenerate the Ninja file if environment inputs have changed. e.g.
  95. // CLI flags, .mk file timestamps, env vars, $(wildcard ..) and some
  96. // $(shell ..) results.
  97. "--regen",
  98. // Skip '-include' directives starting with the specified path. Used to
  99. // ignore generated .mk files.
  100. "--ignore_optional_include=" + filepath.Join(config.OutDir(), "%.P"),
  101. // Detect the use of $(shell echo ...).
  102. "--detect_android_echo",
  103. // Colorful ANSI-based warning and error messages.
  104. "--color_warnings",
  105. // Generate all targets, not just the top level requested ones.
  106. "--gen_all_targets",
  107. // Use the built-in emulator of GNU find for better file finding
  108. // performance. Used with $(shell find ...).
  109. "--use_find_emulator",
  110. // Fail when the find emulator encounters problems.
  111. "--werror_find_emulator",
  112. // Do not provide any built-in rules.
  113. "--no_builtin_rules",
  114. // Fail when suffix rules are used.
  115. "--werror_suffix_rules",
  116. // Fail when a real target depends on a phony target.
  117. "--werror_real_to_phony",
  118. // Makes real_to_phony checks assume that any top-level or leaf
  119. // dependencies that does *not* have a '/' in it is a phony target.
  120. "--top_level_phony",
  121. // Fail when a phony target contains slashes.
  122. "--werror_phony_looks_real",
  123. // Fail when writing to a read-only directory.
  124. "--werror_writable",
  125. // Print Kati's internal statistics, such as the number of variables,
  126. // implicit/explicit/suffix rules, and so on.
  127. "--kati_stats",
  128. }, args...)
  129. // Generate a minimal Ninja file.
  130. //
  131. // Used for build_test and multiproduct_kati, which runs Kati several
  132. // hundred times for different configurations to test file generation logic.
  133. // These can result in generating Ninja files reaching ~1GB or more,
  134. // resulting in ~hundreds of GBs of writes.
  135. //
  136. // Since we don't care about executing the Ninja files in these test cases,
  137. // generating the Ninja file content wastes time, so skip writing any
  138. // information out with --empty_ninja_file.
  139. //
  140. // From https://github.com/google/kati/commit/87b8da7af2c8bea28b1d8ab17679453d859f96e5
  141. if config.EmptyNinjaFile() {
  142. args = append(args, "--empty_ninja_file")
  143. }
  144. // Apply 'local_pool' to to all rules that don't specify a pool.
  145. if config.UseRemoteBuild() {
  146. args = append(args, "--default_pool=local_pool")
  147. }
  148. cmd := Command(ctx, config, "ckati", executable, args...)
  149. // Set up the nsjail sandbox.
  150. cmd.Sandbox = katiSandbox
  151. // Set up stdout and stderr.
  152. pipe, err := cmd.StdoutPipe()
  153. if err != nil {
  154. ctx.Fatalln("Error getting output pipe for ckati:", err)
  155. }
  156. cmd.Stderr = cmd.Stdout
  157. var username string
  158. // Pass on various build environment metadata to Kati.
  159. if usernameFromEnv, ok := cmd.Environment.Get("BUILD_USERNAME"); !ok {
  160. username = "unknown"
  161. if u, err := user.Current(); err == nil {
  162. username = u.Username
  163. } else {
  164. ctx.Println("Failed to get current user:", err)
  165. }
  166. cmd.Environment.Set("BUILD_USERNAME", username)
  167. } else {
  168. username = usernameFromEnv
  169. }
  170. hostname, ok := cmd.Environment.Get("BUILD_HOSTNAME")
  171. if !ok {
  172. hostname, err = os.Hostname()
  173. if err != nil {
  174. ctx.Println("Failed to read hostname:", err)
  175. hostname = "unknown"
  176. }
  177. cmd.Environment.Set("BUILD_HOSTNAME", hostname)
  178. }
  179. writeValueIfChanged(ctx, config, config.SoongOutDir(), "build_hostname.txt", hostname)
  180. // BUILD_NUMBER should be set to the source control value that
  181. // represents the current state of the source code. E.g., a
  182. // perforce changelist number or a git hash. Can be an arbitrary string
  183. // (to allow for source control that uses something other than numbers),
  184. // but must be a single word and a valid file name.
  185. //
  186. // If no BUILD_NUMBER is set, create a useful "I am an engineering build
  187. // from this date/time" value. Make it start with a non-digit so that
  188. // anyone trying to parse it as an integer will probably get "0".
  189. cmd.Environment.Unset("HAS_BUILD_NUMBER")
  190. buildNumber, ok := cmd.Environment.Get("BUILD_NUMBER")
  191. if ok {
  192. cmd.Environment.Set("HAS_BUILD_NUMBER", "true")
  193. writeValueIfChanged(ctx, config, config.OutDir(), "file_name_tag.txt", buildNumber)
  194. } else {
  195. buildNumber = fmt.Sprintf("eng.%.6s.%s", username, time.Now().Format("20060102.150405" /* YYYYMMDD.HHMMSS */))
  196. cmd.Environment.Set("HAS_BUILD_NUMBER", "false")
  197. writeValueIfChanged(ctx, config, config.OutDir(), "file_name_tag.txt", username)
  198. }
  199. // Write the build number to a file so it can be read back in
  200. // without changing the command line every time. Avoids rebuilds
  201. // when using ninja.
  202. writeValueIfChanged(ctx, config, config.SoongOutDir(), "build_number.txt", buildNumber)
  203. // Apply the caller's function closure to mutate the environment variables.
  204. envFunc(cmd.Environment)
  205. cmd.StartOrFatal()
  206. // Set up the ToolStatus command line reader for Kati for a consistent UI
  207. // for the user.
  208. status.KatiReader(ctx.Status.StartTool(), pipe)
  209. cmd.WaitOrFatal()
  210. }
  211. func runKatiBuild(ctx Context, config Config) {
  212. ctx.BeginTrace(metrics.RunKati, "kati build")
  213. defer ctx.EndTrace()
  214. args := []string{
  215. // Mark the output directory as writable.
  216. "--writable", config.OutDir() + "/",
  217. // Fail when encountering implicit rules. e.g.
  218. // %.foo: %.bar
  219. // cp $< $@
  220. "--werror_implicit_rules",
  221. // Entry point for the Kati Ninja file generation.
  222. "-f", "build/make/core/main.mk",
  223. }
  224. if !config.BuildBrokenDupRules() {
  225. // Fail when redefining / duplicating a target.
  226. args = append(args, "--werror_overriding_commands")
  227. }
  228. args = append(args, config.KatiArgs()...)
  229. args = append(args,
  230. // Location of the Make vars .mk file generated by Soong.
  231. "SOONG_MAKEVARS_MK="+config.SoongMakeVarsMk(),
  232. // Location of the Android.mk file generated by Soong. This
  233. // file contains Soong modules represented as Kati modules,
  234. // allowing Kati modules to depend on Soong modules.
  235. "SOONG_ANDROID_MK="+config.SoongAndroidMk(),
  236. // Directory containing outputs for the target device.
  237. "TARGET_DEVICE_DIR="+config.TargetDeviceDir(),
  238. // Directory containing .mk files for packaging purposes, such as
  239. // the dist.mk file, containing dist-for-goals data.
  240. "KATI_PACKAGE_MK_DIR="+config.KatiPackageMkDir())
  241. runKati(ctx, config, katiBuildSuffix, args, func(env *Environment) {})
  242. // compress and dist the main build ninja file.
  243. distGzipFile(ctx, config, config.KatiBuildNinjaFile())
  244. // Cleanup steps.
  245. cleanCopyHeaders(ctx, config)
  246. cleanOldInstalledFiles(ctx, config)
  247. }
  248. // Clean out obsolete header files on the disk that were *not copied* during the
  249. // build with BUILD_COPY_HEADERS and LOCAL_COPY_HEADERS.
  250. //
  251. // These should be increasingly uncommon, as it's a deprecated feature and there
  252. // isn't an equivalent feature in Soong.
  253. func cleanCopyHeaders(ctx Context, config Config) {
  254. ctx.BeginTrace("clean", "clean copy headers")
  255. defer ctx.EndTrace()
  256. // Read and parse the list of copied headers from a file in the product
  257. // output directory.
  258. data, err := ioutil.ReadFile(filepath.Join(config.ProductOut(), ".copied_headers_list"))
  259. if err != nil {
  260. if os.IsNotExist(err) {
  261. return
  262. }
  263. ctx.Fatalf("Failed to read copied headers list: %v", err)
  264. }
  265. headers := strings.Fields(string(data))
  266. if len(headers) < 1 {
  267. ctx.Fatal("Failed to parse copied headers list: %q", string(data))
  268. }
  269. headerDir := headers[0]
  270. headers = headers[1:]
  271. // Walk the tree and remove any headers that are not in the list of copied
  272. // headers in the current build.
  273. filepath.Walk(headerDir,
  274. func(path string, info os.FileInfo, err error) error {
  275. if err != nil {
  276. return nil
  277. }
  278. if info.IsDir() {
  279. return nil
  280. }
  281. if !inList(path, headers) {
  282. ctx.Printf("Removing obsolete header %q", path)
  283. if err := os.Remove(path); err != nil {
  284. ctx.Fatalf("Failed to remove obsolete header %q: %v", path, err)
  285. }
  286. }
  287. return nil
  288. })
  289. }
  290. // Clean out any previously installed files from the disk that are not installed
  291. // in the current build.
  292. func cleanOldInstalledFiles(ctx Context, config Config) {
  293. ctx.BeginTrace("clean", "clean old installed files")
  294. defer ctx.EndTrace()
  295. // We shouldn't be removing files from one side of the two-step asan builds
  296. var suffix string
  297. if v, ok := config.Environment().Get("SANITIZE_TARGET"); ok {
  298. if sanitize := strings.Fields(v); inList("address", sanitize) {
  299. suffix = "_asan"
  300. }
  301. }
  302. cleanOldFiles(ctx, config.ProductOut(), ".installable_files"+suffix)
  303. cleanOldFiles(ctx, config.HostOut(), ".installable_test_files")
  304. }
  305. // Generate the Ninja file containing the packaging command lines for the dist
  306. // dir.
  307. func runKatiPackage(ctx Context, config Config) {
  308. ctx.BeginTrace(metrics.RunKati, "kati package")
  309. defer ctx.EndTrace()
  310. args := []string{
  311. // Mark the dist dir as writable.
  312. "--writable", config.DistDir() + "/",
  313. // Fail when encountering implicit rules. e.g.
  314. "--werror_implicit_rules",
  315. // Fail when redefining / duplicating a target.
  316. "--werror_overriding_commands",
  317. // Entry point.
  318. "-f", "build/make/packaging/main.mk",
  319. // Directory containing .mk files for packaging purposes, such as
  320. // the dist.mk file, containing dist-for-goals data.
  321. "KATI_PACKAGE_MK_DIR=" + config.KatiPackageMkDir(),
  322. }
  323. // Run Kati against a restricted set of environment variables.
  324. runKati(ctx, config, katiPackageSuffix, args, func(env *Environment) {
  325. env.Allow([]string{
  326. // Some generic basics
  327. "LANG",
  328. "LC_MESSAGES",
  329. "PATH",
  330. "PWD",
  331. "TMPDIR",
  332. // Tool configs
  333. "ASAN_SYMBOLIZER_PATH",
  334. "JAVA_HOME",
  335. "PYTHONDONTWRITEBYTECODE",
  336. // Build configuration
  337. "ANDROID_BUILD_SHELL",
  338. "DIST_DIR",
  339. "OUT_DIR",
  340. "FILE_NAME_TAG",
  341. }...)
  342. if config.Dist() {
  343. env.Set("DIST", "true")
  344. env.Set("DIST_DIR", config.DistDir())
  345. }
  346. })
  347. // Compress and dist the packaging Ninja file.
  348. distGzipFile(ctx, config, config.KatiPackageNinjaFile())
  349. }
  350. // Run Kati on the cleanspec files to clean the build.
  351. func runKatiCleanSpec(ctx Context, config Config) {
  352. ctx.BeginTrace(metrics.RunKati, "kati cleanspec")
  353. defer ctx.EndTrace()
  354. runKati(ctx, config, katiCleanspecSuffix, []string{
  355. // Fail when encountering implicit rules. e.g.
  356. "--werror_implicit_rules",
  357. // Fail when redefining / duplicating a target.
  358. "--werror_overriding_commands",
  359. // Entry point.
  360. "-f", "build/make/core/cleanbuild.mk",
  361. "SOONG_MAKEVARS_MK=" + config.SoongMakeVarsMk(),
  362. "TARGET_DEVICE_DIR=" + config.TargetDeviceDir(),
  363. }, func(env *Environment) {})
  364. }