main.go 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598
  1. // Copyright 2017 Google Inc. All rights reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package main
  15. import (
  16. "bufio"
  17. "context"
  18. "flag"
  19. "fmt"
  20. "io"
  21. "io/ioutil"
  22. "log"
  23. "os"
  24. "os/exec"
  25. "path/filepath"
  26. "regexp"
  27. "runtime"
  28. "strings"
  29. "sync"
  30. "syscall"
  31. "time"
  32. "android/soong/ui/logger"
  33. "android/soong/ui/signal"
  34. "android/soong/ui/status"
  35. "android/soong/ui/terminal"
  36. "android/soong/ui/tracer"
  37. "android/soong/zip"
  38. )
  39. var numJobs = flag.Int("j", 0, "number of parallel jobs [0=autodetect]")
  40. var keepArtifacts = flag.Bool("keep", false, "keep archives of artifacts")
  41. var incremental = flag.Bool("incremental", false, "run in incremental mode (saving intermediates)")
  42. var outDir = flag.String("out", "", "path to store output directories (defaults to tmpdir under $OUT when empty)")
  43. var alternateResultDir = flag.Bool("dist", false, "write select results to $DIST_DIR (or <out>/dist when empty)")
  44. var bazelMode = flag.Bool("bazel-mode", false, "use bazel for analysis of certain modules")
  45. var bazelModeStaging = flag.Bool("bazel-mode-staging", false, "use bazel for analysis of certain near-ready modules")
  46. var onlyConfig = flag.Bool("only-config", false, "Only run product config (not Soong or Kati)")
  47. var onlySoong = flag.Bool("only-soong", false, "Only run product config and Soong (not Kati)")
  48. var buildVariant = flag.String("variant", "eng", "build variant to use")
  49. var shardCount = flag.Int("shard-count", 1, "split the products into multiple shards (to spread the build onto multiple machines, etc)")
  50. var shard = flag.Int("shard", 1, "1-indexed shard to execute")
  51. var skipProducts multipleStringArg
  52. var includeProducts multipleStringArg
  53. func init() {
  54. flag.Var(&skipProducts, "skip-products", "comma-separated list of products to skip (known failures, etc)")
  55. flag.Var(&includeProducts, "products", "comma-separated list of products to build")
  56. }
  57. // multipleStringArg is a flag.Value that takes comma separated lists and converts them to a
  58. // []string. The argument can be passed multiple times to append more values.
  59. type multipleStringArg []string
  60. func (m *multipleStringArg) String() string {
  61. return strings.Join(*m, `, `)
  62. }
  63. func (m *multipleStringArg) Set(s string) error {
  64. *m = append(*m, strings.Split(s, ",")...)
  65. return nil
  66. }
  67. const errorLeadingLines = 20
  68. const errorTrailingLines = 20
  69. func errMsgFromLog(filename string) string {
  70. if filename == "" {
  71. return ""
  72. }
  73. data, err := ioutil.ReadFile(filename)
  74. if err != nil {
  75. return ""
  76. }
  77. lines := strings.Split(strings.TrimSpace(string(data)), "\n")
  78. if len(lines) > errorLeadingLines+errorTrailingLines+1 {
  79. lines[errorLeadingLines] = fmt.Sprintf("... skipping %d lines ...",
  80. len(lines)-errorLeadingLines-errorTrailingLines)
  81. lines = append(lines[:errorLeadingLines+1],
  82. lines[len(lines)-errorTrailingLines:]...)
  83. }
  84. var buf strings.Builder
  85. for _, line := range lines {
  86. buf.WriteString("> ")
  87. buf.WriteString(line)
  88. buf.WriteString("\n")
  89. }
  90. return buf.String()
  91. }
  92. // TODO(b/70370883): This tool uses a lot of open files -- over the default
  93. // soft limit of 1024 on some systems. So bump up to the hard limit until I fix
  94. // the algorithm.
  95. func setMaxFiles(log logger.Logger) {
  96. var limits syscall.Rlimit
  97. err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limits)
  98. if err != nil {
  99. log.Println("Failed to get file limit:", err)
  100. return
  101. }
  102. log.Verbosef("Current file limits: %d soft, %d hard", limits.Cur, limits.Max)
  103. if limits.Cur == limits.Max {
  104. return
  105. }
  106. limits.Cur = limits.Max
  107. err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limits)
  108. if err != nil {
  109. log.Println("Failed to increase file limit:", err)
  110. }
  111. }
  112. func inList(str string, list []string) bool {
  113. for _, other := range list {
  114. if str == other {
  115. return true
  116. }
  117. }
  118. return false
  119. }
  120. func copyFile(from, to string) error {
  121. fromFile, err := os.Open(from)
  122. if err != nil {
  123. return err
  124. }
  125. defer fromFile.Close()
  126. toFile, err := os.Create(to)
  127. if err != nil {
  128. return err
  129. }
  130. defer toFile.Close()
  131. _, err = io.Copy(toFile, fromFile)
  132. return err
  133. }
  134. type mpContext struct {
  135. Logger logger.Logger
  136. Status status.ToolStatus
  137. SoongUi string
  138. MainOutDir string
  139. MainLogsDir string
  140. }
  141. func findNamedProducts(soongUi string, log logger.Logger) []string {
  142. cmd := exec.Command(soongUi, "--dumpvars-mode", "--vars=all_named_products")
  143. output, err := cmd.Output()
  144. if err != nil {
  145. log.Fatalf("Cannot determine named products: %v", err)
  146. }
  147. rx := regexp.MustCompile(`^all_named_products='(.*)'$`)
  148. match := rx.FindStringSubmatch(strings.TrimSpace(string(output)))
  149. return strings.Fields(match[1])
  150. }
  151. // ensureEmptyFileExists ensures that the containing directory exists, and the
  152. // specified file exists. If it doesn't exist, it will write an empty file.
  153. func ensureEmptyFileExists(file string, log logger.Logger) {
  154. if _, err := os.Stat(file); os.IsNotExist(err) {
  155. f, err := os.Create(file)
  156. if err != nil {
  157. log.Fatalf("Error creating %s: %q\n", file, err)
  158. }
  159. f.Close()
  160. } else if err != nil {
  161. log.Fatalf("Error checking %s: %q\n", file, err)
  162. }
  163. }
  164. func outDirBase() string {
  165. outDirBase := os.Getenv("OUT_DIR")
  166. if outDirBase == "" {
  167. return "out"
  168. } else {
  169. return outDirBase
  170. }
  171. }
  172. func distDir(outDir string) string {
  173. if distDir := os.Getenv("DIST_DIR"); distDir != "" {
  174. return filepath.Clean(distDir)
  175. } else {
  176. return filepath.Join(outDir, "dist")
  177. }
  178. }
  179. func forceAnsiOutput() bool {
  180. value := os.Getenv("SOONG_UI_ANSI_OUTPUT")
  181. return value == "1" || value == "y" || value == "yes" || value == "on" || value == "true"
  182. }
  183. func getBazelArg() string {
  184. count := 0
  185. str := ""
  186. if *bazelMode {
  187. count++
  188. str = "--bazel-mode"
  189. }
  190. if *bazelModeStaging {
  191. count++
  192. str = "--bazel-mode-staging"
  193. }
  194. if count > 1 {
  195. // Can't set more than one
  196. fmt.Errorf("Only one bazel mode is permitted to be set.")
  197. os.Exit(1)
  198. }
  199. return str
  200. }
  201. func main() {
  202. stdio := terminal.StdioImpl{}
  203. output := terminal.NewStatusOutput(stdio.Stdout(), "", false, false,
  204. forceAnsiOutput())
  205. log := logger.New(output)
  206. defer log.Cleanup()
  207. for _, v := range os.Environ() {
  208. log.Println("Environment: " + v)
  209. }
  210. log.Printf("Argv: %v\n", os.Args)
  211. flag.Parse()
  212. _, cancel := context.WithCancel(context.Background())
  213. defer cancel()
  214. trace := tracer.New(log)
  215. defer trace.Close()
  216. stat := &status.Status{}
  217. defer stat.Finish()
  218. stat.AddOutput(output)
  219. var failures failureCount
  220. stat.AddOutput(&failures)
  221. signal.SetupSignals(log, cancel, func() {
  222. trace.Close()
  223. log.Cleanup()
  224. stat.Finish()
  225. })
  226. soongUi := "build/soong/soong_ui.bash"
  227. var outputDir string
  228. if *outDir != "" {
  229. outputDir = *outDir
  230. } else {
  231. name := "multiproduct"
  232. if !*incremental {
  233. name += "-" + time.Now().Format("20060102150405")
  234. }
  235. outputDir = filepath.Join(outDirBase(), name)
  236. }
  237. log.Println("Output directory:", outputDir)
  238. // The ninja_build file is used by our buildbots to understand that the output
  239. // can be parsed as ninja output.
  240. if err := os.MkdirAll(outputDir, 0777); err != nil {
  241. log.Fatalf("Failed to create output directory: %v", err)
  242. }
  243. ensureEmptyFileExists(filepath.Join(outputDir, "ninja_build"), log)
  244. logsDir := filepath.Join(outputDir, "logs")
  245. os.MkdirAll(logsDir, 0777)
  246. var configLogsDir string
  247. if *alternateResultDir {
  248. configLogsDir = filepath.Join(distDir(outDirBase()), "logs")
  249. } else {
  250. configLogsDir = outputDir
  251. }
  252. log.Println("Logs dir: " + configLogsDir)
  253. os.MkdirAll(configLogsDir, 0777)
  254. log.SetOutput(filepath.Join(configLogsDir, "soong.log"))
  255. trace.SetOutput(filepath.Join(configLogsDir, "build.trace"))
  256. var jobs = *numJobs
  257. if jobs < 1 {
  258. jobs = runtime.NumCPU() / 4
  259. ramGb := int(detectTotalRAM() / (1024 * 1024 * 1024))
  260. if ramJobs := ramGb / 40; ramGb > 0 && jobs > ramJobs {
  261. jobs = ramJobs
  262. }
  263. if jobs < 1 {
  264. jobs = 1
  265. }
  266. }
  267. log.Verbosef("Using %d parallel jobs", jobs)
  268. setMaxFiles(log)
  269. allProducts := findNamedProducts(soongUi, log)
  270. var productsList []string
  271. if len(includeProducts) > 0 {
  272. var missingProducts []string
  273. for _, product := range includeProducts {
  274. if inList(product, allProducts) {
  275. productsList = append(productsList, product)
  276. } else {
  277. missingProducts = append(missingProducts, product)
  278. }
  279. }
  280. if len(missingProducts) > 0 {
  281. log.Fatalf("Products don't exist: %s\n", missingProducts)
  282. }
  283. } else {
  284. productsList = allProducts
  285. }
  286. finalProductsList := make([]string, 0, len(productsList))
  287. skipProduct := func(p string) bool {
  288. for _, s := range skipProducts {
  289. if p == s {
  290. return true
  291. }
  292. }
  293. return false
  294. }
  295. for _, product := range productsList {
  296. if !skipProduct(product) {
  297. finalProductsList = append(finalProductsList, product)
  298. } else {
  299. log.Verbose("Skipping: ", product)
  300. }
  301. }
  302. if *shard < 1 {
  303. log.Fatalf("--shard value must be >= 1, not %d\n", *shard)
  304. } else if *shardCount < 1 {
  305. log.Fatalf("--shard-count value must be >= 1, not %d\n", *shardCount)
  306. } else if *shard > *shardCount {
  307. log.Fatalf("--shard (%d) must not be greater than --shard-count (%d)\n", *shard,
  308. *shardCount)
  309. } else if *shardCount > 1 {
  310. finalProductsList = splitList(finalProductsList, *shardCount)[*shard-1]
  311. }
  312. log.Verbose("Got product list: ", finalProductsList)
  313. s := stat.StartTool()
  314. s.SetTotalActions(len(finalProductsList))
  315. mpCtx := &mpContext{
  316. Logger: log,
  317. Status: s,
  318. SoongUi: soongUi,
  319. MainOutDir: outputDir,
  320. MainLogsDir: logsDir,
  321. }
  322. products := make(chan string, len(productsList))
  323. go func() {
  324. defer close(products)
  325. for _, product := range finalProductsList {
  326. products <- product
  327. }
  328. }()
  329. var wg sync.WaitGroup
  330. for i := 0; i < jobs; i++ {
  331. wg.Add(1)
  332. // To smooth out the spikes in memory usage, skew the
  333. // initial starting time of the jobs by a small amount.
  334. time.Sleep(15 * time.Second)
  335. go func() {
  336. defer wg.Done()
  337. for {
  338. select {
  339. case product := <-products:
  340. if product == "" {
  341. return
  342. }
  343. runSoongUiForProduct(mpCtx, product)
  344. }
  345. }
  346. }()
  347. }
  348. wg.Wait()
  349. if *alternateResultDir {
  350. args := zip.ZipArgs{
  351. FileArgs: []zip.FileArg{
  352. {GlobDir: logsDir, SourcePrefixToStrip: logsDir},
  353. },
  354. OutputFilePath: filepath.Join(distDir(outDirBase()), "logs.zip"),
  355. NumParallelJobs: runtime.NumCPU(),
  356. CompressionLevel: 5,
  357. }
  358. log.Printf("Logs zip: %v\n", args.OutputFilePath)
  359. if err := zip.Zip(args); err != nil {
  360. log.Fatalf("Error zipping logs: %v", err)
  361. }
  362. }
  363. s.Finish()
  364. if failures.count == 1 {
  365. log.Fatal("1 failure")
  366. } else if failures.count > 1 {
  367. log.Fatalf("%d failures %q", failures.count, failures.fails)
  368. } else {
  369. fmt.Fprintln(output, "Success")
  370. }
  371. }
  372. func cleanupAfterProduct(outDir, productZip string) {
  373. if *keepArtifacts {
  374. args := zip.ZipArgs{
  375. FileArgs: []zip.FileArg{
  376. {
  377. GlobDir: outDir,
  378. SourcePrefixToStrip: outDir,
  379. },
  380. },
  381. OutputFilePath: productZip,
  382. NumParallelJobs: runtime.NumCPU(),
  383. CompressionLevel: 5,
  384. }
  385. if err := zip.Zip(args); err != nil {
  386. log.Fatalf("Error zipping artifacts: %v", err)
  387. }
  388. }
  389. if !*incremental {
  390. os.RemoveAll(outDir)
  391. }
  392. }
  393. func runSoongUiForProduct(mpctx *mpContext, product string) {
  394. outDir := filepath.Join(mpctx.MainOutDir, product)
  395. logsDir := filepath.Join(mpctx.MainLogsDir, product)
  396. productZip := filepath.Join(mpctx.MainOutDir, product+".zip")
  397. consoleLogPath := filepath.Join(logsDir, "std.log")
  398. if err := os.MkdirAll(outDir, 0777); err != nil {
  399. mpctx.Logger.Fatalf("Error creating out directory: %v", err)
  400. }
  401. if err := os.MkdirAll(logsDir, 0777); err != nil {
  402. mpctx.Logger.Fatalf("Error creating log directory: %v", err)
  403. }
  404. consoleLogFile, err := os.Create(consoleLogPath)
  405. if err != nil {
  406. mpctx.Logger.Fatalf("Error creating console log file: %v", err)
  407. }
  408. defer consoleLogFile.Close()
  409. consoleLogWriter := bufio.NewWriter(consoleLogFile)
  410. defer consoleLogWriter.Flush()
  411. args := []string{"--make-mode", "--skip-soong-tests", "--skip-ninja"}
  412. if !*keepArtifacts {
  413. args = append(args, "--empty-ninja-file")
  414. }
  415. if *onlyConfig {
  416. args = append(args, "--config-only")
  417. } else if *onlySoong {
  418. args = append(args, "--soong-only")
  419. }
  420. bazelStr := getBazelArg()
  421. if bazelStr != "" {
  422. args = append(args, bazelStr)
  423. }
  424. cmd := exec.Command(mpctx.SoongUi, args...)
  425. cmd.Stdout = consoleLogWriter
  426. cmd.Stderr = consoleLogWriter
  427. cmd.Env = append(os.Environ(),
  428. "OUT_DIR="+outDir,
  429. "TARGET_PRODUCT="+product,
  430. "TARGET_BUILD_VARIANT="+*buildVariant,
  431. "TARGET_BUILD_TYPE=release",
  432. "TARGET_BUILD_APPS=",
  433. "TARGET_BUILD_UNBUNDLED=",
  434. "USE_RBE=false") // Disabling RBE saves ~10 secs per product
  435. if *alternateResultDir {
  436. cmd.Env = append(cmd.Env,
  437. "DIST_DIR="+filepath.Join(distDir(outDirBase()), "products/"+product))
  438. }
  439. action := &status.Action{
  440. Description: product,
  441. Outputs: []string{product},
  442. }
  443. mpctx.Status.StartAction(action)
  444. defer cleanupAfterProduct(outDir, productZip)
  445. before := time.Now()
  446. err = cmd.Run()
  447. if !*onlyConfig && !*onlySoong {
  448. katiBuildNinjaFile := filepath.Join(outDir, "build-"+product+".ninja")
  449. if after, err := os.Stat(katiBuildNinjaFile); err == nil && after.ModTime().After(before) {
  450. err := copyFile(consoleLogPath, filepath.Join(filepath.Dir(consoleLogPath), "std_full.log"))
  451. if err != nil {
  452. log.Fatalf("Error copying log file: %s", err)
  453. }
  454. }
  455. }
  456. var errOutput string
  457. if err == nil {
  458. errOutput = ""
  459. } else {
  460. errOutput = errMsgFromLog(consoleLogPath)
  461. }
  462. mpctx.Status.FinishAction(status.ActionResult{
  463. Action: action,
  464. Error: err,
  465. Output: errOutput,
  466. })
  467. }
  468. type failureCount struct {
  469. count int
  470. fails []string
  471. }
  472. func (f *failureCount) StartAction(action *status.Action, counts status.Counts) {}
  473. func (f *failureCount) FinishAction(result status.ActionResult, counts status.Counts) {
  474. if result.Error != nil {
  475. f.count += 1
  476. f.fails = append(f.fails, result.Action.Description)
  477. }
  478. }
  479. func (f *failureCount) Message(level status.MsgLevel, message string) {
  480. if level >= status.ErrorLvl {
  481. f.count += 1
  482. }
  483. }
  484. func (f *failureCount) Flush() {}
  485. func (f *failureCount) Write(p []byte) (int, error) {
  486. // discard writes
  487. return len(p), nil
  488. }
  489. func splitList(list []string, shardCount int) (ret [][]string) {
  490. each := len(list) / shardCount
  491. extra := len(list) % shardCount
  492. for i := 0; i < shardCount; i++ {
  493. count := each
  494. if extra > 0 {
  495. count += 1
  496. extra -= 1
  497. }
  498. ret = append(ret, list[:count])
  499. list = list[count:]
  500. }
  501. return
  502. }