cuj.go 6.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239
  1. // Copyright 2019 Google Inc. All rights reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. // This executable runs a series of build commands to test and benchmark some critical user journeys.
  15. package main
  16. import (
  17. "context"
  18. "fmt"
  19. "os"
  20. "path/filepath"
  21. "strconv"
  22. "strings"
  23. "time"
  24. "android/soong/ui/build"
  25. "android/soong/ui/logger"
  26. "android/soong/ui/metrics"
  27. "android/soong/ui/signal"
  28. "android/soong/ui/status"
  29. "android/soong/ui/terminal"
  30. "android/soong/ui/tracer"
  31. )
  32. type Test struct {
  33. name string
  34. args []string
  35. before func() error
  36. results TestResults
  37. }
  38. type TestResults struct {
  39. metrics *metrics.Metrics
  40. err error
  41. }
  42. // Run runs a single build command. It emulates the "m" command line by calling into Soong UI directly.
  43. func (t *Test) Run(logsDir string) {
  44. output := terminal.NewStatusOutput(os.Stdout, "", false, false, false)
  45. log := logger.New(output)
  46. defer log.Cleanup()
  47. ctx, cancel := context.WithCancel(context.Background())
  48. defer cancel()
  49. trace := tracer.New(log)
  50. defer trace.Close()
  51. met := metrics.New()
  52. stat := &status.Status{}
  53. defer stat.Finish()
  54. stat.AddOutput(output)
  55. stat.AddOutput(trace.StatusTracer())
  56. signal.SetupSignals(log, cancel, func() {
  57. trace.Close()
  58. log.Cleanup()
  59. stat.Finish()
  60. })
  61. buildCtx := build.Context{ContextImpl: &build.ContextImpl{
  62. Context: ctx,
  63. Logger: log,
  64. Metrics: met,
  65. Tracer: trace,
  66. Writer: output,
  67. Status: stat,
  68. }}
  69. defer logger.Recover(func(err error) {
  70. t.results.err = err
  71. })
  72. config := build.NewConfig(buildCtx, t.args...)
  73. build.SetupOutDir(buildCtx, config)
  74. os.MkdirAll(logsDir, 0777)
  75. log.SetOutput(filepath.Join(logsDir, "soong.log"))
  76. trace.SetOutput(filepath.Join(logsDir, "build.trace"))
  77. stat.AddOutput(status.NewVerboseLog(log, filepath.Join(logsDir, "verbose.log")))
  78. stat.AddOutput(status.NewErrorLog(log, filepath.Join(logsDir, "error.log")))
  79. stat.AddOutput(status.NewProtoErrorLog(log, filepath.Join(logsDir, "build_error")))
  80. stat.AddOutput(status.NewCriticalPathLogger(log, nil))
  81. defer met.Dump(filepath.Join(logsDir, "soong_metrics"))
  82. if start, ok := os.LookupEnv("TRACE_BEGIN_SOONG"); ok {
  83. if !strings.HasSuffix(start, "N") {
  84. if start_time, err := strconv.ParseUint(start, 10, 64); err == nil {
  85. log.Verbosef("Took %dms to start up.",
  86. time.Since(time.Unix(0, int64(start_time))).Nanoseconds()/time.Millisecond.Nanoseconds())
  87. buildCtx.CompleteTrace(metrics.RunSetupTool, "startup", start_time, uint64(time.Now().UnixNano()))
  88. }
  89. }
  90. if executable, err := os.Executable(); err == nil {
  91. trace.ImportMicrofactoryLog(filepath.Join(filepath.Dir(executable), "."+filepath.Base(executable)+".trace"))
  92. }
  93. }
  94. f := build.NewSourceFinder(buildCtx, config)
  95. defer f.Shutdown()
  96. build.FindSources(buildCtx, config, f)
  97. build.Build(buildCtx, config)
  98. t.results.metrics = met
  99. }
  100. // Touch the Intent.java file to cause a rebuild of the frameworks to monitor the
  101. // incremental build speed as mentioned b/152046247. Intent.java file was chosen
  102. // as it is a key component of the framework and is often modified.
  103. func touchIntentFile() error {
  104. const intentFileName = "frameworks/base/core/java/android/content/Intent.java"
  105. currentTime := time.Now().Local()
  106. return os.Chtimes(intentFileName, currentTime, currentTime)
  107. }
  108. func main() {
  109. outDir := os.Getenv("OUT_DIR")
  110. if outDir == "" {
  111. outDir = "out"
  112. }
  113. cujDir := filepath.Join(outDir, "cuj_tests")
  114. wd, _ := os.Getwd()
  115. os.Setenv("TOP", wd)
  116. // Use a subdirectory for the out directory for the tests to keep them isolated.
  117. os.Setenv("OUT_DIR", filepath.Join(cujDir, "out"))
  118. // Each of these tests is run in sequence without resetting the output tree. The state of the output tree will
  119. // affect each successive test. To maintain the validity of the benchmarks across changes, care must be taken
  120. // to avoid changing the state of the tree when a test is run. This is most easily accomplished by adding tests
  121. // at the end.
  122. tests := []Test{
  123. {
  124. // Reset the out directory to get reproducible results.
  125. name: "clean",
  126. args: []string{"clean"},
  127. },
  128. {
  129. // Parse the build files.
  130. name: "nothing",
  131. args: []string{"nothing"},
  132. },
  133. {
  134. // Parse the build files again to monitor issues like globs rerunning.
  135. name: "nothing_rebuild",
  136. args: []string{"nothing"},
  137. },
  138. {
  139. // Parse the build files again, this should always be very short.
  140. name: "nothing_rebuild_twice",
  141. args: []string{"nothing"},
  142. },
  143. {
  144. // Build the framework as a common developer task and one that keeps getting longer.
  145. name: "framework",
  146. args: []string{"framework"},
  147. },
  148. {
  149. // Build the framework again to make sure it doesn't rebuild anything.
  150. name: "framework_rebuild",
  151. args: []string{"framework"},
  152. },
  153. {
  154. // Build the framework again to make sure it doesn't rebuild anything even if it did the second time.
  155. name: "framework_rebuild_twice",
  156. args: []string{"framework"},
  157. },
  158. {
  159. // Scenario major_inc_build (b/152046247): tracking build speed of major incremental build.
  160. name: "major_inc_build_droid",
  161. args: []string{"droid"},
  162. },
  163. {
  164. name: "major_inc_build_framework_minus_apex_after_droid_build",
  165. args: []string{"framework-minus-apex"},
  166. before: touchIntentFile,
  167. },
  168. {
  169. name: "major_inc_build_framework_after_droid_build",
  170. args: []string{"framework"},
  171. before: touchIntentFile,
  172. },
  173. {
  174. name: "major_inc_build_sync_after_droid_build",
  175. args: []string{"sync"},
  176. before: touchIntentFile,
  177. },
  178. {
  179. name: "major_inc_build_droid_rebuild",
  180. args: []string{"droid"},
  181. before: touchIntentFile,
  182. },
  183. {
  184. name: "major_inc_build_update_api_after_droid_rebuild",
  185. args: []string{"update-api"},
  186. before: touchIntentFile,
  187. },
  188. }
  189. cujMetrics := metrics.NewCriticalUserJourneysMetrics()
  190. defer cujMetrics.Dump(filepath.Join(cujDir, "logs", "cuj_metrics.pb"))
  191. for i, t := range tests {
  192. logsSubDir := fmt.Sprintf("%02d_%s", i, t.name)
  193. logsDir := filepath.Join(cujDir, "logs", logsSubDir)
  194. if t.before != nil {
  195. if err := t.before(); err != nil {
  196. fmt.Printf("error running before function on test %q: %v\n", t.name, err)
  197. break
  198. }
  199. }
  200. t.Run(logsDir)
  201. if t.results.err != nil {
  202. fmt.Printf("error running test %q: %s\n", t.name, t.results.err)
  203. break
  204. }
  205. if t.results.metrics != nil {
  206. cujMetrics.Add(t.name, t.results.metrics)
  207. }
  208. }
  209. }