upload.go 4.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140
  1. // Copyright 2020 Google Inc. All rights reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package build
  15. // This file contains the functionality to upload data from one location to
  16. // another.
  17. import (
  18. "fmt"
  19. "io/ioutil"
  20. "os"
  21. "path/filepath"
  22. "time"
  23. "android/soong/ui/metrics"
  24. "google.golang.org/protobuf/proto"
  25. upload_proto "android/soong/ui/metrics/upload_proto"
  26. )
  27. const (
  28. // Used to generate a raw protobuf file that contains information
  29. // of the list of metrics files from host to destination storage.
  30. uploadPbFilename = ".uploader.pb"
  31. )
  32. var (
  33. // For testing purpose.
  34. tmpDir = ioutil.TempDir
  35. )
  36. // pruneMetricsFiles iterates the list of paths, checking if a path exist.
  37. // If a path is a file, it is added to the return list. If the path is a
  38. // directory, a recursive call is made to add the children files of the
  39. // path.
  40. func pruneMetricsFiles(paths []string) []string {
  41. var metricsFiles []string
  42. for _, p := range paths {
  43. fi, err := os.Stat(p)
  44. // Some paths passed may not exist. For example, build errors protobuf
  45. // file may not exist since the build was successful.
  46. if err != nil {
  47. continue
  48. }
  49. if fi.IsDir() {
  50. if l, err := ioutil.ReadDir(p); err != nil {
  51. _, _ = fmt.Fprintf(os.Stderr, "Failed to find files under %s\n", p)
  52. } else {
  53. files := make([]string, 0, len(l))
  54. for _, fi := range l {
  55. files = append(files, filepath.Join(p, fi.Name()))
  56. }
  57. metricsFiles = append(metricsFiles, pruneMetricsFiles(files)...)
  58. }
  59. } else {
  60. metricsFiles = append(metricsFiles, p)
  61. }
  62. }
  63. return metricsFiles
  64. }
  65. // UploadMetrics uploads a set of metrics files to a server for analysis.
  66. // The metrics files are first copied to a temporary directory
  67. // and the uploader is then executed in the background to allow the user/system
  68. // to continue working. Soong communicates to the uploader through the
  69. // upload_proto raw protobuf file.
  70. func UploadMetrics(ctx Context, config Config, simpleOutput bool, buildStarted time.Time, paths ...string) {
  71. ctx.BeginTrace(metrics.RunSetupTool, "upload_metrics")
  72. defer ctx.EndTrace()
  73. uploader := config.MetricsUploaderApp()
  74. if uploader == "" {
  75. // If the uploader path was not specified, no metrics shall be uploaded.
  76. return
  77. }
  78. // Several of the files might be directories.
  79. metricsFiles := pruneMetricsFiles(paths)
  80. if len(metricsFiles) == 0 {
  81. return
  82. }
  83. // The temporary directory cannot be deleted as the metrics uploader is started
  84. // in the background and requires to exist until the operation is done. The
  85. // uploader can delete the directory as it is specified in the upload proto.
  86. tmpDir, err := tmpDir("", "upload_metrics")
  87. if err != nil {
  88. ctx.Fatalf("failed to create a temporary directory to store the list of metrics files: %v\n", err)
  89. }
  90. for i, src := range metricsFiles {
  91. dst := filepath.Join(tmpDir, filepath.Base(src))
  92. if _, err := copyFile(src, dst); err != nil {
  93. ctx.Fatalf("failed to copy %q to %q: %v\n", src, dst, err)
  94. }
  95. metricsFiles[i] = dst
  96. }
  97. // For platform builds, the branch and target name is hardcoded to specific
  98. // values for later extraction of the metrics in the data metrics pipeline.
  99. data, err := proto.Marshal(&upload_proto.Upload{
  100. CreationTimestampMs: proto.Uint64(uint64(buildStarted.UnixNano() / int64(time.Millisecond))),
  101. CompletionTimestampMs: proto.Uint64(uint64(time.Now().UnixNano() / int64(time.Millisecond))),
  102. BranchName: proto.String("developer-metrics"),
  103. TargetName: proto.String("platform-build-systems-metrics"),
  104. MetricsFiles: metricsFiles,
  105. DirectoriesToDelete: []string{tmpDir},
  106. })
  107. if err != nil {
  108. ctx.Fatalf("failed to marshal metrics upload proto buffer message: %v\n", err)
  109. }
  110. pbFile := filepath.Join(tmpDir, uploadPbFilename)
  111. if err := ioutil.WriteFile(pbFile, data, 0644); err != nil {
  112. ctx.Fatalf("failed to write the marshaled metrics upload protobuf to %q: %v\n", pbFile, err)
  113. }
  114. // Start the uploader in the background as it takes several milliseconds to start the uploader
  115. // and prepare the metrics for upload. This affects small shell commands like "lunch".
  116. cmd := Command(ctx, config, "upload metrics", uploader, "--upload-metrics", pbFile)
  117. if simpleOutput {
  118. cmd.RunOrFatal()
  119. } else {
  120. cmd.RunAndStreamOrFatal()
  121. }
  122. }