staging_snapshot.go 5.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246
  1. // Copyright 2023 Google Inc. All rights reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package build
  15. import (
  16. "crypto/sha1"
  17. "encoding/hex"
  18. "encoding/json"
  19. "io"
  20. "io/fs"
  21. "os"
  22. "path/filepath"
  23. "sort"
  24. "strings"
  25. "android/soong/shared"
  26. "android/soong/ui/metrics"
  27. )
  28. // Metadata about a staged file
  29. type fileEntry struct {
  30. Name string `json:"name"`
  31. Mode fs.FileMode `json:"mode"`
  32. Size int64 `json:"size"`
  33. Sha1 string `json:"sha1"`
  34. }
  35. func fileEntryEqual(a fileEntry, b fileEntry) bool {
  36. return a.Name == b.Name && a.Mode == b.Mode && a.Size == b.Size && a.Sha1 == b.Sha1
  37. }
  38. func sha1_hash(filename string) (string, error) {
  39. f, err := os.Open(filename)
  40. if err != nil {
  41. return "", err
  42. }
  43. defer f.Close()
  44. h := sha1.New()
  45. if _, err := io.Copy(h, f); err != nil {
  46. return "", err
  47. }
  48. return hex.EncodeToString(h.Sum(nil)), nil
  49. }
  50. // Subdirs of PRODUCT_OUT to scan
  51. var stagingSubdirs = []string{
  52. "apex",
  53. "cache",
  54. "coverage",
  55. "data",
  56. "debug_ramdisk",
  57. "fake_packages",
  58. "installer",
  59. "oem",
  60. "product",
  61. "ramdisk",
  62. "recovery",
  63. "root",
  64. "sysloader",
  65. "system",
  66. "system_dlkm",
  67. "system_ext",
  68. "system_other",
  69. "testcases",
  70. "test_harness_ramdisk",
  71. "vendor",
  72. "vendor_debug_ramdisk",
  73. "vendor_kernel_ramdisk",
  74. "vendor_ramdisk",
  75. }
  76. // Return an array of stagedFileEntrys, one for each file in the staging directories inside
  77. // productOut
  78. func takeStagingSnapshot(ctx Context, productOut string, subdirs []string) ([]fileEntry, error) {
  79. var outer_err error
  80. if !strings.HasSuffix(productOut, "/") {
  81. productOut += "/"
  82. }
  83. result := []fileEntry{}
  84. for _, subdir := range subdirs {
  85. filepath.WalkDir(productOut+subdir,
  86. func(filename string, dirent fs.DirEntry, err error) error {
  87. // Ignore errors. The most common one is that one of the subdirectories
  88. // hasn't been built, in which case we just report it as empty.
  89. if err != nil {
  90. ctx.Verbosef("scanModifiedStagingOutputs error: %s", err)
  91. return nil
  92. }
  93. if dirent.Type().IsRegular() {
  94. fileInfo, _ := dirent.Info()
  95. relative := strings.TrimPrefix(filename, productOut)
  96. sha, err := sha1_hash(filename)
  97. if err != nil {
  98. outer_err = err
  99. }
  100. result = append(result, fileEntry{
  101. Name: relative,
  102. Mode: fileInfo.Mode(),
  103. Size: fileInfo.Size(),
  104. Sha1: sha,
  105. })
  106. }
  107. return nil
  108. })
  109. }
  110. sort.Slice(result, func(l, r int) bool { return result[l].Name < result[r].Name })
  111. return result, outer_err
  112. }
  113. // Read json into an array of fileEntry. On error return empty array.
  114. func readJson(filename string) ([]fileEntry, error) {
  115. buf, err := os.ReadFile(filename)
  116. if err != nil {
  117. // Not an error, just missing, which is empty.
  118. return []fileEntry{}, nil
  119. }
  120. var result []fileEntry
  121. err = json.Unmarshal(buf, &result)
  122. if err != nil {
  123. // Bad formatting. This is an error
  124. return []fileEntry{}, err
  125. }
  126. return result, nil
  127. }
  128. // Write obj to filename.
  129. func writeJson(filename string, obj interface{}) error {
  130. buf, err := json.MarshalIndent(obj, "", " ")
  131. if err != nil {
  132. return err
  133. }
  134. return os.WriteFile(filename, buf, 0660)
  135. }
  136. type snapshotDiff struct {
  137. Added []string `json:"added"`
  138. Changed []string `json:"changed"`
  139. Removed []string `json:"removed"`
  140. }
  141. // Diff the two snapshots, returning a snapshotDiff.
  142. func diffSnapshots(previous []fileEntry, current []fileEntry) snapshotDiff {
  143. result := snapshotDiff{
  144. Added: []string{},
  145. Changed: []string{},
  146. Removed: []string{},
  147. }
  148. found := make(map[string]bool)
  149. prev := make(map[string]fileEntry)
  150. for _, pre := range previous {
  151. prev[pre.Name] = pre
  152. }
  153. for _, cur := range current {
  154. pre, ok := prev[cur.Name]
  155. found[cur.Name] = true
  156. // Added
  157. if !ok {
  158. result.Added = append(result.Added, cur.Name)
  159. continue
  160. }
  161. // Changed
  162. if !fileEntryEqual(pre, cur) {
  163. result.Changed = append(result.Changed, cur.Name)
  164. }
  165. }
  166. // Removed
  167. for _, pre := range previous {
  168. if !found[pre.Name] {
  169. result.Removed = append(result.Removed, pre.Name)
  170. }
  171. }
  172. // Sort the results
  173. sort.Strings(result.Added)
  174. sort.Strings(result.Changed)
  175. sort.Strings(result.Removed)
  176. return result
  177. }
  178. // Write a json files to dist:
  179. // - A list of which files have changed in this build.
  180. //
  181. // And record in out/soong:
  182. // - A list of all files in the staging directories, including their hashes.
  183. func runStagingSnapshot(ctx Context, config Config) {
  184. ctx.BeginTrace(metrics.RunSoong, "runStagingSnapshot")
  185. defer ctx.EndTrace()
  186. snapshotFilename := shared.JoinPath(config.SoongOutDir(), "staged_files.json")
  187. // Read the existing snapshot file. If it doesn't exist, this is a full
  188. // build, so all files will be treated as new.
  189. previous, err := readJson(snapshotFilename)
  190. if err != nil {
  191. ctx.Fatal(err)
  192. return
  193. }
  194. // Take a snapshot of the current out directory
  195. current, err := takeStagingSnapshot(ctx, config.ProductOut(), stagingSubdirs)
  196. if err != nil {
  197. ctx.Fatal(err)
  198. return
  199. }
  200. // Diff the snapshots
  201. diff := diffSnapshots(previous, current)
  202. // Write the diff (use RealDistDir, not one that might have been faked for bazel)
  203. err = writeJson(shared.JoinPath(config.RealDistDir(), "modified_files.json"), diff)
  204. if err != nil {
  205. ctx.Fatal(err)
  206. return
  207. }
  208. // Update the snapshot
  209. err = writeJson(snapshotFilename, current)
  210. if err != nil {
  211. ctx.Fatal(err)
  212. return
  213. }
  214. }