testing.go 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487
  1. // Copyright 2019 Google Inc. All rights reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package sdk
  15. import (
  16. "fmt"
  17. "path/filepath"
  18. "strings"
  19. "testing"
  20. "android/soong/android"
  21. "android/soong/apex"
  22. "android/soong/cc"
  23. "android/soong/genrule"
  24. "android/soong/java"
  25. "github.com/google/blueprint/proptools"
  26. )
  27. // Prepare for running an sdk test with an apex.
  28. var prepareForSdkTestWithApex = android.GroupFixturePreparers(
  29. apex.PrepareForTestWithApexBuildComponents,
  30. android.FixtureAddTextFile("sdk/tests/Android.bp", `
  31. apex_key {
  32. name: "myapex.key",
  33. public_key: "myapex.avbpubkey",
  34. private_key: "myapex.pem",
  35. }
  36. android_app_certificate {
  37. name: "myapex.cert",
  38. certificate: "myapex",
  39. }
  40. `),
  41. android.FixtureMergeMockFs(map[string][]byte{
  42. "apex_manifest.json": nil,
  43. "system/sepolicy/apex/myapex-file_contexts": nil,
  44. "system/sepolicy/apex/myapex2-file_contexts": nil,
  45. "system/sepolicy/apex/mysdkapex-file_contexts": nil,
  46. "sdk/tests/myapex.avbpubkey": nil,
  47. "sdk/tests/myapex.pem": nil,
  48. "sdk/tests/myapex.x509.pem": nil,
  49. "sdk/tests/myapex.pk8": nil,
  50. }),
  51. )
  52. // Legacy preparer used for running tests within the sdk package.
  53. //
  54. // This includes everything that was needed to run any test in the sdk package prior to the
  55. // introduction of the test fixtures. Tests that are being converted to use fixtures directly
  56. // rather than through the testSdkError() and testSdkWithFs() methods should avoid using this and
  57. // instead should use the various preparers directly using android.GroupFixturePreparers(...) to
  58. // group them when necessary.
  59. //
  60. // deprecated
  61. var prepareForSdkTest = android.GroupFixturePreparers(
  62. cc.PrepareForTestWithCcDefaultModules,
  63. genrule.PrepareForTestWithGenRuleBuildComponents,
  64. java.PrepareForTestWithJavaBuildComponents,
  65. PrepareForTestWithSdkBuildComponents,
  66. prepareForSdkTestWithApex,
  67. cc.PrepareForTestOnWindows,
  68. android.FixtureModifyConfig(func(config android.Config) {
  69. // Add windows as a default disable OS to test behavior when some OS variants
  70. // are disabled.
  71. config.Targets[android.Windows] = []android.Target{
  72. {android.Windows, android.Arch{ArchType: android.X86_64}, android.NativeBridgeDisabled, "", "", true},
  73. }
  74. }),
  75. // Add a build number file.
  76. android.FixtureModifyProductVariables(func(variables android.FixtureProductVariables) {
  77. variables.BuildNumberFile = proptools.StringPtr(BUILD_NUMBER_FILE)
  78. }),
  79. // Make sure that every test provides all the source files.
  80. android.PrepareForTestDisallowNonExistentPaths,
  81. android.MockFS{
  82. "Test.java": nil,
  83. }.AddToFixture(),
  84. )
  85. var PrepareForTestWithSdkBuildComponents = android.GroupFixturePreparers(
  86. android.FixtureRegisterWithContext(registerModuleExportsBuildComponents),
  87. android.FixtureRegisterWithContext(registerSdkBuildComponents),
  88. )
  89. func testSdkWithFs(t *testing.T, bp string, fs android.MockFS) *android.TestResult {
  90. t.Helper()
  91. return android.GroupFixturePreparers(
  92. prepareForSdkTest,
  93. fs.AddToFixture(),
  94. ).RunTestWithBp(t, bp)
  95. }
  96. func testSdkError(t *testing.T, pattern, bp string) {
  97. t.Helper()
  98. prepareForSdkTest.
  99. ExtendWithErrorHandler(android.FixtureExpectsAtLeastOneErrorMatchingPattern(pattern)).
  100. RunTestWithBp(t, bp)
  101. }
  102. func ensureListContains(t *testing.T, result []string, expected string) {
  103. t.Helper()
  104. if !android.InList(expected, result) {
  105. t.Errorf("%q is not found in %v", expected, result)
  106. }
  107. }
  108. // Analyse the sdk build rules to extract information about what it is doing.
  109. //
  110. // e.g. find the src/dest pairs from each cp command, the various zip files
  111. // generated, etc.
  112. func getSdkSnapshotBuildInfo(t *testing.T, result *android.TestResult, sdk *sdk) *snapshotBuildInfo {
  113. info := &snapshotBuildInfo{
  114. t: t,
  115. r: result,
  116. androidBpContents: sdk.GetAndroidBpContentsForTests(),
  117. infoContents: sdk.GetInfoContentsForTests(),
  118. snapshotTestCustomizations: map[snapshotTest]*snapshotTestCustomization{},
  119. targetBuildRelease: sdk.builderForTests.targetBuildRelease,
  120. }
  121. buildParams := sdk.BuildParamsForTests()
  122. copyRules := &strings.Builder{}
  123. otherCopyRules := &strings.Builder{}
  124. snapshotDirPrefix := sdk.builderForTests.snapshotDir.String() + "/"
  125. seenBuildNumberFile := false
  126. for _, bp := range buildParams {
  127. switch bp.Rule.String() {
  128. case android.Cp.String():
  129. output := bp.Output
  130. // Get destination relative to the snapshot root
  131. dest := output.Rel()
  132. src := android.NormalizePathForTesting(bp.Input)
  133. // We differentiate between copy rules for the snapshot, and copy rules for the install file.
  134. if strings.HasPrefix(output.String(), snapshotDirPrefix) {
  135. // Don't include the build-number.txt file in the copy rules as that would break lots of
  136. // tests, just verify that it is copied here as it should appear in every snapshot.
  137. if output.Base() == BUILD_NUMBER_FILE {
  138. seenBuildNumberFile = true
  139. } else {
  140. // Get source relative to build directory.
  141. _, _ = fmt.Fprintf(copyRules, "%s -> %s\n", src, dest)
  142. }
  143. info.snapshotContents = append(info.snapshotContents, dest)
  144. } else {
  145. _, _ = fmt.Fprintf(otherCopyRules, "%s -> %s\n", src, dest)
  146. }
  147. case repackageZip.String():
  148. // Add the destdir to the snapshot contents as that is effectively where
  149. // the content of the repackaged zip is copied.
  150. dest := bp.Args["destdir"]
  151. info.snapshotContents = append(info.snapshotContents, dest)
  152. case zipFiles.String():
  153. // This could be an intermediate zip file and not the actual output zip.
  154. // In that case this will be overridden when the rule to merge the zips
  155. // is processed.
  156. info.outputZip = android.NormalizePathForTesting(bp.Output)
  157. case mergeZips.String():
  158. // Copy the current outputZip to the intermediateZip.
  159. info.intermediateZip = info.outputZip
  160. mergeInput := android.NormalizePathForTesting(bp.Input)
  161. if info.intermediateZip != mergeInput {
  162. t.Errorf("Expected intermediate zip %s to be an input to merge zips but found %s instead",
  163. info.intermediateZip, mergeInput)
  164. }
  165. // Override output zip (which was actually the intermediate zip file) with the actual
  166. // output zip.
  167. info.outputZip = android.NormalizePathForTesting(bp.Output)
  168. // Save the zips to be merged into the intermediate zip.
  169. info.mergeZips = android.NormalizePathsForTesting(bp.Inputs)
  170. }
  171. }
  172. if !seenBuildNumberFile {
  173. panic(fmt.Sprintf("Every snapshot must include the %s file", BUILD_NUMBER_FILE))
  174. }
  175. info.copyRules = copyRules.String()
  176. info.otherCopyRules = otherCopyRules.String()
  177. return info
  178. }
  179. // The enum of different sdk snapshot tests performed by CheckSnapshot.
  180. type snapshotTest int
  181. const (
  182. // The enumeration of the different test configurations.
  183. // A test with the snapshot/Android.bp file but without the original Android.bp file.
  184. checkSnapshotWithoutSource snapshotTest = iota
  185. // A test with both the original source and the snapshot, with the source preferred.
  186. checkSnapshotWithSourcePreferred
  187. // A test with both the original source and the snapshot, with the snapshot preferred.
  188. checkSnapshotPreferredWithSource
  189. // The directory into which the snapshot will be 'unpacked'.
  190. snapshotSubDir = "snapshot"
  191. )
  192. // Check the snapshot build rules.
  193. //
  194. // Takes a list of functions which check different facets of the snapshot build rules.
  195. // Allows each test to customize what is checked without duplicating lots of code
  196. // or proliferating check methods of different flavors.
  197. func CheckSnapshot(t *testing.T, result *android.TestResult, name string, dir string, checkers ...snapshotBuildInfoChecker) {
  198. t.Helper()
  199. // The sdk CommonOS variant is always responsible for generating the snapshot.
  200. variant := android.CommonOS.Name
  201. sdk := result.Module(name, variant).(*sdk)
  202. snapshotBuildInfo := getSdkSnapshotBuildInfo(t, result, sdk)
  203. // Check state of the snapshot build.
  204. for _, checker := range checkers {
  205. checker(snapshotBuildInfo)
  206. }
  207. // Make sure that the generated zip file is in the correct place.
  208. actual := snapshotBuildInfo.outputZip
  209. if dir != "" {
  210. dir = filepath.Clean(dir) + "/"
  211. }
  212. suffix := "-" + soongSdkSnapshotVersionCurrent
  213. expectedZipPath := fmt.Sprintf(".intermediates/%s%s/%s/%s%s.zip", dir, name, variant, name, suffix)
  214. android.AssertStringEquals(t, "Snapshot zip file in wrong place", expectedZipPath, actual)
  215. // Populate a mock filesystem with the files that would have been copied by
  216. // the rules.
  217. fs := android.MockFS{}
  218. for _, dest := range snapshotBuildInfo.snapshotContents {
  219. fs[filepath.Join(snapshotSubDir, dest)] = nil
  220. }
  221. fs[filepath.Join(snapshotSubDir, "Android.bp")] = []byte(snapshotBuildInfo.androidBpContents)
  222. // If the generated snapshot builders not for the current release then it cannot be loaded by
  223. // the current release.
  224. if snapshotBuildInfo.targetBuildRelease != buildReleaseCurrent {
  225. return
  226. }
  227. // The preparers from the original source fixture.
  228. sourcePreparers := result.Preparer()
  229. // Preparer to combine the snapshot and the source.
  230. snapshotPreparer := android.GroupFixturePreparers(sourcePreparers, fs.AddToFixture())
  231. var runSnapshotTestWithCheckers = func(t *testing.T, testConfig snapshotTest, extraPreparer android.FixturePreparer) {
  232. t.Helper()
  233. customization := snapshotBuildInfo.snapshotTestCustomization(testConfig)
  234. customizedPreparers := android.GroupFixturePreparers(customization.preparers...)
  235. // TODO(b/183184375): Set Config.TestAllowNonExistentPaths = false to verify that all the
  236. // files the snapshot needs are actually copied into the snapshot.
  237. // Run the snapshot with the snapshot preparer and the extra preparer, which must come after as
  238. // it may need to modify parts of the MockFS populated by the snapshot preparer.
  239. result := android.GroupFixturePreparers(snapshotPreparer, extraPreparer, customizedPreparers).
  240. ExtendWithErrorHandler(customization.errorHandler).
  241. RunTest(t)
  242. // Perform any additional checks the test need on the result of processing the snapshot.
  243. for _, checker := range customization.checkers {
  244. checker(t, result)
  245. }
  246. }
  247. t.Run("snapshot without source", func(t *testing.T) {
  248. // Remove the source Android.bp file to make sure it works without.
  249. removeSourceAndroidBp := android.FixtureModifyMockFS(func(fs android.MockFS) {
  250. delete(fs, "Android.bp")
  251. })
  252. runSnapshotTestWithCheckers(t, checkSnapshotWithoutSource, removeSourceAndroidBp)
  253. })
  254. t.Run("snapshot with source preferred", func(t *testing.T) {
  255. runSnapshotTestWithCheckers(t, checkSnapshotWithSourcePreferred, android.NullFixturePreparer)
  256. })
  257. t.Run("snapshot preferred with source", func(t *testing.T) {
  258. // Replace the snapshot/Android.bp file with one where "prefer: false," has been replaced with
  259. // "prefer: true,"
  260. preferPrebuilts := android.FixtureModifyMockFS(func(fs android.MockFS) {
  261. snapshotBpFile := filepath.Join(snapshotSubDir, "Android.bp")
  262. unpreferred := string(fs[snapshotBpFile])
  263. fs[snapshotBpFile] = []byte(strings.ReplaceAll(unpreferred, "prefer: false,", "prefer: true,"))
  264. })
  265. runSnapshotTestWithCheckers(t, checkSnapshotPreferredWithSource, preferPrebuilts)
  266. })
  267. }
  268. type snapshotBuildInfoChecker func(info *snapshotBuildInfo)
  269. // Check that the snapshot's generated Android.bp is correct.
  270. //
  271. // Both the expected and actual string are both trimmed before comparing.
  272. func checkAndroidBpContents(expected string) snapshotBuildInfoChecker {
  273. return func(info *snapshotBuildInfo) {
  274. info.t.Helper()
  275. android.AssertTrimmedStringEquals(info.t, "Android.bp contents do not match", expected, info.androidBpContents)
  276. }
  277. }
  278. // Check that the snapshot's copy rules are correct.
  279. //
  280. // The copy rules are formatted as <src> -> <dest>, one per line and then compared
  281. // to the supplied expected string. Both the expected and actual string are trimmed
  282. // before comparing.
  283. func checkAllCopyRules(expected string) snapshotBuildInfoChecker {
  284. return func(info *snapshotBuildInfo) {
  285. info.t.Helper()
  286. android.AssertTrimmedStringEquals(info.t, "Incorrect copy rules", expected, info.copyRules)
  287. }
  288. }
  289. func checkAllOtherCopyRules(expected string) snapshotBuildInfoChecker {
  290. return func(info *snapshotBuildInfo) {
  291. info.t.Helper()
  292. android.AssertTrimmedStringEquals(info.t, "Incorrect copy rules", expected, info.otherCopyRules)
  293. }
  294. }
  295. // Check that the specified paths match the list of zips to merge with the intermediate zip.
  296. func checkMergeZips(expected ...string) snapshotBuildInfoChecker {
  297. return func(info *snapshotBuildInfo) {
  298. info.t.Helper()
  299. if info.intermediateZip == "" {
  300. info.t.Errorf("No intermediate zip file was created")
  301. }
  302. android.AssertDeepEquals(info.t, "mismatching merge zip files", expected, info.mergeZips)
  303. }
  304. }
  305. // Check that the snapshot's info contents are ciorrect.
  306. //
  307. // Both the expected and actual string are both trimmed before comparing.
  308. func checkInfoContents(config android.Config, expected string) snapshotBuildInfoChecker {
  309. return func(info *snapshotBuildInfo) {
  310. info.t.Helper()
  311. android.AssertTrimmedStringEquals(info.t, "info contents do not match",
  312. expected, android.StringRelativeToTop(config, info.infoContents))
  313. }
  314. }
  315. type resultChecker func(t *testing.T, result *android.TestResult)
  316. // snapshotTestPreparer registers a preparer that will be used to customize the specified
  317. // snapshotTest.
  318. func snapshotTestPreparer(snapshotTest snapshotTest, preparer android.FixturePreparer) snapshotBuildInfoChecker {
  319. return func(info *snapshotBuildInfo) {
  320. customization := info.snapshotTestCustomization(snapshotTest)
  321. customization.preparers = append(customization.preparers, preparer)
  322. }
  323. }
  324. // snapshotTestChecker registers a checker that will be run against the result of processing the
  325. // generated snapshot for the specified snapshotTest.
  326. func snapshotTestChecker(snapshotTest snapshotTest, checker resultChecker) snapshotBuildInfoChecker {
  327. return func(info *snapshotBuildInfo) {
  328. customization := info.snapshotTestCustomization(snapshotTest)
  329. customization.checkers = append(customization.checkers, checker)
  330. }
  331. }
  332. // snapshotTestErrorHandler registers an error handler to use when processing the snapshot
  333. // in the specific test case.
  334. //
  335. // Generally, the snapshot should work with all the test cases but some do not and just in case
  336. // there are a lot of issues to resolve, or it will take a lot of time this is a
  337. // get-out-of-jail-free card that allows progress to be made.
  338. //
  339. // deprecated: should only be used as a temporary workaround with an attached to do and bug.
  340. func snapshotTestErrorHandler(snapshotTest snapshotTest, handler android.FixtureErrorHandler) snapshotBuildInfoChecker {
  341. return func(info *snapshotBuildInfo) {
  342. customization := info.snapshotTestCustomization(snapshotTest)
  343. customization.errorHandler = handler
  344. }
  345. }
  346. // Encapsulates information provided by each test to customize a specific snapshotTest.
  347. type snapshotTestCustomization struct {
  348. // Preparers that are used to customize the test fixture before running the test.
  349. preparers []android.FixturePreparer
  350. // Checkers that are run on the result of processing the preferred snapshot in a specific test
  351. // case.
  352. checkers []resultChecker
  353. // Specify an error handler for when processing a specific test case.
  354. //
  355. // In some cases the generated snapshot cannot be used in a test configuration. Those cases are
  356. // invariably bugs that need to be resolved but sometimes that can take a while. This provides a
  357. // mechanism to temporarily ignore that error.
  358. errorHandler android.FixtureErrorHandler
  359. }
  360. // Encapsulates information about the snapshot build structure in order to insulate tests from
  361. // knowing too much about internal structures.
  362. //
  363. // All source/input paths are relative either the build directory. All dest/output paths are
  364. // relative to the snapshot root directory.
  365. type snapshotBuildInfo struct {
  366. t *testing.T
  367. // The result from RunTest()
  368. r *android.TestResult
  369. // The contents of the generated Android.bp file
  370. androidBpContents string
  371. // The contents of the info file.
  372. infoContents string
  373. // The paths, relative to the snapshot root, of all files and directories copied into the
  374. // snapshot.
  375. snapshotContents []string
  376. // A formatted representation of the src/dest pairs for a snapshot, one pair per line,
  377. // of the format src -> dest
  378. copyRules string
  379. // A formatted representation of the src/dest pairs for files not in a snapshot, one pair
  380. // per line, of the format src -> dest
  381. otherCopyRules string
  382. // The path to the intermediate zip, which is a zip created from the source files copied
  383. // into the snapshot directory and which will be merged with other zips to form the final output.
  384. // Is am empty string if there is no intermediate zip because there are no zips to merge in.
  385. intermediateZip string
  386. // The paths to the zips to merge into the output zip, does not include the intermediate
  387. // zip.
  388. mergeZips []string
  389. // The final output zip.
  390. outputZip string
  391. // The target build release.
  392. targetBuildRelease *buildRelease
  393. // The test specific customizations for each snapshot test.
  394. snapshotTestCustomizations map[snapshotTest]*snapshotTestCustomization
  395. }
  396. // snapshotTestCustomization gets the test specific customization for the specified snapshotTest.
  397. //
  398. // If no customization was created previously then it creates a default customization.
  399. func (i *snapshotBuildInfo) snapshotTestCustomization(snapshotTest snapshotTest) *snapshotTestCustomization {
  400. customization := i.snapshotTestCustomizations[snapshotTest]
  401. if customization == nil {
  402. customization = &snapshotTestCustomization{
  403. errorHandler: android.FixtureExpectsNoErrors,
  404. }
  405. i.snapshotTestCustomizations[snapshotTest] = customization
  406. }
  407. return customization
  408. }