run_testlab.go 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472
  1. /*
  2. * Copyright 2018 Google Inc.
  3. *
  4. * Use of this source code is governed by a BSD-style license that can be
  5. * found in the LICENSE file.
  6. */
  7. package main
  8. import (
  9. "bytes"
  10. "context"
  11. "encoding/json"
  12. "flag"
  13. "fmt"
  14. "io"
  15. "io/ioutil"
  16. "net/http"
  17. "os"
  18. "os/exec"
  19. "sort"
  20. "strconv"
  21. "strings"
  22. "syscall"
  23. "time"
  24. "go.skia.org/infra/go/gcs"
  25. "go.skia.org/infra/go/httputils"
  26. "cloud.google.com/go/storage"
  27. "google.golang.org/api/option"
  28. gstorage "google.golang.org/api/storage/v1"
  29. "go.skia.org/infra/go/auth"
  30. "go.skia.org/infra/go/common"
  31. "go.skia.org/infra/go/sklog"
  32. "go.skia.org/infra/go/util"
  33. )
  34. const (
  35. META_DATA_FILENAME = "meta.json"
  36. )
  37. // Command line flags.
  38. var (
  39. devicesFile = flag.String("devices", "", "JSON file that maps device ids to versions to run on. Same format as produced by the dump_devices flag.")
  40. dryRun = flag.Bool("dryrun", false, "Print out the command and quit without triggering tests.")
  41. dumpDevFile = flag.String("dump_devices", "", "Creates a JSON file with all physical devices that are not deprecated.")
  42. minAPIVersion = flag.Int("min_api", 0, "Minimum API version required by device.")
  43. maxAPIVersion = flag.Int("max_api", 99, "Maximum API version required by device.")
  44. properties = flag.String("properties", "", "Custom meta data to be added to the uploaded APK. Comma separated list of key=value pairs, i.e. 'k1=v1,k2=v2,k3=v3.")
  45. uploadGCSPath = flag.String("upload_path", "", "GCS path (bucket/path) to where the APK should be uploaded to. It's assume to a full path (not a directory).")
  46. )
  47. const (
  48. RUN_TESTS_TEMPLATE = `gcloud beta firebase test android run
  49. --type=game-loop
  50. --app=%s
  51. --results-bucket=%s
  52. --results-dir=%s
  53. --directories-to-pull=/sdcard/Android/data/org.skia.skqp
  54. --timeout 30m
  55. %s
  56. `
  57. MODEL_VERSION_TMPL = "--device model=%s,version=%s,orientation=portrait"
  58. RESULT_BUCKET = "skia-firebase-test-lab"
  59. RESULT_DIR_TMPL = "testruns/%s/%s"
  60. RUN_ID_TMPL = "testrun-%d"
  61. CMD_AVAILABLE_DEVICES = "gcloud firebase test android models list --format json"
  62. )
  63. func main() {
  64. common.Init()
  65. // Get the path to the APK. It can be empty if we are dumping the device list.
  66. apkPath := flag.Arg(0)
  67. if *dumpDevFile == "" && apkPath == "" {
  68. sklog.Errorf("Missing APK. The APK file needs to be passed as the positional argument.")
  69. os.Exit(1)
  70. }
  71. // Get the available devices.
  72. fbDevices, deviceList, err := getAvailableDevices()
  73. if err != nil {
  74. sklog.Fatalf("Error retrieving available devices: %s", err)
  75. }
  76. // Dump the device list and exit.
  77. if *dumpDevFile != "" {
  78. if err := writeDeviceList(*dumpDevFile, deviceList); err != nil {
  79. sklog.Fatalf("Unable to write devices: %s", err)
  80. }
  81. return
  82. }
  83. // If no devices are explicitly listed. Use all of them.
  84. whiteList := deviceList
  85. if *devicesFile != "" {
  86. whiteList, err = readDeviceList(*devicesFile)
  87. if err != nil {
  88. sklog.Fatalf("Error reading device file: %s", err)
  89. }
  90. }
  91. // Make sure we can authenticate locally and in the cloud.
  92. ts, err := auth.NewDefaultTokenSource(true, gstorage.CloudPlatformScope, "https://www.googleapis.com/auth/userinfo.email")
  93. if err != nil {
  94. sklog.Fatal(err)
  95. }
  96. client := httputils.DefaultClientConfig().WithTokenSource(ts).With2xxOnly().Client()
  97. // Filter the devices according the white list and other parameters.
  98. devices, ignoredDevices := filterDevices(fbDevices, whiteList, *minAPIVersion, *maxAPIVersion)
  99. sklog.Infof("---\nSelected devices:")
  100. logDevices(devices)
  101. if len(devices) == 0 {
  102. sklog.Errorf("No devices selected. Not running tests.")
  103. os.Exit(1)
  104. }
  105. if err := runTests(apkPath, devices, ignoredDevices, client, *dryRun); err != nil {
  106. sklog.Fatalf("Error running tests on Firebase: %s", err)
  107. }
  108. if !*dryRun && (*uploadGCSPath != "") && (*properties != "") {
  109. if err := uploadAPK(apkPath, *uploadGCSPath, *properties, client); err != nil {
  110. sklog.Fatalf("Error uploading APK to '%s': %s", *uploadGCSPath, err)
  111. }
  112. }
  113. }
  114. // getAvailableDevices queries Firebase Testlab for all physical devices that
  115. // are not deprecated. It returns two lists with the same information.
  116. // The first contains all device information as returned by Firebase while
  117. // the second contains the information necessary to use in a whitelist.
  118. func getAvailableDevices() ([]*DeviceVersions, DeviceList, error) {
  119. // Get the list of all devices in JSON format from Firebase testlab.
  120. var buf bytes.Buffer
  121. var errBuf bytes.Buffer
  122. cmd := parseCommand(CMD_AVAILABLE_DEVICES)
  123. cmd.Stdout = &buf
  124. cmd.Stderr = io.MultiWriter(os.Stdout, &errBuf)
  125. if err := cmd.Run(); err != nil {
  126. return nil, nil, sklog.FmtErrorf("Error running: %s\nError:%s\nStdErr:%s", CMD_AVAILABLE_DEVICES, err, errBuf)
  127. }
  128. // Unmarshal the result.
  129. foundDevices := []*DeviceVersions{}
  130. bufBytes := buf.Bytes()
  131. if err := json.Unmarshal(bufBytes, &foundDevices); err != nil {
  132. return nil, nil, sklog.FmtErrorf("Unmarshal of device information failed: %s \nJSON Input: %s\n", err, string(bufBytes))
  133. }
  134. // Filter the devices and copy them to device list.
  135. devList := DeviceList{}
  136. ret := make([]*DeviceVersions, 0, len(foundDevices))
  137. for _, foundDev := range foundDevices {
  138. // Only consider physical devices and devices that are not deprecated.
  139. if (foundDev.Form == "PHYSICAL") && !util.In("deprecated", foundDev.Tags) {
  140. ret = append(ret, foundDev)
  141. devList = append(devList, &DevInfo{
  142. ID: foundDev.ID,
  143. Name: foundDev.Name,
  144. RunVersions: foundDev.VersionIDs,
  145. })
  146. }
  147. }
  148. return foundDevices, devList, nil
  149. }
  150. // filterDevices filters the given devices by ensuring that they are in the white list
  151. // and within the given api version range.
  152. // It returns two lists: (accepted_devices, ignored_devices)
  153. func filterDevices(foundDevices []*DeviceVersions, whiteList DeviceList, minAPIVersion, maxAPIVersion int) ([]*DeviceVersions, []*DeviceVersions) {
  154. // iterate over the available devices and partition them.
  155. allDevices := make([]*DeviceVersions, 0, len(foundDevices))
  156. ret := make([]*DeviceVersions, 0, len(foundDevices))
  157. ignored := make([]*DeviceVersions, 0, len(foundDevices))
  158. for _, dev := range foundDevices {
  159. // Only include devices that are on the whitelist and have versions defined.
  160. if targetDev := whiteList.find(dev.ID); targetDev != nil && (len(targetDev.RunVersions) > 0) {
  161. versionSet := util.NewStringSet(dev.VersionIDs)
  162. reqVersions := util.NewStringSet(filterVersions(targetDev.RunVersions, minAPIVersion, maxAPIVersion))
  163. whiteListVersions := versionSet.Intersect(reqVersions).Keys()
  164. ignoredVersions := versionSet.Complement(reqVersions).Keys()
  165. sort.Strings(whiteListVersions)
  166. sort.Strings(ignoredVersions)
  167. if len(whiteListVersions) > 0 {
  168. ret = append(ret, &DeviceVersions{FirebaseDevice: dev.FirebaseDevice, RunVersions: whiteListVersions})
  169. }
  170. if len(ignoredVersions) > 0 {
  171. ignored = append(ignored, &DeviceVersions{FirebaseDevice: dev.FirebaseDevice, RunVersions: ignoredVersions})
  172. }
  173. } else {
  174. ignored = append(ignored, &DeviceVersions{FirebaseDevice: dev.FirebaseDevice, RunVersions: dev.VersionIDs})
  175. }
  176. allDevices = append(allDevices, &DeviceVersions{FirebaseDevice: dev.FirebaseDevice, RunVersions: dev.VersionIDs})
  177. }
  178. sklog.Infof("All devices:")
  179. logDevices(allDevices)
  180. return ret, ignored
  181. }
  182. // filterVersions returns the elements in versionIDs where minVersion <= element <= maxVersion.
  183. func filterVersions(versionIDs []string, minVersion, maxVersion int) []string {
  184. ret := make([]string, 0, len(versionIDs))
  185. for _, versionID := range versionIDs {
  186. id, err := strconv.Atoi(versionID)
  187. if err != nil {
  188. sklog.Fatalf("Error parsing version id '%s': %s", versionID, err)
  189. }
  190. if (id >= minVersion) && (id <= maxVersion) {
  191. ret = append(ret, versionID)
  192. }
  193. }
  194. return ret
  195. }
  196. // runTests runs the given apk on the given list of devices.
  197. func runTests(apk_path string, devices, ignoredDevices []*DeviceVersions, client *http.Client, dryRun bool) error {
  198. // Get the model-version we want to test. Assume on average each model has 5 supported versions.
  199. modelSelectors := make([]string, 0, len(devices)*5)
  200. for _, devRec := range devices {
  201. for _, version := range devRec.RunVersions {
  202. modelSelectors = append(modelSelectors, fmt.Sprintf(MODEL_VERSION_TMPL, devRec.FirebaseDevice.ID, version))
  203. }
  204. }
  205. now := time.Now()
  206. nowMs := now.UnixNano() / int64(time.Millisecond)
  207. runID := fmt.Sprintf(RUN_ID_TMPL, nowMs)
  208. resultsDir := fmt.Sprintf(RESULT_DIR_TMPL, now.Format("2006/01/02/15"), runID)
  209. cmdStr := fmt.Sprintf(RUN_TESTS_TEMPLATE, apk_path, RESULT_BUCKET, resultsDir, strings.Join(modelSelectors, "\n"))
  210. cmdStr = strings.TrimSpace(strings.Replace(cmdStr, "\n", " ", -1))
  211. // Run the command.
  212. var errBuf bytes.Buffer
  213. cmd := parseCommand(cmdStr)
  214. cmd.Stdout = os.Stdout
  215. cmd.Stderr = io.MultiWriter(os.Stdout, &errBuf)
  216. exitCode := 0
  217. if dryRun {
  218. fmt.Printf("[dry run]: Would have run this command: %s\n", cmdStr)
  219. return nil
  220. }
  221. if err := cmd.Run(); err != nil {
  222. // Get the exit code.
  223. if exitError, ok := err.(*exec.ExitError); ok {
  224. ws := exitError.Sys().(syscall.WaitStatus)
  225. exitCode = ws.ExitStatus()
  226. }
  227. sklog.Errorf("Error running tests: %s", err)
  228. sklog.Errorf("Exit code: %d", exitCode)
  229. // Exit code 10 means triggering on Testlab succeeded, but but some of the
  230. // runs on devices failed. We consider it a success for this script.
  231. if exitCode != 10 {
  232. return sklog.FmtErrorf("Error running: %s\nError:%s\nStdErr:%s", cmdStr, err, errBuf)
  233. }
  234. }
  235. // Store the result in a meta json file.
  236. meta := &TestRunMeta{
  237. ID: runID,
  238. TS: nowMs,
  239. Devices: devices,
  240. IgnoredDevices: ignoredDevices,
  241. ExitCode: exitCode,
  242. }
  243. targetPath := fmt.Sprintf("%s/%s/%s", RESULT_BUCKET, resultsDir, META_DATA_FILENAME)
  244. if err := meta.writeToGCS(targetPath, client); err != nil {
  245. return err
  246. }
  247. sklog.Infof("Meta data written to gs://%s", targetPath)
  248. return nil
  249. }
  250. // uploadAPK uploads the APK at the given path to the bucket/path in gcsPath.
  251. // The key-value pairs in propStr are set as custom meta data of the APK.
  252. func uploadAPK(apkPath, gcsPath, propStr string, client *http.Client) error {
  253. properties, err := splitProperties(propStr)
  254. if err != nil {
  255. return err
  256. }
  257. apkFile, err := os.Open(apkPath)
  258. if err != nil {
  259. return err
  260. }
  261. defer util.Close(apkFile)
  262. if err := copyReaderToGCS(gcsPath, apkFile, client, "application/vnd.android.package-archive", properties, true, false); err != nil {
  263. return err
  264. }
  265. sklog.Infof("APK uploaded to gs://%s", gcsPath)
  266. return nil
  267. }
  268. // splitProperties receives a comma separated list of 'key=value' pairs and
  269. // returnes them as a map.
  270. func splitProperties(propStr string) (map[string]string, error) {
  271. splitProps := strings.Split(propStr, ",")
  272. properties := make(map[string]string, len(splitProps))
  273. for _, oneProp := range splitProps {
  274. kv := strings.Split(oneProp, "=")
  275. if len(kv) != 2 {
  276. return nil, sklog.FmtErrorf("Inavlid porperties format. Unable to parse '%s'", propStr)
  277. }
  278. properties[strings.TrimSpace(kv[0])] = strings.TrimSpace(kv[1])
  279. }
  280. return properties, nil
  281. }
  282. // logDevices logs the given list of devices.
  283. func logDevices(devices []*DeviceVersions) {
  284. sklog.Infof("Found %d devices.", len(devices))
  285. for _, dev := range devices {
  286. fbDev := dev.FirebaseDevice
  287. sklog.Infof("%-15s %-30s %v / %v", fbDev.ID, fbDev.Name, fbDev.VersionIDs, dev.RunVersions)
  288. }
  289. }
  290. // parseCommad parses a command line and wraps it in an exec.Command instance.
  291. func parseCommand(cmdStr string) *exec.Cmd {
  292. cmdArgs := strings.Split(strings.TrimSpace(cmdStr), " ")
  293. for idx := range cmdArgs {
  294. cmdArgs[idx] = strings.TrimSpace(cmdArgs[idx])
  295. }
  296. return exec.Command(cmdArgs[0], cmdArgs[1:]...)
  297. }
  298. // DeviceList is a simple list of devices, primarily used to define the
  299. // whitelist of devices we want to run on.
  300. type DeviceList []*DevInfo
  301. type DevInfo struct {
  302. ID string `json:"id"`
  303. Name string `json:"name"`
  304. RunVersions []string `json:"runVersions"`
  305. }
  306. func (d DeviceList) find(id string) *DevInfo {
  307. for _, devInfo := range d {
  308. if devInfo.ID == id {
  309. return devInfo
  310. }
  311. }
  312. return nil
  313. }
  314. func writeDeviceList(fileName string, devList DeviceList) error {
  315. jsonBytes, err := json.MarshalIndent(devList, "", " ")
  316. if err != nil {
  317. return sklog.FmtErrorf("Unable to encode JSON: %s", err)
  318. }
  319. if err := ioutil.WriteFile(fileName, jsonBytes, 0644); err != nil {
  320. sklog.FmtErrorf("Unable to write file '%s': %s", fileName, err)
  321. }
  322. return nil
  323. }
  324. func readDeviceList(fileName string) (DeviceList, error) {
  325. inFile, err := os.Open(fileName)
  326. if err != nil {
  327. return nil, sklog.FmtErrorf("Unable to open file '%s': %s", fileName, err)
  328. }
  329. defer util.Close(inFile)
  330. var devList DeviceList
  331. if err := json.NewDecoder(inFile).Decode(&devList); err != nil {
  332. return nil, sklog.FmtErrorf("Unable to decode JSON from '%s': %s", fileName, err)
  333. }
  334. return devList, nil
  335. }
  336. // FirebaseDevice contains the information and JSON tags for device information
  337. // returned by firebase.
  338. type FirebaseDevice struct {
  339. Brand string `json:"brand"`
  340. Form string `json:"form"`
  341. ID string `json:"id"`
  342. Manufacturer string `json:"manufacturer"`
  343. Name string `json:"name"`
  344. VersionIDs []string `json:"supportedVersionIds"`
  345. Tags []string `json:"tags"`
  346. }
  347. // DeviceVersions combines device information from Firebase Testlab with
  348. // a selected list of versions. This is used to define a subset of versions
  349. // used by a devices.
  350. type DeviceVersions struct {
  351. *FirebaseDevice
  352. // RunVersions contains the version ids of interest contained in Device.
  353. RunVersions []string
  354. }
  355. // TestRunMeta contains the meta data of a complete testrun on firebase.
  356. type TestRunMeta struct {
  357. ID string `json:"id"`
  358. TS int64 `json:"timeStamp"`
  359. Devices []*DeviceVersions `json:"devices"`
  360. IgnoredDevices []*DeviceVersions `json:"ignoredDevices"`
  361. ExitCode int `json:"exitCode"`
  362. }
  363. // writeToGCS writes the meta data as JSON to the given bucket and path in
  364. // GCS. It assumes that the provided client has permissions to write to the
  365. // specified location in GCS.
  366. func (t *TestRunMeta) writeToGCS(gcsPath string, client *http.Client) error {
  367. jsonBytes, err := json.Marshal(t)
  368. if err != nil {
  369. return err
  370. }
  371. return copyReaderToGCS(gcsPath, bytes.NewReader(jsonBytes), client, "", nil, false, true)
  372. }
  373. // TODO(stephana): Merge copyReaderToGCS into the go/gcs in
  374. // the infra repository.
  375. // copyReaderToGCS reads all available content from the given reader and writes
  376. // it to the given path in GCS.
  377. func copyReaderToGCS(gcsPath string, reader io.Reader, client *http.Client, contentType string, metaData map[string]string, public bool, gzip bool) error {
  378. storageClient, err := storage.NewClient(context.Background(), option.WithHTTPClient(client))
  379. if err != nil {
  380. return err
  381. }
  382. bucket, path := gcs.SplitGSPath(gcsPath)
  383. w := storageClient.Bucket(bucket).Object(path).NewWriter(context.Background())
  384. // Set the content if requested.
  385. if contentType != "" {
  386. w.ObjectAttrs.ContentType = contentType
  387. }
  388. // Set the meta data if requested
  389. if metaData != nil {
  390. w.Metadata = metaData
  391. }
  392. // Make the object public if requested.
  393. if public {
  394. w.ACL = []storage.ACLRule{{Entity: storage.AllUsers, Role: storage.RoleReader}}
  395. }
  396. // Write the everything the reader can provide to the GCS object. Either
  397. // gzip'ed or plain.
  398. if gzip {
  399. w.ObjectAttrs.ContentEncoding = "gzip"
  400. err = util.WithGzipWriter(w, func(w io.Writer) error {
  401. _, err := io.Copy(w, reader)
  402. return err
  403. })
  404. } else {
  405. _, err = io.Copy(w, reader)
  406. }
  407. // Make sure we return an error when we close the remote object.
  408. if err != nil {
  409. _ = w.CloseWithError(err)
  410. return err
  411. }
  412. return w.Close()
  413. }