extension_query.py 9.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290
  1. #!/usr/bin/env python
  2. # Copyright 2020 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. """Transform CBCM Takeout API Data (Python3)."""
  6. from __future__ import print_function
  7. from __future__ import unicode_literals
  8. import argparse
  9. import csv
  10. import json
  11. import sys
  12. import time
  13. import google_auth_httplib2
  14. from httplib2 import Http
  15. from google.oauth2.service_account import Credentials
  16. from builtins import bytes
  17. from builtins import str
  18. from io import open
  19. def ComputeExtensionsList(extensions_list, data):
  20. """Computes list of machines that have an extension.
  21. This sample function processes the |data| retrieved from the Takeout API and
  22. calculates the list of machines that have installed each extension listed in
  23. the data.
  24. Args:
  25. extensions_list: the extension list dictionary to fill.
  26. data: the data fetched from the Takeout API.
  27. """
  28. for device in data['browsers']:
  29. if 'browsers' not in device:
  30. continue
  31. for browser in device['browsers']:
  32. if 'profiles' not in browser:
  33. continue
  34. for profile in browser['profiles']:
  35. if 'extensions' not in profile:
  36. continue
  37. for extension in profile['extensions']:
  38. key = extension['extensionId']
  39. if 'version' in extension:
  40. key = key + ' @ ' + extension['version']
  41. if key not in extensions_list:
  42. current_extension = {
  43. 'name': extension.get('name', ''),
  44. 'permissions': extension.get('permissions', ''),
  45. 'installed': set(),
  46. 'disabled': set(),
  47. 'forced': set()
  48. }
  49. else:
  50. current_extension = extensions_list[key]
  51. machine_name = device['machineName']
  52. current_extension['installed'].add(machine_name)
  53. if extension.get('installType', '') == 'ADMIN':
  54. current_extension['forced'].add(machine_name)
  55. if extension.get('disabled', False):
  56. current_extension['disabled'].add(machine_name)
  57. extensions_list[key] = current_extension
  58. def DictToList(data, key_name='id'):
  59. """Converts a dict into a list.
  60. The value of each member of |data| must also be a dict. The original key for
  61. the value will be inlined into the value, under the |key_name| key.
  62. Args:
  63. data: a dict where every value is a dict
  64. key_name: the name given to the key that is inlined into the dict's values
  65. Yields:
  66. The values from |data|, with each value's key inlined into the value.
  67. """
  68. assert isinstance(data, dict), '|data| must be a dict'
  69. for key, value in data.items():
  70. assert isinstance(value, dict), '|value| must contain dict items'
  71. value[key_name] = key
  72. yield value
  73. def Flatten(data, all_columns):
  74. """Flattens lists inside |data|, one level deep.
  75. This function will flatten each dictionary key in |data| into a single row
  76. so that it can be written to a CSV file.
  77. Args:
  78. data: the data to be flattened.
  79. all_columns: set of all columns that are found in the result (this will be
  80. filled by the function).
  81. Yields:
  82. A list of dict objects whose lists or sets have been flattened.
  83. """
  84. SEPARATOR = ', '
  85. # Max length of a cell in Excel is technically 32767 characters but if we get
  86. # too close to this limit Excel seems to create weird results when we open
  87. # the CSV file. To protect against this, give a little more buffer to the max
  88. # characters.
  89. MAX_CELL_LENGTH = 32700
  90. for item in data:
  91. added_item = {}
  92. for prop, value in item.items():
  93. # Non-container properties can be added directly.
  94. if not isinstance(value, (list, set)):
  95. added_item[prop] = value
  96. continue
  97. # Otherwise join the container together into a single cell.
  98. num_prop = 'num_' + prop
  99. added_item[num_prop] = len(value)
  100. # For long lists, the cell contents may go over MAX_CELL_LENGTH, so
  101. # split the list into chunks that will fit into MAX_CELL_LENGTH.
  102. flat_list = SEPARATOR.join(sorted(value))
  103. overflow_prop_index = 0
  104. while True:
  105. current_column = prop
  106. if overflow_prop_index:
  107. current_column = prop + '_' + str(overflow_prop_index)
  108. flat_list_len = len(flat_list)
  109. if flat_list_len > MAX_CELL_LENGTH:
  110. last_separator = flat_list.rfind(SEPARATOR, 0,
  111. MAX_CELL_LENGTH - flat_list_len)
  112. if last_separator != -1:
  113. added_item[current_column] = flat_list[0:last_separator]
  114. flat_list = flat_list[last_separator + 2:]
  115. overflow_prop_index = overflow_prop_index + 1
  116. continue
  117. # Fall-through case where no more splitting is possible, this is the
  118. # lass cell to add for this list.
  119. added_item[current_column] = flat_list
  120. break
  121. assert isinstance(added_item[prop],
  122. (int, bool, str)), ('unexpected type for item: %s' %
  123. type(added_item[prop]).__name__)
  124. all_columns.update(added_item.keys())
  125. yield added_item
  126. def ExtensionListAsCsv(extensions_list, csv_filename, sort_column='name'):
  127. """Saves an extensions list to a CSV file.
  128. Args:
  129. extensions_list: an extensions list as returned by ComputeExtensionsList
  130. csv_filename: the name of the CSV file to save
  131. sort_column: the name of the column by which to sort the data
  132. """
  133. all_columns = set()
  134. flattened_list = list(Flatten(DictToList(extensions_list), all_columns))
  135. desired_column_order = [
  136. 'id', 'name', 'num_permissions', 'num_installed', 'num_disabled',
  137. 'num_forced', 'permissions', 'installed', 'disabled', 'forced'
  138. ]
  139. # Order the columns as desired. Columns other than those in
  140. # |desired_column_order| will be in an unspecified order after these columns.
  141. ordered_fieldnames = []
  142. for c in desired_column_order:
  143. matching_columns = []
  144. for f in all_columns:
  145. if f == c or f.startswith(c):
  146. matching_columns.append(f)
  147. ordered_fieldnames.extend(sorted(matching_columns))
  148. ordered_fieldnames.extend(
  149. [x for x in desired_column_order if x not in ordered_fieldnames])
  150. with open(csv_filename, mode='w', newline='', encoding='utf-8') as csv_file:
  151. writer = csv.DictWriter(csv_file, fieldnames=ordered_fieldnames)
  152. writer.writeheader()
  153. for row in sorted(flattened_list, key=lambda ext: ext[sort_column]):
  154. writer.writerow(row)
  155. def main(args):
  156. if not args.admin_email:
  157. print('admin_email must be specified.')
  158. sys.exit(1)
  159. if not args.service_account_key_path:
  160. print('service_account_key_path must be specified.')
  161. sys.exit(1)
  162. # Load the json format key that you downloaded from the Google API
  163. # Console when you created your service account. For p12 keys, use the
  164. # from_p12_keyfile method of ServiceAccountCredentials and specify the
  165. # service account email address, p12 keyfile, and scopes.
  166. service_credentials = Credentials.from_service_account_file(
  167. args.service_account_key_path,
  168. scopes=[
  169. 'https://www.googleapis.com/auth/admin.directory.device.chromebrowsers.readonly'
  170. ],
  171. subject=args.admin_email)
  172. try:
  173. http = google_auth_httplib2.AuthorizedHttp(service_credentials, http=Http())
  174. extensions_list = {}
  175. base_request_url = 'https://admin.googleapis.com/admin/directory/v1.1beta1/customer/my_customer/devices/chromebrowsers'
  176. request_parameters = ''
  177. browsers_processed = 0
  178. while True:
  179. print('Making request to server ...')
  180. retrycount = 0
  181. while retrycount < 5:
  182. response = http.request(base_request_url + '?' + request_parameters,
  183. 'GET')[1]
  184. if isinstance(response, bytes):
  185. response = response.decode('utf-8')
  186. data = json.loads(response)
  187. if 'browsers' not in data:
  188. print('Response error, retrying...')
  189. time.sleep(3)
  190. retrycount += 1
  191. else:
  192. break
  193. browsers_in_data = len(data['browsers'])
  194. print('Request returned %s results, analyzing ...' % (browsers_in_data))
  195. ComputeExtensionsList(extensions_list, data)
  196. browsers_processed += browsers_in_data
  197. if 'nextPageToken' not in data or not data['nextPageToken']:
  198. break
  199. print('%s browsers processed.' % (browsers_processed))
  200. if (args.max_browsers_to_process is not None and
  201. args.max_browsers_to_process <= browsers_processed):
  202. print('Stopping at %s browsers processed.' % (browsers_processed))
  203. break
  204. request_parameters = ('pageToken={}').format(data['nextPageToken'])
  205. finally:
  206. print('Analyze results ...')
  207. ExtensionListAsCsv(extensions_list, args.extension_list_csv)
  208. print("Results written to '%s'" % (args.extension_list_csv))
  209. if __name__ == '__main__':
  210. parser = argparse.ArgumentParser(description='CBCM Extension Analyzer')
  211. parser.add_argument(
  212. '-k',
  213. '--service_account_key_path',
  214. metavar='FILENAME',
  215. required=True,
  216. help='The service account key file used to make API requests.')
  217. parser.add_argument(
  218. '-a',
  219. '--admin_email',
  220. required=True,
  221. help='The admin user used to make the API requests.')
  222. parser.add_argument(
  223. '-x',
  224. '--extension_list_csv',
  225. metavar='FILENAME',
  226. default='./extension_list.csv',
  227. help='Generate an extension list to the specified CSV '
  228. 'file')
  229. parser.add_argument(
  230. '-m',
  231. '--max_browsers_to_process',
  232. type=int,
  233. help='Maximum number of browsers to process. (Must be > 0).')
  234. args = parser.parse_args()
  235. if (args.max_browsers_to_process is not None and
  236. args.max_browsers_to_process <= 0):
  237. print('max_browsers_to_process must be > 0.')
  238. parser.print_help()
  239. sys.exit(1)
  240. main(args)