extension_query_py2.py 9.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292
  1. #!/usr/bin/env python
  2. # Copyright 2020 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. """Transform CBCM Takeout API Data (Python2)."""
  6. from __future__ import print_function
  7. import argparse
  8. import csv
  9. import json
  10. import sys
  11. import google_auth_httplib2
  12. from httplib2 import Http
  13. from google.oauth2.service_account import Credentials
  14. def ComputeExtensionsList(extensions_list, data):
  15. """Computes list of machines that have an extension.
  16. This sample function processes the |data| retrieved from the Takeout API and
  17. calculates the list of machines that have installed each extension listed in
  18. the data.
  19. Args:
  20. extensions_list: the extension list dictionary to fill.
  21. data: the data fetched from the Takeout API.
  22. """
  23. for device in data['browsers']:
  24. if 'browsers' not in device:
  25. continue
  26. for browser in device['browsers']:
  27. if 'profiles' not in browser:
  28. continue
  29. for profile in browser['profiles']:
  30. if 'extensions' not in profile:
  31. continue
  32. for extension in profile['extensions']:
  33. key = extension['extensionId']
  34. if 'version' in extension:
  35. key = key + ' @ ' + extension['version']
  36. if key not in extensions_list:
  37. current_extension = {
  38. 'name': extension.get('name', ''),
  39. 'permissions': extension.get('permissions', ''),
  40. 'installed': set(),
  41. 'disabled': set(),
  42. 'forced': set()
  43. }
  44. else:
  45. current_extension = extensions_list[key]
  46. machine_name = device['machineName']
  47. current_extension['installed'].add(machine_name)
  48. if extension.get('installType', '') == 'ADMIN':
  49. current_extension['forced'].add(machine_name)
  50. if extension.get('disabled', False):
  51. current_extension['disabled'].add(machine_name)
  52. extensions_list[key] = current_extension
  53. def ToUtf8(data):
  54. """Ensures all the values in |data| are encoded as UTF-8.
  55. Expects |data| to be a list of dict objects.
  56. Args:
  57. data: the data to be converted to UTF-8.
  58. Yields:
  59. A list of dict objects whose values have been encoded as UTF-8.
  60. """
  61. for entry in data:
  62. for prop, value in entry.iteritems():
  63. entry[prop] = unicode(value).encode('utf-8')
  64. yield entry
  65. def DictToList(data, key_name='id'):
  66. """Converts a dict into a list.
  67. The value of each member of |data| must also be a dict. The original key for
  68. the value will be inlined into the value, under the |key_name| key.
  69. Args:
  70. data: a dict where every value is a dict
  71. key_name: the name given to the key that is inlined into the dict's values
  72. Yields:
  73. The values from |data|, with each value's key inlined into the value.
  74. """
  75. assert isinstance(data, dict), '|data| must be a dict'
  76. for key, value in data.items():
  77. assert isinstance(value, dict), '|value| must contain dict items'
  78. value[key_name] = key
  79. yield value
  80. def Flatten(data, all_columns):
  81. """Flattens lists inside |data|, one level deep.
  82. This function will flatten each dictionary key in |data| into a single row
  83. so that it can be written to a CSV file.
  84. Args:
  85. data: the data to be flattened.
  86. all_columns: set of all columns that are found in the result (this will be
  87. filled by the function).
  88. Yields:
  89. A list of dict objects whose lists or sets have been flattened.
  90. """
  91. SEPARATOR = ', '
  92. # Max length of a cell in Excel is technically 32767 characters but if we get
  93. # too close to this limit Excel seems to create weird results when we open
  94. # the CSV file. To protect against this, give a little more buffer to the max
  95. # characters.
  96. MAX_CELL_LENGTH = 32700
  97. for item in data:
  98. added_item = {}
  99. for prop, value in item.items():
  100. # Non-container properties can be added directly.
  101. if not isinstance(value, (list, set)):
  102. added_item[prop] = value
  103. continue
  104. # Otherwise join the container together into a single cell.
  105. num_prop = 'num_' + prop
  106. added_item[num_prop] = len(value)
  107. # For long lists, the cell contents may go over MAX_CELL_LENGTH, so
  108. # split the list into chunks that will fit into MAX_CELL_LENGTH.
  109. flat_list = SEPARATOR.join(sorted(value))
  110. overflow_prop_index = 0
  111. while True:
  112. current_column = prop
  113. if overflow_prop_index:
  114. current_column = prop + '_' + str(overflow_prop_index)
  115. flat_list_len = len(flat_list)
  116. if flat_list_len > MAX_CELL_LENGTH:
  117. last_separator = flat_list.rfind(SEPARATOR, 0,
  118. MAX_CELL_LENGTH - flat_list_len)
  119. if last_separator != -1:
  120. added_item[current_column] = flat_list[0:last_separator]
  121. flat_list = flat_list[last_separator + 2:]
  122. overflow_prop_index = overflow_prop_index + 1
  123. continue
  124. # Fall-through case where no more splitting is possible, this is the
  125. # lass cell to add for this list.
  126. added_item[current_column] = flat_list
  127. break
  128. assert isinstance(
  129. added_item[prop],
  130. (int, bool, str, unicode)), ('unexpected type for item: %s' %
  131. type(added_item[prop]).__name__)
  132. all_columns.update(added_item.keys())
  133. yield added_item
  134. def ExtensionListAsCsv(extensions_list, csv_filename, sort_column='name'):
  135. """Saves an extensions list to a CSV file.
  136. Args:
  137. extensions_list: an extensions list as returned by ComputeExtensionsList
  138. csv_filename: the name of the CSV file to save
  139. sort_column: the name of the column by which to sort the data
  140. """
  141. all_columns = set()
  142. flattened_list = [
  143. x for x in ToUtf8(Flatten(DictToList(extensions_list), all_columns))
  144. ]
  145. desired_column_order = [
  146. 'id', 'name', 'num_permissions', 'num_installed', 'num_disabled',
  147. 'num_forced', 'permissions', 'installed', 'disabled', 'forced'
  148. ]
  149. # Order the columns as desired. Columns other than those in
  150. # |desired_column_order| will be in an unspecified order after these columns.
  151. ordered_fieldnames = []
  152. for c in desired_column_order:
  153. matching_columns = []
  154. for f in all_columns:
  155. if f == c or f.startswith(c):
  156. matching_columns.append(f)
  157. ordered_fieldnames.extend(sorted(matching_columns))
  158. ordered_fieldnames.extend(
  159. [x for x in desired_column_order if x not in ordered_fieldnames])
  160. with open(csv_filename, mode='w') as csv_file:
  161. writer = csv.DictWriter(csv_file, fieldnames=ordered_fieldnames)
  162. writer.writeheader()
  163. for row in sorted(flattened_list, key=lambda ext: ext[sort_column]):
  164. writer.writerow(row)
  165. def main(args):
  166. # Load the json format key that you downloaded from the Google API
  167. # Console when you created your service account. For p12 keys, use the
  168. # from_p12_keyfile method of ServiceAccountCredentials and specify the
  169. # service account email address, p12 keyfile, and scopes.
  170. service_credentials = Credentials.from_service_account_file(
  171. args.service_account_key_path,
  172. scopes=[
  173. 'https://www.googleapis.com/auth/admin.directory.device.chromebrowsers.readonly'
  174. ],
  175. subject=args.admin_email)
  176. try:
  177. http = google_auth_httplib2.AuthorizedHttp(service_credentials, http=Http())
  178. extensions_list = {}
  179. base_request_url = 'https://admin.googleapis.com/admin/directory/v1.1beta1/customer/my_customer/devices/chromebrowsers'
  180. request_parameters = ''
  181. browsers_processed = 0
  182. while True:
  183. print('Making request to server ...')
  184. retrycount = 0
  185. while retrycount < 5:
  186. data = json.loads(
  187. http.request(base_request_url + '?' + request_parameters, 'GET')[1])
  188. if 'browsers' not in data:
  189. print('Response error, retrying...')
  190. time.sleep(3)
  191. retrycount += 1
  192. else:
  193. break
  194. browsers_in_data = len(data['browsers'])
  195. print('Request returned %s results, analyzing ...' % (browsers_in_data))
  196. ComputeExtensionsList(extensions_list, data)
  197. browsers_processed += browsers_in_data
  198. if 'nextPageToken' not in data or not data['nextPageToken']:
  199. break
  200. print('%s browsers processed.' % (browsers_processed))
  201. if (args.max_browsers_to_process is not None and
  202. args.max_browsers_to_process <= browsers_processed):
  203. print('Stopping at %s browsers processed.' % (browsers_processed))
  204. break
  205. request_parameters = ('pageToken={}').format(data['nextPageToken'])
  206. finally:
  207. print('Analyze results ...')
  208. ExtensionListAsCsv(extensions_list, args.extension_list_csv)
  209. print("Results written to '%s'" % (args.extension_list_csv))
  210. if __name__ == '__main__':
  211. parser = argparse.ArgumentParser(description='CBCM Extension Analyzer')
  212. parser.add_argument(
  213. '-k',
  214. '--service_account_key_path',
  215. metavar='FILENAME',
  216. required=True,
  217. help='The service account key file used to make API requests.')
  218. parser.add_argument(
  219. '-a',
  220. '--admin_email',
  221. required=True,
  222. help='The admin user used to make the API requests.')
  223. parser.add_argument(
  224. '-x',
  225. '--extension_list_csv',
  226. metavar='FILENAME',
  227. default='./extension_list.csv',
  228. help='Generate an extension list to the specified CSV '
  229. 'file')
  230. parser.add_argument(
  231. '-m',
  232. '--max_browsers_to_process',
  233. type=int,
  234. help='Maximum number of browsers to process. (Must be > 0).')
  235. args = parser.parse_args()
  236. if (args.max_browsers_to_process is not None and
  237. args.max_browsers_to_process <= 0):
  238. print('max_browsers_to_process must be > 0.')
  239. parser.print_help()
  240. sys.exit(1)
  241. main(args)