locale_tool.py 49 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512
  1. #!/usr/bin/env vpython3
  2. # Copyright 2019 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. """Helper script used to manage locale-related files in Chromium.
  6. This script is used to check, and potentially fix, many locale-related files
  7. in your Chromium workspace, such as:
  8. - GRIT input files (.grd) and the corresponding translations (.xtb).
  9. - BUILD.gn files listing Android localized resource string resource .xml
  10. generated by GRIT for all supported Chrome locales. These correspond to
  11. <output> elements that use the type="android" attribute.
  12. The --scan-dir <dir> option can be used to check for all files under a specific
  13. directory, and the --fix-inplace option can be used to try fixing any file
  14. that doesn't pass the check.
  15. This can be very handy to avoid tedious and repetitive work when adding new
  16. translations / locales to the Chrome code base, since this script can update
  17. said input files for you.
  18. Important note: checks and fix may fail on some input files. For example
  19. remoting/resources/remoting_strings.grd contains an in-line comment element
  20. inside its <outputs> section that breaks the script. The check will fail, and
  21. trying to fix it too, but at least the file will not be modified.
  22. """
  23. from __future__ import print_function
  24. import argparse
  25. import json
  26. import os
  27. import re
  28. import shutil
  29. import subprocess
  30. import sys
  31. import unittest
  32. # Assume this script is under build/
  33. _SCRIPT_DIR = os.path.dirname(__file__)
  34. _SCRIPT_NAME = os.path.join(_SCRIPT_DIR, os.path.basename(__file__))
  35. _TOP_SRC_DIR = os.path.join(_SCRIPT_DIR, '..')
  36. # Need to import android/gyp/util/resource_utils.py here.
  37. sys.path.insert(0, os.path.join(_SCRIPT_DIR, 'android/gyp'))
  38. from util import build_utils
  39. from util import resource_utils
  40. # This locale is the default and doesn't have translations.
  41. _DEFAULT_LOCALE = 'en-US'
  42. # Misc terminal codes to provide human friendly progress output.
  43. _CONSOLE_CODE_MOVE_CURSOR_TO_COLUMN_0 = '\x1b[0G'
  44. _CONSOLE_CODE_ERASE_LINE = '\x1b[K'
  45. _CONSOLE_START_LINE = (
  46. _CONSOLE_CODE_MOVE_CURSOR_TO_COLUMN_0 + _CONSOLE_CODE_ERASE_LINE)
  47. ##########################################################################
  48. ##########################################################################
  49. #####
  50. ##### G E N E R I C H E L P E R F U N C T I O N S
  51. #####
  52. ##########################################################################
  53. ##########################################################################
  54. def _FixChromiumLangAttribute(lang):
  55. """Map XML "lang" attribute values to Chromium locale names."""
  56. _CHROMIUM_LANG_FIXES = {
  57. 'en': 'en-US', # For now, Chromium doesn't have an 'en' locale.
  58. 'iw': 'he', # 'iw' is the obsolete form of ISO 639-1 for Hebrew
  59. 'no': 'nb', # 'no' is used by the Translation Console for Norwegian (nb).
  60. }
  61. return _CHROMIUM_LANG_FIXES.get(lang, lang)
  62. def _FixTranslationConsoleLocaleName(locale):
  63. _FIXES = {
  64. 'nb': 'no', # Norwegian.
  65. 'he': 'iw', # Hebrew
  66. }
  67. return _FIXES.get(locale, locale)
  68. def _CompareLocaleLists(list_a, list_expected, list_name):
  69. """Compare two lists of locale names. Print errors if they differ.
  70. Args:
  71. list_a: First list of locales.
  72. list_expected: Second list of locales, as expected.
  73. list_name: Name of list printed in error messages.
  74. Returns:
  75. On success, return False. On error, print error messages and return True.
  76. """
  77. errors = []
  78. missing_locales = sorted(set(list_a) - set(list_expected))
  79. if missing_locales:
  80. errors.append('Missing locales: %s' % missing_locales)
  81. extra_locales = sorted(set(list_expected) - set(list_a))
  82. if extra_locales:
  83. errors.append('Unexpected locales: %s' % extra_locales)
  84. if errors:
  85. print('Errors in %s definition:' % list_name)
  86. for error in errors:
  87. print(' %s\n' % error)
  88. return True
  89. return False
  90. def _BuildIntervalList(input_list, predicate):
  91. """Find ranges of contiguous list items that pass a given predicate.
  92. Args:
  93. input_list: An input list of items of any type.
  94. predicate: A function that takes a list item and return True if it
  95. passes a given test.
  96. Returns:
  97. A list of (start_pos, end_pos) tuples, where all items in
  98. [start_pos, end_pos) pass the predicate.
  99. """
  100. result = []
  101. size = len(input_list)
  102. start = 0
  103. while True:
  104. # Find first item in list that passes the predicate.
  105. while start < size and not predicate(input_list[start]):
  106. start += 1
  107. if start >= size:
  108. return result
  109. # Find first item in the rest of the list that does not pass the
  110. # predicate.
  111. end = start + 1
  112. while end < size and predicate(input_list[end]):
  113. end += 1
  114. result.append((start, end))
  115. start = end + 1
  116. def _SortListSubRange(input_list, start, end, key_func):
  117. """Sort an input list's sub-range according to a specific key function.
  118. Args:
  119. input_list: An input list.
  120. start: Sub-range starting position in list.
  121. end: Sub-range limit position in list.
  122. key_func: A function that extracts a sort key from a line.
  123. Returns:
  124. A copy of |input_list|, with all items in [|start|, |end|) sorted
  125. according to |key_func|.
  126. """
  127. result = input_list[:start]
  128. inputs = []
  129. for pos in xrange(start, end):
  130. line = input_list[pos]
  131. key = key_func(line)
  132. inputs.append((key, line))
  133. for _, line in sorted(inputs):
  134. result.append(line)
  135. result += input_list[end:]
  136. return result
  137. def _SortElementsRanges(lines, element_predicate, element_key):
  138. """Sort all elements of a given type in a list of lines by a given key.
  139. Args:
  140. lines: input lines.
  141. element_predicate: predicate function to select elements to sort.
  142. element_key: lambda returning a comparison key for each element that
  143. passes the predicate.
  144. Returns:
  145. A new list of input lines, with lines [start..end) sorted.
  146. """
  147. intervals = _BuildIntervalList(lines, element_predicate)
  148. for start, end in intervals:
  149. lines = _SortListSubRange(lines, start, end, element_key)
  150. return lines
  151. def _ProcessFile(input_file, locales, check_func, fix_func):
  152. """Process a given input file, potentially fixing it.
  153. Args:
  154. input_file: Input file path.
  155. locales: List of Chrome locales to consider / expect.
  156. check_func: A lambda called to check the input file lines with
  157. (input_lines, locales) argument. It must return an list of error
  158. messages, or None on success.
  159. fix_func: None, or a lambda called to fix the input file lines with
  160. (input_lines, locales). It must return the new list of lines for
  161. the input file, and may raise an Exception in case of error.
  162. Returns:
  163. True at the moment.
  164. """
  165. print('%sProcessing %s...' % (_CONSOLE_START_LINE, input_file), end=' ')
  166. sys.stdout.flush()
  167. with open(input_file) as f:
  168. input_lines = f.readlines()
  169. errors = check_func(input_file, input_lines, locales)
  170. if errors:
  171. print('\n%s%s' % (_CONSOLE_START_LINE, '\n'.join(errors)))
  172. if fix_func:
  173. try:
  174. input_lines = fix_func(input_file, input_lines, locales)
  175. output = ''.join(input_lines)
  176. with open(input_file, 'wt') as f:
  177. f.write(output)
  178. print('Fixed %s.' % input_file)
  179. except Exception as e: # pylint: disable=broad-except
  180. print('Skipped %s: %s' % (input_file, e))
  181. return True
  182. def _ScanDirectoriesForFiles(scan_dirs, file_predicate):
  183. """Scan a directory for files that match a given predicate.
  184. Args:
  185. scan_dir: A list of top-level directories to start scan in.
  186. file_predicate: lambda function which is passed the file's base name
  187. and returns True if its full path, relative to |scan_dir|, should be
  188. passed in the result.
  189. Returns:
  190. A list of file full paths.
  191. """
  192. result = []
  193. for src_dir in scan_dirs:
  194. for root, _, files in os.walk(src_dir):
  195. result.extend(os.path.join(root, f) for f in files if file_predicate(f))
  196. return result
  197. def _WriteFile(file_path, file_data):
  198. """Write |file_data| to |file_path|."""
  199. with open(file_path, 'w') as f:
  200. f.write(file_data)
  201. def _FindGnExecutable():
  202. """Locate the real GN executable used by this Chromium checkout.
  203. This is needed because the depot_tools 'gn' wrapper script will look
  204. for .gclient and other things we really don't need here.
  205. Returns:
  206. Path of real host GN executable from current Chromium src/ checkout.
  207. """
  208. # Simply scan buildtools/*/gn and return the first one found so we don't
  209. # have to guess the platform-specific sub-directory name (e.g. 'linux64'
  210. # for 64-bit Linux machines).
  211. buildtools_dir = os.path.join(_TOP_SRC_DIR, 'buildtools')
  212. for subdir in os.listdir(buildtools_dir):
  213. subdir_path = os.path.join(buildtools_dir, subdir)
  214. if not os.path.isdir(subdir_path):
  215. continue
  216. gn_path = os.path.join(subdir_path, 'gn')
  217. if os.path.exists(gn_path):
  218. return gn_path
  219. return None
  220. def _PrettyPrintListAsLines(input_list, available_width, trailing_comma=False):
  221. result = []
  222. input_str = ', '.join(input_list)
  223. while len(input_str) > available_width:
  224. pos = input_str.rfind(',', 0, available_width)
  225. result.append(input_str[:pos + 1])
  226. input_str = input_str[pos + 1:].lstrip()
  227. if trailing_comma and input_str:
  228. input_str += ','
  229. result.append(input_str)
  230. return result
  231. class _PrettyPrintListAsLinesTest(unittest.TestCase):
  232. def test_empty_list(self):
  233. self.assertListEqual([''], _PrettyPrintListAsLines([], 10))
  234. def test_wrapping(self):
  235. input_list = ['foo', 'bar', 'zoo', 'tool']
  236. self.assertListEqual(
  237. _PrettyPrintListAsLines(input_list, 8),
  238. ['foo,', 'bar,', 'zoo,', 'tool'])
  239. self.assertListEqual(
  240. _PrettyPrintListAsLines(input_list, 12), ['foo, bar,', 'zoo, tool'])
  241. self.assertListEqual(
  242. _PrettyPrintListAsLines(input_list, 79), ['foo, bar, zoo, tool'])
  243. def test_trailing_comma(self):
  244. input_list = ['foo', 'bar', 'zoo', 'tool']
  245. self.assertListEqual(
  246. _PrettyPrintListAsLines(input_list, 8, trailing_comma=True),
  247. ['foo,', 'bar,', 'zoo,', 'tool,'])
  248. self.assertListEqual(
  249. _PrettyPrintListAsLines(input_list, 12, trailing_comma=True),
  250. ['foo, bar,', 'zoo, tool,'])
  251. self.assertListEqual(
  252. _PrettyPrintListAsLines(input_list, 79, trailing_comma=True),
  253. ['foo, bar, zoo, tool,'])
  254. ##########################################################################
  255. ##########################################################################
  256. #####
  257. ##### L O C A L E S L I S T S
  258. #####
  259. ##########################################################################
  260. ##########################################################################
  261. # Various list of locales that will be extracted from build/config/locales.gni
  262. # Do not use these directly, use ChromeLocales(), and IosUnsupportedLocales()
  263. # instead to access these lists.
  264. _INTERNAL_CHROME_LOCALES = []
  265. _INTERNAL_IOS_UNSUPPORTED_LOCALES = []
  266. def ChromeLocales():
  267. """Return the list of all locales supported by Chrome."""
  268. if not _INTERNAL_CHROME_LOCALES:
  269. _ExtractAllChromeLocalesLists()
  270. return _INTERNAL_CHROME_LOCALES
  271. def IosUnsupportedLocales():
  272. """Return the list of locales that are unsupported on iOS."""
  273. if not _INTERNAL_IOS_UNSUPPORTED_LOCALES:
  274. _ExtractAllChromeLocalesLists()
  275. return _INTERNAL_IOS_UNSUPPORTED_LOCALES
  276. def _PrepareTinyGnWorkspace(work_dir, out_subdir_name='out'):
  277. """Populate an empty directory with a tiny set of working GN config files.
  278. This allows us to run 'gn gen <out> --root <work_dir>' as fast as possible
  279. to generate files containing the locales list. This takes about 300ms on
  280. a decent machine, instead of more than 5 seconds when running the equivalent
  281. commands from a real Chromium workspace, which requires regenerating more
  282. than 23k targets.
  283. Args:
  284. work_dir: target working directory.
  285. out_subdir_name: Name of output sub-directory.
  286. Returns:
  287. Full path of output directory created inside |work_dir|.
  288. """
  289. # Create top-level .gn file that must point to the BUILDCONFIG.gn.
  290. _WriteFile(os.path.join(work_dir, '.gn'),
  291. 'buildconfig = "//BUILDCONFIG.gn"\n')
  292. # Create BUILDCONFIG.gn which must set a default toolchain. Also add
  293. # all variables that may be used in locales.gni in a declare_args() block.
  294. _WriteFile(
  295. os.path.join(work_dir, 'BUILDCONFIG.gn'),
  296. r'''set_default_toolchain("toolchain")
  297. declare_args () {
  298. is_ios = false
  299. is_android = true
  300. }
  301. ''')
  302. # Create fake toolchain required by BUILDCONFIG.gn.
  303. os.mkdir(os.path.join(work_dir, 'toolchain'))
  304. _WriteFile(os.path.join(work_dir, 'toolchain', 'BUILD.gn'),
  305. r'''toolchain("toolchain") {
  306. tool("stamp") {
  307. command = "touch {{output}}" # Required by action()
  308. }
  309. }
  310. ''')
  311. # Create top-level BUILD.gn, GN requires at least one target to build so do
  312. # that with a fake action which will never be invoked. Also write the locales
  313. # to misc files in the output directory.
  314. _WriteFile(
  315. os.path.join(work_dir, 'BUILD.gn'), r'''import("//locales.gni")
  316. action("create_foo") { # fake action to avoid GN complaints.
  317. script = "//build/create_foo.py"
  318. inputs = []
  319. outputs = [ "$target_out_dir/$target_name" ]
  320. }
  321. # Write the locales lists to files in the output directory.
  322. _filename = root_build_dir + "/foo"
  323. write_file(_filename + ".locales", locales, "json")
  324. write_file(_filename + ".ios_unsupported_locales",
  325. ios_unsupported_locales,
  326. "json")
  327. ''')
  328. # Copy build/config/locales.gni to the workspace, as required by BUILD.gn.
  329. shutil.copyfile(os.path.join(_TOP_SRC_DIR, 'build', 'config', 'locales.gni'),
  330. os.path.join(work_dir, 'locales.gni'))
  331. # Create output directory.
  332. out_path = os.path.join(work_dir, out_subdir_name)
  333. os.mkdir(out_path)
  334. # And ... we're good.
  335. return out_path
  336. # Set this global variable to the path of a given temporary directory
  337. # before calling _ExtractAllChromeLocalesLists() if you want to debug
  338. # the locales list extraction process.
  339. _DEBUG_LOCALES_WORK_DIR = None
  340. def _ReadJsonList(file_path):
  341. """Read a JSON file that must contain a list, and return it."""
  342. with open(file_path) as f:
  343. data = json.load(f)
  344. assert isinstance(data, list), "JSON file %s is not a list!" % file_path
  345. return [item.encode('utf8') for item in data]
  346. def _ExtractAllChromeLocalesLists():
  347. with build_utils.TempDir() as tmp_path:
  348. if _DEBUG_LOCALES_WORK_DIR:
  349. tmp_path = _DEBUG_LOCALES_WORK_DIR
  350. build_utils.DeleteDirectory(tmp_path)
  351. build_utils.MakeDirectory(tmp_path)
  352. out_path = _PrepareTinyGnWorkspace(tmp_path, 'out')
  353. # NOTE: The file suffixes used here should be kept in sync with
  354. # build/config/locales.gni
  355. gn_executable = _FindGnExecutable()
  356. try:
  357. subprocess.check_output(
  358. [gn_executable, 'gen', out_path, '--root=' + tmp_path])
  359. except subprocess.CalledProcessError as e:
  360. print(e.output)
  361. raise e
  362. global _INTERNAL_CHROME_LOCALES
  363. _INTERNAL_CHROME_LOCALES = _ReadJsonList(
  364. os.path.join(out_path, 'foo.locales'))
  365. global _INTERNAL_IOS_UNSUPPORTED_LOCALES
  366. _INTERNAL_IOS_UNSUPPORTED_LOCALES = _ReadJsonList(
  367. os.path.join(out_path, 'foo.ios_unsupported_locales'))
  368. ##########################################################################
  369. ##########################################################################
  370. #####
  371. ##### G R D H E L P E R F U N C T I O N S
  372. #####
  373. ##########################################################################
  374. ##########################################################################
  375. # Technical note:
  376. #
  377. # Even though .grd files are XML, an xml parser library is not used in order
  378. # to preserve the original file's structure after modification. ElementTree
  379. # tends to re-order attributes in each element when re-writing an XML
  380. # document tree, which is undesirable here.
  381. #
  382. # Thus simple line-based regular expression matching is used instead.
  383. #
  384. # Misc regular expressions used to match elements and their attributes.
  385. _RE_OUTPUT_ELEMENT = re.compile(r'<output (.*)\s*/>')
  386. _RE_TRANSLATION_ELEMENT = re.compile(r'<file( | .* )path="(.*\.xtb)".*/>')
  387. _RE_FILENAME_ATTRIBUTE = re.compile(r'filename="([^"]*)"')
  388. _RE_LANG_ATTRIBUTE = re.compile(r'lang="([^"]*)"')
  389. _RE_PATH_ATTRIBUTE = re.compile(r'path="([^"]*)"')
  390. _RE_TYPE_ANDROID_ATTRIBUTE = re.compile(r'type="android"')
  391. def _IsGritInputFile(input_file):
  392. """Returns True iff this is a GRIT input file."""
  393. return input_file.endswith('.grd')
  394. def _GetXmlLangAttribute(xml_line):
  395. """Extract the lang attribute value from an XML input line."""
  396. m = _RE_LANG_ATTRIBUTE.search(xml_line)
  397. if not m:
  398. return None
  399. return m.group(1)
  400. class _GetXmlLangAttributeTest(unittest.TestCase):
  401. TEST_DATA = {
  402. '': None,
  403. 'foo': None,
  404. 'lang=foo': None,
  405. 'lang="foo"': 'foo',
  406. '<something lang="foo bar" />': 'foo bar',
  407. '<file lang="fr-CA" path="path/to/strings_fr-CA.xtb" />': 'fr-CA',
  408. }
  409. def test_GetXmlLangAttribute(self):
  410. for test_line, expected in self.TEST_DATA.iteritems():
  411. self.assertEquals(_GetXmlLangAttribute(test_line), expected)
  412. def _SortGrdElementsRanges(grd_lines, element_predicate):
  413. """Sort all .grd elements of a given type by their lang attribute."""
  414. return _SortElementsRanges(grd_lines, element_predicate, _GetXmlLangAttribute)
  415. def _CheckGrdElementRangeLang(grd_lines, start, end, wanted_locales):
  416. """Check the element 'lang' attributes in specific .grd lines range.
  417. This really checks the following:
  418. - Each item has a correct 'lang' attribute.
  419. - There are no duplicated lines for the same 'lang' attribute.
  420. - That there are no extra locales that Chromium doesn't want.
  421. - That no wanted locale is missing.
  422. Args:
  423. grd_lines: Input .grd lines.
  424. start: Sub-range start position in input line list.
  425. end: Sub-range limit position in input line list.
  426. wanted_locales: Set of wanted Chromium locale names.
  427. Returns:
  428. List of error message strings for this input. Empty on success.
  429. """
  430. errors = []
  431. locales = set()
  432. for pos in xrange(start, end):
  433. line = grd_lines[pos]
  434. lang = _GetXmlLangAttribute(line)
  435. if not lang:
  436. errors.append('%d: Missing "lang" attribute in <output> element' % pos +
  437. 1)
  438. continue
  439. cr_locale = _FixChromiumLangAttribute(lang)
  440. if cr_locale in locales:
  441. errors.append(
  442. '%d: Redefinition of <output> for "%s" locale' % (pos + 1, lang))
  443. locales.add(cr_locale)
  444. extra_locales = locales.difference(wanted_locales)
  445. if extra_locales:
  446. errors.append('%d-%d: Extra locales found: %s' % (start + 1, end + 1,
  447. sorted(extra_locales)))
  448. missing_locales = wanted_locales.difference(locales)
  449. if missing_locales:
  450. errors.append('%d-%d: Missing locales: %s' % (start + 1, end + 1,
  451. sorted(missing_locales)))
  452. return errors
  453. ##########################################################################
  454. ##########################################################################
  455. #####
  456. ##### G R D A N D R O I D O U T P U T S
  457. #####
  458. ##########################################################################
  459. ##########################################################################
  460. def _IsGrdAndroidOutputLine(line):
  461. """Returns True iff this is an Android-specific <output> line."""
  462. m = _RE_OUTPUT_ELEMENT.search(line)
  463. if m:
  464. return 'type="android"' in m.group(1)
  465. return False
  466. assert _IsGrdAndroidOutputLine(' <output type="android"/>')
  467. # Many of the functions below have unused arguments due to genericity.
  468. # pylint: disable=unused-argument
  469. def _CheckGrdElementRangeAndroidOutputFilename(grd_lines, start, end,
  470. wanted_locales):
  471. """Check all <output> elements in specific input .grd lines range.
  472. This really checks the following:
  473. - Filenames exist for each listed locale.
  474. - Filenames are well-formed.
  475. Args:
  476. grd_lines: Input .grd lines.
  477. start: Sub-range start position in input line list.
  478. end: Sub-range limit position in input line list.
  479. wanted_locales: Set of wanted Chromium locale names.
  480. Returns:
  481. List of error message strings for this input. Empty on success.
  482. """
  483. errors = []
  484. for pos in xrange(start, end):
  485. line = grd_lines[pos]
  486. lang = _GetXmlLangAttribute(line)
  487. if not lang:
  488. continue
  489. cr_locale = _FixChromiumLangAttribute(lang)
  490. m = _RE_FILENAME_ATTRIBUTE.search(line)
  491. if not m:
  492. errors.append('%d: Missing filename attribute in <output> element' % pos +
  493. 1)
  494. else:
  495. filename = m.group(1)
  496. if not filename.endswith('.xml'):
  497. errors.append(
  498. '%d: Filename should end with ".xml": %s' % (pos + 1, filename))
  499. dirname = os.path.basename(os.path.dirname(filename))
  500. prefix = ('values-%s' % resource_utils.ToAndroidLocaleName(cr_locale)
  501. if cr_locale != _DEFAULT_LOCALE else 'values')
  502. if dirname != prefix:
  503. errors.append(
  504. '%s: Directory name should be %s: %s' % (pos + 1, prefix, filename))
  505. return errors
  506. def _CheckGrdAndroidOutputElements(grd_file, grd_lines, wanted_locales):
  507. """Check all <output> elements related to Android.
  508. Args:
  509. grd_file: Input .grd file path.
  510. grd_lines: List of input .grd lines.
  511. wanted_locales: set of wanted Chromium locale names.
  512. Returns:
  513. List of error message strings. Empty on success.
  514. """
  515. intervals = _BuildIntervalList(grd_lines, _IsGrdAndroidOutputLine)
  516. errors = []
  517. for start, end in intervals:
  518. errors += _CheckGrdElementRangeLang(grd_lines, start, end, wanted_locales)
  519. errors += _CheckGrdElementRangeAndroidOutputFilename(grd_lines, start, end,
  520. wanted_locales)
  521. return errors
  522. def _AddMissingLocalesInGrdAndroidOutputs(grd_file, grd_lines, wanted_locales):
  523. """Fix an input .grd line by adding missing Android outputs.
  524. Args:
  525. grd_file: Input .grd file path.
  526. grd_lines: Input .grd line list.
  527. wanted_locales: set of Chromium locale names.
  528. Returns:
  529. A new list of .grd lines, containing new <output> elements when needed
  530. for locales from |wanted_locales| that were not part of the input.
  531. """
  532. intervals = _BuildIntervalList(grd_lines, _IsGrdAndroidOutputLine)
  533. for start, end in reversed(intervals):
  534. locales = set()
  535. for pos in xrange(start, end):
  536. lang = _GetXmlLangAttribute(grd_lines[pos])
  537. locale = _FixChromiumLangAttribute(lang)
  538. locales.add(locale)
  539. missing_locales = wanted_locales.difference(locales)
  540. if not missing_locales:
  541. continue
  542. src_locale = 'bg'
  543. src_lang_attribute = 'lang="%s"' % src_locale
  544. src_line = None
  545. for pos in xrange(start, end):
  546. if src_lang_attribute in grd_lines[pos]:
  547. src_line = grd_lines[pos]
  548. break
  549. if not src_line:
  550. raise Exception(
  551. 'Cannot find <output> element with "%s" lang attribute' % src_locale)
  552. line_count = end - 1
  553. for locale in missing_locales:
  554. android_locale = resource_utils.ToAndroidLocaleName(locale)
  555. dst_line = src_line.replace(
  556. 'lang="%s"' % src_locale, 'lang="%s"' % locale).replace(
  557. 'values-%s/' % src_locale, 'values-%s/' % android_locale)
  558. grd_lines.insert(line_count, dst_line)
  559. line_count += 1
  560. # Sort the new <output> elements.
  561. return _SortGrdElementsRanges(grd_lines, _IsGrdAndroidOutputLine)
  562. ##########################################################################
  563. ##########################################################################
  564. #####
  565. ##### G R D T R A N S L A T I O N S
  566. #####
  567. ##########################################################################
  568. ##########################################################################
  569. def _IsTranslationGrdOutputLine(line):
  570. """Returns True iff this is an output .xtb <file> element."""
  571. m = _RE_TRANSLATION_ELEMENT.search(line)
  572. return m is not None
  573. class _IsTranslationGrdOutputLineTest(unittest.TestCase):
  574. def test_GrdTranslationOutputLines(self):
  575. _VALID_INPUT_LINES = [
  576. '<file path="foo/bar.xtb" />',
  577. '<file path="foo/bar.xtb"/>',
  578. '<file lang="fr-CA" path="translations/aw_strings_fr-CA.xtb"/>',
  579. '<file lang="fr-CA" path="translations/aw_strings_fr-CA.xtb" />',
  580. ' <file path="translations/aw_strings_ar.xtb" lang="ar" />',
  581. ]
  582. _INVALID_INPUT_LINES = ['<file path="foo/bar.xml" />']
  583. for line in _VALID_INPUT_LINES:
  584. self.assertTrue(
  585. _IsTranslationGrdOutputLine(line),
  586. '_IsTranslationGrdOutputLine() returned False for [%s]' % line)
  587. for line in _INVALID_INPUT_LINES:
  588. self.assertFalse(
  589. _IsTranslationGrdOutputLine(line),
  590. '_IsTranslationGrdOutputLine() returned True for [%s]' % line)
  591. def _CheckGrdTranslationElementRange(grd_lines, start, end,
  592. wanted_locales):
  593. """Check all <translations> sub-elements in specific input .grd lines range.
  594. This really checks the following:
  595. - Each item has a 'path' attribute.
  596. - Each such path value ends up with '.xtb'.
  597. Args:
  598. grd_lines: Input .grd lines.
  599. start: Sub-range start position in input line list.
  600. end: Sub-range limit position in input line list.
  601. wanted_locales: Set of wanted Chromium locale names.
  602. Returns:
  603. List of error message strings for this input. Empty on success.
  604. """
  605. errors = []
  606. for pos in xrange(start, end):
  607. line = grd_lines[pos]
  608. lang = _GetXmlLangAttribute(line)
  609. if not lang:
  610. continue
  611. m = _RE_PATH_ATTRIBUTE.search(line)
  612. if not m:
  613. errors.append('%d: Missing path attribute in <file> element' % pos +
  614. 1)
  615. else:
  616. filename = m.group(1)
  617. if not filename.endswith('.xtb'):
  618. errors.append(
  619. '%d: Path should end with ".xtb": %s' % (pos + 1, filename))
  620. return errors
  621. def _CheckGrdTranslations(grd_file, grd_lines, wanted_locales):
  622. """Check all <file> elements that correspond to an .xtb output file.
  623. Args:
  624. grd_file: Input .grd file path.
  625. grd_lines: List of input .grd lines.
  626. wanted_locales: set of wanted Chromium locale names.
  627. Returns:
  628. List of error message strings. Empty on success.
  629. """
  630. wanted_locales = wanted_locales - set([_DEFAULT_LOCALE])
  631. intervals = _BuildIntervalList(grd_lines, _IsTranslationGrdOutputLine)
  632. errors = []
  633. for start, end in intervals:
  634. errors += _CheckGrdElementRangeLang(grd_lines, start, end, wanted_locales)
  635. errors += _CheckGrdTranslationElementRange(grd_lines, start, end,
  636. wanted_locales)
  637. return errors
  638. # Regular expression used to replace the lang attribute inside .xtb files.
  639. _RE_TRANSLATIONBUNDLE = re.compile('<translationbundle lang="(.*)">')
  640. def _CreateFakeXtbFileFrom(src_xtb_path, dst_xtb_path, dst_locale):
  641. """Create a fake .xtb file.
  642. Args:
  643. src_xtb_path: Path to source .xtb file to copy from.
  644. dst_xtb_path: Path to destination .xtb file to write to.
  645. dst_locale: Destination locale, the lang attribute in the source file
  646. will be substituted with this value before its lines are written
  647. to the destination file.
  648. """
  649. with open(src_xtb_path) as f:
  650. src_xtb_lines = f.readlines()
  651. def replace_xtb_lang_attribute(line):
  652. m = _RE_TRANSLATIONBUNDLE.search(line)
  653. if not m:
  654. return line
  655. return line[:m.start(1)] + dst_locale + line[m.end(1):]
  656. dst_xtb_lines = [replace_xtb_lang_attribute(line) for line in src_xtb_lines]
  657. with build_utils.AtomicOutput(dst_xtb_path) as tmp:
  658. tmp.writelines(dst_xtb_lines)
  659. def _AddMissingLocalesInGrdTranslations(grd_file, grd_lines, wanted_locales):
  660. """Fix an input .grd line by adding missing Android outputs.
  661. This also creates fake .xtb files from the one provided for 'en-GB'.
  662. Args:
  663. grd_file: Input .grd file path.
  664. grd_lines: Input .grd line list.
  665. wanted_locales: set of Chromium locale names.
  666. Returns:
  667. A new list of .grd lines, containing new <output> elements when needed
  668. for locales from |wanted_locales| that were not part of the input.
  669. """
  670. wanted_locales = wanted_locales - set([_DEFAULT_LOCALE])
  671. intervals = _BuildIntervalList(grd_lines, _IsTranslationGrdOutputLine)
  672. for start, end in reversed(intervals):
  673. locales = set()
  674. for pos in xrange(start, end):
  675. lang = _GetXmlLangAttribute(grd_lines[pos])
  676. locale = _FixChromiumLangAttribute(lang)
  677. locales.add(locale)
  678. missing_locales = wanted_locales.difference(locales)
  679. if not missing_locales:
  680. continue
  681. src_locale = 'en-GB'
  682. src_lang_attribute = 'lang="%s"' % src_locale
  683. src_line = None
  684. for pos in xrange(start, end):
  685. if src_lang_attribute in grd_lines[pos]:
  686. src_line = grd_lines[pos]
  687. break
  688. if not src_line:
  689. raise Exception(
  690. 'Cannot find <file> element with "%s" lang attribute' % src_locale)
  691. src_path = os.path.join(
  692. os.path.dirname(grd_file),
  693. _RE_PATH_ATTRIBUTE.search(src_line).group(1))
  694. line_count = end - 1
  695. for locale in missing_locales:
  696. dst_line = src_line.replace(
  697. 'lang="%s"' % src_locale, 'lang="%s"' % locale).replace(
  698. '_%s.xtb' % src_locale, '_%s.xtb' % locale)
  699. grd_lines.insert(line_count, dst_line)
  700. line_count += 1
  701. dst_path = src_path.replace('_%s.xtb' % src_locale, '_%s.xtb' % locale)
  702. _CreateFakeXtbFileFrom(src_path, dst_path, locale)
  703. # Sort the new <output> elements.
  704. return _SortGrdElementsRanges(grd_lines, _IsTranslationGrdOutputLine)
  705. ##########################################################################
  706. ##########################################################################
  707. #####
  708. ##### G N A N D R O I D O U T P U T S
  709. #####
  710. ##########################################################################
  711. ##########################################################################
  712. _RE_GN_VALUES_LIST_LINE = re.compile(
  713. r'^\s*".*values(\-([A-Za-z0-9-]+))?/.*\.xml",\s*$')
  714. def _IsBuildGnInputFile(input_file):
  715. """Returns True iff this is a BUILD.gn file."""
  716. return os.path.basename(input_file) == 'BUILD.gn'
  717. def _GetAndroidGnOutputLocale(line):
  718. """Check a GN list, and return its Android locale if it is an output .xml"""
  719. m = _RE_GN_VALUES_LIST_LINE.match(line)
  720. if not m:
  721. return None
  722. if m.group(1): # First group is optional and contains group 2.
  723. return m.group(2)
  724. return resource_utils.ToAndroidLocaleName(_DEFAULT_LOCALE)
  725. def _IsAndroidGnOutputLine(line):
  726. """Returns True iff this is an Android-specific localized .xml output."""
  727. return _GetAndroidGnOutputLocale(line) != None
  728. def _CheckGnOutputsRangeForLocalizedStrings(gn_lines, start, end):
  729. """Check that a range of GN lines corresponds to localized strings.
  730. Special case: Some BUILD.gn files list several non-localized .xml files
  731. that should be ignored by this function, e.g. in
  732. components/cronet/android/BUILD.gn, the following appears:
  733. inputs = [
  734. ...
  735. "sample/res/layout/activity_main.xml",
  736. "sample/res/layout/dialog_url.xml",
  737. "sample/res/values/dimens.xml",
  738. "sample/res/values/strings.xml",
  739. ...
  740. ]
  741. These are non-localized strings, and should be ignored. This function is
  742. used to detect them quickly.
  743. """
  744. for pos in xrange(start, end):
  745. if not 'values/' in gn_lines[pos]:
  746. return True
  747. return False
  748. def _CheckGnOutputsRange(gn_lines, start, end, wanted_locales):
  749. if not _CheckGnOutputsRangeForLocalizedStrings(gn_lines, start, end):
  750. return []
  751. errors = []
  752. locales = set()
  753. for pos in xrange(start, end):
  754. line = gn_lines[pos]
  755. android_locale = _GetAndroidGnOutputLocale(line)
  756. assert android_locale != None
  757. cr_locale = resource_utils.ToChromiumLocaleName(android_locale)
  758. if cr_locale in locales:
  759. errors.append('%s: Redefinition of output for "%s" locale' %
  760. (pos + 1, android_locale))
  761. locales.add(cr_locale)
  762. extra_locales = locales.difference(wanted_locales)
  763. if extra_locales:
  764. errors.append('%d-%d: Extra locales: %s' % (start + 1, end + 1,
  765. sorted(extra_locales)))
  766. missing_locales = wanted_locales.difference(locales)
  767. if missing_locales:
  768. errors.append('%d-%d: Missing locales: %s' % (start + 1, end + 1,
  769. sorted(missing_locales)))
  770. return errors
  771. def _CheckGnAndroidOutputs(gn_file, gn_lines, wanted_locales):
  772. intervals = _BuildIntervalList(gn_lines, _IsAndroidGnOutputLine)
  773. errors = []
  774. for start, end in intervals:
  775. errors += _CheckGnOutputsRange(gn_lines, start, end, wanted_locales)
  776. return errors
  777. def _AddMissingLocalesInGnAndroidOutputs(gn_file, gn_lines, wanted_locales):
  778. intervals = _BuildIntervalList(gn_lines, _IsAndroidGnOutputLine)
  779. # NOTE: Since this may insert new lines to each interval, process the
  780. # list in reverse order to maintain valid (start,end) positions during
  781. # the iteration.
  782. for start, end in reversed(intervals):
  783. if not _CheckGnOutputsRangeForLocalizedStrings(gn_lines, start, end):
  784. continue
  785. locales = set()
  786. for pos in xrange(start, end):
  787. lang = _GetAndroidGnOutputLocale(gn_lines[pos])
  788. locale = resource_utils.ToChromiumLocaleName(lang)
  789. locales.add(locale)
  790. missing_locales = wanted_locales.difference(locales)
  791. if not missing_locales:
  792. continue
  793. src_locale = 'bg'
  794. src_values = 'values-%s/' % resource_utils.ToAndroidLocaleName(src_locale)
  795. src_line = None
  796. for pos in xrange(start, end):
  797. if src_values in gn_lines[pos]:
  798. src_line = gn_lines[pos]
  799. break
  800. if not src_line:
  801. raise Exception(
  802. 'Cannot find output list item with "%s" locale' % src_locale)
  803. line_count = end - 1
  804. for locale in missing_locales:
  805. if locale == _DEFAULT_LOCALE:
  806. dst_line = src_line.replace('values-%s/' % src_locale, 'values/')
  807. else:
  808. dst_line = src_line.replace(
  809. 'values-%s/' % src_locale,
  810. 'values-%s/' % resource_utils.ToAndroidLocaleName(locale))
  811. gn_lines.insert(line_count, dst_line)
  812. line_count += 1
  813. gn_lines = _SortListSubRange(
  814. gn_lines, start, line_count,
  815. lambda line: _RE_GN_VALUES_LIST_LINE.match(line).group(1))
  816. return gn_lines
  817. ##########################################################################
  818. ##########################################################################
  819. #####
  820. ##### T R A N S L A T I O N E X P E C T A T I O N S
  821. #####
  822. ##########################################################################
  823. ##########################################################################
  824. _EXPECTATIONS_FILENAME = 'translation_expectations.pyl'
  825. # Technical note: the format of translation_expectations.pyl
  826. # is a 'Python literal', which defines a python dictionary, so should
  827. # be easy to parse. However, when modifying it, care should be taken
  828. # to respect the line comments and the order of keys within the text
  829. # file.
  830. def _ReadPythonLiteralFile(pyl_path):
  831. """Read a .pyl file into a Python data structure."""
  832. with open(pyl_path) as f:
  833. pyl_content = f.read()
  834. # Evaluate as a Python data structure, use an empty global
  835. # and local dictionary.
  836. return eval(pyl_content, dict(), dict())
  837. def _UpdateLocalesInExpectationLines(pyl_lines,
  838. wanted_locales,
  839. available_width=79):
  840. """Update the locales list(s) found in an expectations file.
  841. Args:
  842. pyl_lines: Iterable of input lines from the file.
  843. wanted_locales: Set or list of new locale names.
  844. available_width: Optional, number of character colums used
  845. to word-wrap the new list items.
  846. Returns:
  847. New list of updated lines.
  848. """
  849. locales_list = ['"%s"' % loc for loc in sorted(wanted_locales)]
  850. result = []
  851. line_count = len(pyl_lines)
  852. line_num = 0
  853. DICT_START = '"languages": ['
  854. while line_num < line_count:
  855. line = pyl_lines[line_num]
  856. line_num += 1
  857. result.append(line)
  858. # Look for start of "languages" dictionary.
  859. pos = line.find(DICT_START)
  860. if pos < 0:
  861. continue
  862. start_margin = pos
  863. start_line = line_num
  864. # Skip over all lines from the list.
  865. while (line_num < line_count and
  866. not pyl_lines[line_num].rstrip().endswith('],')):
  867. line_num += 1
  868. continue
  869. if line_num == line_count:
  870. raise Exception('%d: Missing list termination!' % start_line)
  871. # Format the new list according to the new margin.
  872. locale_width = available_width - (start_margin + 2)
  873. locale_lines = _PrettyPrintListAsLines(
  874. locales_list, locale_width, trailing_comma=True)
  875. for locale_line in locale_lines:
  876. result.append(' ' * (start_margin + 2) + locale_line)
  877. result.append(' ' * start_margin + '],')
  878. line_num += 1
  879. return result
  880. class _UpdateLocalesInExpectationLinesTest(unittest.TestCase):
  881. def test_simple(self):
  882. self.maxDiff = 1000
  883. input_text = r'''
  884. # This comment should be preserved
  885. # 23456789012345678901234567890123456789
  886. {
  887. "android_grd": {
  888. "languages": [
  889. "aa", "bb", "cc", "dd", "ee",
  890. "ff", "gg", "hh", "ii", "jj",
  891. "kk"],
  892. },
  893. # Example with bad indentation in input.
  894. "another_grd": {
  895. "languages": [
  896. "aa", "bb", "cc", "dd", "ee", "ff", "gg", "hh", "ii", "jj", "kk",
  897. ],
  898. },
  899. }
  900. '''
  901. expected_text = r'''
  902. # This comment should be preserved
  903. # 23456789012345678901234567890123456789
  904. {
  905. "android_grd": {
  906. "languages": [
  907. "A2", "AA", "BB", "CC", "DD",
  908. "E2", "EE", "FF", "GG", "HH",
  909. "I2", "II", "JJ", "KK",
  910. ],
  911. },
  912. # Example with bad indentation in input.
  913. "another_grd": {
  914. "languages": [
  915. "A2", "AA", "BB", "CC", "DD",
  916. "E2", "EE", "FF", "GG", "HH",
  917. "I2", "II", "JJ", "KK",
  918. ],
  919. },
  920. }
  921. '''
  922. input_lines = input_text.splitlines()
  923. test_locales = ([
  924. 'AA', 'BB', 'CC', 'DD', 'EE', 'FF', 'GG', 'HH', 'II', 'JJ', 'KK', 'A2',
  925. 'E2', 'I2'
  926. ])
  927. expected_lines = expected_text.splitlines()
  928. self.assertListEqual(
  929. _UpdateLocalesInExpectationLines(input_lines, test_locales, 40),
  930. expected_lines)
  931. def test_missing_list_termination(self):
  932. input_lines = r'''
  933. "languages": ['
  934. "aa", "bb", "cc", "dd"
  935. '''.splitlines()
  936. with self.assertRaises(Exception) as cm:
  937. _UpdateLocalesInExpectationLines(input_lines, ['a', 'b'], 40)
  938. self.assertEqual(str(cm.exception), '2: Missing list termination!')
  939. def _UpdateLocalesInExpectationFile(pyl_path, wanted_locales):
  940. """Update all locales listed in a given expectations file.
  941. Args:
  942. pyl_path: Path to .pyl file to update.
  943. wanted_locales: List of locales that need to be written to
  944. the file.
  945. """
  946. tc_locales = {
  947. _FixTranslationConsoleLocaleName(locale)
  948. for locale in set(wanted_locales) - set([_DEFAULT_LOCALE])
  949. }
  950. with open(pyl_path) as f:
  951. input_lines = [l.rstrip() for l in f.readlines()]
  952. updated_lines = _UpdateLocalesInExpectationLines(input_lines, tc_locales)
  953. with build_utils.AtomicOutput(pyl_path) as f:
  954. f.writelines('\n'.join(updated_lines) + '\n')
  955. ##########################################################################
  956. ##########################################################################
  957. #####
  958. ##### C H E C K E V E R Y T H I N G
  959. #####
  960. ##########################################################################
  961. ##########################################################################
  962. # pylint: enable=unused-argument
  963. def _IsAllInputFile(input_file):
  964. return _IsGritInputFile(input_file) or _IsBuildGnInputFile(input_file)
  965. def _CheckAllFiles(input_file, input_lines, wanted_locales):
  966. errors = []
  967. if _IsGritInputFile(input_file):
  968. errors += _CheckGrdTranslations(input_file, input_lines, wanted_locales)
  969. errors += _CheckGrdAndroidOutputElements(
  970. input_file, input_lines, wanted_locales)
  971. elif _IsBuildGnInputFile(input_file):
  972. errors += _CheckGnAndroidOutputs(input_file, input_lines, wanted_locales)
  973. return errors
  974. def _AddMissingLocalesInAllFiles(input_file, input_lines, wanted_locales):
  975. if _IsGritInputFile(input_file):
  976. lines = _AddMissingLocalesInGrdTranslations(
  977. input_file, input_lines, wanted_locales)
  978. lines = _AddMissingLocalesInGrdAndroidOutputs(
  979. input_file, lines, wanted_locales)
  980. elif _IsBuildGnInputFile(input_file):
  981. lines = _AddMissingLocalesInGnAndroidOutputs(
  982. input_file, input_lines, wanted_locales)
  983. return lines
  984. ##########################################################################
  985. ##########################################################################
  986. #####
  987. ##### C O M M A N D H A N D L I N G
  988. #####
  989. ##########################################################################
  990. ##########################################################################
  991. class _Command(object):
  992. """A base class for all commands recognized by this script.
  993. Usage is the following:
  994. 1) Derived classes must re-define the following class-based fields:
  995. - name: Command name (e.g. 'list-locales')
  996. - description: Command short description.
  997. - long_description: Optional. Command long description.
  998. NOTE: As a convenience, if the first character is a newline,
  999. it will be omitted in the help output.
  1000. 2) Derived classes for commands that take arguments should override
  1001. RegisterExtraArgs(), which receives a corresponding argparse
  1002. sub-parser as argument.
  1003. 3) Derived classes should implement a Run() command, which can read
  1004. the current arguments from self.args.
  1005. """
  1006. name = None
  1007. description = None
  1008. long_description = None
  1009. def __init__(self):
  1010. self._parser = None
  1011. self.args = None
  1012. def RegisterExtraArgs(self, subparser):
  1013. pass
  1014. def RegisterArgs(self, parser):
  1015. subp = parser.add_parser(
  1016. self.name, help=self.description,
  1017. description=self.long_description or self.description,
  1018. formatter_class=argparse.RawDescriptionHelpFormatter)
  1019. self._parser = subp
  1020. subp.set_defaults(command=self)
  1021. group = subp.add_argument_group('%s arguments' % self.name)
  1022. self.RegisterExtraArgs(group)
  1023. def ProcessArgs(self, args):
  1024. self.args = args
  1025. class _ListLocalesCommand(_Command):
  1026. """Implement the 'list-locales' command to list locale lists of interest."""
  1027. name = 'list-locales'
  1028. description = 'List supported Chrome locales'
  1029. long_description = r'''
  1030. List locales of interest, by default this prints all locales supported by
  1031. Chrome, but `--type=ios_unsupported` can be used for the list of locales
  1032. unsupported on iOS.
  1033. These values are extracted directly from build/config/locales.gni.
  1034. Additionally, use the --as-json argument to print the list as a JSON list,
  1035. instead of the default format (which is a space-separated list of locale names).
  1036. '''
  1037. # Maps type argument to a function returning the corresponding locales list.
  1038. TYPE_MAP = {
  1039. 'all': ChromeLocales,
  1040. 'ios_unsupported': IosUnsupportedLocales,
  1041. }
  1042. def RegisterExtraArgs(self, group):
  1043. group.add_argument(
  1044. '--as-json',
  1045. action='store_true',
  1046. help='Output as JSON list.')
  1047. group.add_argument(
  1048. '--type',
  1049. choices=tuple(self.TYPE_MAP.viewkeys()),
  1050. default='all',
  1051. help='Select type of locale list to print.')
  1052. def Run(self):
  1053. locale_list = self.TYPE_MAP[self.args.type]()
  1054. if self.args.as_json:
  1055. print('[%s]' % ", ".join("'%s'" % loc for loc in locale_list))
  1056. else:
  1057. print(' '.join(locale_list))
  1058. class _CheckInputFileBaseCommand(_Command):
  1059. """Used as a base for other _Command subclasses that check input files.
  1060. Subclasses should also define the following class-level variables:
  1061. - select_file_func:
  1062. A predicate that receives a file name (not path) and return True if it
  1063. should be selected for inspection. Used when scanning directories with
  1064. '--scan-dir <dir>'.
  1065. - check_func:
  1066. - fix_func:
  1067. Two functions passed as parameters to _ProcessFile(), see relevant
  1068. documentation in this function's definition.
  1069. """
  1070. select_file_func = None
  1071. check_func = None
  1072. fix_func = None
  1073. def RegisterExtraArgs(self, group):
  1074. group.add_argument(
  1075. '--scan-dir',
  1076. action='append',
  1077. help='Optional directory to scan for input files recursively.')
  1078. group.add_argument(
  1079. 'input',
  1080. nargs='*',
  1081. help='Input file(s) to check.')
  1082. group.add_argument(
  1083. '--fix-inplace',
  1084. action='store_true',
  1085. help='Try to fix the files in-place too.')
  1086. group.add_argument(
  1087. '--add-locales',
  1088. help='Space-separated list of additional locales to use')
  1089. def Run(self):
  1090. args = self.args
  1091. input_files = []
  1092. if args.input:
  1093. input_files = args.input
  1094. if args.scan_dir:
  1095. input_files.extend(_ScanDirectoriesForFiles(
  1096. args.scan_dir, self.select_file_func.__func__))
  1097. locales = ChromeLocales()
  1098. if args.add_locales:
  1099. locales.extend(args.add_locales.split(' '))
  1100. locales = set(locales)
  1101. for input_file in input_files:
  1102. _ProcessFile(input_file,
  1103. locales,
  1104. self.check_func.__func__,
  1105. self.fix_func.__func__ if args.fix_inplace else None)
  1106. print('%sDone.' % (_CONSOLE_START_LINE))
  1107. class _CheckGrdAndroidOutputsCommand(_CheckInputFileBaseCommand):
  1108. name = 'check-grd-android-outputs'
  1109. description = (
  1110. 'Check the Android resource (.xml) files outputs in GRIT input files.')
  1111. long_description = r'''
  1112. Check the Android .xml files outputs in one or more input GRIT (.grd) files
  1113. for the following conditions:
  1114. - Each item has a correct 'lang' attribute.
  1115. - There are no duplicated lines for the same 'lang' attribute.
  1116. - That there are no extra locales that Chromium doesn't want.
  1117. - That no wanted locale is missing.
  1118. - Filenames exist for each listed locale.
  1119. - Filenames are well-formed.
  1120. '''
  1121. select_file_func = _IsGritInputFile
  1122. check_func = _CheckGrdAndroidOutputElements
  1123. fix_func = _AddMissingLocalesInGrdAndroidOutputs
  1124. class _CheckGrdTranslationsCommand(_CheckInputFileBaseCommand):
  1125. name = 'check-grd-translations'
  1126. description = (
  1127. 'Check the translation (.xtb) files outputted by .grd input files.')
  1128. long_description = r'''
  1129. Check the translation (.xtb) file outputs in one or more input GRIT (.grd) files
  1130. for the following conditions:
  1131. - Each item has a correct 'lang' attribute.
  1132. - There are no duplicated lines for the same 'lang' attribute.
  1133. - That there are no extra locales that Chromium doesn't want.
  1134. - That no wanted locale is missing.
  1135. - Each item has a 'path' attribute.
  1136. - Each such path value ends up with '.xtb'.
  1137. '''
  1138. select_file_func = _IsGritInputFile
  1139. check_func = _CheckGrdTranslations
  1140. fix_func = _AddMissingLocalesInGrdTranslations
  1141. class _CheckGnAndroidOutputsCommand(_CheckInputFileBaseCommand):
  1142. name = 'check-gn-android-outputs'
  1143. description = 'Check the Android .xml file lists in GN build files.'
  1144. long_description = r'''
  1145. Check one or more BUILD.gn file, looking for lists of Android resource .xml
  1146. files, and checking that:
  1147. - There are no duplicated output files in the list.
  1148. - Each output file belongs to a wanted Chromium locale.
  1149. - There are no output files for unwanted Chromium locales.
  1150. '''
  1151. select_file_func = _IsBuildGnInputFile
  1152. check_func = _CheckGnAndroidOutputs
  1153. fix_func = _AddMissingLocalesInGnAndroidOutputs
  1154. class _CheckAllCommand(_CheckInputFileBaseCommand):
  1155. name = 'check-all'
  1156. description = 'Check everything.'
  1157. long_description = 'Equivalent to calling all other check-xxx commands.'
  1158. select_file_func = _IsAllInputFile
  1159. check_func = _CheckAllFiles
  1160. fix_func = _AddMissingLocalesInAllFiles
  1161. class _UpdateExpectationsCommand(_Command):
  1162. name = 'update-expectations'
  1163. description = 'Update translation expectations file.'
  1164. long_description = r'''
  1165. Update %s files to match the current list of locales supported by Chromium.
  1166. This is especially useful to add new locales before updating any GRIT or GN
  1167. input file with the --add-locales option.
  1168. ''' % _EXPECTATIONS_FILENAME
  1169. def RegisterExtraArgs(self, group):
  1170. group.add_argument(
  1171. '--add-locales',
  1172. help='Space-separated list of additional locales to use.')
  1173. def Run(self):
  1174. locales = ChromeLocales()
  1175. add_locales = self.args.add_locales
  1176. if add_locales:
  1177. locales.extend(add_locales.split(' '))
  1178. expectation_paths = [
  1179. 'tools/gritsettings/translation_expectations.pyl',
  1180. 'clank/tools/translation_expectations.pyl',
  1181. ]
  1182. missing_expectation_files = []
  1183. for path in enumerate(expectation_paths):
  1184. file_path = os.path.join(_TOP_SRC_DIR, path)
  1185. if not os.path.exists(file_path):
  1186. missing_expectation_files.append(file_path)
  1187. continue
  1188. _UpdateLocalesInExpectationFile(file_path, locales)
  1189. if missing_expectation_files:
  1190. sys.stderr.write('WARNING: Missing file(s): %s\n' %
  1191. (', '.join(missing_expectation_files)))
  1192. class _UnitTestsCommand(_Command):
  1193. name = 'unit-tests'
  1194. description = 'Run internal unit-tests for this script'
  1195. def RegisterExtraArgs(self, group):
  1196. group.add_argument(
  1197. '-v', '--verbose', action='count', help='Increase test verbosity.')
  1198. group.add_argument('args', nargs=argparse.REMAINDER)
  1199. def Run(self):
  1200. argv = [_SCRIPT_NAME] + self.args.args
  1201. unittest.main(argv=argv, verbosity=self.args.verbose)
  1202. # List of all commands supported by this script.
  1203. _COMMANDS = [
  1204. _ListLocalesCommand,
  1205. _CheckGrdAndroidOutputsCommand,
  1206. _CheckGrdTranslationsCommand,
  1207. _CheckGnAndroidOutputsCommand,
  1208. _CheckAllCommand,
  1209. _UpdateExpectationsCommand,
  1210. _UnitTestsCommand,
  1211. ]
  1212. def main(argv):
  1213. parser = argparse.ArgumentParser(
  1214. description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
  1215. subparsers = parser.add_subparsers()
  1216. commands = [clazz() for clazz in _COMMANDS]
  1217. for command in commands:
  1218. command.RegisterArgs(subparsers)
  1219. if not argv:
  1220. argv = ['--help']
  1221. args = parser.parse_args(argv)
  1222. args.command.ProcessArgs(args)
  1223. args.command.Run()
  1224. if __name__ == "__main__":
  1225. main(sys.argv[1:])