base.py 104 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505
  1. # Copyright (C) 2010 Google Inc. All rights reserved.
  2. #
  3. # Redistribution and use in source and binary forms, with or without
  4. # modification, are permitted provided that the following conditions are
  5. # met:
  6. #
  7. # * Redistributions of source code must retain the above copyright
  8. # notice, this list of conditions and the following disclaimer.
  9. # * Redistributions in binary form must reproduce the above
  10. # copyright notice, this list of conditions and the following disclaimer
  11. # in the documentation and/or other materials provided with the
  12. # distribution.
  13. # * Neither the Google name nor the names of its
  14. # contributors may be used to endorse or promote products derived from
  15. # this software without specific prior written permission.
  16. #
  17. # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18. # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19. # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20. # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21. # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22. # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23. # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24. # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25. # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26. # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27. # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28. """Abstract base class for Port classes.
  29. The Port classes encapsulate Port-specific (platform-specific) behavior
  30. in the web test infrastructure.
  31. """
  32. import time
  33. import collections
  34. import json
  35. import logging
  36. import optparse
  37. import re
  38. import sys
  39. import tempfile
  40. from collections import defaultdict
  41. import six
  42. from six.moves import zip_longest
  43. from urllib.parse import urljoin
  44. from blinkpy.common import exit_codes
  45. from blinkpy.common import find_files
  46. from blinkpy.common import read_checksum_from_png
  47. from blinkpy.common import path_finder
  48. from blinkpy.common.memoized import memoized
  49. from blinkpy.common.system.executive import ScriptError
  50. from blinkpy.common.system.path import abspath_to_uri
  51. from blinkpy.w3c.wpt_manifest import WPTManifest, MANIFEST_NAME
  52. from blinkpy.web_tests.layout_package.bot_test_expectations import BotTestExpectationsFactory
  53. from blinkpy.web_tests.models.test_configuration import TestConfiguration
  54. from blinkpy.web_tests.models.test_run_results import TestRunException
  55. from blinkpy.web_tests.models.typ_types import (
  56. TestExpectations,
  57. ResultType,
  58. SerializableTypHost,
  59. )
  60. from blinkpy.web_tests.port import driver
  61. from blinkpy.web_tests.port import server_process
  62. from blinkpy.web_tests.port.factory import PortFactory
  63. from blinkpy.web_tests.servers import apache_http
  64. from blinkpy.web_tests.servers import pywebsocket
  65. from blinkpy.web_tests.servers import wptserve
  66. from blinkpy.web_tests.skia_gold import blink_skia_gold_properties as sgp
  67. from blinkpy.web_tests.skia_gold import blink_skia_gold_session_manager as sgsm
  68. _log = logging.getLogger(__name__)
  69. # Path relative to the build directory.
  70. CONTENT_SHELL_FONTS_DIR = "test_fonts"
  71. FONT_FILES = [
  72. [[CONTENT_SHELL_FONTS_DIR], 'Ahem.ttf', None],
  73. [[CONTENT_SHELL_FONTS_DIR], 'Arimo-Bold.ttf', None],
  74. [[CONTENT_SHELL_FONTS_DIR], 'Arimo-BoldItalic.ttf', None],
  75. [[CONTENT_SHELL_FONTS_DIR], 'Arimo-Italic.ttf', None],
  76. [[CONTENT_SHELL_FONTS_DIR], 'Arimo-Regular.ttf', None],
  77. [[CONTENT_SHELL_FONTS_DIR], 'Cousine-Bold.ttf', None],
  78. [[CONTENT_SHELL_FONTS_DIR], 'Cousine-BoldItalic.ttf', None],
  79. [[CONTENT_SHELL_FONTS_DIR], 'Cousine-Italic.ttf', None],
  80. [[CONTENT_SHELL_FONTS_DIR], 'Cousine-Regular.ttf', None],
  81. [[CONTENT_SHELL_FONTS_DIR], 'DejaVuSans.ttf', None],
  82. [[CONTENT_SHELL_FONTS_DIR], 'GardinerModBug.ttf', None],
  83. [[CONTENT_SHELL_FONTS_DIR], 'GardinerModCat.ttf', None],
  84. [[CONTENT_SHELL_FONTS_DIR], 'Garuda.ttf', None],
  85. [[CONTENT_SHELL_FONTS_DIR], 'Gelasio-Bold.ttf', None],
  86. [[CONTENT_SHELL_FONTS_DIR], 'Gelasio-BoldItalic.ttf', None],
  87. [[CONTENT_SHELL_FONTS_DIR], 'Gelasio-Italic.ttf', None],
  88. [[CONTENT_SHELL_FONTS_DIR], 'Gelasio-Regular.ttf', None],
  89. [[CONTENT_SHELL_FONTS_DIR], 'Lohit-Devanagari.ttf', None],
  90. [[CONTENT_SHELL_FONTS_DIR], 'Lohit-Gurmukhi.ttf', None],
  91. [[CONTENT_SHELL_FONTS_DIR], 'Lohit-Tamil.ttf', None],
  92. [[CONTENT_SHELL_FONTS_DIR], 'MuktiNarrow.ttf', None],
  93. [[CONTENT_SHELL_FONTS_DIR], 'NotoColorEmoji.ttf', None],
  94. [[CONTENT_SHELL_FONTS_DIR], 'NotoSansCJK-VF.otf.ttc', None],
  95. [[CONTENT_SHELL_FONTS_DIR], 'NotoSansKhmer-Regular.ttf', None],
  96. [[CONTENT_SHELL_FONTS_DIR], 'NotoSansSymbols2-Regular.ttf', None],
  97. [[CONTENT_SHELL_FONTS_DIR], 'NotoSansTibetan-Regular.ttf', None],
  98. [[CONTENT_SHELL_FONTS_DIR], 'Tinos-Bold.ttf', None],
  99. [[CONTENT_SHELL_FONTS_DIR], 'Tinos-BoldItalic.ttf', None],
  100. [[CONTENT_SHELL_FONTS_DIR], 'Tinos-Italic.ttf', None],
  101. [[CONTENT_SHELL_FONTS_DIR], 'Tinos-Regular.ttf', None],
  102. ]
  103. # This is the fingerprint of wpt's certificate found in
  104. # blinkpy/third_party/wpt/certs. The following line is updated by
  105. # update_cert.py.
  106. WPT_FINGERPRINT = 'Nxvaj3+bY3oVrTc+Jp7m3E3sB1n3lXtnMDCyBsqEXiY='
  107. # One for 127.0.0.1.sxg.pem
  108. SXG_FINGERPRINT = '55qC1nKu2A88ESbFmk5sTPQS/ScG+8DD7P+2bgFA9iM='
  109. # And one for external/wpt/signed-exchange/resources/127.0.0.1.sxg.pem
  110. SXG_WPT_FINGERPRINT = '0Rt4mT6SJXojEMHTnKnlJ/hBKMBcI4kteBlhR1eTTdk='
  111. # A convervative rule for names that are valid for file or directory names.
  112. VALID_FILE_NAME_REGEX = re.compile(r'^[\w\-=]+$')
  113. # This sub directory will be inside the results directory and it will
  114. # contain all the disc artifacts created by web tests
  115. ARTIFACTS_SUB_DIR = 'layout-test-results'
  116. class Port(object):
  117. """Abstract class for Port-specific hooks for the web_test package."""
  118. # Subclasses override this. This should indicate the basic implementation
  119. # part of the port name, e.g., 'mac', 'win', 'gtk'; there is one unique
  120. # value per class.
  121. # FIXME: Rename this to avoid confusion with the "full port name".
  122. port_name = None
  123. # Test paths use forward slash as separator on all platforms.
  124. TEST_PATH_SEPARATOR = '/'
  125. ALL_BUILD_TYPES = ('debug', 'release')
  126. CONTENT_SHELL_NAME = 'content_shell'
  127. # Update the first line in third_party/blink/web_tests/TestExpectations and
  128. # the documentation in docs/testing/web_test_expectations.md when this list
  129. # changes.
  130. ALL_SYSTEMS = (
  131. ('mac10.13', 'x86'),
  132. ('mac10.14', 'x86'),
  133. ('mac10.15', 'x86'),
  134. ('mac11', 'x86'),
  135. ('mac11-arm64', 'arm64'),
  136. ('mac12', 'x86_64'),
  137. ('mac12-arm64', 'arm64'),
  138. ('win10.20h2', 'x86'),
  139. ('win11', 'x64'),
  140. ('trusty', 'x86_64'),
  141. ('fuchsia', 'x86_64'),
  142. )
  143. CONFIGURATION_SPECIFIER_MACROS = {
  144. 'mac': [
  145. 'mac10.13', 'mac10.14', 'mac10.15', 'mac11', 'mac11-arm64',
  146. 'mac12', 'mac12-arm64'
  147. ],
  148. 'win': ['win10.20h2', 'win11'],
  149. 'linux': ['trusty'],
  150. 'fuchsia': ['fuchsia'],
  151. }
  152. # List of ports open on the host that the tests will connect to. When tests
  153. # run on a separate machine (Android and Fuchsia) these ports need to be
  154. # forwarded back to the host.
  155. # 8000, 8080 and 8443 are for http/https tests;
  156. # 8880 is for websocket tests (see apache_http.py and pywebsocket.py).
  157. # 8001, 8081, 8444, and 8445 are for http/https WPT;
  158. # 9001 and 9444 are for websocket WPT (see wptserve.py).
  159. SERVER_PORTS = [8000, 8001, 8080, 8081, 8443, 8444, 8445, 8880, 9001, 9444]
  160. FALLBACK_PATHS = {}
  161. SUPPORTED_VERSIONS = []
  162. # URL to the build requirements page.
  163. BUILD_REQUIREMENTS_URL = ''
  164. # The suffixes of baseline files (not extensions).
  165. BASELINE_SUFFIX = '-expected'
  166. BASELINE_MISMATCH_SUFFIX = '-expected-mismatch'
  167. # All of the non-reftest baseline extensions we use.
  168. BASELINE_EXTENSIONS = ('.wav', '.txt', '.png')
  169. FLAG_EXPECTATIONS_PREFIX = 'FlagExpectations'
  170. # The following is used for concetenating WebDriver test names.
  171. WEBDRIVER_SUBTEST_SEPARATOR = '>>'
  172. # The following is used for concetenating WebDriver test names in pytest format.
  173. WEBDRIVER_SUBTEST_PYTEST_SEPARATOR = '::'
  174. # The following two constants must match. When adding a new WPT root, also
  175. # remember to add an alias rule to //third_party/wpt_tools/wpt.config.json.
  176. # WPT_DIRS maps WPT roots on the file system to URL prefixes on wptserve.
  177. # The order matters: '/' MUST be the last URL prefix.
  178. WPT_DIRS = collections.OrderedDict([
  179. ('wpt_internal', '/wpt_internal/'),
  180. ('external/wpt', '/'),
  181. ])
  182. # WPT_REGEX captures: 1. the root directory of WPT relative to web_tests
  183. # (without a trailing slash), 2. the path of the test within WPT (without a
  184. # leading slash).
  185. WPT_REGEX = re.compile(
  186. r'^(?:virtual/[^/]+/)?(external/wpt|wpt_internal)/(.*)$')
  187. # This regex parses the WPT-style style fuzzy match syntax. For actual WPT
  188. # tests, this is not needed since this information is contained in the
  189. # manifest. However, we reuse this syntax for some non-WPT tests as well.
  190. WPT_FUZZY_REGEX = re.compile(
  191. r'<(?:html:)?meta\s+name=(?:fuzzy|"fuzzy")\s+content='
  192. r'"(?:(.+):)?(?:maxDifference=)?(?:(\d+)-)?(\d+);(?:totalPixels=)?(?:(\d+)-)?(\d+)"\s*/?>'
  193. )
  194. # Because this is an abstract base class, arguments to functions may be
  195. # unused in this class - pylint: disable=unused-argument
  196. @classmethod
  197. def latest_platform_fallback_path(cls):
  198. return cls.FALLBACK_PATHS[cls.SUPPORTED_VERSIONS[-1]]
  199. @classmethod
  200. def determine_full_port_name(cls, host, options, port_name):
  201. """Return a fully-specified port name that can be used to construct objects."""
  202. # Subclasses will usually override this.
  203. assert port_name.startswith(cls.port_name)
  204. return port_name
  205. def __init__(self, host, port_name, options=None, **kwargs):
  206. # This value is the "full port name", and may be different from
  207. # cls.port_name by having version modifiers appended to it.
  208. self._name = port_name
  209. # These are default values that should be overridden in a subclasses.
  210. self._version = ''
  211. self._architecture = 'x86'
  212. # FIXME: Ideally we'd have a package-wide way to get a well-formed
  213. # options object that had all of the necessary options defined on it.
  214. self._options = options or optparse.Values()
  215. self.host = host
  216. self._executive = host.executive
  217. self._filesystem = host.filesystem
  218. self._path_finder = path_finder.PathFinder(host.filesystem)
  219. self._http_server = None
  220. self._websocket_server = None
  221. self._wpt_server = None
  222. self._image_differ = None
  223. self.server_process_constructor = server_process.ServerProcess # This can be overridden for testing.
  224. self._http_lock = None # FIXME: Why does this live on the port object?
  225. self._dump_reader = None
  226. if not hasattr(options, 'configuration') or not options.configuration:
  227. self.set_option_default('configuration',
  228. self.default_configuration())
  229. if not hasattr(options, 'target') or not options.target:
  230. self.set_option_default('target', self._options.configuration)
  231. if not hasattr(options, 'no_virtual_tests'):
  232. self.set_option_default('no_virtual_tests', False)
  233. self._test_configuration = None
  234. self._results_directory = None
  235. self._virtual_test_suites = None
  236. self._used_expectation_files = None
  237. self._skia_gold_temp_dir = None
  238. self._skia_gold_session_manager = None
  239. self._skia_gold_properties = None
  240. def __del__(self):
  241. if self._skia_gold_temp_dir:
  242. self._filesystem.rmtree(self._skia_gold_temp_dir,
  243. ignore_errors=True)
  244. def __str__(self):
  245. return 'Port{name=%s, version=%s, architecture=%s, test_configuration=%s}' % (
  246. self._name, self._version, self._architecture,
  247. self._test_configuration)
  248. def get_platform_tags(self):
  249. """Returns system condition tags that are used to find active expectations
  250. for a test run on a specific system"""
  251. return frozenset([
  252. self._options.configuration.lower(), self._version, self.port_name,
  253. self._architecture
  254. ])
  255. @memoized
  256. def flag_specific_config_name(self):
  257. """Returns the name of the flag-specific configuration if it's specified in
  258. --flag-specific option, or None. The name must be defined in
  259. FlagSpecificConfig or an AssertionError will be raised.
  260. """
  261. config_name = self.get_option('flag_specific')
  262. if config_name:
  263. configs = self._flag_specific_configs()
  264. assert config_name in configs, '{} is not defined in FlagSpecificConfig'.format(
  265. config_name)
  266. return config_name
  267. return None
  268. @memoized
  269. def _flag_specific_configs(self):
  270. """Reads configuration from FlagSpecificConfig and returns a dictionary from name to args."""
  271. config_file = self._filesystem.join(self.web_tests_dir(),
  272. 'FlagSpecificConfig')
  273. if not self._filesystem.exists(config_file):
  274. return {}
  275. try:
  276. json_configs = json.loads(
  277. self._filesystem.read_text_file(config_file))
  278. except ValueError as error:
  279. raise ValueError('{} is not a valid JSON file: {}'.format(
  280. config_file, error))
  281. configs = {}
  282. for config in json_configs:
  283. name = config['name']
  284. args = config['args']
  285. if not VALID_FILE_NAME_REGEX.match(name):
  286. raise ValueError(
  287. '{}: name "{}" contains invalid characters'.format(
  288. config_file, name))
  289. if name in configs:
  290. raise ValueError('{} contains duplicated name {}.'.format(
  291. config_file, name))
  292. if args in configs.values():
  293. raise ValueError(
  294. '{}: name "{}" has the same args as another entry.'.format(
  295. config_file, name))
  296. configs[name] = args
  297. return configs
  298. def _specified_additional_driver_flags(self):
  299. """Returns the list of additional driver flags specified by the user in
  300. the following ways, concatenated:
  301. 1. Flags in web_tests/additional-driver-flag.setting.
  302. 2. flags expanded from --flag-specific=<name> based on flag-specific config.
  303. 3. Zero or more flags passed by --additional-driver-flag.
  304. """
  305. flags = []
  306. flag_file = self._filesystem.join(self.web_tests_dir(),
  307. 'additional-driver-flag.setting')
  308. if self._filesystem.exists(flag_file):
  309. flags = self._filesystem.read_text_file(flag_file).split()
  310. flag_specific_option = self.flag_specific_config_name()
  311. if flag_specific_option:
  312. flags += self._flag_specific_configs()[flag_specific_option]
  313. flags += self.get_option('additional_driver_flag', [])
  314. return flags
  315. def additional_driver_flags(self):
  316. flags = self._specified_additional_driver_flags()
  317. if self.driver_name() == self.CONTENT_SHELL_NAME:
  318. flags += [
  319. '--run-web-tests',
  320. '--ignore-certificate-errors-spki-list=' + WPT_FINGERPRINT +
  321. ',' + SXG_FINGERPRINT + ',' + SXG_WPT_FINGERPRINT,
  322. # Required for WebTransport tests.
  323. '--origin-to-force-quic-on=web-platform.test:11000',
  324. '--user-data-dir'
  325. ]
  326. if self.get_option('nocheck_sys_deps', False):
  327. flags.append('--disable-system-font-check')
  328. # If we're already repeating the tests more than once, then we're not
  329. # particularly concerned with speed. Resetting the shell between tests
  330. # increases test run time by 2-5X, but provides more consistent results
  331. # [less state leaks between tests].
  332. if (self.get_option('reset_shell_between_tests')
  333. or (self.get_option('repeat_each')
  334. and self.get_option('repeat_each') > 1)
  335. or (self.get_option('iterations')
  336. and self.get_option('iterations') > 1)):
  337. flags += ['--reset-shell-between-tests']
  338. return flags
  339. def supports_per_test_timeout(self):
  340. return False
  341. def default_smoke_test_only(self):
  342. return False
  343. def _default_timeout_ms(self):
  344. return 6000
  345. def timeout_ms(self):
  346. timeout_ms = self._default_timeout_ms()
  347. if self.get_option('configuration') == 'Debug':
  348. # Debug is about 5x slower than Release.
  349. return 5 * timeout_ms
  350. if self._build_has_dcheck_always_on():
  351. # Release with DCHECK is also slower than pure Release.
  352. return 2 * timeout_ms
  353. return timeout_ms
  354. def skia_gold_temp_dir(self):
  355. return self._skia_gold_temp_dir
  356. def skia_gold_properties(self):
  357. if not self._skia_gold_properties:
  358. self._skia_gold_properties = sgp.BlinkSkiaGoldProperties(
  359. self._options)
  360. return self._skia_gold_properties
  361. def skia_gold_session_manager(self):
  362. if not self._skia_gold_session_manager:
  363. self._skia_gold_temp_dir = self._filesystem.mkdtemp()
  364. self._skia_gold_session_manager = sgsm.BlinkSkiaGoldSessionManager(
  365. str(self._skia_gold_temp_dir), self.skia_gold_properties())
  366. return self._skia_gold_session_manager
  367. def skia_gold_json_keys(self):
  368. return {
  369. 'configuration': self._options.configuration.lower(),
  370. 'version': self._version,
  371. 'port': self.port_name,
  372. 'architecture': self._architecture,
  373. 'ignore': '1',
  374. }
  375. @memoized
  376. def _build_has_dcheck_always_on(self):
  377. args_gn_file = self._build_path('args.gn')
  378. if not self._filesystem.exists(args_gn_file):
  379. _log.error('Unable to find %s', args_gn_file)
  380. return False
  381. contents = self._filesystem.read_text_file(args_gn_file)
  382. return bool(
  383. re.search(r'^\s*dcheck_always_on\s*=\s*true\s*(#.*)?$', contents,
  384. re.MULTILINE))
  385. def driver_stop_timeout(self):
  386. """Returns the amount of time in seconds to wait before killing the process in driver.stop()."""
  387. # We want to wait for at least 3 seconds, but if we are really slow, we
  388. # want to be slow on cleanup as well (for things like ASAN, Valgrind, etc.)
  389. return (3.0 * float(self.get_option('timeout_ms', '0')) /
  390. self._default_timeout_ms())
  391. def default_batch_size(self):
  392. """Returns the default batch size to use for this port."""
  393. if self.get_option('enable_sanitizer'):
  394. # ASAN/MSAN/TSAN use more memory than regular content_shell. Their
  395. # memory usage may also grow over time, up to a certain point.
  396. # Relaunching the driver periodically helps keep it under control.
  397. return 40
  398. # The default batch size now is 100, to battle against resource leak.
  399. return 100
  400. def default_child_processes(self):
  401. """Returns the number of child processes to use for this port."""
  402. return self._executive.cpu_count()
  403. def default_max_locked_shards(self):
  404. """Returns the number of "locked" shards to run in parallel (like the http tests)."""
  405. max_locked_shards = int(self.default_child_processes()) // 4
  406. if not max_locked_shards:
  407. return 1
  408. return max_locked_shards
  409. def baseline_version_dir(self):
  410. """Returns the absolute path to the platform-and-version-specific results."""
  411. baseline_search_paths = self.baseline_search_path()
  412. return baseline_search_paths[0]
  413. def baseline_flag_specific_dir(self):
  414. """If --flag-specific is specified, returns the absolute path to the flag-specific
  415. platform-independent results. Otherwise returns None."""
  416. config_name = self.flag_specific_config_name()
  417. if not config_name:
  418. return None
  419. return self._filesystem.join(self.web_tests_dir(), 'flag-specific',
  420. config_name)
  421. def baseline_search_path(self):
  422. return (self.get_option('additional_platform_directory', []) +
  423. self._flag_specific_baseline_search_path() +
  424. self._compare_baseline() +
  425. list(self.default_baseline_search_path()))
  426. def default_baseline_search_path(self):
  427. """Returns a list of absolute paths to directories to search under for baselines.
  428. The directories are searched in order.
  429. """
  430. return map(self._absolute_baseline_path,
  431. self.FALLBACK_PATHS[self.version()])
  432. @memoized
  433. def _compare_baseline(self):
  434. factory = PortFactory(self.host)
  435. target_port = self.get_option('compare_port')
  436. if target_port:
  437. return factory.get(target_port).default_baseline_search_path()
  438. return []
  439. def _check_file_exists(self,
  440. path_to_file,
  441. file_description,
  442. override_step=None,
  443. more_logging=True):
  444. """Verifies that the file is present where expected, or logs an error.
  445. Args:
  446. file_name: The (human friendly) name or description of the file
  447. you're looking for (e.g., "HTTP Server"). Used for error logging.
  448. override_step: An optional string to be logged if the check fails.
  449. more_logging: Whether or not to log the error messages.
  450. Returns:
  451. True if the file exists, else False.
  452. """
  453. if not self._filesystem.exists(path_to_file):
  454. if more_logging:
  455. _log.error('Unable to find %s', file_description)
  456. _log.error(' at %s', path_to_file)
  457. if override_step:
  458. _log.error(' %s', override_step)
  459. _log.error('')
  460. return False
  461. return True
  462. def check_build(self, needs_http, printer):
  463. if not self._check_file_exists(self._path_to_driver(), 'test driver'):
  464. return exit_codes.UNEXPECTED_ERROR_EXIT_STATUS
  465. if not self._check_driver_build_up_to_date(
  466. self.get_option('configuration')):
  467. return exit_codes.UNEXPECTED_ERROR_EXIT_STATUS
  468. if not self._check_file_exists(self._path_to_image_diff(),
  469. 'image_diff'):
  470. return exit_codes.UNEXPECTED_ERROR_EXIT_STATUS
  471. if self._dump_reader and not self._dump_reader.check_is_functional():
  472. return exit_codes.UNEXPECTED_ERROR_EXIT_STATUS
  473. if needs_http and not self.check_httpd():
  474. return exit_codes.UNEXPECTED_ERROR_EXIT_STATUS
  475. return exit_codes.OK_EXIT_STATUS
  476. def check_sys_deps(self):
  477. """Checks whether the system is properly configured.
  478. Most checks happen during invocation of the driver prior to running
  479. tests. This can be overridden to run custom checks.
  480. Returns:
  481. An exit status code.
  482. """
  483. return exit_codes.OK_EXIT_STATUS
  484. def check_httpd(self):
  485. httpd_path = self.path_to_apache()
  486. if httpd_path:
  487. try:
  488. env = self.setup_environ_for_server()
  489. if self._executive.run_command(
  490. [httpd_path, '-v'], env=env, return_exit_code=True) != 0:
  491. _log.error('httpd seems broken. Cannot run http tests.')
  492. return False
  493. return True
  494. except OSError as e:
  495. _log.error('while trying to run: ' + httpd_path)
  496. _log.error('httpd launch error: ' + repr(e))
  497. _log.error('No httpd found. Cannot run http tests.')
  498. return False
  499. def do_text_results_differ(self, expected_text, actual_text):
  500. return expected_text != actual_text
  501. def do_audio_results_differ(self, expected_audio, actual_audio):
  502. return expected_audio != actual_audio
  503. def diff_image(self,
  504. expected_contents,
  505. actual_contents,
  506. max_channel_diff=None,
  507. max_pixels_diff=None):
  508. """Compares two images and returns an (image diff, error string) pair.
  509. If an error occurs (like image_diff isn't found, or crashes), we log an
  510. error and return True (for a diff).
  511. """
  512. # If only one of them exists, return that one.
  513. if not actual_contents and not expected_contents:
  514. return (None, None, None)
  515. if not actual_contents:
  516. return (expected_contents, None, None)
  517. if not expected_contents:
  518. return (actual_contents, None, None)
  519. tempdir = self._filesystem.mkdtemp()
  520. expected_filename = self._filesystem.join(str(tempdir), 'expected.png')
  521. self._filesystem.write_binary_file(expected_filename,
  522. expected_contents)
  523. actual_filename = self._filesystem.join(str(tempdir), 'actual.png')
  524. self._filesystem.write_binary_file(actual_filename, actual_contents)
  525. diff_filename = self._filesystem.join(str(tempdir), 'diff.png')
  526. executable = self._path_to_image_diff()
  527. # Although we are handed 'old', 'new', image_diff wants 'new', 'old'.
  528. command = [
  529. executable, '--diff', actual_filename, expected_filename,
  530. diff_filename
  531. ]
  532. # Notifies image_diff to allow a tolerance when calculating the pixel
  533. # diff. To account for variances when the tests are ran on an actual
  534. # GPU.
  535. if self.get_option('fuzzy_diff'):
  536. command.append('--fuzzy-diff')
  537. # The max_channel_diff and max_pixels_diff arguments are used by WPT
  538. # tests for fuzzy reftests. See
  539. # https://web-platform-tests.org/writing-tests/reftests.html#fuzzy-matching
  540. if max_channel_diff is not None:
  541. command.append('--fuzzy-max-channel-diff={}'.format('-'.join(
  542. map(str, max_channel_diff))))
  543. if max_pixels_diff is not None:
  544. command.append('--fuzzy-max-pixels-diff={}'.format('-'.join(
  545. map(str, max_pixels_diff))))
  546. result = None
  547. stats = None
  548. err_str = None
  549. def handle_output(output):
  550. if output:
  551. match = re.search(
  552. "Found pixels_different: (\d+), max_channel_diff: (\d+)",
  553. output)
  554. _log.debug(output)
  555. if match:
  556. return {
  557. "maxDifference": int(match.group(2)),
  558. "totalPixels": int(match.group(1))
  559. }
  560. return None
  561. try:
  562. output = self._executive.run_command(command)
  563. stats = handle_output(output)
  564. except ScriptError as error:
  565. if error.exit_code == 1:
  566. result = self._filesystem.read_binary_file(diff_filename)
  567. stats = handle_output(error.output)
  568. else:
  569. err_str = 'Image diff returned an exit code of %s. See http://crbug.com/278596' % error.exit_code
  570. except OSError as error:
  571. err_str = 'error running image diff: %s' % error
  572. finally:
  573. self._filesystem.rmtree(str(tempdir))
  574. return (result, stats, err_str or None)
  575. def driver_name(self):
  576. if self.get_option('driver_name'):
  577. return self.get_option('driver_name')
  578. return self.CONTENT_SHELL_NAME
  579. def expected_baselines_by_extension(self, test_name):
  580. """Returns a dict mapping baseline suffix to relative path for each baseline in a test.
  581. For reftests, it returns ".==" or ".!=" instead of the suffix.
  582. """
  583. # FIXME: The name similarity between this and expected_baselines()
  584. # below, is unfortunate. We should probably rename them both.
  585. baseline_dict = {}
  586. reference_files = self.reference_files(test_name)
  587. if reference_files:
  588. # FIXME: How should this handle more than one type of reftest?
  589. baseline_dict['.' + reference_files[0][0]] = \
  590. self.relative_test_filename(reference_files[0][1])
  591. for extension in self.BASELINE_EXTENSIONS:
  592. path = self.expected_filename(
  593. test_name, extension, return_default=False)
  594. baseline_dict[extension] = self.relative_test_filename(
  595. path) if path else path
  596. return baseline_dict
  597. def output_filename(self, test_name, suffix, extension):
  598. """Generates the output filename for a test.
  599. This method gives a proper filename for various outputs of a test,
  600. including baselines and actual results. Usually, the output filename
  601. follows the pattern: test_name_without_ext+suffix+extension, but when
  602. the test name contains query strings, e.g. external/wpt/foo.html?wss,
  603. test_name_without_ext is mangled to be external/wpt/foo_wss.
  604. It is encouraged to use this method instead of writing another mangling.
  605. Args:
  606. test_name: The name of a test.
  607. suffix: A suffix string to add before the extension
  608. (e.g. "-expected").
  609. extension: The extension of the output file (starting with .).
  610. Returns:
  611. A string, the output filename.
  612. """
  613. # WPT names might contain query strings, e.g. external/wpt/foo.html?wss,
  614. # in which case we mangle test_name_root (the part of a path before the
  615. # last extension point) to external/wpt/foo_wss, and the output filename
  616. # becomes external/wpt/foo_wss-expected.txt.
  617. index = test_name.find('?')
  618. if index != -1:
  619. test_name_root, _ = self._filesystem.splitext(test_name[:index])
  620. query_part = test_name[index:]
  621. test_name_root += self._filesystem.sanitize_filename(query_part)
  622. else:
  623. test_name_root, _ = self._filesystem.splitext(test_name)
  624. return test_name_root + suffix + extension
  625. def expected_baselines(self,
  626. test_name,
  627. extension,
  628. all_baselines=False,
  629. match=True):
  630. """Given a test name, finds where the baseline results are located.
  631. Return values will be in the format appropriate for the current
  632. platform (e.g., "\\" for path separators on Windows). If the results
  633. file is not found, then None will be returned for the directory,
  634. but the expected relative pathname will still be returned.
  635. This routine is generic but lives here since it is used in
  636. conjunction with the other baseline and filename routines that are
  637. platform specific.
  638. Args:
  639. test_name: Name of test file (usually a relative path under web_tests/)
  640. extension: File extension of the expected results, including dot;
  641. e.g. '.txt' or '.png'. This should not be None, but may be an
  642. empty string.
  643. all_baselines: If True, return an ordered list of all baseline paths
  644. for the given platform. If False, return only the first one.
  645. match: Whether the baseline is a match or a mismatch.
  646. Returns:
  647. A list of (baseline_dir, results_filename) pairs, where
  648. baseline_dir - abs path to the top of the results tree (or test
  649. tree)
  650. results_filename - relative path from top of tree to the results
  651. file
  652. (port.join() of the two gives you the full path to the file,
  653. unless None was returned.)
  654. """
  655. baseline_filename = self.output_filename(
  656. test_name,
  657. self.BASELINE_SUFFIX if match else self.BASELINE_MISMATCH_SUFFIX,
  658. extension)
  659. baseline_search_path = self.baseline_search_path()
  660. baselines = []
  661. for baseline_dir in baseline_search_path:
  662. if self._filesystem.exists(
  663. self._filesystem.join(baseline_dir, baseline_filename)):
  664. baselines.append((baseline_dir, baseline_filename))
  665. if not all_baselines and baselines:
  666. return baselines
  667. baseline_dir = self.generic_baselines_dir()
  668. if self._filesystem.exists(
  669. self._filesystem.join(baseline_dir, baseline_filename)):
  670. baselines.append((baseline_dir, baseline_filename))
  671. if baselines:
  672. return baselines
  673. return [(None, baseline_filename)]
  674. def expected_filename(self,
  675. test_name,
  676. extension,
  677. return_default=True,
  678. fallback_base_for_virtual=True,
  679. match=True,
  680. look_for_same_folder_reference_file=False):
  681. """Given a test name, returns an absolute path to its expected results.
  682. If no expected results are found in any of the searched directories,
  683. the directory in which the test itself is located will be returned.
  684. The return value is in the format appropriate for the platform
  685. (e.g., "\\" for path separators on windows).
  686. This routine is generic but is implemented here to live alongside
  687. the other baseline and filename manipulation routines.
  688. Args:
  689. test_name: Name of test file (usually a relative path under web_tests/)
  690. extension: File extension of the expected results, including dot;
  691. e.g. '.txt' or '.png'. This should not be None, but may be an
  692. empty string.
  693. return_default: If True, returns the path to the generic expectation
  694. if nothing else is found; if False, returns None.
  695. fallback_base_for_virtual: For virtual test only. When no virtual
  696. specific baseline is found, if this parameter is True, fallback
  697. to find baselines of the base test; if False, depending on
  698. |return_default|, returns the generic virtual baseline or None.
  699. match: Whether the baseline is a match or a mismatch.
  700. look_for_same_folder_reference_file: For reference test only. Returns
  701. the reference file if found in the same folder of the test file.
  702. Returns:
  703. An absolute path to its expected results, or None if not found.
  704. """
  705. # The [0] means the first expected baseline (which is the one to be
  706. # used) in the fallback paths.
  707. baseline_dir, baseline_filename = self.expected_baselines(
  708. test_name, extension, match=match)[0]
  709. if baseline_dir:
  710. return self._filesystem.join(baseline_dir, baseline_filename)
  711. if look_for_same_folder_reference_file:
  712. path = self._filesystem.join(self.web_tests_dir(),
  713. baseline_filename)
  714. if self._filesystem.exists(path):
  715. return path
  716. if fallback_base_for_virtual:
  717. actual_test_name = self.lookup_virtual_test_base(test_name)
  718. if actual_test_name:
  719. return self.expected_filename(
  720. actual_test_name,
  721. extension,
  722. return_default,
  723. match=match,
  724. look_for_same_folder_reference_file=look_for_same_folder_reference_file
  725. )
  726. if return_default:
  727. return self._filesystem.join(self.generic_baselines_dir(),
  728. baseline_filename)
  729. return None
  730. def fallback_expected_filename(self, test_name, extension):
  731. """Given a test name, returns an absolute path to its next fallback baseline.
  732. Args:
  733. same as expected_filename()
  734. Returns:
  735. An absolute path to the next fallback baseline, or None if not found.
  736. """
  737. baselines = self.expected_baselines(
  738. test_name, extension, all_baselines=True)
  739. if len(baselines) < 2:
  740. actual_test_name = self.lookup_virtual_test_base(test_name)
  741. if actual_test_name:
  742. if len(baselines) == 0:
  743. return self.fallback_expected_filename(
  744. actual_test_name, extension)
  745. # In this case, baselines[0] is the current baseline of the
  746. # virtual test, so the first base test baseline is the fallback
  747. # baseline of the virtual test.
  748. return self.expected_filename(
  749. actual_test_name, extension, return_default=False)
  750. return None
  751. baseline_dir, baseline_filename = baselines[1]
  752. if baseline_dir:
  753. return self._filesystem.join(baseline_dir, baseline_filename)
  754. return None
  755. def expected_checksum(self, test_name):
  756. """Returns the checksum of the image we expect the test to produce,
  757. or None if it is a text-only test.
  758. """
  759. png_path = self.expected_filename(test_name, '.png')
  760. if self._filesystem.exists(png_path):
  761. with self._filesystem.open_binary_file_for_reading(
  762. png_path) as filehandle:
  763. return read_checksum_from_png.read_checksum(filehandle)
  764. return None
  765. def expected_image(self, test_name):
  766. """Returns the image we expect the test to produce."""
  767. baseline_path = self.expected_filename(test_name, '.png')
  768. if not self._filesystem.exists(baseline_path):
  769. return None
  770. return self._filesystem.read_binary_file(baseline_path)
  771. def expected_audio(self, test_name):
  772. baseline_path = self.expected_filename(test_name, '.wav')
  773. if not self._filesystem.exists(baseline_path):
  774. return None
  775. return self._filesystem.read_binary_file(baseline_path)
  776. def expected_text(self, test_name):
  777. """Returns the text output we expect the test to produce, or None
  778. if we don't expect there to be any text output.
  779. End-of-line characters are normalized to '\n'.
  780. """
  781. # FIXME: DRT output is actually utf-8, but since we don't decode the
  782. # output from DRT (instead treating it as a binary string), we read the
  783. # baselines as a binary string, too.
  784. baseline_path = self.expected_filename(test_name, '.txt')
  785. if not self._filesystem.exists(baseline_path):
  786. return None
  787. text = self._filesystem.read_binary_file(baseline_path)
  788. return text.replace(b'\r\n', b'\n')
  789. def expected_subtest_failure(self, test_name):
  790. baseline = self.expected_text(test_name)
  791. if baseline:
  792. baseline = baseline.decode('utf8', 'replace')
  793. if re.search(r"^(FAIL|NOTRUN|TIMEOUT)", baseline, re.MULTILINE):
  794. return True
  795. return False
  796. def expected_harness_error(self, test_name):
  797. baseline = self.expected_text(test_name)
  798. if baseline:
  799. baseline = baseline.decode('utf8', 'replace')
  800. if re.search(r"^Harness Error\.", baseline, re.MULTILINE):
  801. return True
  802. return False
  803. def reference_files(self, test_name):
  804. """Returns a list of expectation (== or !=) and filename pairs"""
  805. # Try to find -expected.* or -expected-mismatch.* in the same directory.
  806. reftest_list = []
  807. for expectation in ('==', '!='):
  808. for extension in Port.supported_file_extensions:
  809. path = self.expected_filename(
  810. test_name,
  811. extension,
  812. match=(expectation == '=='),
  813. look_for_same_folder_reference_file=True
  814. )
  815. if self._filesystem.exists(path):
  816. reftest_list.append((expectation, path))
  817. if reftest_list:
  818. return reftest_list
  819. # Try to extract information from MANIFEST.json.
  820. match = self.WPT_REGEX.match(test_name)
  821. if not match:
  822. return []
  823. wpt_path = match.group(1)
  824. path_in_wpt = match.group(2)
  825. for expectation, ref_path_in_wpt in self.wpt_manifest(
  826. wpt_path).extract_reference_list(path_in_wpt):
  827. ref_absolute_path = self._filesystem.join(
  828. self.web_tests_dir(), wpt_path + ref_path_in_wpt)
  829. reftest_list.append((expectation, ref_absolute_path))
  830. return reftest_list
  831. def tests(self, paths=None):
  832. """Returns all tests or tests matching supplied paths.
  833. Args:
  834. paths: Array of paths to match. If supplied, this function will only
  835. return tests matching at least one path in paths.
  836. Returns:
  837. An array of test paths and test names. The latter are web platform
  838. tests that don't correspond to file paths but are valid tests,
  839. for instance a file path test.any.js could correspond to two test
  840. names: test.any.html and test.any.worker.html.
  841. """
  842. tests = self.real_tests(paths)
  843. if paths:
  844. if not self._options.no_virtual_tests:
  845. tests.extend(self._virtual_tests_matching_paths(paths))
  846. if (any(wpt_path in path for wpt_path in self.WPT_DIRS
  847. for path in paths)
  848. # TODO(robertma): Remove this special case when external/wpt is moved to wpt.
  849. or any('external' in path for path in paths)):
  850. tests.extend(self._wpt_test_urls_matching_paths(paths))
  851. else:
  852. # '/' is used instead of filesystem.sep as the WPT manifest always
  853. # uses '/' for paths (it is not OS dependent).
  854. wpt_tests = [
  855. wpt_path + '/' + test for wpt_path in self.WPT_DIRS
  856. for test in self.wpt_manifest(wpt_path).all_urls()
  857. ]
  858. tests_by_dir = defaultdict(list)
  859. for test in tests + wpt_tests:
  860. dirname = self._filesystem.dirname(test) + '/'
  861. tests_by_dir[dirname].append(test)
  862. if not self._options.no_virtual_tests:
  863. tests.extend(self._all_virtual_tests(tests_by_dir))
  864. tests.extend(wpt_tests)
  865. return tests
  866. def real_tests_from_dict(self, paths, tests_by_dir):
  867. """Find all real tests in paths, using results saved in dict."""
  868. files = []
  869. for path in paths:
  870. if self._has_supported_extension_for_all(path):
  871. # only append the file when it is in tests_by_dir
  872. dirname = self._filesystem.dirname(path) + '/'
  873. if path in tests_by_dir.get(dirname, []):
  874. files.append(path)
  875. continue
  876. path = path + '/' if path[-1] != '/' else path
  877. for key, value in tests_by_dir.items():
  878. if key.startswith(path):
  879. files.extend(value)
  880. return files
  881. def real_tests(self, paths):
  882. """Find all real tests in paths except WPT."""
  883. # When collecting test cases, skip these directories.
  884. skipped_directories = set([
  885. 'platform', 'resources', 'support', 'script-tests', 'reference',
  886. 'reftest', 'SmokeTests'
  887. ])
  888. # Also ignore all WPT directories. Note that this is only an
  889. # optimization; is_non_wpt_test_file should skip WPT regardless.
  890. skipped_directories |= set(self.WPT_DIRS)
  891. files = find_files.find(self._filesystem, self.web_tests_dir(), paths, skipped_directories,
  892. lambda _, dirname, filename: self.is_non_wpt_test_file(dirname, filename),
  893. self.test_key)
  894. return [self.relative_test_filename(f) for f in files]
  895. @staticmethod
  896. def is_reference_html_file(filesystem, dirname, filename):
  897. # TODO(robertma): We probably do not need prefixes/suffixes other than
  898. # -expected{-mismatch} any more. Or worse, there might be actual tests
  899. # with these prefixes/suffixes.
  900. if filename.startswith('ref-') or filename.startswith('notref-'):
  901. return True
  902. filename_without_ext, _ = filesystem.splitext(filename)
  903. for suffix in ['-expected', '-expected-mismatch', '-ref', '-notref']:
  904. if filename_without_ext.endswith(suffix):
  905. return True
  906. return False
  907. # When collecting test cases, we include any file with these extensions.
  908. supported_file_extensions = set([
  909. '.html',
  910. '.xml',
  911. '.xhtml',
  912. '.xht',
  913. '.pl',
  914. '.htm',
  915. '.php',
  916. '.svg',
  917. '.mht',
  918. '.pdf',
  919. ])
  920. def _has_supported_extension_for_all(self, filename):
  921. extension = self._filesystem.splitext(filename)[1]
  922. if 'inspector-protocol' in filename and extension == '.js':
  923. return True
  924. if 'devtools' in filename and extension == '.js':
  925. return True
  926. return extension in self.supported_file_extensions
  927. def _has_supported_extension(self, filename):
  928. """Returns True if filename is one of the file extensions we want to run a test on."""
  929. extension = self._filesystem.splitext(filename)[1]
  930. return extension in self.supported_file_extensions
  931. def is_non_wpt_test_file(self, dirname, filename):
  932. # Convert dirname to a relative path to web_tests with slashes
  933. # normalized and ensure it has a trailing slash.
  934. normalized_test_dir = self.relative_test_filename(
  935. dirname) + self.TEST_PATH_SEPARATOR
  936. if any(
  937. normalized_test_dir.startswith(d + self.TEST_PATH_SEPARATOR)
  938. for d in self.WPT_DIRS):
  939. return False
  940. extension = self._filesystem.splitext(filename)[1]
  941. if 'inspector-protocol' in dirname and extension == '.js':
  942. return True
  943. if 'devtools' in dirname and extension == '.js':
  944. return True
  945. return (self._has_supported_extension(filename)
  946. and not Port.is_reference_html_file(self._filesystem, dirname,
  947. filename))
  948. @memoized
  949. def wpt_manifest(self, path):
  950. assert path in self.WPT_DIRS
  951. # Convert '/' to the platform-specific separator.
  952. path = self._filesystem.normpath(path)
  953. self._filesystem.maybe_make_directory(
  954. self._filesystem.join(self.web_tests_dir(), path))
  955. manifest_path = self._filesystem.join(self.web_tests_dir(), path,
  956. MANIFEST_NAME)
  957. if not self._filesystem.exists(manifest_path) or self.get_option(
  958. 'manifest_update', False):
  959. _log.debug('Generating MANIFEST.json for %s...', path)
  960. WPTManifest.ensure_manifest(self, path)
  961. return WPTManifest(self.host, manifest_path)
  962. def is_wpt_crash_test(self, test_name):
  963. """Returns whether a WPT test is a crashtest.
  964. See https://web-platform-tests.org/writing-tests/crashtest.html.
  965. """
  966. match = self.WPT_REGEX.match(test_name)
  967. if not match:
  968. return False
  969. wpt_path = match.group(1)
  970. path_in_wpt = match.group(2)
  971. return self.wpt_manifest(wpt_path).is_crash_test(path_in_wpt)
  972. def is_slow_wpt_test(self, test_name):
  973. # When DCHECK is enabled, idlharness tests run 5-6x slower due to the
  974. # amount of JavaScript they use (most web_tests run very little JS).
  975. # This causes flaky timeouts for a lot of them, as a 0.5-1s test becomes
  976. # close to the default 6s timeout.
  977. if (self.is_wpt_idlharness_test(test_name)
  978. and self._build_has_dcheck_always_on()):
  979. return True
  980. match = self.WPT_REGEX.match(test_name)
  981. if not match:
  982. return False
  983. wpt_path = match.group(1)
  984. path_in_wpt = match.group(2)
  985. return self.wpt_manifest(wpt_path).is_slow_test(path_in_wpt)
  986. def extract_wpt_pac(self, test_name):
  987. match = self.WPT_REGEX.match(test_name)
  988. if not match:
  989. return None
  990. wpt_path = match.group(1)
  991. path_in_wpt = match.group(2)
  992. pac = self.wpt_manifest(wpt_path).extract_test_pac(path_in_wpt)
  993. if pac is None:
  994. return None
  995. hosts_and_ports = self.create_driver(0).WPT_HOST_AND_PORTS
  996. return urljoin(
  997. "http://{}:{}".format(hosts_and_ports[0], hosts_and_ports[1]),
  998. urljoin(path_in_wpt, pac))
  999. def get_wpt_fuzzy_metadata(self, test_name):
  1000. """Returns the WPT-style fuzzy metadata for the given test.
  1001. The metadata is a pair of lists, (maxDifference, totalPixels), where
  1002. each list is a [min, max] range, inclusive. If the test has no fuzzy metadata,
  1003. returns (None, None).
  1004. See https://web-platform-tests.org/writing-tests/reftests.html#fuzzy-matching
  1005. """
  1006. match = self.WPT_REGEX.match(test_name)
  1007. if match:
  1008. # This is an actual WPT test, so we can get the metadata from the manifest.
  1009. wpt_path = match.group(1)
  1010. path_in_wpt = match.group(2)
  1011. return self.wpt_manifest(wpt_path).extract_fuzzy_metadata(
  1012. path_in_wpt)
  1013. # This is not a WPT test, so we will parse the metadata ourselves.
  1014. if not self.test_isfile(test_name):
  1015. return (None, None)
  1016. # We use a safe encoding because some test files are incompatible with utf-8.
  1017. test_file = self.read_test(test_name, "latin-1")
  1018. if not test_file:
  1019. return (None, None)
  1020. # We only take the first match which is in line with what we do for WPT tests.
  1021. fuzzy_match = self.WPT_FUZZY_REGEX.search(test_file)
  1022. if not fuzzy_match:
  1023. return (None, None)
  1024. _, max_diff_min, max_diff_max, tot_pix_min, tot_pix_max = \
  1025. fuzzy_match.groups()
  1026. if not max_diff_min:
  1027. max_diff_min = max_diff_max
  1028. if not tot_pix_min:
  1029. tot_pix_min = tot_pix_max
  1030. return ([int(max_diff_min),
  1031. int(max_diff_max)], [int(tot_pix_min),
  1032. int(tot_pix_max)])
  1033. def get_file_path_for_wpt_test(self, test_name):
  1034. """Returns the real file path for the given WPT test.
  1035. Or None if the test is not a WPT.
  1036. """
  1037. match = self.WPT_REGEX.match(test_name)
  1038. if not match:
  1039. return None
  1040. wpt_path = match.group(1)
  1041. path_in_wpt = match.group(2)
  1042. file_path_in_wpt = self.wpt_manifest(wpt_path).file_path_for_test_url(
  1043. path_in_wpt)
  1044. if not file_path_in_wpt:
  1045. return None
  1046. return self._filesystem.join(wpt_path, file_path_in_wpt)
  1047. def test_key(self, test_name):
  1048. """Turns a test name into a pair of sublists: the natural sort key of the
  1049. dirname, and the natural sort key of the basename.
  1050. This can be used when sorting paths so that files in a directory.
  1051. directory are kept together rather than being mixed in with files in
  1052. subdirectories.
  1053. """
  1054. dirname, basename = self.split_test(test_name)
  1055. return (self._natural_sort_key(dirname + self.TEST_PATH_SEPARATOR),
  1056. self._natural_sort_key(basename))
  1057. def _natural_sort_key(self, string_to_split):
  1058. """Turns a string into a list of string and number chunks.
  1059. For example: "z23a" -> ["z", 23, "a"]
  1060. This can be used to implement "natural sort" order. See:
  1061. http://www.codinghorror.com/blog/2007/12/sorting-for-humans-natural-sort-order.html
  1062. http://nedbatchelder.com/blog/200712.html#e20071211T054956
  1063. """
  1064. def tryint(val):
  1065. try:
  1066. return int(val)
  1067. except ValueError:
  1068. return val
  1069. return [tryint(chunk) for chunk in re.split(r'(\d+)', string_to_split)]
  1070. def read_test(self, test_name, encoding="utf8"):
  1071. """Returns the contents of the given test according to the given encoding.
  1072. If no corresponding file can be found, returns None instead.
  1073. Warning: some tests are in utf8-incompatible encodings.
  1074. """
  1075. path = self.abspath_for_test(test_name)
  1076. if self._filesystem.isfile(path):
  1077. return self._filesystem.read_binary_file(path).decode(encoding)
  1078. base = self.lookup_virtual_test_base(test_name)
  1079. if not base:
  1080. return None
  1081. path = self.abspath_for_test(base)
  1082. if self._filesystem.isfile(path):
  1083. return self._filesystem.read_binary_file(path).decode(encoding)
  1084. return None
  1085. @memoized
  1086. def test_isfile(self, test_name):
  1087. """Returns True if the test name refers to an existing test file."""
  1088. # Used by test_expectations.py to apply rules to a file.
  1089. if self._filesystem.isfile(self.abspath_for_test(test_name)):
  1090. return True
  1091. base = self.lookup_virtual_test_base(test_name)
  1092. return base and self._filesystem.isfile(self.abspath_for_test(base))
  1093. @memoized
  1094. def test_isdir(self, test_name):
  1095. """Returns True if the test name refers to an existing directory of tests."""
  1096. # Used by test_expectations.py to apply rules to whole directories.
  1097. if self._filesystem.isdir(self.abspath_for_test(test_name)):
  1098. return True
  1099. base = self.lookup_virtual_test_base(test_name)
  1100. return base and self._filesystem.isdir(self.abspath_for_test(base))
  1101. @memoized
  1102. def test_exists(self, test_name):
  1103. """Returns True if the test name refers to an existing test directory or file."""
  1104. # Used by lint_test_expectations.py to determine if an entry refers to a
  1105. # valid test.
  1106. if self.is_wpt_test(test_name):
  1107. # A virtual WPT test must have valid virtual prefix and base.
  1108. if test_name.startswith('virtual/'):
  1109. return bool(self.lookup_virtual_test_base(test_name))
  1110. # Otherwise treat any WPT test as existing regardless of their real
  1111. # existence on the file system.
  1112. # TODO(crbug.com/959958): Actually check existence of WPT tests.
  1113. return True
  1114. return self.test_isfile(test_name) or self.test_isdir(test_name)
  1115. def split_test(self, test_name):
  1116. """Splits a test name into the 'directory' part and the 'basename' part."""
  1117. index = test_name.rfind(self.TEST_PATH_SEPARATOR)
  1118. if index < 1:
  1119. return ('', test_name)
  1120. return (test_name[0:index], test_name[index:])
  1121. def normalize_test_name(self, test_name):
  1122. """Returns a normalized version of the test name or test directory."""
  1123. if test_name.endswith('/'):
  1124. return test_name
  1125. if self.test_isdir(test_name):
  1126. return test_name + '/'
  1127. return test_name
  1128. def driver_cmd_line(self):
  1129. """Prints the DRT (DumpRenderTree) command that will be used."""
  1130. return self.create_driver(0).cmd_line([])
  1131. def update_baseline(self, baseline_path, data):
  1132. """Updates the baseline for a test.
  1133. Args:
  1134. baseline_path: the actual path to use for baseline, not the path to
  1135. the test. This function is used to update either generic or
  1136. platform-specific baselines, but we can't infer which here.
  1137. data: contents of the baseline.
  1138. """
  1139. self._filesystem.write_binary_file(baseline_path, data)
  1140. def _path_from_chromium_base(self, *comps):
  1141. return self._path_finder.path_from_chromium_base(*comps)
  1142. def _perf_tests_dir(self):
  1143. return self._path_finder.perf_tests_dir()
  1144. def web_tests_dir(self):
  1145. custom_web_tests_dir = self.get_option('layout_tests_directory')
  1146. if custom_web_tests_dir:
  1147. return self._filesystem.abspath(custom_web_tests_dir)
  1148. return self._path_finder.web_tests_dir()
  1149. def generic_baselines_dir(self):
  1150. return self._filesystem.join(self.web_tests_dir(), "platform", "generic")
  1151. def skips_test(self, test):
  1152. """Checks whether the given test is skipped for this port.
  1153. Returns True if:
  1154. - the test is a manual test
  1155. - the port runs smoke tests only and the test is not in the list
  1156. - the test is marked as Skip in NeverFixTest
  1157. - the test is a virtual test not intended to run on this platform.
  1158. """
  1159. return (self.is_manual_test(test)
  1160. or self.skipped_due_to_smoke_tests(test)
  1161. or self.skipped_in_never_fix_tests(test)
  1162. or self.virtual_test_skipped_due_to_platform_config(test))
  1163. @memoized
  1164. def _tests_from_file(self, filename):
  1165. tests = set()
  1166. file_contents = self._filesystem.read_text_file(filename)
  1167. for line in file_contents.splitlines():
  1168. line = line.strip()
  1169. if line.startswith('#') or not line:
  1170. continue
  1171. tests.add(line)
  1172. return tests
  1173. def is_manual_test(self, test):
  1174. """Skip the test if it is a WPT manual test"""
  1175. return self.is_wpt_test(test) and '-manual.' in test
  1176. def skipped_due_to_smoke_tests(self, test):
  1177. """Checks if the test is skipped based on the set of Smoke tests.
  1178. Returns True if this port runs only smoke tests, and the test is not
  1179. in the smoke tests file; returns False otherwise.
  1180. """
  1181. if not self.default_smoke_test_only():
  1182. return False
  1183. smoke_test_filename = self.path_to_smoke_tests_file()
  1184. if not self._filesystem.exists(smoke_test_filename):
  1185. return False
  1186. smoke_tests = self._tests_from_file(smoke_test_filename)
  1187. return test not in smoke_tests
  1188. def path_to_smoke_tests_file(self):
  1189. # Historically we only have one smoke tests list. That one now becomes
  1190. # the default
  1191. return self._filesystem.join(self.web_tests_dir(), 'SmokeTests',
  1192. 'Default.txt')
  1193. def skipped_in_never_fix_tests(self, test):
  1194. """Checks if the test is marked as Skip in NeverFixTests for this port.
  1195. Skip in NeverFixTests indicate we will never fix the failure and
  1196. permanently skip the test. Only Skip lines are allowed in NeverFixTests.
  1197. Some lines in NeverFixTests are platform-specific.
  1198. Note: this will not work with skipped directories. See also the same
  1199. issue with update_all_test_expectations_files in test_importer.py.
  1200. """
  1201. # Note: The parsing logic here (reading the file, constructing a
  1202. # parser, etc.) is very similar to blinkpy/w3c/test_copier.py.
  1203. path = self.path_to_never_fix_tests_file()
  1204. contents = self._filesystem.read_text_file(path)
  1205. test_expectations = TestExpectations(tags=self.get_platform_tags())
  1206. test_expectations.parse_tagged_list(contents)
  1207. return ResultType.Skip in test_expectations.expectations_for(
  1208. test).results
  1209. def path_to_never_fix_tests_file(self):
  1210. return self._filesystem.join(self.web_tests_dir(), 'NeverFixTests')
  1211. def virtual_test_skipped_due_to_platform_config(self, test):
  1212. """Checks if the virtual test is skipped based on the platform config.
  1213. Returns True if the virtual test is not intend to run on this port, due
  1214. to the platform config in VirtualTestSuites; returns False otherwise.
  1215. """
  1216. suite = self._lookup_virtual_suite(test)
  1217. if suite is not None:
  1218. return self.operating_system() not in suite.platforms
  1219. return False
  1220. def name(self):
  1221. """Returns a name that uniquely identifies this particular type of port.
  1222. This is the full port name including both base port name and version,
  1223. and can be passed to PortFactory.get() to instantiate a port.
  1224. """
  1225. return self._name
  1226. def operating_system(self):
  1227. raise NotImplementedError
  1228. def version(self):
  1229. """Returns a string indicating the version of a given platform
  1230. For example, "win10" or "trusty". This is used to help identify the
  1231. exact port when parsing test expectations, determining search paths,
  1232. and logging information.
  1233. """
  1234. return self._version
  1235. def architecture(self):
  1236. return self._architecture
  1237. def python3_command(self):
  1238. """Returns the correct command to use to run python3.
  1239. This exists because Windows has inconsistent behavior between the bots
  1240. and local developer machines, such that determining which python3 name
  1241. to use is non-trivial. See https://crbug.com/1155616.
  1242. Once blinkpy runs under python3, this can be removed in favour of
  1243. callers using sys.executable.
  1244. """
  1245. if six.PY3:
  1246. # Prefer sys.executable when the current script runs under python3.
  1247. # The current script might be running with vpython3 and in that case
  1248. # using the same executable will share the same virtualenv.
  1249. return sys.executable
  1250. return 'python3'
  1251. def get_option(self, name, default_value=None):
  1252. return getattr(self._options, name, default_value)
  1253. def set_option_default(self, name, default_value):
  1254. return self._options.ensure_value(name, default_value)
  1255. def relative_test_filename(self, filename):
  1256. """Returns a Unix-style path for a filename relative to web_tests.
  1257. Ports may legitimately return absolute paths here if no relative path
  1258. makes sense.
  1259. """
  1260. # Ports that run on windows need to override this method to deal with
  1261. # filenames with backslashes in them.
  1262. if filename.startswith(self.web_tests_dir()):
  1263. return self.host.filesystem.relpath(filename, self.web_tests_dir())
  1264. else:
  1265. return self.host.filesystem.abspath(filename)
  1266. @memoized
  1267. def abspath_for_test(self, test_name):
  1268. """Returns the full path to the file for a given test name.
  1269. This is the inverse of relative_test_filename().
  1270. """
  1271. return self._filesystem.join(self.web_tests_dir(), test_name)
  1272. @memoized
  1273. def args_for_test(self, test_name):
  1274. args = self._lookup_virtual_test_args(test_name)
  1275. pac_url = self.extract_wpt_pac(test_name)
  1276. if pac_url is not None:
  1277. args.append("--proxy-pac-url=" + pac_url)
  1278. tracing_categories = self.get_option('enable_tracing')
  1279. if tracing_categories:
  1280. args.append('--trace-startup=' + tracing_categories)
  1281. # Do not finish the trace until the test is finished.
  1282. args.append('--trace-startup-duration=0')
  1283. # Append the current time to the output file name to ensure that
  1284. # the subsequent repetitions of the test do not overwrite older
  1285. # trace files.
  1286. current_time = time.strftime("%Y-%m-%d-%H-%M-%S")
  1287. file_name = 'trace_layout_test_{}_{}.json'.format(
  1288. self._filesystem.sanitize_filename(test_name), current_time)
  1289. args.append('--trace-startup-file=' + file_name)
  1290. return args
  1291. @memoized
  1292. def name_for_test(self, test_name):
  1293. test_base = self.lookup_virtual_test_base(test_name)
  1294. if test_base and not self._filesystem.exists(
  1295. self.abspath_for_test(test_name)):
  1296. return test_base
  1297. return test_name
  1298. def bot_test_times_path(self):
  1299. # TODO(crbug.com/1030434): For the not_site_per_process_blink_web_tests step on linux,
  1300. # an exception is raised when merging the bot times json files. This happens whenever they
  1301. # are outputted into the results directory. Temporarily we will return the bot times json
  1302. # file relative to the target directory.
  1303. return self._build_path('webkit_test_times', 'bot_times_ms.json')
  1304. def results_directory(self):
  1305. """Returns the absolute path directory which will store all web tests outputted
  1306. files. It may include a sub directory for artifacts and it may store performance test results."""
  1307. if not self._results_directory:
  1308. option_val = self.get_option(
  1309. 'results_directory') or self.default_results_directory()
  1310. assert not self._filesystem.basename(option_val) == 'layout-test-results', (
  1311. 'crbug.com/1026494, crbug.com/1027708: The layout-test-results sub directory should '
  1312. 'not be passed as part of the --results-directory command line argument.')
  1313. self._results_directory = self._filesystem.abspath(option_val)
  1314. return self._results_directory
  1315. def artifacts_directory(self):
  1316. """Returns path to artifacts sub directory of the results directory. This
  1317. directory will store test artifacts, which may include actual and expected
  1318. output from web tests."""
  1319. return self._filesystem.join(self.results_directory(),
  1320. ARTIFACTS_SUB_DIR)
  1321. def perf_results_directory(self):
  1322. return self.results_directory()
  1323. def inspector_build_directory(self):
  1324. return self._build_path('gen', 'third_party', 'devtools-frontend',
  1325. 'src', 'front_end')
  1326. def generated_sources_directory(self):
  1327. return self._build_path('gen')
  1328. def apache_config_directory(self):
  1329. return self._path_finder.path_from_blink_tools('apache_config')
  1330. def default_results_directory(self):
  1331. """Returns the absolute path to the build directory."""
  1332. return self._build_path()
  1333. @memoized
  1334. def typ_host(self):
  1335. return SerializableTypHost()
  1336. def setup_test_run(self):
  1337. """Performs port-specific work at the beginning of a test run."""
  1338. # Delete the disk cache if any to ensure a clean test run.
  1339. dump_render_tree_binary_path = self._path_to_driver()
  1340. cachedir = self._filesystem.dirname(dump_render_tree_binary_path)
  1341. cachedir = self._filesystem.join(cachedir, 'cache')
  1342. if self._filesystem.exists(cachedir):
  1343. self._filesystem.rmtree(cachedir)
  1344. if self._dump_reader:
  1345. self._filesystem.maybe_make_directory(
  1346. self._dump_reader.crash_dumps_directory())
  1347. def num_workers(self, requested_num_workers):
  1348. """Returns the number of available workers (possibly less than the number requested)."""
  1349. return requested_num_workers
  1350. def clean_up_test_run(self):
  1351. """Performs port-specific work at the end of a test run."""
  1352. if self._image_differ:
  1353. self._image_differ.stop()
  1354. self._image_differ = None
  1355. def setup_environ_for_server(self):
  1356. # We intentionally copy only a subset of the environment when
  1357. # launching subprocesses to ensure consistent test results.
  1358. clean_env = {}
  1359. variables_to_copy = [
  1360. 'CHROME_DEVEL_SANDBOX',
  1361. 'CHROME_IPC_LOGGING',
  1362. 'ASAN_OPTIONS',
  1363. 'TSAN_OPTIONS',
  1364. 'MSAN_OPTIONS',
  1365. 'LSAN_OPTIONS',
  1366. 'UBSAN_OPTIONS',
  1367. 'VALGRIND_LIB',
  1368. 'VALGRIND_LIB_INNER',
  1369. 'TMPDIR',
  1370. ]
  1371. if 'TMPDIR' not in self.host.environ:
  1372. self.host.environ['TMPDIR'] = tempfile.gettempdir()
  1373. # CGIs are run directory-relative so they need an absolute TMPDIR
  1374. self.host.environ['TMPDIR'] = self._filesystem.abspath(
  1375. self.host.environ['TMPDIR'])
  1376. if self.host.platform.is_linux() or self.host.platform.is_freebsd():
  1377. variables_to_copy += [
  1378. 'XAUTHORITY', 'HOME', 'LANG', 'LD_LIBRARY_PATH',
  1379. 'DBUS_SESSION_BUS_ADDRESS', 'XDG_DATA_DIRS', 'XDG_RUNTIME_DIR'
  1380. ]
  1381. clean_env['DISPLAY'] = self.host.environ.get('DISPLAY', ':1')
  1382. if self.host.platform.is_mac():
  1383. variables_to_copy += [
  1384. 'HOME',
  1385. ]
  1386. if self.host.platform.is_win():
  1387. variables_to_copy += [
  1388. 'PATH',
  1389. ]
  1390. for variable in variables_to_copy:
  1391. if variable in self.host.environ:
  1392. clean_env[variable] = self.host.environ[variable]
  1393. for string_variable in self.get_option('additional_env_var', []):
  1394. [name, value] = string_variable.split('=', 1)
  1395. clean_env[name] = value
  1396. if self.host.platform.is_linux() and not self.use_system_httpd():
  1397. # set up LD_LIBRARY_PATH when we are using httpd built from 3pp.
  1398. path_to_libs = self._filesystem.join(self.apache_server_root(), 'lib')
  1399. if clean_env.get('LD_LIBRARY_PATH'):
  1400. clean_env['LD_LIBRARY_PATH'] = path_to_libs + ':' + clean_env['LD_LIBRARY_PATH']
  1401. else:
  1402. clean_env['LD_LIBRARY_PATH'] = path_to_libs
  1403. return clean_env
  1404. def show_results_html_file(self, results_filename):
  1405. """Displays the given HTML file in a user's browser."""
  1406. return self.host.user.open_url(
  1407. abspath_to_uri(self.host.platform, results_filename))
  1408. def create_driver(self, worker_number, no_timeout=False):
  1409. """Returns a newly created Driver subclass for starting/stopping the
  1410. test driver.
  1411. """
  1412. return self._driver_class()(self, worker_number, no_timeout=no_timeout)
  1413. def requires_http_server(self):
  1414. # Does the port require an HTTP server for running tests? This could
  1415. # be the case when the tests aren't run on the host platform.
  1416. return False
  1417. def start_http_server(self,
  1418. additional_dirs,
  1419. number_of_drivers,
  1420. output_dir=''):
  1421. """Start a web server. Raise an error if it can't start or is already running.
  1422. Ports can stub this out if they don't need a web server to be running.
  1423. """
  1424. assert not self._http_server, 'Already running an http server.'
  1425. output_dir = output_dir or self.artifacts_directory()
  1426. server = apache_http.ApacheHTTP(
  1427. self,
  1428. output_dir,
  1429. additional_dirs=additional_dirs,
  1430. number_of_servers=(number_of_drivers * 4))
  1431. server.start()
  1432. self._http_server = server
  1433. def start_websocket_server(self, output_dir=''):
  1434. """Start a web server. Raise an error if it can't start or is already running.
  1435. Ports can stub this out if they don't need a websocket server to be running.
  1436. """
  1437. assert not self._websocket_server, 'Already running a websocket server.'
  1438. output_dir = output_dir or self.artifacts_directory()
  1439. server = pywebsocket.PyWebSocket(
  1440. self,
  1441. output_dir,
  1442. python_executable=self._options.python_executable)
  1443. server.start()
  1444. self._websocket_server = server
  1445. @staticmethod
  1446. def is_wpt_test(test):
  1447. """Whether a test is considered a web-platform-tests test."""
  1448. return Port.WPT_REGEX.match(test)
  1449. @staticmethod
  1450. def is_wpt_idlharness_test(test_file):
  1451. """Returns whether a WPT test is (probably) an idlharness test.
  1452. There are no rules in WPT that can be used to identify idlharness tests
  1453. without examining the file contents (which would be expensive). This
  1454. method utilizes a filename heuristic, based on the convention of
  1455. including 'idlharness' in the appropriate test names.
  1456. """
  1457. match = Port.WPT_REGEX.match(test_file)
  1458. if not match:
  1459. return False
  1460. filename = match.group(2).split('/')[-1]
  1461. return 'idlharness' in filename
  1462. @staticmethod
  1463. def should_use_wptserve(test):
  1464. return Port.is_wpt_test(test)
  1465. def start_wptserve(self, output_dir=''):
  1466. """Starts a WPT web server.
  1467. Raises an error if it can't start or is already running.
  1468. """
  1469. assert not self._wpt_server, 'Already running a WPT server.'
  1470. output_dir = output_dir or self.artifacts_directory()
  1471. # We currently don't support any output mechanism for the WPT server.
  1472. server = wptserve.WPTServe(self, output_dir)
  1473. server.start()
  1474. self._wpt_server = server
  1475. def stop_wptserve(self):
  1476. """Shuts down the WPT server if it is running."""
  1477. if self._wpt_server:
  1478. self._wpt_server.stop()
  1479. self._wpt_server = None
  1480. def http_server_requires_http_protocol_options_unsafe(self):
  1481. httpd_path = self.path_to_apache()
  1482. intentional_syntax_error = 'INTENTIONAL_SYNTAX_ERROR'
  1483. # yapf: disable
  1484. cmd = [
  1485. httpd_path,
  1486. '-t',
  1487. '-f', self.path_to_apache_config_file(),
  1488. '-C', 'ServerRoot "%s"' % self.apache_server_root(),
  1489. '-C', 'HttpProtocolOptions Unsafe',
  1490. '-C', intentional_syntax_error
  1491. ]
  1492. # yapf: enable
  1493. env = self.setup_environ_for_server()
  1494. def error_handler(err):
  1495. pass
  1496. output = self._executive.run_command(
  1497. cmd, env=env, error_handler=error_handler)
  1498. # If apache complains about the intentional error, it apparently
  1499. # accepted the HttpProtocolOptions directive, and we should add it.
  1500. return intentional_syntax_error in output
  1501. def http_server_supports_ipv6(self):
  1502. # Apache < 2.4 on win32 does not support IPv6.
  1503. return not self.host.platform.is_win()
  1504. def stop_http_server(self):
  1505. """Shuts down the http server if it is running."""
  1506. if self._http_server:
  1507. self._http_server.stop()
  1508. self._http_server = None
  1509. def stop_websocket_server(self):
  1510. """Shuts down the websocket server if it is running."""
  1511. if self._websocket_server:
  1512. self._websocket_server.stop()
  1513. self._websocket_server = None
  1514. #
  1515. # TEST EXPECTATION-RELATED METHODS
  1516. #
  1517. def test_configuration(self):
  1518. """Returns the current TestConfiguration for the port."""
  1519. if not self._test_configuration:
  1520. self._test_configuration = TestConfiguration(
  1521. self._version, self._architecture,
  1522. self._options.configuration.lower())
  1523. return self._test_configuration
  1524. # FIXME: Belongs on a Platform object.
  1525. @memoized
  1526. def all_test_configurations(self):
  1527. """Returns a list of TestConfiguration instances, representing all available
  1528. test configurations for this port.
  1529. """
  1530. return self._generate_all_test_configurations()
  1531. # FIXME: Belongs on a Platform object.
  1532. def configuration_specifier_macros(self):
  1533. """Ports may provide a way to abbreviate configuration specifiers to conveniently
  1534. refer to them as one term or alias specific values to more generic ones. For example:
  1535. (win10, win11) -> win # Abbreviate all Windows versions into one namesake.
  1536. (precise, trusty) -> linux # Change specific name of Linux distro to a more generic term.
  1537. Returns a dictionary, each key representing a macro term ('win', for example),
  1538. and value being a list of valid configuration specifiers (such as ['win10', 'win11']).
  1539. """
  1540. return self.CONFIGURATION_SPECIFIER_MACROS
  1541. def _generate_all_test_configurations(self):
  1542. """Returns a sequence of the TestConfigurations the port supports."""
  1543. # By default, we assume we want to test every graphics type in
  1544. # every configuration on every system.
  1545. test_configurations = []
  1546. for version, architecture in self.ALL_SYSTEMS:
  1547. for build_type in self.ALL_BUILD_TYPES:
  1548. test_configurations.append(
  1549. TestConfiguration(version, architecture, build_type))
  1550. return test_configurations
  1551. def _flag_specific_expectations_path(self):
  1552. config_name = self.flag_specific_config_name()
  1553. if config_name:
  1554. return self.path_to_flag_specific_expectations_file(config_name)
  1555. def _flag_specific_baseline_search_path(self):
  1556. dir = self.baseline_flag_specific_dir()
  1557. return [dir] if dir else []
  1558. def expectations_dict(self):
  1559. """Returns an OrderedDict of name -> expectations strings.
  1560. The names are expected to be (but not required to be) paths in the
  1561. filesystem. If the name is a path, the file can be considered updatable
  1562. for things like rebaselining, so don't use names that are paths if
  1563. they're not paths.
  1564. Generally speaking the ordering should be files in the filesystem in
  1565. cascade order (TestExpectations followed by Skipped, if the port honors
  1566. both formats), then any built-in expectations (e.g., from compile-time
  1567. exclusions), then --additional-expectations options.
  1568. """
  1569. # FIXME: rename this to test_expectations() once all the callers are
  1570. # updated to know about the ordered dict.
  1571. expectations = collections.OrderedDict()
  1572. default_expectations_files = set(self.default_expectations_files())
  1573. ignore_default = self.get_option('ignore_default_expectations', False)
  1574. for path in self.used_expectations_files():
  1575. is_default = path in default_expectations_files
  1576. if ignore_default and is_default:
  1577. continue
  1578. path_exists = self._filesystem.exists(path)
  1579. if is_default:
  1580. if path_exists:
  1581. expectations[path] = self._filesystem.read_text_file(path)
  1582. else:
  1583. if path_exists:
  1584. _log.debug(
  1585. "reading additional_expectations from path '%s'", path)
  1586. expectations[path] = self._filesystem.read_text_file(path)
  1587. else:
  1588. # TODO(rmhasan): Fix additional expectation paths for
  1589. # not_site_per_process_blink_web_tests, then change this
  1590. # back to raising exceptions for incorrect expectation
  1591. # paths.
  1592. _log.warning(
  1593. "additional_expectations path '%s' does not exist",
  1594. path)
  1595. return expectations
  1596. def all_expectations_dict(self):
  1597. """Returns an OrderedDict of name -> expectations strings."""
  1598. expectations = self.expectations_dict()
  1599. flag_path = self._filesystem.join(self.web_tests_dir(),
  1600. self.FLAG_EXPECTATIONS_PREFIX)
  1601. if not self._filesystem.exists(flag_path):
  1602. return expectations
  1603. for (_, _, filenames) in self._filesystem.walk(flag_path):
  1604. if 'README.txt' in filenames:
  1605. filenames.remove('README.txt')
  1606. if 'PRESUBMIT.py' in filenames:
  1607. filenames.remove('PRESUBMIT.py')
  1608. for filename in filenames:
  1609. path = self._filesystem.join(flag_path, filename)
  1610. try:
  1611. expectations[path] = self._filesystem.read_text_file(path)
  1612. except UnicodeDecodeError:
  1613. _log.error('Failed to read expectations file: \'%s\'',
  1614. path)
  1615. raise
  1616. return expectations
  1617. def bot_expectations(self):
  1618. if not self.get_option('ignore_flaky_tests'):
  1619. return {}
  1620. full_port_name = self.determine_full_port_name(
  1621. self.host, self._options, self.port_name)
  1622. builder_category = self.get_option('ignore_builder_category', 'layout')
  1623. step_names = ['blink_web_tests', 'blink_wpt_tests']
  1624. retval = {}
  1625. for step_name in step_names:
  1626. factory = BotTestExpectationsFactory(self.host.builders, step_name)
  1627. # FIXME: This only grabs release builder's flakiness data. If we're running debug,
  1628. # when we should grab the debug builder's data.
  1629. expectations = factory.expectations_for_port(full_port_name,
  1630. builder_category)
  1631. if not expectations:
  1632. continue
  1633. ignore_mode = self.get_option('ignore_flaky_tests')
  1634. if ignore_mode == 'very-flaky' or ignore_mode == 'maybe-flaky':
  1635. retval.update(expectations.flakes_by_path(ignore_mode == 'very-flaky'))
  1636. elif ignore_mode == 'unexpected':
  1637. retval.update(expectations.unexpected_results_by_path())
  1638. else:
  1639. _log.warning("Unexpected ignore mode: '%s'.", ignore_mode)
  1640. return retval
  1641. def default_expectations_files(self):
  1642. """Returns a list of paths to expectations files that apply by default.
  1643. There are other "test expectations" files that may be applied if
  1644. the --additional-expectations flag is passed; those aren't included
  1645. here.
  1646. """
  1647. return filter(None, [
  1648. self.path_to_generic_test_expectations_file(),
  1649. self.path_to_webdriver_expectations_file(),
  1650. self._filesystem.join(self.web_tests_dir(), 'NeverFixTests'),
  1651. self._filesystem.join(self.web_tests_dir(),
  1652. 'StaleTestExpectations'),
  1653. self._filesystem.join(self.web_tests_dir(), 'SlowTests')
  1654. ])
  1655. def used_expectations_files(self):
  1656. """Returns a list of paths to expectation files that are used."""
  1657. if self._used_expectation_files is None:
  1658. self._used_expectation_files = list(
  1659. self.default_expectations_files())
  1660. flag_specific = self._flag_specific_expectations_path()
  1661. if flag_specific:
  1662. self._used_expectation_files.append(flag_specific)
  1663. for path in self.get_option('additional_expectations', []):
  1664. expanded_path = self._filesystem.expanduser(path)
  1665. abs_path = self._filesystem.abspath(expanded_path)
  1666. self._used_expectation_files.append(abs_path)
  1667. return self._used_expectation_files
  1668. def extra_expectations_files(self):
  1669. """Returns a list of paths to test expectations not loaded by default.
  1670. These paths are passed via --additional-expectations on some builders.
  1671. """
  1672. return [
  1673. self._filesystem.join(self.web_tests_dir(), 'ASANExpectations'),
  1674. self._filesystem.join(self.web_tests_dir(), 'LeakExpectations'),
  1675. self._filesystem.join(self.web_tests_dir(), 'MSANExpectations'),
  1676. ]
  1677. @memoized
  1678. def path_to_generic_test_expectations_file(self):
  1679. return self._filesystem.join(self.web_tests_dir(), 'TestExpectations')
  1680. @memoized
  1681. def path_to_webdriver_expectations_file(self):
  1682. return self._filesystem.join(self.web_tests_dir(),
  1683. 'WebDriverExpectations')
  1684. def path_to_flag_specific_expectations_file(self, flag_specific):
  1685. return self._filesystem.join(self.web_tests_dir(),
  1686. self.FLAG_EXPECTATIONS_PREFIX,
  1687. flag_specific)
  1688. def repository_path(self):
  1689. """Returns the repository path for the chromium code base."""
  1690. return self._path_from_chromium_base('build')
  1691. def default_configuration(self):
  1692. return 'Release'
  1693. def clobber_old_port_specific_results(self):
  1694. pass
  1695. def use_system_httpd(self):
  1696. # We use system httpd on linux-arm64 and BSD
  1697. return False
  1698. # FIXME: This does not belong on the port object.
  1699. @memoized
  1700. def path_to_apache(self):
  1701. """Returns the full path to the apache binary.
  1702. This is needed only by ports that use the apache_http_server module.
  1703. """
  1704. raise NotImplementedError('Port.path_to_apache')
  1705. def apache_server_root(self):
  1706. """Returns the root that the apache binary is installed to.
  1707. This is used for the ServerRoot directive.
  1708. """
  1709. executable = self.path_to_apache()
  1710. return self._filesystem.dirname(self._filesystem.dirname(executable))
  1711. def path_to_apache_config_file(self):
  1712. """Returns the full path to the apache configuration file.
  1713. If the WEBKIT_HTTP_SERVER_CONF_PATH environment variable is set, its
  1714. contents will be used instead.
  1715. This is needed only by ports that use the apache_http_server module.
  1716. """
  1717. config_file_from_env = self.host.environ.get(
  1718. 'WEBKIT_HTTP_SERVER_CONF_PATH')
  1719. if config_file_from_env:
  1720. if not self._filesystem.exists(config_file_from_env):
  1721. raise IOError(
  1722. '%s was not found on the system' % config_file_from_env)
  1723. return config_file_from_env
  1724. config_file_name = self._apache_config_file_name_for_platform()
  1725. return self._filesystem.join(self.apache_config_directory(),
  1726. config_file_name)
  1727. def _apache_version(self):
  1728. env = self.setup_environ_for_server()
  1729. config = self._executive.run_command([self.path_to_apache(), '-v'], env=env)
  1730. # Log version including patch level.
  1731. _log.debug(
  1732. 'Found apache version %s',
  1733. re.sub(
  1734. r'(?:.|\n)*Server version: Apache/(\d+\.\d+(?:\.\d+)?)(?:.|\n)*',
  1735. r'\1', config))
  1736. return re.sub(r'(?:.|\n)*Server version: Apache/(\d+\.\d+)(?:.|\n)*',
  1737. r'\1', config)
  1738. def _apache_config_file_name_for_platform(self):
  1739. # Keep the logic to use apache version even though we only have
  1740. # configuration file for 2.4 now, in case we will have newer version in
  1741. # future.
  1742. return 'apache2-httpd-' + self._apache_version() + '-php7.conf'
  1743. def _path_to_driver(self, target=None):
  1744. """Returns the full path to the test driver."""
  1745. return self._build_path(target, self.driver_name())
  1746. def _path_to_image_diff(self):
  1747. """Returns the full path to the image_diff binary, or None if it is not available.
  1748. This is likely used only by diff_image()
  1749. """
  1750. return self._build_path('image_diff')
  1751. def _absolute_baseline_path(self, platform_dir):
  1752. """Return the absolute path to the top of the baseline tree for a
  1753. given platform directory.
  1754. """
  1755. return self._filesystem.join(self.web_tests_dir(), 'platform',
  1756. platform_dir)
  1757. def _driver_class(self):
  1758. """Returns the port's driver implementation."""
  1759. return driver.Driver
  1760. def output_contains_sanitizer_messages(self, output):
  1761. if not output:
  1762. return None
  1763. if (b'AddressSanitizer' in output) or (b'MemorySanitizer' in output):
  1764. return True
  1765. return False
  1766. def _get_crash_log(self, name, pid, stdout, stderr, newer_than):
  1767. if self.output_contains_sanitizer_messages(stderr):
  1768. # Running the symbolizer script can take a lot of memory, so we need to
  1769. # serialize access to it across all the concurrently running drivers.
  1770. llvm_symbolizer_path = self._path_from_chromium_base(
  1771. 'third_party', 'llvm-build', 'Release+Asserts', 'bin',
  1772. 'llvm-symbolizer')
  1773. if self._filesystem.exists(llvm_symbolizer_path):
  1774. env = self.host.environ.copy()
  1775. env['LLVM_SYMBOLIZER_PATH'] = llvm_symbolizer_path
  1776. else:
  1777. env = None
  1778. sanitizer_filter_path = self._path_from_chromium_base(
  1779. 'tools', 'valgrind', 'asan', 'asan_symbolize.py')
  1780. sanitizer_strip_path_prefix = 'Release/../../'
  1781. if self._filesystem.exists(sanitizer_filter_path):
  1782. stderr = self._executive.run_command([
  1783. 'flock', sys.executable, sanitizer_filter_path,
  1784. sanitizer_strip_path_prefix
  1785. ],
  1786. input=stderr,
  1787. decode_output=False,
  1788. env=env)
  1789. name_str = name or '<unknown process name>'
  1790. pid_str = str(pid or '<unknown>')
  1791. # We require stdout and stderr to be bytestrings, not character strings.
  1792. if stdout:
  1793. stdout_lines = stdout.decode('utf8', 'replace').splitlines()
  1794. else:
  1795. stdout_lines = [u'<empty>']
  1796. if stderr:
  1797. stderr_lines = stderr.decode('utf8', 'replace').splitlines()
  1798. else:
  1799. stderr_lines = [u'<empty>']
  1800. return (stderr,
  1801. ('crash log for %s (pid %s):\n%s\n%s\n' %
  1802. (name_str, pid_str, '\n'.join(
  1803. ('STDOUT: ' + l) for l in stdout_lines), '\n'.join(
  1804. ('STDERR: ' + l)
  1805. for l in stderr_lines))).encode('utf8', 'replace'),
  1806. self._get_crash_site(stderr_lines))
  1807. def _get_crash_site(self, stderr_lines):
  1808. # [blah:blah:blah:FATAL:
  1809. prefix_re = r'\[[\w:/.]*FATAL:'
  1810. # crash_file.ext(line)
  1811. site_re = r'(?P<site>[\w_]*\.[\w_]*\(\d*\))'
  1812. # ] blah failed
  1813. suffix_re = r'\]\s*(Check failed|Security DCHECK failed)'
  1814. pattern = re.compile(prefix_re + site_re + suffix_re)
  1815. for line in stderr_lines:
  1816. match = pattern.search(line)
  1817. if match:
  1818. return match.group('site')
  1819. return None
  1820. def look_for_new_crash_logs(self, crashed_processes, start_time):
  1821. pass
  1822. def look_for_new_samples(self, unresponsive_processes, start_time):
  1823. pass
  1824. def sample_process(self, name, pid):
  1825. pass
  1826. def virtual_test_suites(self):
  1827. if self._virtual_test_suites is None:
  1828. path_to_virtual_test_suites = self._filesystem.join(
  1829. self.web_tests_dir(), 'VirtualTestSuites')
  1830. assert self._filesystem.exists(path_to_virtual_test_suites), \
  1831. path_to_virtual_test_suites + ' not found'
  1832. try:
  1833. test_suite_json = json.loads(
  1834. self._filesystem.read_text_file(
  1835. path_to_virtual_test_suites))
  1836. self._virtual_test_suites = []
  1837. for json_config in test_suite_json:
  1838. vts = VirtualTestSuite(**json_config)
  1839. if any(vts.full_prefix == s.full_prefix
  1840. for s in self._virtual_test_suites):
  1841. raise ValueError(
  1842. '{} contains entries with the same prefix: {!r}. Please combine them'
  1843. .format(path_to_virtual_test_suites, json_config))
  1844. self._virtual_test_suites.append(vts)
  1845. except ValueError as error:
  1846. raise ValueError('{} is not a valid JSON file: {}'.format(
  1847. path_to_virtual_test_suites, error))
  1848. return self._virtual_test_suites
  1849. def _all_virtual_tests(self, tests_by_dir):
  1850. tests = []
  1851. for suite in self.virtual_test_suites():
  1852. if suite.bases:
  1853. tests.extend(map(lambda x: suite.full_prefix + x,
  1854. self.real_tests_from_dict(suite.bases, tests_by_dir)))
  1855. return tests
  1856. def _get_bases_for_suite_with_paths(self, suite, paths):
  1857. """Returns a set of bases of the virutual suite that are referenced by
  1858. paths. E.g. given a virtual test suite `foo` with the following bases:
  1859. bar/baz
  1860. bar/quu
  1861. qux
  1862. and given paths of [virtual/foo/bar], this method would return
  1863. [bar/baz, bar/quu]
  1864. Given paths of [virtual/foo/bar/baz/test.html], the return would be
  1865. [bar/baz]
  1866. """
  1867. real_paths = [p.replace(suite.full_prefix, '', 1) for p in paths \
  1868. if p.startswith(suite.full_prefix)]
  1869. # Test for paths that are under the suite's bases, so that we don't run
  1870. # a non-existent test.
  1871. bases = set()
  1872. for real_path in real_paths:
  1873. for base in suite.bases:
  1874. if real_path.startswith(base) or base.startswith(real_path):
  1875. bases.add(base)
  1876. return list(bases)
  1877. def _virtual_tests_for_suite_with_paths(self, suite, paths):
  1878. if not suite.bases:
  1879. return []
  1880. bases = self._get_bases_for_suite_with_paths(suite, paths)
  1881. if not bases:
  1882. return []
  1883. tests = []
  1884. tests.extend(
  1885. map(lambda x: suite.full_prefix + x, self.real_tests(bases)))
  1886. wpt_bases = []
  1887. for base in bases:
  1888. if any(base.startswith(wpt_dir) for wpt_dir in self.WPT_DIRS):
  1889. wpt_bases.append(base)
  1890. if wpt_bases:
  1891. tests.extend(
  1892. self._wpt_test_urls_matching_paths(
  1893. wpt_bases, [suite.full_prefix] * len(wpt_bases)))
  1894. return tests
  1895. def _virtual_tests_matching_paths(self, paths):
  1896. tests = []
  1897. normalized_paths = [self.normalize_test_name(p) for p in paths]
  1898. for suite in self.virtual_test_suites():
  1899. virtual_paths = [
  1900. p for p in normalized_paths if p.startswith(suite.full_prefix)
  1901. ]
  1902. if not virtual_paths:
  1903. continue
  1904. for test in self._virtual_tests_for_suite_with_paths(
  1905. suite, virtual_paths):
  1906. if any(test.startswith(p) for p in normalized_paths):
  1907. tests.append(test)
  1908. if any(self._path_has_wildcard(path) for path in paths):
  1909. _log.warning(
  1910. 'WARNING: Wildcards in paths are not supported for virtual test suites.'
  1911. )
  1912. return tests
  1913. def _path_has_wildcard(self, path):
  1914. return '*' in path
  1915. def _wpt_test_urls_matching_paths(self, filter_paths, virtual_prefixes=[]):
  1916. """Returns a set of paths that are tests to be run from the
  1917. web-platform-test manifest files.
  1918. filter_paths: A list of strings that are prefix matched against the
  1919. list of tests in the WPT manifests. Only tests that match are returned.
  1920. virtual_prefixes: A list of prefixes corresponding to paths in |filter_paths|.
  1921. If present, each test path output should have its virtual prefix
  1922. prepended to the resulting path to the test.
  1923. """
  1924. # Generate the manifest files if needed and then read them. Do this once
  1925. # for this whole method as the file is large and generation/loading is
  1926. # slow.
  1927. wpts = [(wpt_path, self.wpt_manifest(wpt_path))
  1928. for wpt_path in self.WPT_DIRS]
  1929. tests = []
  1930. # This walks through the set of paths where we should look for tests.
  1931. # For each path, a map can be provided that we replace 'path' with in
  1932. # the result.
  1933. for filter_path, virtual_prefix in zip_longest(filter_paths,
  1934. virtual_prefixes):
  1935. # This is to make sure "external[\\/]?" can also match to
  1936. # external/wpt.
  1937. # TODO(robertma): Remove this special case when external/wpt is
  1938. # moved to wpt.
  1939. if filter_path.rstrip('\\/').endswith('external'):
  1940. filter_path = self._filesystem.join(filter_path, 'wpt')
  1941. # '/' is used throughout this function instead of filesystem.sep as
  1942. # the WPT manifest always uses '/' for paths (it is not OS
  1943. # dependent).
  1944. if self._filesystem.sep != '/':
  1945. filter_path = filter_path.replace(self._filesystem.sep, '/')
  1946. # Drop empty path components.
  1947. filter_path = filter_path.replace('//', '/')
  1948. # We now have in |filter_path| a path to an actual test directory or file
  1949. # on disk, in unix format, relative to the root of the web_tests
  1950. # directory.
  1951. for wpt_path, wpt_manifest in wpts:
  1952. # If the |filter_path| is not inside a WPT dir, then we will
  1953. # match no tests in the manifest.
  1954. if not filter_path.startswith(wpt_path):
  1955. continue
  1956. # Drop the WPT prefix (including the joining '/') from |path|.
  1957. filter_path_from_wpt = filter_path[len(wpt_path) + 1:]
  1958. # An empty filter matches everything.
  1959. if filter_path_from_wpt:
  1960. # If the filter is to a specific test file that ends with .js,
  1961. # we match that against tests with any extension by dropping
  1962. # the extension from the filter.
  1963. #
  1964. # Else, when matching a directory, ensure the filter ends in '/'
  1965. # to only match the exact directory name and not directories
  1966. # with the filter as a prefix.
  1967. if wpt_manifest.is_test_file(filter_path_from_wpt):
  1968. filter_path_from_wpt = re.sub(r'\.js$', '.',
  1969. filter_path_from_wpt)
  1970. elif not wpt_manifest.is_test_url(filter_path_from_wpt):
  1971. filter_path_from_wpt = filter_path_from_wpt.rstrip(
  1972. '/') + '/'
  1973. # We now have a path to an actual test directory or file on
  1974. # disk, in unix format, relative to the WPT directory.
  1975. #
  1976. # Look for all tests in the manifest that are under the relative
  1977. # |filter_path_from_wpt|.
  1978. for test_path_from_wpt in wpt_manifest.all_urls():
  1979. assert not test_path_from_wpt.startswith('/')
  1980. assert not test_path_from_wpt.endswith('/')
  1981. # Drop empty path components.
  1982. test_path_from_wpt = test_path_from_wpt.replace('//', '/')
  1983. if test_path_from_wpt.startswith(filter_path_from_wpt):
  1984. # The result is a test path from the root web test
  1985. # directory. If a |virtual_prefix| was given, we prepend
  1986. # that to the result.
  1987. prefix = virtual_prefix if virtual_prefix else ''
  1988. tests.append(prefix + wpt_path + '/' +
  1989. test_path_from_wpt)
  1990. return tests
  1991. def _lookup_virtual_suite(self, test_name):
  1992. if not test_name.startswith('virtual/'):
  1993. return None
  1994. for suite in self.virtual_test_suites():
  1995. if test_name.startswith(suite.full_prefix):
  1996. return suite
  1997. return None
  1998. def lookup_virtual_test_base(self, test_name):
  1999. suite = self._lookup_virtual_suite(test_name)
  2000. if not suite:
  2001. return None
  2002. assert test_name.startswith(suite.full_prefix)
  2003. maybe_base = self.normalize_test_name(
  2004. test_name[len(suite.full_prefix):])
  2005. for base in suite.bases:
  2006. normalized_base = self.normalize_test_name(base)
  2007. if normalized_base.startswith(maybe_base) or maybe_base.startswith(
  2008. normalized_base):
  2009. return maybe_base
  2010. return None
  2011. def _lookup_virtual_test_args(self, test_name):
  2012. normalized_test_name = self.normalize_test_name(test_name)
  2013. for suite in self.virtual_test_suites():
  2014. if normalized_test_name.startswith(suite.full_prefix):
  2015. return suite.args
  2016. return []
  2017. def _build_path(self, *comps):
  2018. """Returns a path from the build directory."""
  2019. return self._build_path_with_target(self._options.target, *comps)
  2020. def _build_path_with_target(self, target, *comps):
  2021. target = target or self.get_option('target')
  2022. return self._filesystem.join(
  2023. self._path_from_chromium_base(),
  2024. self.get_option('build_directory') or 'out', target, *comps)
  2025. def _check_driver_build_up_to_date(self, target):
  2026. # FIXME: We should probably get rid of this check altogether as it has
  2027. # outlived its usefulness in a GN-based world, but for the moment we
  2028. # will just check things if they are using the standard Debug or Release
  2029. # target directories.
  2030. if target not in ('Debug', 'Release'):
  2031. return True
  2032. try:
  2033. debug_path = self._path_to_driver('Debug')
  2034. release_path = self._path_to_driver('Release')
  2035. debug_mtime = self._filesystem.mtime(debug_path)
  2036. release_mtime = self._filesystem.mtime(release_path)
  2037. if (debug_mtime > release_mtime and target == 'Release'
  2038. or release_mtime > debug_mtime and target == 'Debug'):
  2039. most_recent_binary = 'Release' if target == 'Debug' else 'Debug'
  2040. _log.warning(
  2041. 'You are running the %s binary. However the %s binary appears to be more recent. '
  2042. 'Please pass --%s.', target, most_recent_binary,
  2043. most_recent_binary.lower())
  2044. _log.warning('')
  2045. # This will fail if we don't have both a debug and release binary.
  2046. # That's fine because, in this case, we must already be running the
  2047. # most up-to-date one.
  2048. except OSError:
  2049. pass
  2050. return True
  2051. def _get_font_files(self):
  2052. """Returns list of font files that should be used by the test."""
  2053. # TODO(sergeyu): Currently FONT_FILES is valid only on Linux. Make it
  2054. # usable on other platforms if necessary.
  2055. result = []
  2056. for (font_dirs, font_file, package) in FONT_FILES:
  2057. exists = False
  2058. for font_dir in font_dirs:
  2059. font_path = self._filesystem.join(font_dir, font_file)
  2060. if not self._filesystem.isabs(font_path):
  2061. font_path = self._build_path(font_path)
  2062. if self._check_file_exists(font_path, '', more_logging=False):
  2063. result.append(font_path)
  2064. exists = True
  2065. break
  2066. if not exists:
  2067. message = 'You are missing %s under %s.' % (font_file,
  2068. font_dirs)
  2069. if package:
  2070. message += ' Try installing %s. See build instructions.' % package
  2071. _log.error(message)
  2072. raise TestRunException(exit_codes.SYS_DEPS_EXIT_STATUS,
  2073. message)
  2074. return result
  2075. @staticmethod
  2076. def split_webdriver_test_name(test_name):
  2077. """Splits a WebDriver test name into a filename and a subtest name and
  2078. returns both of them. E.g.
  2079. test.py>>foo.html -> (test.py, foo.html)
  2080. test.py -> (test.py, None)
  2081. """
  2082. separator_index = test_name.find(Port.WEBDRIVER_SUBTEST_SEPARATOR)
  2083. if separator_index == -1:
  2084. return (test_name, None)
  2085. webdriver_test_name = test_name[:separator_index]
  2086. separator_len = len(Port.WEBDRIVER_SUBTEST_SEPARATOR)
  2087. subtest_suffix = test_name[separator_index + separator_len:]
  2088. return (webdriver_test_name, subtest_suffix)
  2089. @staticmethod
  2090. def add_webdriver_subtest_suffix(test_name, subtest_name):
  2091. """Appends a subtest name to a WebDriver test name. E.g.
  2092. (test.py, foo.html) -> test.py>>foo.html
  2093. (test.py, None) -> test.py
  2094. """
  2095. if subtest_name:
  2096. return test_name + Port.WEBDRIVER_SUBTEST_SEPARATOR + subtest_name
  2097. return test_name
  2098. @staticmethod
  2099. def split_webdriver_subtest_pytest_name(test_name):
  2100. """Splits a WebDriver test name in pytest format into a filename and a subtest name and
  2101. returns both of them. E.g.
  2102. test.py::foo.html -> (test.py, foo.html)
  2103. test.py -> (test.py, None)
  2104. """
  2105. names_after_split = test_name.split(
  2106. Port.WEBDRIVER_SUBTEST_PYTEST_SEPARATOR)
  2107. assert len(names_after_split) <= 2, \
  2108. "%s has a length greater than 2 after split by ::" % (test_name)
  2109. if len(names_after_split) == 1:
  2110. return (names_after_split[0], None)
  2111. return (names_after_split[0], names_after_split[1])
  2112. @staticmethod
  2113. def add_webdriver_subtest_pytest_suffix(test_name, subtest_name):
  2114. if subtest_name is None:
  2115. return test_name
  2116. return test_name + Port.WEBDRIVER_SUBTEST_PYTEST_SEPARATOR + subtest_name
  2117. class VirtualTestSuite(object):
  2118. def __init__(self, prefix=None, platforms=None, bases=None, args=None):
  2119. assert VALID_FILE_NAME_REGEX.match(prefix), \
  2120. "Virtual test suite prefix '{}' contains invalid characters".format(prefix)
  2121. assert isinstance(platforms, list)
  2122. assert isinstance(bases, list)
  2123. assert args
  2124. assert isinstance(args, list)
  2125. self.full_prefix = 'virtual/' + prefix + '/'
  2126. self.platforms = [x.lower() for x in platforms]
  2127. self.bases = bases
  2128. self.args = args
  2129. def __repr__(self):
  2130. return "VirtualTestSuite('%s', %s, %s, %s)" % (self.full_prefix,
  2131. self.platforms,
  2132. self.bases, self.args)