gpu_test_expectations_parser_unittest.cc 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383
  1. // Copyright (c) 2012 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include <stdint.h>
  5. #include "base/format_macros.h"
  6. #include "base/strings/stringprintf.h"
  7. #include "gpu/config/gpu_test_expectations_parser.h"
  8. #include "testing/gtest/include/gtest/gtest.h"
  9. namespace gpu {
  10. struct TestOSEntry {
  11. const char* name;
  12. GPUTestConfig::OS os;
  13. };
  14. static const TestOSEntry kOsFamilyWin = { "WIN", GPUTestConfig::kOsWin };
  15. static const TestOSEntry kOsFamilyMac = { "MAC", GPUTestConfig::kOsMac };
  16. static const struct TestOsWithFamily {
  17. TestOSEntry version;
  18. TestOSEntry family;
  19. } kOSVersionsWithFamily[] = {
  20. {{"XP", GPUTestConfig::kOsWinXP}, kOsFamilyWin},
  21. {{"VISTA", GPUTestConfig::kOsWinVista}, kOsFamilyWin},
  22. {{"WIN7", GPUTestConfig::kOsWin7}, kOsFamilyWin},
  23. {{"WIN8", GPUTestConfig::kOsWin8}, kOsFamilyWin},
  24. {{"WIN10", GPUTestConfig::kOsWin10}, kOsFamilyWin},
  25. {{"LEOPARD", GPUTestConfig::kOsMacLeopard}, kOsFamilyMac},
  26. {{"SNOWLEOPARD", GPUTestConfig::kOsMacSnowLeopard}, kOsFamilyMac},
  27. {{"LION", GPUTestConfig::kOsMacLion}, kOsFamilyMac},
  28. {{"MOUNTAINLION", GPUTestConfig::kOsMacMountainLion}, kOsFamilyMac},
  29. {{"MAVERICKS", GPUTestConfig::kOsMacMavericks}, kOsFamilyMac},
  30. {{"YOSEMITE", GPUTestConfig::kOsMacYosemite}, kOsFamilyMac},
  31. {{"ELCAPITAN", GPUTestConfig::kOsMacElCapitan}, kOsFamilyMac},
  32. {{"SIERRA", GPUTestConfig::kOsMacSierra}, kOsFamilyMac},
  33. {{"HIGHSIERRA", GPUTestConfig::kOsMacHighSierra}, kOsFamilyMac},
  34. {{"MOJAVE", GPUTestConfig::kOsMacMojave}, kOsFamilyMac},
  35. {{"CATALINA", GPUTestConfig::kOsMacCatalina}, kOsFamilyMac},
  36. {{"BIGSUR", GPUTestConfig::kOsMacBigSur}, kOsFamilyMac},
  37. {{"MONTEREY", GPUTestConfig::kOsMacMonterey}, kOsFamilyMac},
  38. {{"LINUX", GPUTestConfig::kOsLinux}, {"LINUX", GPUTestConfig::kOsLinux}},
  39. {{"CHROMEOS", GPUTestConfig::kOsChromeOS},
  40. {"CHROMEOS", GPUTestConfig::kOsChromeOS}},
  41. {{"ANDROID", GPUTestConfig::kOsAndroid},
  42. {"ANDROID", GPUTestConfig::kOsAndroid}}};
  43. TestOSEntry GetUnrelatedOS(const TestOSEntry& os) {
  44. return (os.os & kOsFamilyWin.os) ? kOsFamilyMac : kOsFamilyWin;
  45. }
  46. // Prints test parameter details.
  47. std::ostream& operator << (std::ostream& out, const TestOsWithFamily& os) {
  48. out << "{ os_name: \"" << os.version.name
  49. << "\", os_flag: " << os.version.os
  50. << ", os_family: \"" << os.family.name
  51. << "\", os_family_flag: " << os.family.os
  52. << " }";
  53. return out;
  54. }
  55. class GPUTestExpectationsParserTest : public testing::Test {
  56. public:
  57. GPUTestExpectationsParserTest() = default;
  58. ~GPUTestExpectationsParserTest() override = default;
  59. const GPUTestBotConfig& bot_config() const {
  60. return bot_config_;
  61. }
  62. protected:
  63. void SetUp() override {
  64. bot_config_.set_os(GPUTestConfig::kOsWin7);
  65. bot_config_.set_build_type(GPUTestConfig::kBuildTypeRelease);
  66. bot_config_.AddGPUVendor(0x10de);
  67. bot_config_.set_gpu_device_id(0x0640);
  68. bot_config_.set_api(GPUTestConfig::kAPID3D11);
  69. bot_config_.set_command_decoder(GPUTestConfig::kCommandDecoderPassthrough);
  70. ASSERT_TRUE(bot_config_.IsValid());
  71. }
  72. void TearDown() override {}
  73. void set_os(int32_t os) {
  74. bot_config_.set_os(os);
  75. ASSERT_TRUE(bot_config_.IsValid());
  76. }
  77. private:
  78. GPUTestBotConfig bot_config_;
  79. };
  80. class GPUTestExpectationsParserParamTest
  81. : public GPUTestExpectationsParserTest,
  82. public testing::WithParamInterface<TestOsWithFamily> {
  83. public:
  84. GPUTestExpectationsParserParamTest() = default;
  85. ~GPUTestExpectationsParserParamTest() override = default;
  86. protected:
  87. const GPUTestBotConfig& GetBotConfig() {
  88. set_os(GetParam().version.os);
  89. return bot_config();
  90. }
  91. private:
  92. // Restrict access to bot_config() function.
  93. // GetBotConfig() should be used instead.
  94. using GPUTestExpectationsParserTest::bot_config;
  95. };
  96. TEST_F(GPUTestExpectationsParserTest, CommentOnly) {
  97. const std::string text =
  98. " \n"
  99. "// This is just some comment\n"
  100. "";
  101. GPUTestExpectationsParser parser;
  102. EXPECT_TRUE(parser.LoadTestExpectations(text));
  103. EXPECT_EQ(0u, parser.GetErrorMessages().size());
  104. EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass,
  105. parser.GetTestExpectation("some_test", bot_config()));
  106. }
  107. TEST_P(GPUTestExpectationsParserParamTest, ValidFullEntry) {
  108. const std::string text =
  109. base::StringPrintf("BUG12345 %s RELEASE NVIDIA 0x0640 : MyTest = FAIL",
  110. GetParam().version.name);
  111. GPUTestExpectationsParser parser;
  112. EXPECT_TRUE(parser.LoadTestExpectations(text));
  113. EXPECT_EQ(0u, parser.GetErrorMessages().size());
  114. EXPECT_EQ(GPUTestExpectationsParser::kGpuTestFail,
  115. parser.GetTestExpectation("MyTest", GetBotConfig()));
  116. }
  117. TEST_P(GPUTestExpectationsParserParamTest, ValidPartialEntry) {
  118. const std::string text =
  119. base::StringPrintf("BUG12345 %s NVIDIA : MyTest = TIMEOUT",
  120. GetParam().family.name);
  121. GPUTestExpectationsParser parser;
  122. EXPECT_TRUE(parser.LoadTestExpectations(text));
  123. EXPECT_EQ(0u, parser.GetErrorMessages().size());
  124. EXPECT_EQ(GPUTestExpectationsParser::kGpuTestTimeout,
  125. parser.GetTestExpectation("MyTest", GetBotConfig()));
  126. }
  127. TEST_P(GPUTestExpectationsParserParamTest, ValidUnrelatedOsEntry) {
  128. const std::string text = base::StringPrintf(
  129. "BUG12345 %s : MyTest = TIMEOUT",
  130. GetUnrelatedOS(GetParam().version).name);
  131. GPUTestExpectationsParser parser;
  132. EXPECT_TRUE(parser.LoadTestExpectations(text));
  133. EXPECT_EQ(0u, parser.GetErrorMessages().size());
  134. EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass,
  135. parser.GetTestExpectation("MyTest", GetBotConfig()));
  136. }
  137. TEST_F(GPUTestExpectationsParserTest, ValidUnrelatedTestEntry) {
  138. const std::string text =
  139. "BUG12345 WIN7 RELEASE NVIDIA 0x0640 : AnotherTest = FAIL";
  140. GPUTestExpectationsParser parser;
  141. EXPECT_TRUE(parser.LoadTestExpectations(text));
  142. EXPECT_EQ(0u, parser.GetErrorMessages().size());
  143. EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass,
  144. parser.GetTestExpectation("MyTest", bot_config()));
  145. }
  146. TEST_F(GPUTestExpectationsParserTest, AllModifiers) {
  147. const std::string text =
  148. "BUG12345 XP VISTA WIN7 WIN8 WIN10 LEOPARD SNOWLEOPARD LION MOUNTAINLION "
  149. "MAVERICKS LINUX CHROMEOS ANDROID "
  150. "NVIDIA INTEL AMD VMWARE RELEASE DEBUG : MyTest = "
  151. "PASS FAIL FLAKY TIMEOUT SKIP";
  152. GPUTestExpectationsParser parser;
  153. EXPECT_TRUE(parser.LoadTestExpectations(text));
  154. EXPECT_EQ(0u, parser.GetErrorMessages().size());
  155. EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass |
  156. GPUTestExpectationsParser::kGpuTestFail |
  157. GPUTestExpectationsParser::kGpuTestFlaky |
  158. GPUTestExpectationsParser::kGpuTestTimeout |
  159. GPUTestExpectationsParser::kGpuTestSkip,
  160. parser.GetTestExpectation("MyTest", bot_config()));
  161. }
  162. TEST_P(GPUTestExpectationsParserParamTest, DuplicateModifiers) {
  163. const std::string text = base::StringPrintf(
  164. "BUG12345 %s %s RELEASE NVIDIA 0x0640 : MyTest = FAIL",
  165. GetParam().version.name,
  166. GetParam().version.name);
  167. GPUTestExpectationsParser parser;
  168. EXPECT_FALSE(parser.LoadTestExpectations(text));
  169. EXPECT_NE(0u, parser.GetErrorMessages().size());
  170. }
  171. TEST_F(GPUTestExpectationsParserTest, AllModifiersLowerCase) {
  172. const std::string text =
  173. "BUG12345 xp vista win7 win8 win10 leopard snowleopard lion linux "
  174. "chromeos android nvidia intel amd vmware release debug : MyTest = "
  175. "pass fail flaky timeout skip";
  176. GPUTestExpectationsParser parser;
  177. EXPECT_TRUE(parser.LoadTestExpectations(text));
  178. EXPECT_EQ(0u, parser.GetErrorMessages().size());
  179. EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass |
  180. GPUTestExpectationsParser::kGpuTestFail |
  181. GPUTestExpectationsParser::kGpuTestFlaky |
  182. GPUTestExpectationsParser::kGpuTestTimeout |
  183. GPUTestExpectationsParser::kGpuTestSkip,
  184. parser.GetTestExpectation("MyTest", bot_config()));
  185. }
  186. TEST_F(GPUTestExpectationsParserTest, MissingColon) {
  187. const std::string text =
  188. "BUG12345 XP MyTest = FAIL";
  189. GPUTestExpectationsParser parser;
  190. EXPECT_FALSE(parser.LoadTestExpectations(text));
  191. EXPECT_NE(0u, parser.GetErrorMessages().size());
  192. }
  193. TEST_F(GPUTestExpectationsParserTest, MissingEqual) {
  194. const std::string text =
  195. "BUG12345 XP : MyTest FAIL";
  196. GPUTestExpectationsParser parser;
  197. EXPECT_FALSE(parser.LoadTestExpectations(text));
  198. EXPECT_NE(0u, parser.GetErrorMessages().size());
  199. }
  200. TEST_F(GPUTestExpectationsParserTest, IllegalModifier) {
  201. const std::string text =
  202. "BUG12345 XP XXX : MyTest = FAIL";
  203. GPUTestExpectationsParser parser;
  204. EXPECT_FALSE(parser.LoadTestExpectations(text));
  205. EXPECT_NE(0u, parser.GetErrorMessages().size());
  206. }
  207. TEST_P(GPUTestExpectationsParserParamTest, OsConflicts) {
  208. const std::string text = base::StringPrintf("BUG12345 %s %s : MyTest = FAIL",
  209. GetParam().version.name,
  210. GetParam().family.name);
  211. GPUTestExpectationsParser parser;
  212. EXPECT_FALSE(parser.LoadTestExpectations(text));
  213. EXPECT_NE(0u, parser.GetErrorMessages().size());
  214. }
  215. TEST_F(GPUTestExpectationsParserTest, InvalidModifierCombination) {
  216. const std::string text =
  217. "BUG12345 XP NVIDIA INTEL 0x0640 : MyTest = FAIL";
  218. GPUTestExpectationsParser parser;
  219. EXPECT_FALSE(parser.LoadTestExpectations(text));
  220. EXPECT_NE(0u, parser.GetErrorMessages().size());
  221. }
  222. TEST_F(GPUTestExpectationsParserTest, BadGpuDeviceID) {
  223. const std::string text =
  224. "BUG12345 XP NVIDIA 0xU07X : MyTest = FAIL";
  225. GPUTestExpectationsParser parser;
  226. EXPECT_FALSE(parser.LoadTestExpectations(text));
  227. EXPECT_NE(0u, parser.GetErrorMessages().size());
  228. }
  229. TEST_F(GPUTestExpectationsParserTest, MoreThanOneGpuDeviceID) {
  230. const std::string text =
  231. "BUG12345 XP NVIDIA 0x0640 0x0641 : MyTest = FAIL";
  232. GPUTestExpectationsParser parser;
  233. EXPECT_FALSE(parser.LoadTestExpectations(text));
  234. EXPECT_NE(0u, parser.GetErrorMessages().size());
  235. }
  236. TEST_P(GPUTestExpectationsParserParamTest, MultipleEntriesConflicts) {
  237. const std::string text = base::StringPrintf(
  238. "BUG12345 %s RELEASE NVIDIA 0x0640 : MyTest = FAIL\n"
  239. "BUG12345 %s : MyTest = FAIL",
  240. GetParam().version.name,
  241. GetParam().family.name);
  242. GPUTestExpectationsParser parser;
  243. EXPECT_FALSE(parser.LoadTestExpectations(text));
  244. EXPECT_NE(0u, parser.GetErrorMessages().size());
  245. }
  246. TEST_F(GPUTestExpectationsParserTest, MultipleTests) {
  247. const std::string text =
  248. "BUG12345 WIN7 RELEASE NVIDIA 0x0640 : MyTest = FAIL\n"
  249. "BUG12345 WIN : AnotherTest = FAIL";
  250. GPUTestExpectationsParser parser;
  251. EXPECT_TRUE(parser.LoadTestExpectations(text));
  252. EXPECT_EQ(0u, parser.GetErrorMessages().size());
  253. }
  254. TEST_F(GPUTestExpectationsParserTest, ValidMultipleEntries) {
  255. const std::string text =
  256. "BUG12345 WIN7 RELEASE NVIDIA 0x0640 : MyTest = FAIL\n"
  257. "BUG12345 LINUX : MyTest = TIMEOUT";
  258. GPUTestExpectationsParser parser;
  259. EXPECT_TRUE(parser.LoadTestExpectations(text));
  260. EXPECT_EQ(0u, parser.GetErrorMessages().size());
  261. EXPECT_EQ(GPUTestExpectationsParser::kGpuTestFail,
  262. parser.GetTestExpectation("MyTest", bot_config()));
  263. }
  264. TEST_F(GPUTestExpectationsParserTest, StarMatching) {
  265. const std::string text =
  266. "BUG12345 WIN7 RELEASE NVIDIA 0x0640 : MyTest* = FAIL";
  267. GPUTestExpectationsParser parser;
  268. EXPECT_TRUE(parser.LoadTestExpectations(text));
  269. EXPECT_EQ(0u, parser.GetErrorMessages().size());
  270. EXPECT_EQ(GPUTestExpectationsParser::kGpuTestFail,
  271. parser.GetTestExpectation("MyTest0", bot_config()));
  272. EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass,
  273. parser.GetTestExpectation("OtherTest", bot_config()));
  274. }
  275. TEST_F(GPUTestExpectationsParserTest, ValidAPI) {
  276. const std::string text =
  277. "BUG12345 WIN7 NVIDIA D3D9 D3D11 OPENGL GLES : MyTest = FAIL";
  278. GPUTestExpectationsParser parser;
  279. EXPECT_TRUE(parser.LoadTestExpectations(text));
  280. EXPECT_EQ(0u, parser.GetErrorMessages().size());
  281. EXPECT_EQ(GPUTestExpectationsParser::kGpuTestFail,
  282. parser.GetTestExpectation("MyTest", bot_config()));
  283. }
  284. TEST_F(GPUTestExpectationsParserTest, MultipleAPIsConflict) {
  285. const std::string text = "BUG12345 WIN7 NVIDIA D3D9 D3D9 : MyTest = FAIL";
  286. GPUTestExpectationsParser parser;
  287. EXPECT_FALSE(parser.LoadTestExpectations(text));
  288. EXPECT_NE(0u, parser.GetErrorMessages().size());
  289. }
  290. TEST_F(GPUTestExpectationsParserTest, PassthroughCommandDecoder) {
  291. const std::string text = "BUG12345 PASSTHROUGH : MyTest = FAIL";
  292. GPUTestExpectationsParser parser;
  293. EXPECT_TRUE(parser.LoadTestExpectations(text));
  294. EXPECT_EQ(0u, parser.GetErrorMessages().size());
  295. EXPECT_EQ(GPUTestExpectationsParser::kGpuTestFail,
  296. parser.GetTestExpectation("MyTest", bot_config()));
  297. }
  298. TEST_F(GPUTestExpectationsParserTest, ValidatingCommandDecoder) {
  299. const std::string text = "BUG12345 VALIDATING : MyTest = FAIL";
  300. GPUTestExpectationsParser parser;
  301. EXPECT_TRUE(parser.LoadTestExpectations(text));
  302. EXPECT_EQ(0u, parser.GetErrorMessages().size());
  303. EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass,
  304. parser.GetTestExpectation("MyTest", bot_config()));
  305. }
  306. TEST_F(GPUTestExpectationsParserTest, MultipleCommandDecodersConflict) {
  307. const std::string text = "BUG12345 VALIDATING VALIDATING : MyTest = FAIL";
  308. GPUTestExpectationsParser parser;
  309. EXPECT_FALSE(parser.LoadTestExpectations(text));
  310. EXPECT_NE(0u, parser.GetErrorMessages().size());
  311. }
  312. INSTANTIATE_TEST_SUITE_P(GPUTestExpectationsParser,
  313. GPUTestExpectationsParserParamTest,
  314. ::testing::ValuesIn(kOSVersionsWithFamily));
  315. } // namespace gpu