// Copyright (c) 2012 The Chromium Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. #include #include "base/format_macros.h" #include "base/strings/stringprintf.h" #include "gpu/config/gpu_test_expectations_parser.h" #include "testing/gtest/include/gtest/gtest.h" namespace gpu { struct TestOSEntry { const char* name; GPUTestConfig::OS os; }; static const TestOSEntry kOsFamilyWin = { "WIN", GPUTestConfig::kOsWin }; static const TestOSEntry kOsFamilyMac = { "MAC", GPUTestConfig::kOsMac }; static const struct TestOsWithFamily { TestOSEntry version; TestOSEntry family; } kOSVersionsWithFamily[] = { {{"XP", GPUTestConfig::kOsWinXP}, kOsFamilyWin}, {{"VISTA", GPUTestConfig::kOsWinVista}, kOsFamilyWin}, {{"WIN7", GPUTestConfig::kOsWin7}, kOsFamilyWin}, {{"WIN8", GPUTestConfig::kOsWin8}, kOsFamilyWin}, {{"WIN10", GPUTestConfig::kOsWin10}, kOsFamilyWin}, {{"LEOPARD", GPUTestConfig::kOsMacLeopard}, kOsFamilyMac}, {{"SNOWLEOPARD", GPUTestConfig::kOsMacSnowLeopard}, kOsFamilyMac}, {{"LION", GPUTestConfig::kOsMacLion}, kOsFamilyMac}, {{"MOUNTAINLION", GPUTestConfig::kOsMacMountainLion}, kOsFamilyMac}, {{"MAVERICKS", GPUTestConfig::kOsMacMavericks}, kOsFamilyMac}, {{"YOSEMITE", GPUTestConfig::kOsMacYosemite}, kOsFamilyMac}, {{"ELCAPITAN", GPUTestConfig::kOsMacElCapitan}, kOsFamilyMac}, {{"SIERRA", GPUTestConfig::kOsMacSierra}, kOsFamilyMac}, {{"HIGHSIERRA", GPUTestConfig::kOsMacHighSierra}, kOsFamilyMac}, {{"MOJAVE", GPUTestConfig::kOsMacMojave}, kOsFamilyMac}, {{"CATALINA", GPUTestConfig::kOsMacCatalina}, kOsFamilyMac}, {{"BIGSUR", GPUTestConfig::kOsMacBigSur}, kOsFamilyMac}, {{"MONTEREY", GPUTestConfig::kOsMacMonterey}, kOsFamilyMac}, {{"LINUX", GPUTestConfig::kOsLinux}, {"LINUX", GPUTestConfig::kOsLinux}}, {{"CHROMEOS", GPUTestConfig::kOsChromeOS}, {"CHROMEOS", GPUTestConfig::kOsChromeOS}}, {{"ANDROID", GPUTestConfig::kOsAndroid}, {"ANDROID", GPUTestConfig::kOsAndroid}}}; TestOSEntry GetUnrelatedOS(const TestOSEntry& os) { return (os.os & kOsFamilyWin.os) ? kOsFamilyMac : kOsFamilyWin; } // Prints test parameter details. std::ostream& operator << (std::ostream& out, const TestOsWithFamily& os) { out << "{ os_name: \"" << os.version.name << "\", os_flag: " << os.version.os << ", os_family: \"" << os.family.name << "\", os_family_flag: " << os.family.os << " }"; return out; } class GPUTestExpectationsParserTest : public testing::Test { public: GPUTestExpectationsParserTest() = default; ~GPUTestExpectationsParserTest() override = default; const GPUTestBotConfig& bot_config() const { return bot_config_; } protected: void SetUp() override { bot_config_.set_os(GPUTestConfig::kOsWin7); bot_config_.set_build_type(GPUTestConfig::kBuildTypeRelease); bot_config_.AddGPUVendor(0x10de); bot_config_.set_gpu_device_id(0x0640); bot_config_.set_api(GPUTestConfig::kAPID3D11); bot_config_.set_command_decoder(GPUTestConfig::kCommandDecoderPassthrough); ASSERT_TRUE(bot_config_.IsValid()); } void TearDown() override {} void set_os(int32_t os) { bot_config_.set_os(os); ASSERT_TRUE(bot_config_.IsValid()); } private: GPUTestBotConfig bot_config_; }; class GPUTestExpectationsParserParamTest : public GPUTestExpectationsParserTest, public testing::WithParamInterface { public: GPUTestExpectationsParserParamTest() = default; ~GPUTestExpectationsParserParamTest() override = default; protected: const GPUTestBotConfig& GetBotConfig() { set_os(GetParam().version.os); return bot_config(); } private: // Restrict access to bot_config() function. // GetBotConfig() should be used instead. using GPUTestExpectationsParserTest::bot_config; }; TEST_F(GPUTestExpectationsParserTest, CommentOnly) { const std::string text = " \n" "// This is just some comment\n" ""; GPUTestExpectationsParser parser; EXPECT_TRUE(parser.LoadTestExpectations(text)); EXPECT_EQ(0u, parser.GetErrorMessages().size()); EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass, parser.GetTestExpectation("some_test", bot_config())); } TEST_P(GPUTestExpectationsParserParamTest, ValidFullEntry) { const std::string text = base::StringPrintf("BUG12345 %s RELEASE NVIDIA 0x0640 : MyTest = FAIL", GetParam().version.name); GPUTestExpectationsParser parser; EXPECT_TRUE(parser.LoadTestExpectations(text)); EXPECT_EQ(0u, parser.GetErrorMessages().size()); EXPECT_EQ(GPUTestExpectationsParser::kGpuTestFail, parser.GetTestExpectation("MyTest", GetBotConfig())); } TEST_P(GPUTestExpectationsParserParamTest, ValidPartialEntry) { const std::string text = base::StringPrintf("BUG12345 %s NVIDIA : MyTest = TIMEOUT", GetParam().family.name); GPUTestExpectationsParser parser; EXPECT_TRUE(parser.LoadTestExpectations(text)); EXPECT_EQ(0u, parser.GetErrorMessages().size()); EXPECT_EQ(GPUTestExpectationsParser::kGpuTestTimeout, parser.GetTestExpectation("MyTest", GetBotConfig())); } TEST_P(GPUTestExpectationsParserParamTest, ValidUnrelatedOsEntry) { const std::string text = base::StringPrintf( "BUG12345 %s : MyTest = TIMEOUT", GetUnrelatedOS(GetParam().version).name); GPUTestExpectationsParser parser; EXPECT_TRUE(parser.LoadTestExpectations(text)); EXPECT_EQ(0u, parser.GetErrorMessages().size()); EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass, parser.GetTestExpectation("MyTest", GetBotConfig())); } TEST_F(GPUTestExpectationsParserTest, ValidUnrelatedTestEntry) { const std::string text = "BUG12345 WIN7 RELEASE NVIDIA 0x0640 : AnotherTest = FAIL"; GPUTestExpectationsParser parser; EXPECT_TRUE(parser.LoadTestExpectations(text)); EXPECT_EQ(0u, parser.GetErrorMessages().size()); EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass, parser.GetTestExpectation("MyTest", bot_config())); } TEST_F(GPUTestExpectationsParserTest, AllModifiers) { const std::string text = "BUG12345 XP VISTA WIN7 WIN8 WIN10 LEOPARD SNOWLEOPARD LION MOUNTAINLION " "MAVERICKS LINUX CHROMEOS ANDROID " "NVIDIA INTEL AMD VMWARE RELEASE DEBUG : MyTest = " "PASS FAIL FLAKY TIMEOUT SKIP"; GPUTestExpectationsParser parser; EXPECT_TRUE(parser.LoadTestExpectations(text)); EXPECT_EQ(0u, parser.GetErrorMessages().size()); EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass | GPUTestExpectationsParser::kGpuTestFail | GPUTestExpectationsParser::kGpuTestFlaky | GPUTestExpectationsParser::kGpuTestTimeout | GPUTestExpectationsParser::kGpuTestSkip, parser.GetTestExpectation("MyTest", bot_config())); } TEST_P(GPUTestExpectationsParserParamTest, DuplicateModifiers) { const std::string text = base::StringPrintf( "BUG12345 %s %s RELEASE NVIDIA 0x0640 : MyTest = FAIL", GetParam().version.name, GetParam().version.name); GPUTestExpectationsParser parser; EXPECT_FALSE(parser.LoadTestExpectations(text)); EXPECT_NE(0u, parser.GetErrorMessages().size()); } TEST_F(GPUTestExpectationsParserTest, AllModifiersLowerCase) { const std::string text = "BUG12345 xp vista win7 win8 win10 leopard snowleopard lion linux " "chromeos android nvidia intel amd vmware release debug : MyTest = " "pass fail flaky timeout skip"; GPUTestExpectationsParser parser; EXPECT_TRUE(parser.LoadTestExpectations(text)); EXPECT_EQ(0u, parser.GetErrorMessages().size()); EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass | GPUTestExpectationsParser::kGpuTestFail | GPUTestExpectationsParser::kGpuTestFlaky | GPUTestExpectationsParser::kGpuTestTimeout | GPUTestExpectationsParser::kGpuTestSkip, parser.GetTestExpectation("MyTest", bot_config())); } TEST_F(GPUTestExpectationsParserTest, MissingColon) { const std::string text = "BUG12345 XP MyTest = FAIL"; GPUTestExpectationsParser parser; EXPECT_FALSE(parser.LoadTestExpectations(text)); EXPECT_NE(0u, parser.GetErrorMessages().size()); } TEST_F(GPUTestExpectationsParserTest, MissingEqual) { const std::string text = "BUG12345 XP : MyTest FAIL"; GPUTestExpectationsParser parser; EXPECT_FALSE(parser.LoadTestExpectations(text)); EXPECT_NE(0u, parser.GetErrorMessages().size()); } TEST_F(GPUTestExpectationsParserTest, IllegalModifier) { const std::string text = "BUG12345 XP XXX : MyTest = FAIL"; GPUTestExpectationsParser parser; EXPECT_FALSE(parser.LoadTestExpectations(text)); EXPECT_NE(0u, parser.GetErrorMessages().size()); } TEST_P(GPUTestExpectationsParserParamTest, OsConflicts) { const std::string text = base::StringPrintf("BUG12345 %s %s : MyTest = FAIL", GetParam().version.name, GetParam().family.name); GPUTestExpectationsParser parser; EXPECT_FALSE(parser.LoadTestExpectations(text)); EXPECT_NE(0u, parser.GetErrorMessages().size()); } TEST_F(GPUTestExpectationsParserTest, InvalidModifierCombination) { const std::string text = "BUG12345 XP NVIDIA INTEL 0x0640 : MyTest = FAIL"; GPUTestExpectationsParser parser; EXPECT_FALSE(parser.LoadTestExpectations(text)); EXPECT_NE(0u, parser.GetErrorMessages().size()); } TEST_F(GPUTestExpectationsParserTest, BadGpuDeviceID) { const std::string text = "BUG12345 XP NVIDIA 0xU07X : MyTest = FAIL"; GPUTestExpectationsParser parser; EXPECT_FALSE(parser.LoadTestExpectations(text)); EXPECT_NE(0u, parser.GetErrorMessages().size()); } TEST_F(GPUTestExpectationsParserTest, MoreThanOneGpuDeviceID) { const std::string text = "BUG12345 XP NVIDIA 0x0640 0x0641 : MyTest = FAIL"; GPUTestExpectationsParser parser; EXPECT_FALSE(parser.LoadTestExpectations(text)); EXPECT_NE(0u, parser.GetErrorMessages().size()); } TEST_P(GPUTestExpectationsParserParamTest, MultipleEntriesConflicts) { const std::string text = base::StringPrintf( "BUG12345 %s RELEASE NVIDIA 0x0640 : MyTest = FAIL\n" "BUG12345 %s : MyTest = FAIL", GetParam().version.name, GetParam().family.name); GPUTestExpectationsParser parser; EXPECT_FALSE(parser.LoadTestExpectations(text)); EXPECT_NE(0u, parser.GetErrorMessages().size()); } TEST_F(GPUTestExpectationsParserTest, MultipleTests) { const std::string text = "BUG12345 WIN7 RELEASE NVIDIA 0x0640 : MyTest = FAIL\n" "BUG12345 WIN : AnotherTest = FAIL"; GPUTestExpectationsParser parser; EXPECT_TRUE(parser.LoadTestExpectations(text)); EXPECT_EQ(0u, parser.GetErrorMessages().size()); } TEST_F(GPUTestExpectationsParserTest, ValidMultipleEntries) { const std::string text = "BUG12345 WIN7 RELEASE NVIDIA 0x0640 : MyTest = FAIL\n" "BUG12345 LINUX : MyTest = TIMEOUT"; GPUTestExpectationsParser parser; EXPECT_TRUE(parser.LoadTestExpectations(text)); EXPECT_EQ(0u, parser.GetErrorMessages().size()); EXPECT_EQ(GPUTestExpectationsParser::kGpuTestFail, parser.GetTestExpectation("MyTest", bot_config())); } TEST_F(GPUTestExpectationsParserTest, StarMatching) { const std::string text = "BUG12345 WIN7 RELEASE NVIDIA 0x0640 : MyTest* = FAIL"; GPUTestExpectationsParser parser; EXPECT_TRUE(parser.LoadTestExpectations(text)); EXPECT_EQ(0u, parser.GetErrorMessages().size()); EXPECT_EQ(GPUTestExpectationsParser::kGpuTestFail, parser.GetTestExpectation("MyTest0", bot_config())); EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass, parser.GetTestExpectation("OtherTest", bot_config())); } TEST_F(GPUTestExpectationsParserTest, ValidAPI) { const std::string text = "BUG12345 WIN7 NVIDIA D3D9 D3D11 OPENGL GLES : MyTest = FAIL"; GPUTestExpectationsParser parser; EXPECT_TRUE(parser.LoadTestExpectations(text)); EXPECT_EQ(0u, parser.GetErrorMessages().size()); EXPECT_EQ(GPUTestExpectationsParser::kGpuTestFail, parser.GetTestExpectation("MyTest", bot_config())); } TEST_F(GPUTestExpectationsParserTest, MultipleAPIsConflict) { const std::string text = "BUG12345 WIN7 NVIDIA D3D9 D3D9 : MyTest = FAIL"; GPUTestExpectationsParser parser; EXPECT_FALSE(parser.LoadTestExpectations(text)); EXPECT_NE(0u, parser.GetErrorMessages().size()); } TEST_F(GPUTestExpectationsParserTest, PassthroughCommandDecoder) { const std::string text = "BUG12345 PASSTHROUGH : MyTest = FAIL"; GPUTestExpectationsParser parser; EXPECT_TRUE(parser.LoadTestExpectations(text)); EXPECT_EQ(0u, parser.GetErrorMessages().size()); EXPECT_EQ(GPUTestExpectationsParser::kGpuTestFail, parser.GetTestExpectation("MyTest", bot_config())); } TEST_F(GPUTestExpectationsParserTest, ValidatingCommandDecoder) { const std::string text = "BUG12345 VALIDATING : MyTest = FAIL"; GPUTestExpectationsParser parser; EXPECT_TRUE(parser.LoadTestExpectations(text)); EXPECT_EQ(0u, parser.GetErrorMessages().size()); EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass, parser.GetTestExpectation("MyTest", bot_config())); } TEST_F(GPUTestExpectationsParserTest, MultipleCommandDecodersConflict) { const std::string text = "BUG12345 VALIDATING VALIDATING : MyTest = FAIL"; GPUTestExpectationsParser parser; EXPECT_FALSE(parser.LoadTestExpectations(text)); EXPECT_NE(0u, parser.GetErrorMessages().size()); } INSTANTIATE_TEST_SUITE_P(GPUTestExpectationsParser, GPUTestExpectationsParserParamTest, ::testing::ValuesIn(kOSVersionsWithFamily)); } // namespace gpu