123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383 |
- // Copyright (c) 2012 The Chromium Authors. All rights reserved.
- // Use of this source code is governed by a BSD-style license that can be
- // found in the LICENSE file.
- #include <stdint.h>
- #include "base/format_macros.h"
- #include "base/strings/stringprintf.h"
- #include "gpu/config/gpu_test_expectations_parser.h"
- #include "testing/gtest/include/gtest/gtest.h"
- namespace gpu {
- struct TestOSEntry {
- const char* name;
- GPUTestConfig::OS os;
- };
- static const TestOSEntry kOsFamilyWin = { "WIN", GPUTestConfig::kOsWin };
- static const TestOSEntry kOsFamilyMac = { "MAC", GPUTestConfig::kOsMac };
- static const struct TestOsWithFamily {
- TestOSEntry version;
- TestOSEntry family;
- } kOSVersionsWithFamily[] = {
- {{"XP", GPUTestConfig::kOsWinXP}, kOsFamilyWin},
- {{"VISTA", GPUTestConfig::kOsWinVista}, kOsFamilyWin},
- {{"WIN7", GPUTestConfig::kOsWin7}, kOsFamilyWin},
- {{"WIN8", GPUTestConfig::kOsWin8}, kOsFamilyWin},
- {{"WIN10", GPUTestConfig::kOsWin10}, kOsFamilyWin},
- {{"LEOPARD", GPUTestConfig::kOsMacLeopard}, kOsFamilyMac},
- {{"SNOWLEOPARD", GPUTestConfig::kOsMacSnowLeopard}, kOsFamilyMac},
- {{"LION", GPUTestConfig::kOsMacLion}, kOsFamilyMac},
- {{"MOUNTAINLION", GPUTestConfig::kOsMacMountainLion}, kOsFamilyMac},
- {{"MAVERICKS", GPUTestConfig::kOsMacMavericks}, kOsFamilyMac},
- {{"YOSEMITE", GPUTestConfig::kOsMacYosemite}, kOsFamilyMac},
- {{"ELCAPITAN", GPUTestConfig::kOsMacElCapitan}, kOsFamilyMac},
- {{"SIERRA", GPUTestConfig::kOsMacSierra}, kOsFamilyMac},
- {{"HIGHSIERRA", GPUTestConfig::kOsMacHighSierra}, kOsFamilyMac},
- {{"MOJAVE", GPUTestConfig::kOsMacMojave}, kOsFamilyMac},
- {{"CATALINA", GPUTestConfig::kOsMacCatalina}, kOsFamilyMac},
- {{"BIGSUR", GPUTestConfig::kOsMacBigSur}, kOsFamilyMac},
- {{"MONTEREY", GPUTestConfig::kOsMacMonterey}, kOsFamilyMac},
- {{"LINUX", GPUTestConfig::kOsLinux}, {"LINUX", GPUTestConfig::kOsLinux}},
- {{"CHROMEOS", GPUTestConfig::kOsChromeOS},
- {"CHROMEOS", GPUTestConfig::kOsChromeOS}},
- {{"ANDROID", GPUTestConfig::kOsAndroid},
- {"ANDROID", GPUTestConfig::kOsAndroid}}};
- TestOSEntry GetUnrelatedOS(const TestOSEntry& os) {
- return (os.os & kOsFamilyWin.os) ? kOsFamilyMac : kOsFamilyWin;
- }
- // Prints test parameter details.
- std::ostream& operator << (std::ostream& out, const TestOsWithFamily& os) {
- out << "{ os_name: \"" << os.version.name
- << "\", os_flag: " << os.version.os
- << ", os_family: \"" << os.family.name
- << "\", os_family_flag: " << os.family.os
- << " }";
- return out;
- }
- class GPUTestExpectationsParserTest : public testing::Test {
- public:
- GPUTestExpectationsParserTest() = default;
- ~GPUTestExpectationsParserTest() override = default;
- const GPUTestBotConfig& bot_config() const {
- return bot_config_;
- }
- protected:
- void SetUp() override {
- bot_config_.set_os(GPUTestConfig::kOsWin7);
- bot_config_.set_build_type(GPUTestConfig::kBuildTypeRelease);
- bot_config_.AddGPUVendor(0x10de);
- bot_config_.set_gpu_device_id(0x0640);
- bot_config_.set_api(GPUTestConfig::kAPID3D11);
- bot_config_.set_command_decoder(GPUTestConfig::kCommandDecoderPassthrough);
- ASSERT_TRUE(bot_config_.IsValid());
- }
- void TearDown() override {}
- void set_os(int32_t os) {
- bot_config_.set_os(os);
- ASSERT_TRUE(bot_config_.IsValid());
- }
- private:
- GPUTestBotConfig bot_config_;
- };
- class GPUTestExpectationsParserParamTest
- : public GPUTestExpectationsParserTest,
- public testing::WithParamInterface<TestOsWithFamily> {
- public:
- GPUTestExpectationsParserParamTest() = default;
- ~GPUTestExpectationsParserParamTest() override = default;
- protected:
- const GPUTestBotConfig& GetBotConfig() {
- set_os(GetParam().version.os);
- return bot_config();
- }
- private:
- // Restrict access to bot_config() function.
- // GetBotConfig() should be used instead.
- using GPUTestExpectationsParserTest::bot_config;
- };
- TEST_F(GPUTestExpectationsParserTest, CommentOnly) {
- const std::string text =
- " \n"
- "// This is just some comment\n"
- "";
- GPUTestExpectationsParser parser;
- EXPECT_TRUE(parser.LoadTestExpectations(text));
- EXPECT_EQ(0u, parser.GetErrorMessages().size());
- EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass,
- parser.GetTestExpectation("some_test", bot_config()));
- }
- TEST_P(GPUTestExpectationsParserParamTest, ValidFullEntry) {
- const std::string text =
- base::StringPrintf("BUG12345 %s RELEASE NVIDIA 0x0640 : MyTest = FAIL",
- GetParam().version.name);
- GPUTestExpectationsParser parser;
- EXPECT_TRUE(parser.LoadTestExpectations(text));
- EXPECT_EQ(0u, parser.GetErrorMessages().size());
- EXPECT_EQ(GPUTestExpectationsParser::kGpuTestFail,
- parser.GetTestExpectation("MyTest", GetBotConfig()));
- }
- TEST_P(GPUTestExpectationsParserParamTest, ValidPartialEntry) {
- const std::string text =
- base::StringPrintf("BUG12345 %s NVIDIA : MyTest = TIMEOUT",
- GetParam().family.name);
- GPUTestExpectationsParser parser;
- EXPECT_TRUE(parser.LoadTestExpectations(text));
- EXPECT_EQ(0u, parser.GetErrorMessages().size());
- EXPECT_EQ(GPUTestExpectationsParser::kGpuTestTimeout,
- parser.GetTestExpectation("MyTest", GetBotConfig()));
- }
- TEST_P(GPUTestExpectationsParserParamTest, ValidUnrelatedOsEntry) {
- const std::string text = base::StringPrintf(
- "BUG12345 %s : MyTest = TIMEOUT",
- GetUnrelatedOS(GetParam().version).name);
- GPUTestExpectationsParser parser;
- EXPECT_TRUE(parser.LoadTestExpectations(text));
- EXPECT_EQ(0u, parser.GetErrorMessages().size());
- EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass,
- parser.GetTestExpectation("MyTest", GetBotConfig()));
- }
- TEST_F(GPUTestExpectationsParserTest, ValidUnrelatedTestEntry) {
- const std::string text =
- "BUG12345 WIN7 RELEASE NVIDIA 0x0640 : AnotherTest = FAIL";
- GPUTestExpectationsParser parser;
- EXPECT_TRUE(parser.LoadTestExpectations(text));
- EXPECT_EQ(0u, parser.GetErrorMessages().size());
- EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass,
- parser.GetTestExpectation("MyTest", bot_config()));
- }
- TEST_F(GPUTestExpectationsParserTest, AllModifiers) {
- const std::string text =
- "BUG12345 XP VISTA WIN7 WIN8 WIN10 LEOPARD SNOWLEOPARD LION MOUNTAINLION "
- "MAVERICKS LINUX CHROMEOS ANDROID "
- "NVIDIA INTEL AMD VMWARE RELEASE DEBUG : MyTest = "
- "PASS FAIL FLAKY TIMEOUT SKIP";
- GPUTestExpectationsParser parser;
- EXPECT_TRUE(parser.LoadTestExpectations(text));
- EXPECT_EQ(0u, parser.GetErrorMessages().size());
- EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass |
- GPUTestExpectationsParser::kGpuTestFail |
- GPUTestExpectationsParser::kGpuTestFlaky |
- GPUTestExpectationsParser::kGpuTestTimeout |
- GPUTestExpectationsParser::kGpuTestSkip,
- parser.GetTestExpectation("MyTest", bot_config()));
- }
- TEST_P(GPUTestExpectationsParserParamTest, DuplicateModifiers) {
- const std::string text = base::StringPrintf(
- "BUG12345 %s %s RELEASE NVIDIA 0x0640 : MyTest = FAIL",
- GetParam().version.name,
- GetParam().version.name);
- GPUTestExpectationsParser parser;
- EXPECT_FALSE(parser.LoadTestExpectations(text));
- EXPECT_NE(0u, parser.GetErrorMessages().size());
- }
- TEST_F(GPUTestExpectationsParserTest, AllModifiersLowerCase) {
- const std::string text =
- "BUG12345 xp vista win7 win8 win10 leopard snowleopard lion linux "
- "chromeos android nvidia intel amd vmware release debug : MyTest = "
- "pass fail flaky timeout skip";
- GPUTestExpectationsParser parser;
- EXPECT_TRUE(parser.LoadTestExpectations(text));
- EXPECT_EQ(0u, parser.GetErrorMessages().size());
- EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass |
- GPUTestExpectationsParser::kGpuTestFail |
- GPUTestExpectationsParser::kGpuTestFlaky |
- GPUTestExpectationsParser::kGpuTestTimeout |
- GPUTestExpectationsParser::kGpuTestSkip,
- parser.GetTestExpectation("MyTest", bot_config()));
- }
- TEST_F(GPUTestExpectationsParserTest, MissingColon) {
- const std::string text =
- "BUG12345 XP MyTest = FAIL";
- GPUTestExpectationsParser parser;
- EXPECT_FALSE(parser.LoadTestExpectations(text));
- EXPECT_NE(0u, parser.GetErrorMessages().size());
- }
- TEST_F(GPUTestExpectationsParserTest, MissingEqual) {
- const std::string text =
- "BUG12345 XP : MyTest FAIL";
- GPUTestExpectationsParser parser;
- EXPECT_FALSE(parser.LoadTestExpectations(text));
- EXPECT_NE(0u, parser.GetErrorMessages().size());
- }
- TEST_F(GPUTestExpectationsParserTest, IllegalModifier) {
- const std::string text =
- "BUG12345 XP XXX : MyTest = FAIL";
- GPUTestExpectationsParser parser;
- EXPECT_FALSE(parser.LoadTestExpectations(text));
- EXPECT_NE(0u, parser.GetErrorMessages().size());
- }
- TEST_P(GPUTestExpectationsParserParamTest, OsConflicts) {
- const std::string text = base::StringPrintf("BUG12345 %s %s : MyTest = FAIL",
- GetParam().version.name,
- GetParam().family.name);
- GPUTestExpectationsParser parser;
- EXPECT_FALSE(parser.LoadTestExpectations(text));
- EXPECT_NE(0u, parser.GetErrorMessages().size());
- }
- TEST_F(GPUTestExpectationsParserTest, InvalidModifierCombination) {
- const std::string text =
- "BUG12345 XP NVIDIA INTEL 0x0640 : MyTest = FAIL";
- GPUTestExpectationsParser parser;
- EXPECT_FALSE(parser.LoadTestExpectations(text));
- EXPECT_NE(0u, parser.GetErrorMessages().size());
- }
- TEST_F(GPUTestExpectationsParserTest, BadGpuDeviceID) {
- const std::string text =
- "BUG12345 XP NVIDIA 0xU07X : MyTest = FAIL";
- GPUTestExpectationsParser parser;
- EXPECT_FALSE(parser.LoadTestExpectations(text));
- EXPECT_NE(0u, parser.GetErrorMessages().size());
- }
- TEST_F(GPUTestExpectationsParserTest, MoreThanOneGpuDeviceID) {
- const std::string text =
- "BUG12345 XP NVIDIA 0x0640 0x0641 : MyTest = FAIL";
- GPUTestExpectationsParser parser;
- EXPECT_FALSE(parser.LoadTestExpectations(text));
- EXPECT_NE(0u, parser.GetErrorMessages().size());
- }
- TEST_P(GPUTestExpectationsParserParamTest, MultipleEntriesConflicts) {
- const std::string text = base::StringPrintf(
- "BUG12345 %s RELEASE NVIDIA 0x0640 : MyTest = FAIL\n"
- "BUG12345 %s : MyTest = FAIL",
- GetParam().version.name,
- GetParam().family.name);
- GPUTestExpectationsParser parser;
- EXPECT_FALSE(parser.LoadTestExpectations(text));
- EXPECT_NE(0u, parser.GetErrorMessages().size());
- }
- TEST_F(GPUTestExpectationsParserTest, MultipleTests) {
- const std::string text =
- "BUG12345 WIN7 RELEASE NVIDIA 0x0640 : MyTest = FAIL\n"
- "BUG12345 WIN : AnotherTest = FAIL";
- GPUTestExpectationsParser parser;
- EXPECT_TRUE(parser.LoadTestExpectations(text));
- EXPECT_EQ(0u, parser.GetErrorMessages().size());
- }
- TEST_F(GPUTestExpectationsParserTest, ValidMultipleEntries) {
- const std::string text =
- "BUG12345 WIN7 RELEASE NVIDIA 0x0640 : MyTest = FAIL\n"
- "BUG12345 LINUX : MyTest = TIMEOUT";
- GPUTestExpectationsParser parser;
- EXPECT_TRUE(parser.LoadTestExpectations(text));
- EXPECT_EQ(0u, parser.GetErrorMessages().size());
- EXPECT_EQ(GPUTestExpectationsParser::kGpuTestFail,
- parser.GetTestExpectation("MyTest", bot_config()));
- }
- TEST_F(GPUTestExpectationsParserTest, StarMatching) {
- const std::string text =
- "BUG12345 WIN7 RELEASE NVIDIA 0x0640 : MyTest* = FAIL";
- GPUTestExpectationsParser parser;
- EXPECT_TRUE(parser.LoadTestExpectations(text));
- EXPECT_EQ(0u, parser.GetErrorMessages().size());
- EXPECT_EQ(GPUTestExpectationsParser::kGpuTestFail,
- parser.GetTestExpectation("MyTest0", bot_config()));
- EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass,
- parser.GetTestExpectation("OtherTest", bot_config()));
- }
- TEST_F(GPUTestExpectationsParserTest, ValidAPI) {
- const std::string text =
- "BUG12345 WIN7 NVIDIA D3D9 D3D11 OPENGL GLES : MyTest = FAIL";
- GPUTestExpectationsParser parser;
- EXPECT_TRUE(parser.LoadTestExpectations(text));
- EXPECT_EQ(0u, parser.GetErrorMessages().size());
- EXPECT_EQ(GPUTestExpectationsParser::kGpuTestFail,
- parser.GetTestExpectation("MyTest", bot_config()));
- }
- TEST_F(GPUTestExpectationsParserTest, MultipleAPIsConflict) {
- const std::string text = "BUG12345 WIN7 NVIDIA D3D9 D3D9 : MyTest = FAIL";
- GPUTestExpectationsParser parser;
- EXPECT_FALSE(parser.LoadTestExpectations(text));
- EXPECT_NE(0u, parser.GetErrorMessages().size());
- }
- TEST_F(GPUTestExpectationsParserTest, PassthroughCommandDecoder) {
- const std::string text = "BUG12345 PASSTHROUGH : MyTest = FAIL";
- GPUTestExpectationsParser parser;
- EXPECT_TRUE(parser.LoadTestExpectations(text));
- EXPECT_EQ(0u, parser.GetErrorMessages().size());
- EXPECT_EQ(GPUTestExpectationsParser::kGpuTestFail,
- parser.GetTestExpectation("MyTest", bot_config()));
- }
- TEST_F(GPUTestExpectationsParserTest, ValidatingCommandDecoder) {
- const std::string text = "BUG12345 VALIDATING : MyTest = FAIL";
- GPUTestExpectationsParser parser;
- EXPECT_TRUE(parser.LoadTestExpectations(text));
- EXPECT_EQ(0u, parser.GetErrorMessages().size());
- EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass,
- parser.GetTestExpectation("MyTest", bot_config()));
- }
- TEST_F(GPUTestExpectationsParserTest, MultipleCommandDecodersConflict) {
- const std::string text = "BUG12345 VALIDATING VALIDATING : MyTest = FAIL";
- GPUTestExpectationsParser parser;
- EXPECT_FALSE(parser.LoadTestExpectations(text));
- EXPECT_NE(0u, parser.GetErrorMessages().size());
- }
- INSTANTIATE_TEST_SUITE_P(GPUTestExpectationsParser,
- GPUTestExpectationsParserParamTest,
- ::testing::ValuesIn(kOSVersionsWithFamily));
- } // namespace gpu
|