query_parser_unittest.cc 7.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214
  1. // Copyright 2014 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "components/query_parser/query_parser.h"
  5. #include <stddef.h>
  6. #include "base/strings/utf_string_conversions.h"
  7. #include "testing/gtest/include/gtest/gtest.h"
  8. namespace query_parser {
  9. class QueryParserTest : public testing::Test {
  10. public:
  11. struct TestData {
  12. const char* input;
  13. const int expected_word_count;
  14. };
  15. std::string QueryToString(const std::string& query);
  16. };
  17. // Test helper: Convert a user query string in 8-bit (for hardcoding
  18. // convenience) to a SQLite query string.
  19. std::string QueryParserTest::QueryToString(const std::string& query) {
  20. std::u16string sqlite_query;
  21. QueryParser::ParseQuery(base::UTF8ToUTF16(query), MatchingAlgorithm::DEFAULT,
  22. &sqlite_query);
  23. return base::UTF16ToUTF8(sqlite_query);
  24. }
  25. // Basic multi-word queries, including prefix matching.
  26. TEST_F(QueryParserTest, SimpleQueries) {
  27. EXPECT_EQ("", QueryToString(" "));
  28. EXPECT_EQ("singleword*", QueryToString("singleword"));
  29. EXPECT_EQ("spacedout*", QueryToString(" spacedout "));
  30. EXPECT_EQ("foo* bar*", QueryToString("foo bar"));
  31. // Short words aren't prefix matches. For Korean Hangul
  32. // the minimum is 2 while for other scripts, it's 3.
  33. EXPECT_EQ("f b", QueryToString(" f b"));
  34. // KA JANG
  35. EXPECT_EQ(base::WideToUTF8(L"\xAC00 \xC7A5"),
  36. QueryToString(base::WideToUTF8(L" \xAC00 \xC7A5")));
  37. EXPECT_EQ("foo* bar*", QueryToString(" foo bar "));
  38. // KA-JANG BICH-GO
  39. EXPECT_EQ(base::WideToUTF8(L"\xAC00\xC7A5* \xBE5B\xACE0*"),
  40. QueryToString(base::WideToUTF8(L"\xAC00\xC7A5 \xBE5B\xACE0")));
  41. }
  42. // Quoted substring parsing.
  43. TEST_F(QueryParserTest, Quoted) {
  44. // ASCII quotes
  45. EXPECT_EQ("\"Quoted\"", QueryToString("\"Quoted\""));
  46. // Missing end quotes
  47. EXPECT_EQ("\"miss end\"", QueryToString("\"miss end"));
  48. // Missing begin quotes
  49. EXPECT_EQ("miss* beg*", QueryToString("miss beg\""));
  50. // Weird formatting
  51. EXPECT_EQ("\"Many\" \"quotes\"", QueryToString("\"Many \"\"quotes"));
  52. }
  53. // Apostrophes within words should be preserved, but otherwise stripped.
  54. TEST_F(QueryParserTest, Apostrophes) {
  55. EXPECT_EQ("foo* bar's*", QueryToString("foo bar's"));
  56. EXPECT_EQ("l'foo*", QueryToString("l'foo"));
  57. EXPECT_EQ("foo*", QueryToString("'foo"));
  58. }
  59. // Special characters.
  60. TEST_F(QueryParserTest, SpecialChars) {
  61. EXPECT_EQ("foo* the* bar*", QueryToString("!#:/*foo#$*;'* the!#:/*bar"));
  62. }
  63. TEST_F(QueryParserTest, NumWords) {
  64. TestData data[] = {
  65. { "blah", 1 },
  66. { "foo \"bar baz\"", 3 },
  67. { "foo \"baz\"", 2 },
  68. { "foo \"bar baz\" blah", 4 },
  69. };
  70. for (size_t i = 0; i < std::size(data); ++i) {
  71. std::u16string query_string;
  72. EXPECT_EQ(
  73. data[i].expected_word_count,
  74. QueryParser::ParseQuery(base::UTF8ToUTF16(data[i].input),
  75. MatchingAlgorithm::DEFAULT, &query_string));
  76. }
  77. }
  78. TEST_F(QueryParserTest, ParseQueryNodesAndMatch) {
  79. struct TestData2 {
  80. const std::string query;
  81. const std::string text;
  82. const bool matches;
  83. const size_t m1_start;
  84. const size_t m1_end;
  85. const size_t m2_start;
  86. const size_t m2_end;
  87. } data[] = {
  88. { "foo", "fooey foo", true, 0, 3, 6, 9 },
  89. { "foo foo", "foo", true, 0, 3, 0, 0 },
  90. { "foo fooey", "fooey", true, 0, 5, 0, 0 },
  91. { "fooey foo", "fooey", true, 0, 5, 0, 0 },
  92. { "foo fooey bar", "bar fooey", true, 0, 3, 4, 9 },
  93. { "blah", "blah", true, 0, 4, 0, 0 },
  94. { "blah", "foo", false, 0, 0, 0, 0 },
  95. { "blah", "blahblah", true, 0, 4, 0, 0 },
  96. { "blah", "foo blah", true, 4, 8, 0, 0 },
  97. { "foo blah", "blah", false, 0, 0, 0, 0 },
  98. { "foo blah", "blahx foobar", true, 0, 4, 6, 9 },
  99. { "\"foo blah\"", "foo blah", true, 0, 8, 0, 0 },
  100. { "\"foo blah\"", "foox blahx", false, 0, 0, 0, 0 },
  101. { "\"foo blah\"", "foo blah", true, 0, 8, 0, 0 },
  102. { "\"foo blah\"", "\"foo blah\"", true, 1, 9, 0, 0 },
  103. { "foo blah", "\"foo bar blah\"", true, 1, 4, 9, 13 },
  104. };
  105. for (size_t i = 0; i < std::size(data); ++i) {
  106. query_parser::QueryNodeVector query_nodes;
  107. QueryParser::ParseQueryNodes(base::UTF8ToUTF16(data[i].query),
  108. MatchingAlgorithm::DEFAULT, &query_nodes);
  109. Snippet::MatchPositions match_positions;
  110. ASSERT_EQ(data[i].matches,
  111. QueryParser::DoesQueryMatch(base::UTF8ToUTF16(data[i].text),
  112. query_nodes, &match_positions));
  113. size_t offset = 0;
  114. if (data[i].m1_start != 0 || data[i].m1_end != 0) {
  115. ASSERT_TRUE(match_positions.size() >= 1);
  116. EXPECT_EQ(data[i].m1_start, match_positions[0].first);
  117. EXPECT_EQ(data[i].m1_end, match_positions[0].second);
  118. offset++;
  119. }
  120. if (data[i].m2_start != 0 || data[i].m2_end != 0) {
  121. ASSERT_TRUE(match_positions.size() == 1 + offset);
  122. EXPECT_EQ(data[i].m2_start, match_positions[offset].first);
  123. EXPECT_EQ(data[i].m2_end, match_positions[offset].second);
  124. }
  125. }
  126. }
  127. TEST_F(QueryParserTest, ParseQueryWords) {
  128. struct TestData2 {
  129. const std::string text;
  130. const std::string w1;
  131. const std::string w2;
  132. const std::string w3;
  133. const size_t word_count;
  134. } data[] = {
  135. { "foo", "foo", "", "", 1 },
  136. { "foo bar", "foo", "bar", "", 2 },
  137. { "\"foo bar\"", "foo", "bar", "", 2 },
  138. { "\"foo bar\" a", "foo", "bar", "a", 3 },
  139. };
  140. for (size_t i = 0; i < std::size(data); ++i) {
  141. std::vector<std::u16string> results;
  142. QueryParser::ParseQueryWords(base::UTF8ToUTF16(data[i].text),
  143. MatchingAlgorithm::DEFAULT, &results);
  144. ASSERT_EQ(data[i].word_count, results.size());
  145. EXPECT_EQ(data[i].w1, base::UTF16ToUTF8(results[0]));
  146. if (results.size() == 2)
  147. EXPECT_EQ(data[i].w2, base::UTF16ToUTF8(results[1]));
  148. if (results.size() == 3)
  149. EXPECT_EQ(data[i].w3, base::UTF16ToUTF8(results[2]));
  150. }
  151. }
  152. TEST_F(QueryParserTest, ParseQueryNodesAndMatchExact) {
  153. struct TestData2 {
  154. const std::string query;
  155. const std::string find_in_text;
  156. const bool matches;
  157. } data[] = {
  158. // Trivial cases.
  159. {"blah", "blah", true},
  160. {"blah", "foo", false},
  161. {"blah", "blahblah", false},
  162. {"blah", "foo blah", true},
  163. {"foo blah", "blah", false},
  164. {"foo blah", "blahx foobar", false},
  165. // Verify some prefix-match edge cases.
  166. {"foo", "fooey foo", true},
  167. {"foo foo", "foo", true},
  168. // Query contains a term that doesn't have an exact match in the text.
  169. {"foo fooey", "fooey", false},
  170. {"fooey foo", "fooey", false},
  171. {"foo fooey bar", "bar fooey", false},
  172. // Verify that quotes still work correctly.
  173. {"\"foo blah\"", "foo blah", true},
  174. {"\"foo blah\"", "foox blahx", false},
  175. {"\"foo blah\"", "\"foo blah\"", true},
  176. {"foo blah", "\"foo bar blah\"", true},
  177. };
  178. for (size_t i = 0; i < std::size(data); ++i) {
  179. SCOPED_TRACE(::testing::Message()
  180. << " Testing case i=" << i << " query=" << data[i].query
  181. << " find_in_text=" << data[i].find_in_text);
  182. QueryWordVector find_in_words;
  183. QueryParser::ExtractQueryWords(base::UTF8ToUTF16(data[i].find_in_text),
  184. &find_in_words);
  185. query_parser::QueryNodeVector query_nodes;
  186. QueryParser::ParseQueryNodes(base::UTF8ToUTF16(data[i].query),
  187. MatchingAlgorithm::DEFAULT, &query_nodes);
  188. EXPECT_EQ(data[i].matches, QueryParser::DoesQueryMatch(
  189. find_in_words, query_nodes, /*exact=*/true));
  190. }
  191. }
  192. } // namespace query_parser