page_features.cc 9.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247
  1. // Copyright 2015 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "components/dom_distiller/core/page_features.h"
  5. #include <stddef.h>
  6. #include <memory>
  7. #include <string>
  8. #include "base/json/json_reader.h"
  9. #include "third_party/re2/src/re2/re2.h"
  10. #include "url/gurl.h"
  11. namespace dom_distiller {
  12. /* This code needs to derive features in the same way and order in which they
  13. * are derived when training the model. Parts of that code are reproduced in the
  14. * comments below.
  15. */
  16. namespace {
  17. std::string GetLastSegment(const std::string& path) {
  18. // return re.search('[^/]*\/?$', path).group(0)
  19. if (path.size() == 0)
  20. return "";
  21. if (path.size() == 1) {
  22. DCHECK(path[0] == '/');
  23. return path;
  24. }
  25. size_t start = path.rfind("/", path.size() - 2);
  26. return start == std::string::npos ? "" : path.substr(start + 1);
  27. }
  28. int CountMatches(const std::string& s, const std::string& p) {
  29. // return len(re.findall(p, s))
  30. re2::StringPiece sp(s);
  31. re2::RE2 regexp(p);
  32. int count = 0;
  33. while (re2::RE2::FindAndConsume(&sp, regexp))
  34. count++;
  35. return count;
  36. }
  37. int GetWordCount(const std::string& s) {
  38. return CountMatches(s, "\\w+");
  39. }
  40. bool Contains(const std::string& n, const std::string& h) {
  41. return h.find(n) != std::string::npos;
  42. }
  43. bool EndsWith(const std::string& t, const std::string& s) {
  44. return s.size() >= t.size() &&
  45. s.compare(s.size() - t.size(), std::string::npos, t) == 0;
  46. }
  47. } // namespace
  48. int kDerivedFeaturesCount = 29;
  49. std::vector<double> CalculateDerivedFeatures(bool isOGArticle,
  50. const GURL& url,
  51. double numElements,
  52. double numAnchors,
  53. double numForms,
  54. const std::string& innerText,
  55. const std::string& textContent,
  56. const std::string& innerHTML) {
  57. // In the training pipeline, the strings are explicitly encoded in utf-8 (as
  58. // they are here).
  59. const std::string& path = url.path();
  60. int innerTextWords = GetWordCount(innerText);
  61. int textContentWords = GetWordCount(textContent);
  62. int innerHTMLWords = GetWordCount(innerHTML);
  63. std::vector<double> features;
  64. // 'opengraph', opengraph,
  65. features.push_back(isOGArticle);
  66. // 'forum', 'forum' in path,
  67. features.push_back(Contains("forum", path));
  68. // 'index', 'index' in path,
  69. features.push_back(Contains("index", path));
  70. // 'view', 'view' in path,
  71. features.push_back(Contains("view", path));
  72. // 'asp', '.asp' in path,
  73. features.push_back(Contains(".asp", path));
  74. // 'phpbb', 'phpbb' in path,
  75. features.push_back(Contains("phpbb", path));
  76. // 'php', path.endswith('.php'),
  77. features.push_back(EndsWith(".php", path));
  78. // 'pathlength', len(path),
  79. features.push_back(path.size());
  80. // 'domain', len(path) < 2,
  81. features.push_back(path.size() < 2);
  82. // 'pathcomponents', CountMatches(path, r'\/.'),
  83. features.push_back(CountMatches(path, "\\/."));
  84. // 'slugdetector', CountMatches(path, r'[^\w/]'),
  85. features.push_back(CountMatches(path, "[^\\w/]"));
  86. // 'pathnumbers', CountMatches(path, r'\d+'),
  87. features.push_back(CountMatches(path, "\\d+"));
  88. // 'lastSegmentLength', len(GetLastSegment(path)),
  89. features.push_back(GetLastSegment(path).size());
  90. // 'formcount', numForms,
  91. features.push_back(numForms);
  92. // 'anchorcount', numAnchors,
  93. features.push_back(numAnchors);
  94. // 'elementcount', numElements,
  95. features.push_back(numElements);
  96. // 'anchorratio', float(numAnchors) / max(1, numElements),
  97. features.push_back(double(numAnchors) / std::max<double>(1, numElements));
  98. // 'innertextlength', len(innerText),
  99. features.push_back(innerText.size());
  100. // 'textcontentlength', len(textContent),
  101. features.push_back(textContent.size());
  102. // 'innerhtmllength', len(innerHTML),
  103. features.push_back(innerHTML.size());
  104. // 'innertextlengthratio', float(len(innerText)) / max(1, len(innerHTML)),
  105. features.push_back(double(innerText.size()) /
  106. std::max<double>(1.0, innerHTML.size()));
  107. // 'textcontentlengthratio', float(len(textContent)) / max(1, len(innerHTML)),
  108. features.push_back(double(textContent.size()) /
  109. std::max<double>(1.0, innerHTML.size()));
  110. // 'innertexttextcontentlengthratio',
  111. // float(len(innerText)) / max(1, len(textContent)),
  112. features.push_back(double(innerText.size()) /
  113. std::max<double>(1.0, textContent.size()));
  114. // 'innertextwordcount', innerTextWords,
  115. features.push_back(innerTextWords);
  116. // 'textcontentwordcount', textContentWords,
  117. features.push_back(textContentWords);
  118. // 'innerhtmlwordcount', innerHTMLWords,
  119. features.push_back(innerHTMLWords);
  120. // 'innertextwordcountratio', float(innerTextWords) / max(1, innerHTMLWords),
  121. features.push_back(double(innerTextWords) /
  122. std::max<int>(1.0, innerHTMLWords));
  123. // 'textcontentwordcountratio',
  124. // float(textContentWords) / max(1, innerHTMLWords),
  125. features.push_back(double(textContentWords) /
  126. std::max<int>(1.0, innerHTMLWords));
  127. // 'innertexttextcontentwordcountratio',
  128. // float(innerTextWords) / max(1, textContentWords),
  129. features.push_back(double(innerTextWords) /
  130. std::max<int>(1.0, textContentWords));
  131. return features;
  132. }
  133. std::vector<double> CalculateDerivedFeaturesFromJSON(
  134. const base::Value* stringified_json) {
  135. if (!stringified_json->is_string()) {
  136. return std::vector<double>();
  137. }
  138. absl::optional<base::Value> json =
  139. base::JSONReader::Read(stringified_json->GetString());
  140. if (!json) {
  141. return std::vector<double>();
  142. }
  143. if (!json->is_dict()) {
  144. return std::vector<double>();
  145. }
  146. auto& dict = json->GetDict();
  147. absl::optional<double> numElements = dict.FindDouble("numElements");
  148. absl::optional<double> numAnchors = dict.FindDouble("numAnchors");
  149. absl::optional<double> numForms = dict.FindDouble("numForms");
  150. absl::optional<bool> isOGArticle = dict.FindBool("opengraph");
  151. std::string* url = dict.FindString("url");
  152. std::string* innerText = dict.FindString("innerText");
  153. std::string* textContent = dict.FindString("textContent");
  154. std::string* innerHTML = dict.FindString("innerHTML");
  155. if (!(isOGArticle.has_value() && url && numElements && numAnchors &&
  156. numForms && innerText && textContent && innerHTML)) {
  157. return std::vector<double>();
  158. }
  159. GURL parsed_url(*url);
  160. if (!parsed_url.is_valid()) {
  161. return std::vector<double>();
  162. }
  163. return CalculateDerivedFeatures(isOGArticle.value(), parsed_url, *numElements,
  164. *numAnchors, *numForms, *innerText,
  165. *textContent, *innerHTML);
  166. }
  167. std::vector<double> CalculateDerivedFeatures(bool openGraph,
  168. const GURL& url,
  169. unsigned elementCount,
  170. unsigned anchorCount,
  171. unsigned formCount,
  172. double mozScore,
  173. double mozScoreAllSqrt,
  174. double mozScoreAllLinear) {
  175. const std::string& path = url.path();
  176. std::vector<double> features;
  177. // 'opengraph', opengraph,
  178. features.push_back(openGraph);
  179. // 'forum', 'forum' in path,
  180. features.push_back(Contains("forum", path));
  181. // 'index', 'index' in path,
  182. features.push_back(Contains("index", path));
  183. // 'search', 'search' in path,
  184. features.push_back(Contains("search", path));
  185. // 'view', 'view' in path,
  186. features.push_back(Contains("view", path));
  187. // 'archive', 'archive' in path,
  188. features.push_back(Contains("archive", path));
  189. // 'asp', '.asp' in path,
  190. features.push_back(Contains(".asp", path));
  191. // 'phpbb', 'phpbb' in path,
  192. features.push_back(Contains("phpbb", path));
  193. // 'php', path.endswith('.php'),
  194. features.push_back(EndsWith(".php", path));
  195. // 'pathLength', len(path),
  196. features.push_back(path.size());
  197. // 'domain', len(path) < 2,
  198. features.push_back(path.size() < 2);
  199. // 'pathComponents', CountMatches(path, r'\/.'),
  200. features.push_back(CountMatches(path, "\\/."));
  201. // 'slugDetector', CountMatches(path, r'[^\w/]'),
  202. features.push_back(CountMatches(path, "[^\\w/]"));
  203. // 'pathNumbers', CountMatches(path, r'\d+'),
  204. features.push_back(CountMatches(path, "\\d+"));
  205. // 'lastSegmentLength', len(GetLastSegment(path)),
  206. features.push_back(GetLastSegment(path).size());
  207. // 'formCount', numForms,
  208. features.push_back(formCount);
  209. // 'anchorCount', numAnchors,
  210. features.push_back(anchorCount);
  211. // 'elementCount', numElements,
  212. features.push_back(elementCount);
  213. // 'anchorRatio', float(numAnchors) / max(1, numElements),
  214. features.push_back(double(anchorCount) / std::max<double>(1, elementCount));
  215. // 'mozScore'
  216. features.push_back(mozScore);
  217. // 'mozScoreAllSqrt'
  218. features.push_back(mozScoreAllSqrt);
  219. // 'mozScoreAllLinear'
  220. features.push_back(mozScoreAllLinear);
  221. return features;
  222. }
  223. } // namespace dom_distiller