123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247 |
- // Copyright 2015 The Chromium Authors. All rights reserved.
- // Use of this source code is governed by a BSD-style license that can be
- // found in the LICENSE file.
- #include "components/dom_distiller/core/page_features.h"
- #include <stddef.h>
- #include <memory>
- #include <string>
- #include "base/json/json_reader.h"
- #include "third_party/re2/src/re2/re2.h"
- #include "url/gurl.h"
- namespace dom_distiller {
- /* This code needs to derive features in the same way and order in which they
- * are derived when training the model. Parts of that code are reproduced in the
- * comments below.
- */
- namespace {
- std::string GetLastSegment(const std::string& path) {
- // return re.search('[^/]*\/?$', path).group(0)
- if (path.size() == 0)
- return "";
- if (path.size() == 1) {
- DCHECK(path[0] == '/');
- return path;
- }
- size_t start = path.rfind("/", path.size() - 2);
- return start == std::string::npos ? "" : path.substr(start + 1);
- }
- int CountMatches(const std::string& s, const std::string& p) {
- // return len(re.findall(p, s))
- re2::StringPiece sp(s);
- re2::RE2 regexp(p);
- int count = 0;
- while (re2::RE2::FindAndConsume(&sp, regexp))
- count++;
- return count;
- }
- int GetWordCount(const std::string& s) {
- return CountMatches(s, "\\w+");
- }
- bool Contains(const std::string& n, const std::string& h) {
- return h.find(n) != std::string::npos;
- }
- bool EndsWith(const std::string& t, const std::string& s) {
- return s.size() >= t.size() &&
- s.compare(s.size() - t.size(), std::string::npos, t) == 0;
- }
- } // namespace
- int kDerivedFeaturesCount = 29;
- std::vector<double> CalculateDerivedFeatures(bool isOGArticle,
- const GURL& url,
- double numElements,
- double numAnchors,
- double numForms,
- const std::string& innerText,
- const std::string& textContent,
- const std::string& innerHTML) {
- // In the training pipeline, the strings are explicitly encoded in utf-8 (as
- // they are here).
- const std::string& path = url.path();
- int innerTextWords = GetWordCount(innerText);
- int textContentWords = GetWordCount(textContent);
- int innerHTMLWords = GetWordCount(innerHTML);
- std::vector<double> features;
- // 'opengraph', opengraph,
- features.push_back(isOGArticle);
- // 'forum', 'forum' in path,
- features.push_back(Contains("forum", path));
- // 'index', 'index' in path,
- features.push_back(Contains("index", path));
- // 'view', 'view' in path,
- features.push_back(Contains("view", path));
- // 'asp', '.asp' in path,
- features.push_back(Contains(".asp", path));
- // 'phpbb', 'phpbb' in path,
- features.push_back(Contains("phpbb", path));
- // 'php', path.endswith('.php'),
- features.push_back(EndsWith(".php", path));
- // 'pathlength', len(path),
- features.push_back(path.size());
- // 'domain', len(path) < 2,
- features.push_back(path.size() < 2);
- // 'pathcomponents', CountMatches(path, r'\/.'),
- features.push_back(CountMatches(path, "\\/."));
- // 'slugdetector', CountMatches(path, r'[^\w/]'),
- features.push_back(CountMatches(path, "[^\\w/]"));
- // 'pathnumbers', CountMatches(path, r'\d+'),
- features.push_back(CountMatches(path, "\\d+"));
- // 'lastSegmentLength', len(GetLastSegment(path)),
- features.push_back(GetLastSegment(path).size());
- // 'formcount', numForms,
- features.push_back(numForms);
- // 'anchorcount', numAnchors,
- features.push_back(numAnchors);
- // 'elementcount', numElements,
- features.push_back(numElements);
- // 'anchorratio', float(numAnchors) / max(1, numElements),
- features.push_back(double(numAnchors) / std::max<double>(1, numElements));
- // 'innertextlength', len(innerText),
- features.push_back(innerText.size());
- // 'textcontentlength', len(textContent),
- features.push_back(textContent.size());
- // 'innerhtmllength', len(innerHTML),
- features.push_back(innerHTML.size());
- // 'innertextlengthratio', float(len(innerText)) / max(1, len(innerHTML)),
- features.push_back(double(innerText.size()) /
- std::max<double>(1.0, innerHTML.size()));
- // 'textcontentlengthratio', float(len(textContent)) / max(1, len(innerHTML)),
- features.push_back(double(textContent.size()) /
- std::max<double>(1.0, innerHTML.size()));
- // 'innertexttextcontentlengthratio',
- // float(len(innerText)) / max(1, len(textContent)),
- features.push_back(double(innerText.size()) /
- std::max<double>(1.0, textContent.size()));
- // 'innertextwordcount', innerTextWords,
- features.push_back(innerTextWords);
- // 'textcontentwordcount', textContentWords,
- features.push_back(textContentWords);
- // 'innerhtmlwordcount', innerHTMLWords,
- features.push_back(innerHTMLWords);
- // 'innertextwordcountratio', float(innerTextWords) / max(1, innerHTMLWords),
- features.push_back(double(innerTextWords) /
- std::max<int>(1.0, innerHTMLWords));
- // 'textcontentwordcountratio',
- // float(textContentWords) / max(1, innerHTMLWords),
- features.push_back(double(textContentWords) /
- std::max<int>(1.0, innerHTMLWords));
- // 'innertexttextcontentwordcountratio',
- // float(innerTextWords) / max(1, textContentWords),
- features.push_back(double(innerTextWords) /
- std::max<int>(1.0, textContentWords));
- return features;
- }
- std::vector<double> CalculateDerivedFeaturesFromJSON(
- const base::Value* stringified_json) {
- if (!stringified_json->is_string()) {
- return std::vector<double>();
- }
- absl::optional<base::Value> json =
- base::JSONReader::Read(stringified_json->GetString());
- if (!json) {
- return std::vector<double>();
- }
- if (!json->is_dict()) {
- return std::vector<double>();
- }
- auto& dict = json->GetDict();
- absl::optional<double> numElements = dict.FindDouble("numElements");
- absl::optional<double> numAnchors = dict.FindDouble("numAnchors");
- absl::optional<double> numForms = dict.FindDouble("numForms");
- absl::optional<bool> isOGArticle = dict.FindBool("opengraph");
- std::string* url = dict.FindString("url");
- std::string* innerText = dict.FindString("innerText");
- std::string* textContent = dict.FindString("textContent");
- std::string* innerHTML = dict.FindString("innerHTML");
- if (!(isOGArticle.has_value() && url && numElements && numAnchors &&
- numForms && innerText && textContent && innerHTML)) {
- return std::vector<double>();
- }
- GURL parsed_url(*url);
- if (!parsed_url.is_valid()) {
- return std::vector<double>();
- }
- return CalculateDerivedFeatures(isOGArticle.value(), parsed_url, *numElements,
- *numAnchors, *numForms, *innerText,
- *textContent, *innerHTML);
- }
- std::vector<double> CalculateDerivedFeatures(bool openGraph,
- const GURL& url,
- unsigned elementCount,
- unsigned anchorCount,
- unsigned formCount,
- double mozScore,
- double mozScoreAllSqrt,
- double mozScoreAllLinear) {
- const std::string& path = url.path();
- std::vector<double> features;
- // 'opengraph', opengraph,
- features.push_back(openGraph);
- // 'forum', 'forum' in path,
- features.push_back(Contains("forum", path));
- // 'index', 'index' in path,
- features.push_back(Contains("index", path));
- // 'search', 'search' in path,
- features.push_back(Contains("search", path));
- // 'view', 'view' in path,
- features.push_back(Contains("view", path));
- // 'archive', 'archive' in path,
- features.push_back(Contains("archive", path));
- // 'asp', '.asp' in path,
- features.push_back(Contains(".asp", path));
- // 'phpbb', 'phpbb' in path,
- features.push_back(Contains("phpbb", path));
- // 'php', path.endswith('.php'),
- features.push_back(EndsWith(".php", path));
- // 'pathLength', len(path),
- features.push_back(path.size());
- // 'domain', len(path) < 2,
- features.push_back(path.size() < 2);
- // 'pathComponents', CountMatches(path, r'\/.'),
- features.push_back(CountMatches(path, "\\/."));
- // 'slugDetector', CountMatches(path, r'[^\w/]'),
- features.push_back(CountMatches(path, "[^\\w/]"));
- // 'pathNumbers', CountMatches(path, r'\d+'),
- features.push_back(CountMatches(path, "\\d+"));
- // 'lastSegmentLength', len(GetLastSegment(path)),
- features.push_back(GetLastSegment(path).size());
- // 'formCount', numForms,
- features.push_back(formCount);
- // 'anchorCount', numAnchors,
- features.push_back(anchorCount);
- // 'elementCount', numElements,
- features.push_back(elementCount);
- // 'anchorRatio', float(numAnchors) / max(1, numElements),
- features.push_back(double(anchorCount) / std::max<double>(1, elementCount));
- // 'mozScore'
- features.push_back(mozScore);
- // 'mozScoreAllSqrt'
- features.push_back(mozScoreAllSqrt);
- // 'mozScoreAllLinear'
- features.push_back(mozScoreAllLinear);
- return features;
- }
- } // namespace dom_distiller
|