url_util.cc 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490
  1. // Copyright 2018 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "components/url_matcher/url_util.h"
  5. #include <memory>
  6. #include <string>
  7. #include "base/files/file_path.h"
  8. #include "base/logging.h"
  9. #include "base/no_destructor.h"
  10. #include "base/strings/escape.h"
  11. #include "base/strings/string_number_conversions.h"
  12. #include "base/strings/string_util.h"
  13. #include "components/google/core/common/google_util.h"
  14. #include "components/url_formatter/url_fixer.h"
  15. #include "components/url_matcher/url_matcher.h"
  16. #include "net/base/filename_util.h"
  17. #include "net/base/url_util.h"
  18. #include "third_party/re2/src/re2/re2.h"
  19. #include "url/gurl.h"
  20. using url_matcher::URLMatcher;
  21. using url_matcher::URLMatcherCondition;
  22. using url_matcher::URLMatcherConditionFactory;
  23. using url_matcher::URLMatcherConditionSet;
  24. using url_matcher::URLMatcherPortFilter;
  25. using url_matcher::URLMatcherSchemeFilter;
  26. using url_matcher::URLQueryElementMatcherCondition;
  27. namespace url_matcher {
  28. namespace util {
  29. namespace {
  30. // Host/regex pattern for Google AMP Cache URLs.
  31. // See https://developers.google.com/amp/cache/overview#amp-cache-url-format
  32. // for a definition of the format of AMP Cache URLs.
  33. const char kGoogleAmpCacheHost[] = "cdn.ampproject.org";
  34. const char kGoogleAmpCachePathPattern[] = "/[a-z]/(s/)?(.*)";
  35. // Regex pattern for the path of Google AMP Viewer URLs.
  36. const char kGoogleAmpViewerPathPattern[] = "/amp/(s/)?(.*)";
  37. // Host, path prefix, and query regex pattern for Google web cache URLs.
  38. const char kGoogleWebCacheHost[] = "webcache.googleusercontent.com";
  39. const char kGoogleWebCachePathPrefix[] = "/search";
  40. const char kGoogleWebCacheQueryPattern[] =
  41. "cache:(.{12}:)?(https?://)?([^ :]*)( [^:]*)?";
  42. const char kGoogleTranslateSubdomain[] = "translate.";
  43. const char kAlternateGoogleTranslateHost[] = "translate.googleusercontent.com";
  44. // Maximum filters allowed. Filters over this index are ignored.
  45. const size_t kMaxFiltersAllowed = 1000;
  46. // Returns a full URL using either "http" or "https" as the scheme.
  47. GURL BuildURL(bool is_https, const std::string& host_and_path) {
  48. std::string scheme = is_https ? url::kHttpsScheme : url::kHttpScheme;
  49. return GURL(scheme + "://" + host_and_path);
  50. }
  51. void ProcessQueryToConditions(
  52. url_matcher::URLMatcherConditionFactory* condition_factory,
  53. const std::string& query,
  54. bool allow,
  55. std::set<URLQueryElementMatcherCondition>* query_conditions) {
  56. url::Component query_left = url::MakeRange(0, query.length());
  57. url::Component key;
  58. url::Component value;
  59. // Depending on the filter type being block-list or allow-list, the matcher
  60. // choose any or every match. The idea is a URL should be blocked if
  61. // there is any occurrence of the key value pair. It should be allowed
  62. // only if every occurrence of the key is followed by the value. This avoids
  63. // situations such as a user appending an allowed video parameter in the
  64. // end of the query and watching a video of their choice (the last parameter
  65. // is ignored by some web servers like youtube's).
  66. URLQueryElementMatcherCondition::Type match_type =
  67. allow ? URLQueryElementMatcherCondition::MATCH_ALL
  68. : URLQueryElementMatcherCondition::MATCH_ANY;
  69. while (ExtractQueryKeyValue(query.data(), &query_left, &key, &value)) {
  70. URLQueryElementMatcherCondition::QueryElementType query_element_type =
  71. value.len ? URLQueryElementMatcherCondition::ELEMENT_TYPE_KEY_VALUE
  72. : URLQueryElementMatcherCondition::ELEMENT_TYPE_KEY;
  73. URLQueryElementMatcherCondition::QueryValueMatchType query_value_match_type;
  74. if (!value.len && key.len && query[key.end() - 1] == '*') {
  75. --key.len;
  76. query_value_match_type =
  77. URLQueryElementMatcherCondition::QUERY_VALUE_MATCH_PREFIX;
  78. } else if (value.len && query[value.end() - 1] == '*') {
  79. --value.len;
  80. query_value_match_type =
  81. URLQueryElementMatcherCondition::QUERY_VALUE_MATCH_PREFIX;
  82. } else {
  83. query_value_match_type =
  84. URLQueryElementMatcherCondition::QUERY_VALUE_MATCH_EXACT;
  85. }
  86. query_conditions->insert(URLQueryElementMatcherCondition(
  87. query.substr(key.begin, key.len), query.substr(value.begin, value.len),
  88. query_value_match_type, query_element_type, match_type,
  89. condition_factory));
  90. }
  91. }
  92. // Helper class for testing the URL against precompiled regexes. This is a
  93. // singleton so the cached regexes are only created once.
  94. class EmbeddedURLExtractor {
  95. public:
  96. EmbeddedURLExtractor(const EmbeddedURLExtractor&) = delete;
  97. EmbeddedURLExtractor& operator=(const EmbeddedURLExtractor&) = delete;
  98. static EmbeddedURLExtractor* GetInstance() {
  99. static base::NoDestructor<EmbeddedURLExtractor> instance;
  100. return instance.get();
  101. }
  102. // Implements url_filter::GetEmbeddedURL().
  103. GURL GetEmbeddedURL(const GURL& url) {
  104. // Check for "*.cdn.ampproject.org" URLs.
  105. if (url.DomainIs(kGoogleAmpCacheHost)) {
  106. std::string s;
  107. std::string embedded;
  108. if (re2::RE2::FullMatch(url.path(), google_amp_cache_path_regex_, &s,
  109. &embedded)) {
  110. if (url.has_query())
  111. embedded += "?" + url.query();
  112. return BuildURL(!s.empty(), embedded);
  113. }
  114. }
  115. // Check for "www.google.TLD/amp/" URLs.
  116. if (google_util::IsGoogleDomainUrl(
  117. url, google_util::DISALLOW_SUBDOMAIN,
  118. google_util::DISALLOW_NON_STANDARD_PORTS)) {
  119. std::string s;
  120. std::string embedded;
  121. if (re2::RE2::FullMatch(url.path(), google_amp_viewer_path_regex_, &s,
  122. &embedded)) {
  123. // The embedded URL may be percent-encoded. Undo that.
  124. embedded = base::UnescapeBinaryURLComponent(embedded);
  125. return BuildURL(!s.empty(), embedded);
  126. }
  127. }
  128. // Check for Google web cache URLs
  129. // ("webcache.googleusercontent.com/search?q=cache:...").
  130. std::string query;
  131. if (url.host_piece() == kGoogleWebCacheHost &&
  132. base::StartsWith(url.path_piece(), kGoogleWebCachePathPrefix) &&
  133. net::GetValueForKeyInQuery(url, "q", &query)) {
  134. std::string fingerprint;
  135. std::string scheme;
  136. std::string embedded;
  137. if (re2::RE2::FullMatch(query, google_web_cache_query_regex_,
  138. &fingerprint, &scheme, &embedded)) {
  139. return BuildURL(scheme == "https://", embedded);
  140. }
  141. }
  142. // Check for Google translate URLs ("translate.google.TLD/...?...&u=URL" or
  143. // "translate.googleusercontent.com/...?...&u=URL").
  144. bool is_translate = false;
  145. if (base::StartsWith(url.host_piece(), kGoogleTranslateSubdomain)) {
  146. // Remove the "translate." prefix.
  147. GURL::Replacements replace;
  148. replace.SetHostStr(
  149. url.host_piece().substr(strlen(kGoogleTranslateSubdomain)));
  150. GURL trimmed = url.ReplaceComponents(replace);
  151. // Check that the remainder is a Google URL. Note: IsGoogleDomainUrl
  152. // checks for [www.]google.TLD, but we don't want the "www.", so
  153. // explicitly exclude that.
  154. // TODO(treib,pam): Instead of excluding "www." manually, teach
  155. // IsGoogleDomainUrl a mode that doesn't allow it.
  156. is_translate = google_util::IsGoogleDomainUrl(
  157. trimmed, google_util::DISALLOW_SUBDOMAIN,
  158. google_util::DISALLOW_NON_STANDARD_PORTS) &&
  159. !base::StartsWith(trimmed.host_piece(), "www.");
  160. }
  161. bool is_alternate_translate =
  162. url.host_piece() == kAlternateGoogleTranslateHost;
  163. if (is_translate || is_alternate_translate) {
  164. std::string embedded;
  165. if (net::GetValueForKeyInQuery(url, "u", &embedded)) {
  166. // The embedded URL may or may not include a scheme. Fix it if
  167. // necessary.
  168. return url_formatter::FixupURL(embedded, /*desired_tld=*/std::string());
  169. }
  170. }
  171. return GURL();
  172. }
  173. private:
  174. friend class base::NoDestructor<EmbeddedURLExtractor>;
  175. EmbeddedURLExtractor()
  176. : google_amp_cache_path_regex_(kGoogleAmpCachePathPattern),
  177. google_amp_viewer_path_regex_(kGoogleAmpViewerPathPattern),
  178. google_web_cache_query_regex_(kGoogleWebCacheQueryPattern) {
  179. DCHECK(google_amp_cache_path_regex_.ok());
  180. DCHECK(google_amp_viewer_path_regex_.ok());
  181. DCHECK(google_web_cache_query_regex_.ok());
  182. }
  183. ~EmbeddedURLExtractor() = default;
  184. const re2::RE2 google_amp_cache_path_regex_;
  185. const re2::RE2 google_amp_viewer_path_regex_;
  186. const re2::RE2 google_web_cache_query_regex_;
  187. };
  188. } // namespace
  189. // Converts a ValueList |value| of strings into a vector. Returns true if
  190. // successful.
  191. bool GetAsStringVector(const base::Value* value,
  192. std::vector<std::string>* out) {
  193. if (!value->is_list())
  194. return false;
  195. for (const base::Value& item : value->GetListDeprecated()) {
  196. if (!item.is_string())
  197. return false;
  198. out->push_back(item.GetString());
  199. }
  200. return true;
  201. }
  202. GURL Normalize(const GURL& url) {
  203. GURL normalized_url = url;
  204. GURL::Replacements replacements;
  205. // Strip username, password, query, and ref.
  206. replacements.ClearUsername();
  207. replacements.ClearPassword();
  208. replacements.ClearQuery();
  209. replacements.ClearRef();
  210. return url.ReplaceComponents(replacements);
  211. }
  212. GURL GetEmbeddedURL(const GURL& url) {
  213. return EmbeddedURLExtractor::GetInstance()->GetEmbeddedURL(url);
  214. }
  215. size_t GetMaxFiltersAllowed() {
  216. return kMaxFiltersAllowed;
  217. }
  218. FilterComponents::FilterComponents() = default;
  219. FilterComponents::~FilterComponents() = default;
  220. FilterComponents::FilterComponents(FilterComponents&&) = default;
  221. bool FilterComponents::IsWildcard() const {
  222. return host.empty() && scheme.empty() && path.empty() && query.empty() &&
  223. port == 0 && number_of_url_matching_conditions == 0 &&
  224. match_subdomains;
  225. }
  226. scoped_refptr<URLMatcherConditionSet> CreateConditionSet(
  227. URLMatcher* url_matcher,
  228. base::MatcherStringPattern::ID id,
  229. const std::string& scheme,
  230. const std::string& host,
  231. bool match_subdomains,
  232. uint16_t port,
  233. const std::string& path,
  234. const std::string& query,
  235. bool allow) {
  236. URLMatcherConditionFactory* condition_factory =
  237. url_matcher->condition_factory();
  238. std::set<URLMatcherCondition> conditions;
  239. conditions.insert(
  240. match_subdomains
  241. ? condition_factory->CreateHostSuffixPathPrefixCondition(host, path)
  242. : condition_factory->CreateHostEqualsPathPrefixCondition(host, path));
  243. std::set<URLQueryElementMatcherCondition> query_conditions;
  244. if (!query.empty()) {
  245. ProcessQueryToConditions(condition_factory, query, allow,
  246. &query_conditions);
  247. }
  248. std::unique_ptr<URLMatcherSchemeFilter> scheme_filter;
  249. if (!scheme.empty())
  250. scheme_filter = std::make_unique<URLMatcherSchemeFilter>(scheme);
  251. std::unique_ptr<URLMatcherPortFilter> port_filter;
  252. if (port != 0) {
  253. std::vector<URLMatcherPortFilter::Range> ranges;
  254. ranges.push_back(URLMatcherPortFilter::CreateRange(port));
  255. port_filter = std::make_unique<URLMatcherPortFilter>(ranges);
  256. }
  257. return base::MakeRefCounted<URLMatcherConditionSet>(
  258. id, conditions, query_conditions, std::move(scheme_filter),
  259. std::move(port_filter));
  260. }
  261. bool FilterToComponents(const std::string& filter,
  262. std::string* scheme,
  263. std::string* host,
  264. bool* match_subdomains,
  265. uint16_t* port,
  266. std::string* path,
  267. std::string* query) {
  268. DCHECK(scheme);
  269. DCHECK(host);
  270. DCHECK(match_subdomains);
  271. DCHECK(port);
  272. DCHECK(path);
  273. DCHECK(query);
  274. url::Parsed parsed;
  275. std::string lc_filter = base::ToLowerASCII(filter);
  276. const std::string url_scheme = url_formatter::SegmentURL(filter, &parsed);
  277. // Check if it's a scheme wildcard pattern. We support both versions
  278. // (scheme:* and scheme://*) the later being consistent with old filter
  279. // definitions.
  280. if (lc_filter == url_scheme + ":*" || lc_filter == url_scheme + "://*") {
  281. scheme->assign(url_scheme);
  282. host->clear();
  283. *match_subdomains = true;
  284. *port = 0;
  285. path->clear();
  286. query->clear();
  287. return true;
  288. }
  289. if (url_scheme == url::kFileScheme) {
  290. base::FilePath file_path;
  291. if (!net::FileURLToFilePath(GURL(filter), &file_path))
  292. return false;
  293. *scheme = url::kFileScheme;
  294. host->clear();
  295. *match_subdomains = true;
  296. *port = 0;
  297. *path = file_path.AsUTF8Unsafe();
  298. #if defined(FILE_PATH_USES_WIN_SEPARATORS)
  299. // Separators have to be canonicalized on Windows.
  300. std::replace(path->begin(), path->end(), '\\', '/');
  301. *path = "/" + *path;
  302. #endif
  303. return true;
  304. }
  305. // According to documentation host can't be empty.
  306. if (!parsed.host.is_nonempty())
  307. return false;
  308. if (parsed.scheme.is_nonempty())
  309. scheme->assign(url_scheme);
  310. else
  311. scheme->clear();
  312. host->assign(filter, parsed.host.begin, parsed.host.len);
  313. *host = base::ToLowerASCII(*host);
  314. // Special '*' host, matches all hosts.
  315. if (*host == "*") {
  316. host->clear();
  317. *match_subdomains = true;
  318. } else if (host->at(0) == '.') {
  319. // A leading dot in the pattern syntax means that we don't want to match
  320. // subdomains.
  321. host->erase(0, 1);
  322. *match_subdomains = false;
  323. } else {
  324. url::RawCanonOutputT<char> output;
  325. url::CanonHostInfo host_info;
  326. url::CanonicalizeHostVerbose(filter.c_str(), parsed.host, &output,
  327. &host_info);
  328. if (host_info.family == url::CanonHostInfo::NEUTRAL) {
  329. // We want to match subdomains. Add a dot in front to make sure we only
  330. // match at domain component boundaries.
  331. *host = "." + *host;
  332. *match_subdomains = true;
  333. } else {
  334. *match_subdomains = false;
  335. }
  336. }
  337. if (parsed.port.is_nonempty()) {
  338. int int_port;
  339. if (!base::StringToInt(filter.substr(parsed.port.begin, parsed.port.len),
  340. &int_port)) {
  341. return false;
  342. }
  343. if (int_port <= 0 || int_port > std::numeric_limits<uint16_t>::max())
  344. return false;
  345. *port = int_port;
  346. } else {
  347. // Match any port.
  348. *port = 0;
  349. }
  350. if (parsed.path.is_nonempty())
  351. path->assign(filter, parsed.path.begin, parsed.path.len);
  352. else
  353. path->clear();
  354. if (parsed.query.is_nonempty())
  355. query->assign(filter, parsed.query.begin, parsed.query.len);
  356. else
  357. query->clear();
  358. return true;
  359. }
  360. void AddFilters(URLMatcher* matcher,
  361. bool allow,
  362. base::MatcherStringPattern::ID* id,
  363. const base::Value::List& patterns,
  364. std::map<base::MatcherStringPattern::ID,
  365. url_matcher::util::FilterComponents>* filters) {
  366. URLMatcherConditionSet::Vector all_conditions;
  367. size_t size = std::min(kMaxFiltersAllowed, patterns.size());
  368. scoped_refptr<URLMatcherConditionSet> condition_set;
  369. for (size_t i = 0; i < size; ++i) {
  370. DCHECK(patterns[i].is_string());
  371. const std::string pattern = patterns[i].GetString();
  372. FilterComponents components;
  373. components.allow = allow;
  374. if (!FilterToComponents(pattern, &components.scheme, &components.host,
  375. &components.match_subdomains, &components.port,
  376. &components.path, &components.query)) {
  377. LOG(ERROR) << "Invalid pattern " << pattern;
  378. continue;
  379. }
  380. condition_set =
  381. CreateConditionSet(matcher, ++(*id), components.scheme, components.host,
  382. components.match_subdomains, components.port,
  383. components.path, components.query, allow);
  384. if (filters) {
  385. components.number_of_url_matching_conditions =
  386. condition_set->query_conditions().size();
  387. (*filters)[*id] = std::move(components);
  388. }
  389. all_conditions.push_back(std::move(condition_set));
  390. }
  391. matcher->AddConditionSets(all_conditions);
  392. }
  393. void AddFilters(URLMatcher* matcher,
  394. bool allow,
  395. base::MatcherStringPattern::ID* id,
  396. const std::vector<std::string>& patterns,
  397. std::map<base::MatcherStringPattern::ID,
  398. url_matcher::util::FilterComponents>* filters) {
  399. URLMatcherConditionSet::Vector all_conditions;
  400. size_t size = std::min(kMaxFiltersAllowed, patterns.size());
  401. scoped_refptr<URLMatcherConditionSet> condition_set;
  402. for (size_t i = 0; i < size; ++i) {
  403. FilterComponents components;
  404. components.allow = allow;
  405. if (!FilterToComponents(patterns[i], &components.scheme, &components.host,
  406. &components.match_subdomains, &components.port,
  407. &components.path, &components.query)) {
  408. LOG(ERROR) << "Invalid pattern " << patterns[i];
  409. continue;
  410. }
  411. condition_set =
  412. CreateConditionSet(matcher, ++(*id), components.scheme, components.host,
  413. components.match_subdomains, components.port,
  414. components.path, components.query, allow);
  415. if (filters) {
  416. components.number_of_url_matching_conditions =
  417. condition_set->query_conditions().size();
  418. (*filters)[*id] = std::move(components);
  419. }
  420. all_conditions.push_back(std::move(condition_set));
  421. }
  422. matcher->AddConditionSets(all_conditions);
  423. }
  424. void AddAllowFilters(url_matcher::URLMatcher* matcher,
  425. const base::Value::List& patterns) {
  426. base::MatcherStringPattern::ID id(0);
  427. AddFilters(matcher, true, &id, patterns);
  428. }
  429. void AddAllowFilters(url_matcher::URLMatcher* matcher,
  430. const std::vector<std::string>& patterns) {
  431. base::MatcherStringPattern::ID id(0);
  432. AddFilters(matcher, true, &id, patterns);
  433. }
  434. } // namespace util
  435. } // namespace url_matcher