url_util.cc 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492
  1. // Copyright 2013 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "net/base/url_util.h"
  5. #include "build/build_config.h"
  6. #if BUILDFLAG(IS_POSIX)
  7. #include <netinet/in.h>
  8. #elif BUILDFLAG(IS_WIN)
  9. #include <ws2tcpip.h>
  10. #endif
  11. #include "base/check_op.h"
  12. #include "base/strings/escape.h"
  13. #include "base/strings/string_piece.h"
  14. #include "base/strings/string_util.h"
  15. #include "base/strings/stringprintf.h"
  16. #include "base/strings/utf_string_conversions.h"
  17. #include "net/base/ip_address.h"
  18. #include "net/base/registry_controlled_domains/registry_controlled_domain.h"
  19. #include "url/gurl.h"
  20. #include "url/scheme_host_port.h"
  21. #include "url/url_canon.h"
  22. #include "url/url_canon_ip.h"
  23. #include "url/url_constants.h"
  24. #include "url/url_util.h"
  25. namespace net {
  26. namespace {
  27. bool IsHostCharAlphanumeric(char c) {
  28. // We can just check lowercase because uppercase characters have already been
  29. // normalized.
  30. return ((c >= 'a') && (c <= 'z')) || ((c >= '0') && (c <= '9'));
  31. }
  32. bool IsNormalizedLocalhostTLD(const std::string& host) {
  33. return base::EndsWith(host, ".localhost");
  34. }
  35. // Helper function used by GetIdentityFromURL. If |escaped_text| can be "safely
  36. // unescaped" to a valid UTF-8 string, return that string, as UTF-16. Otherwise,
  37. // convert it as-is to UTF-16. "Safely unescaped" is defined as having no
  38. // escaped character between '0x00' and '0x1F', inclusive.
  39. std::u16string UnescapeIdentityString(base::StringPiece escaped_text) {
  40. std::string unescaped_text;
  41. if (base::UnescapeBinaryURLComponentSafe(
  42. escaped_text, false /* fail_on_path_separators */, &unescaped_text)) {
  43. std::u16string result;
  44. if (base::UTF8ToUTF16(unescaped_text.data(), unescaped_text.length(),
  45. &result)) {
  46. return result;
  47. }
  48. }
  49. return base::UTF8ToUTF16(escaped_text);
  50. }
  51. } // namespace
  52. GURL AppendQueryParameter(const GURL& url,
  53. const std::string& name,
  54. const std::string& value) {
  55. std::string query(url.query());
  56. if (!query.empty())
  57. query += "&";
  58. query += (base::EscapeQueryParamValue(name, true) + "=" +
  59. base::EscapeQueryParamValue(value, true));
  60. GURL::Replacements replacements;
  61. replacements.SetQueryStr(query);
  62. return url.ReplaceComponents(replacements);
  63. }
  64. GURL AppendOrReplaceQueryParameter(const GURL& url,
  65. const std::string& name,
  66. const std::string& value) {
  67. bool replaced = false;
  68. std::string param_name = base::EscapeQueryParamValue(name, true);
  69. std::string param_value = base::EscapeQueryParamValue(value, true);
  70. const std::string input = url.query();
  71. url::Component cursor(0, input.size());
  72. std::string output;
  73. url::Component key_range, value_range;
  74. while (url::ExtractQueryKeyValue(input.data(), &cursor, &key_range,
  75. &value_range)) {
  76. const base::StringPiece key(
  77. input.data() + key_range.begin, key_range.len);
  78. std::string key_value_pair;
  79. // Check |replaced| as only the first pair should be replaced.
  80. if (!replaced && key == param_name) {
  81. replaced = true;
  82. key_value_pair = (param_name + "=" + param_value);
  83. } else {
  84. key_value_pair.assign(input, key_range.begin,
  85. value_range.end() - key_range.begin);
  86. }
  87. if (!output.empty())
  88. output += "&";
  89. output += key_value_pair;
  90. }
  91. if (!replaced) {
  92. if (!output.empty())
  93. output += "&";
  94. output += (param_name + "=" + param_value);
  95. }
  96. GURL::Replacements replacements;
  97. replacements.SetQueryStr(output);
  98. return url.ReplaceComponents(replacements);
  99. }
  100. QueryIterator::QueryIterator(const GURL& url)
  101. : url_(url),
  102. at_end_(!url.is_valid()) {
  103. if (!at_end_) {
  104. query_ = url.parsed_for_possibly_invalid_spec().query;
  105. Advance();
  106. }
  107. }
  108. QueryIterator::~QueryIterator() = default;
  109. base::StringPiece QueryIterator::GetKey() const {
  110. DCHECK(!at_end_);
  111. if (key_.is_nonempty())
  112. return base::StringPiece(&url_.spec()[key_.begin], key_.len);
  113. return base::StringPiece();
  114. }
  115. base::StringPiece QueryIterator::GetValue() const {
  116. DCHECK(!at_end_);
  117. if (value_.is_nonempty())
  118. return base::StringPiece(&url_.spec()[value_.begin], value_.len);
  119. return base::StringPiece();
  120. }
  121. const std::string& QueryIterator::GetUnescapedValue() {
  122. DCHECK(!at_end_);
  123. if (value_.is_nonempty() && unescaped_value_.empty()) {
  124. unescaped_value_ = base::UnescapeURLComponent(
  125. GetValue(),
  126. base::UnescapeRule::SPACES | base::UnescapeRule::PATH_SEPARATORS |
  127. base::UnescapeRule::URL_SPECIAL_CHARS_EXCEPT_PATH_SEPARATORS |
  128. base::UnescapeRule::REPLACE_PLUS_WITH_SPACE);
  129. }
  130. return unescaped_value_;
  131. }
  132. bool QueryIterator::IsAtEnd() const {
  133. return at_end_;
  134. }
  135. void QueryIterator::Advance() {
  136. DCHECK (!at_end_);
  137. key_.reset();
  138. value_.reset();
  139. unescaped_value_.clear();
  140. at_end_ =
  141. !url::ExtractQueryKeyValue(url_.spec().c_str(), &query_, &key_, &value_);
  142. }
  143. bool GetValueForKeyInQuery(const GURL& url,
  144. const std::string& search_key,
  145. std::string* out_value) {
  146. for (QueryIterator it(url); !it.IsAtEnd(); it.Advance()) {
  147. if (it.GetKey() == search_key) {
  148. *out_value = it.GetUnescapedValue();
  149. return true;
  150. }
  151. }
  152. return false;
  153. }
  154. bool ParseHostAndPort(base::StringPiece input, std::string* host, int* port) {
  155. if (input.empty())
  156. return false;
  157. url::Component auth_component(0, input.size());
  158. url::Component username_component;
  159. url::Component password_component;
  160. url::Component hostname_component;
  161. url::Component port_component;
  162. url::ParseAuthority(input.data(), auth_component, &username_component,
  163. &password_component, &hostname_component,
  164. &port_component);
  165. // There shouldn't be a username/password.
  166. if (username_component.is_valid() || password_component.is_valid())
  167. return false;
  168. if (!hostname_component.is_nonempty())
  169. return false; // Failed parsing.
  170. int parsed_port_number = -1;
  171. if (port_component.is_nonempty()) {
  172. parsed_port_number = url::ParsePort(input.data(), port_component);
  173. // If parsing failed, port_number will be either PORT_INVALID or
  174. // PORT_UNSPECIFIED, both of which are negative.
  175. if (parsed_port_number < 0)
  176. return false; // Failed parsing the port number.
  177. }
  178. if (port_component.len == 0)
  179. return false; // Reject inputs like "foo:"
  180. unsigned char tmp_ipv6_addr[16];
  181. // If the hostname starts with a bracket, it is either an IPv6 literal or
  182. // invalid. If it is an IPv6 literal then strip the brackets.
  183. if (hostname_component.len > 0 && input[hostname_component.begin] == '[') {
  184. if (input[hostname_component.end() - 1] == ']' &&
  185. url::IPv6AddressToNumber(input.data(), hostname_component,
  186. tmp_ipv6_addr)) {
  187. // Strip the brackets.
  188. hostname_component.begin++;
  189. hostname_component.len -= 2;
  190. } else {
  191. return false;
  192. }
  193. }
  194. // Pass results back to caller.
  195. host->assign(input.data() + hostname_component.begin, hostname_component.len);
  196. *port = parsed_port_number;
  197. return true; // Success.
  198. }
  199. std::string GetHostAndPort(const GURL& url) {
  200. // For IPv6 literals, GURL::host() already includes the brackets so it is
  201. // safe to just append a colon.
  202. return base::StringPrintf("%s:%d", url.host().c_str(),
  203. url.EffectiveIntPort());
  204. }
  205. std::string GetHostAndOptionalPort(const GURL& url) {
  206. // For IPv6 literals, GURL::host() already includes the brackets
  207. // so it is safe to just append a colon.
  208. if (url.has_port())
  209. return base::StringPrintf("%s:%s", url.host().c_str(), url.port().c_str());
  210. return url.host();
  211. }
  212. NET_EXPORT std::string GetHostAndOptionalPort(
  213. const url::SchemeHostPort& scheme_host_port) {
  214. int default_port = url::DefaultPortForScheme(
  215. scheme_host_port.scheme().data(),
  216. static_cast<int>(scheme_host_port.scheme().length()));
  217. if (default_port != scheme_host_port.port()) {
  218. return base::StringPrintf("%s:%i", scheme_host_port.host().c_str(),
  219. scheme_host_port.port());
  220. }
  221. return scheme_host_port.host();
  222. }
  223. std::string TrimEndingDot(base::StringPiece host) {
  224. base::StringPiece host_trimmed = host;
  225. size_t len = host_trimmed.length();
  226. if (len > 1 && host_trimmed[len - 1] == '.') {
  227. host_trimmed.remove_suffix(1);
  228. }
  229. return std::string(host_trimmed);
  230. }
  231. std::string GetHostOrSpecFromURL(const GURL& url) {
  232. return url.has_host() ? TrimEndingDot(url.host_piece()) : url.spec();
  233. }
  234. std::string GetSuperdomain(base::StringPiece domain) {
  235. size_t dot_pos = domain.find('.');
  236. if (dot_pos == std::string::npos)
  237. return "";
  238. return std::string(domain.substr(dot_pos + 1));
  239. }
  240. bool IsSubdomainOf(base::StringPiece subdomain, base::StringPiece superdomain) {
  241. // Subdomain must be identical or have strictly more labels than the
  242. // superdomain.
  243. if (subdomain.length() <= superdomain.length())
  244. return subdomain == superdomain;
  245. // Superdomain must be suffix of subdomain, and the last character not
  246. // included in the matching substring must be a dot.
  247. if (!base::EndsWith(subdomain, superdomain))
  248. return false;
  249. subdomain.remove_suffix(superdomain.length());
  250. return subdomain.back() == '.';
  251. }
  252. std::string CanonicalizeHost(base::StringPiece host,
  253. url::CanonHostInfo* host_info) {
  254. // Try to canonicalize the host.
  255. const url::Component raw_host_component(0, static_cast<int>(host.length()));
  256. std::string canon_host;
  257. url::StdStringCanonOutput canon_host_output(&canon_host);
  258. // A url::StdStringCanonOutput starts off with a zero length buffer. The
  259. // first time through Grow() immediately resizes it to 32 bytes, incurring
  260. // a malloc. With libcxx a 22 byte or smaller request can be accommodated
  261. // within the std::string itself (i.e. no malloc occurs). Start the buffer
  262. // off at the max size to avoid a malloc on short strings.
  263. // NOTE: To ensure the final size is correctly reflected, it's necessary
  264. // to call Complete() which will adjust the size to the actual bytes written.
  265. // This is handled below for success cases, while failure cases discard all
  266. // the output.
  267. const int kCxxMaxStringBufferSizeWithoutMalloc = 22;
  268. canon_host_output.Resize(kCxxMaxStringBufferSizeWithoutMalloc);
  269. url::CanonicalizeHostVerbose(host.data(), raw_host_component,
  270. &canon_host_output, host_info);
  271. if (host_info->out_host.is_nonempty() &&
  272. host_info->family != url::CanonHostInfo::BROKEN) {
  273. // Success! Assert that there's no extra garbage.
  274. canon_host_output.Complete();
  275. DCHECK_EQ(host_info->out_host.len, static_cast<int>(canon_host.length()));
  276. } else {
  277. // Empty host, or canonicalization failed. We'll return empty.
  278. canon_host.clear();
  279. }
  280. return canon_host;
  281. }
  282. bool IsCanonicalizedHostCompliant(const std::string& host) {
  283. if (host.empty())
  284. return false;
  285. bool in_component = false;
  286. bool most_recent_component_started_alphanumeric = false;
  287. for (char c : host) {
  288. if (!in_component) {
  289. most_recent_component_started_alphanumeric = IsHostCharAlphanumeric(c);
  290. if (!most_recent_component_started_alphanumeric && (c != '-') &&
  291. (c != '_')) {
  292. return false;
  293. }
  294. in_component = true;
  295. } else if (c == '.') {
  296. in_component = false;
  297. } else if (!IsHostCharAlphanumeric(c) && (c != '-') && (c != '_')) {
  298. return false;
  299. }
  300. }
  301. return most_recent_component_started_alphanumeric;
  302. }
  303. bool IsHostnameNonUnique(const std::string& hostname) {
  304. // CanonicalizeHost requires surrounding brackets to parse an IPv6 address.
  305. const std::string host_or_ip = hostname.find(':') != std::string::npos ?
  306. "[" + hostname + "]" : hostname;
  307. url::CanonHostInfo host_info;
  308. std::string canonical_name = CanonicalizeHost(host_or_ip, &host_info);
  309. // If canonicalization fails, then the input is truly malformed. However,
  310. // to avoid mis-reporting bad inputs as "non-unique", treat them as unique.
  311. if (canonical_name.empty())
  312. return false;
  313. // If |hostname| is an IP address, check to see if it's in an IANA-reserved
  314. // range reserved for non-publicly routable networks.
  315. if (host_info.IsIPAddress()) {
  316. IPAddress host_addr;
  317. if (!host_addr.AssignFromIPLiteral(hostname.substr(
  318. host_info.out_host.begin, host_info.out_host.len))) {
  319. return false;
  320. }
  321. switch (host_info.family) {
  322. case url::CanonHostInfo::IPV4:
  323. case url::CanonHostInfo::IPV6:
  324. return !host_addr.IsPubliclyRoutable();
  325. case url::CanonHostInfo::NEUTRAL:
  326. case url::CanonHostInfo::BROKEN:
  327. return false;
  328. }
  329. }
  330. // Check for a registry controlled portion of |hostname|, ignoring private
  331. // registries, as they already chain to ICANN-administered registries,
  332. // and explicitly ignoring unknown registries.
  333. //
  334. // Note: This means that as new gTLDs are introduced on the Internet, they
  335. // will be treated as non-unique until the registry controlled domain list
  336. // is updated. However, because gTLDs are expected to provide significant
  337. // advance notice to deprecate older versions of this code, this an
  338. // acceptable tradeoff.
  339. return !registry_controlled_domains::HostHasRegistryControlledDomain(
  340. canonical_name, registry_controlled_domains::EXCLUDE_UNKNOWN_REGISTRIES,
  341. registry_controlled_domains::EXCLUDE_PRIVATE_REGISTRIES);
  342. }
  343. bool IsLocalhost(const GURL& url) {
  344. return HostStringIsLocalhost(url.HostNoBracketsPiece());
  345. }
  346. bool HostStringIsLocalhost(base::StringPiece host) {
  347. IPAddress ip_address;
  348. if (ip_address.AssignFromIPLiteral(host))
  349. return ip_address.IsLoopback();
  350. return IsLocalHostname(host);
  351. }
  352. GURL SimplifyUrlForRequest(const GURL& url) {
  353. DCHECK(url.is_valid());
  354. // Fast path to avoid re-canonicalization via ReplaceComponents.
  355. if (!url.has_username() && !url.has_password() && !url.has_ref())
  356. return url;
  357. GURL::Replacements replacements;
  358. replacements.ClearUsername();
  359. replacements.ClearPassword();
  360. replacements.ClearRef();
  361. return url.ReplaceComponents(replacements);
  362. }
  363. GURL ChangeWebSocketSchemeToHttpScheme(const GURL& url) {
  364. DCHECK(url.SchemeIsWSOrWSS());
  365. GURL::Replacements replace_scheme;
  366. replace_scheme.SetSchemeStr(url.SchemeIs(url::kWssScheme) ? url::kHttpsScheme
  367. : url::kHttpScheme);
  368. return url.ReplaceComponents(replace_scheme);
  369. }
  370. bool IsStandardSchemeWithNetworkHost(base::StringPiece scheme) {
  371. // file scheme is special. Windows file share origins can have network hosts.
  372. if (scheme == url::kFileScheme)
  373. return true;
  374. url::SchemeType scheme_type;
  375. if (!url::GetStandardSchemeType(
  376. scheme.data(), url::Component(0, scheme.length()), &scheme_type)) {
  377. return false;
  378. }
  379. return scheme_type == url::SCHEME_WITH_HOST_PORT_AND_USER_INFORMATION ||
  380. scheme_type == url::SCHEME_WITH_HOST_AND_PORT;
  381. }
  382. void GetIdentityFromURL(const GURL& url,
  383. std::u16string* username,
  384. std::u16string* password) {
  385. *username = UnescapeIdentityString(url.username());
  386. *password = UnescapeIdentityString(url.password());
  387. }
  388. bool HasGoogleHost(const GURL& url) {
  389. return IsGoogleHost(url.host_piece());
  390. }
  391. bool IsGoogleHost(base::StringPiece host) {
  392. static const char* kGoogleHostSuffixes[] = {
  393. ".google.com",
  394. ".youtube.com",
  395. ".gmail.com",
  396. ".doubleclick.net",
  397. ".gstatic.com",
  398. ".googlevideo.com",
  399. ".googleusercontent.com",
  400. ".googlesyndication.com",
  401. ".google-analytics.com",
  402. ".googleadservices.com",
  403. ".googleapis.com",
  404. ".ytimg.com",
  405. };
  406. for (const char* suffix : kGoogleHostSuffixes) {
  407. // Here it's possible to get away with faster case-sensitive comparisons
  408. // because the list above is all lowercase, and a GURL's host name will
  409. // always be canonicalized to lowercase as well.
  410. if (base::EndsWith(host, suffix))
  411. return true;
  412. }
  413. return false;
  414. }
  415. bool IsLocalHostname(base::StringPiece host) {
  416. std::string normalized_host = base::ToLowerASCII(host);
  417. // Remove any trailing '.'.
  418. if (!normalized_host.empty() && *normalized_host.rbegin() == '.')
  419. normalized_host.resize(normalized_host.size() - 1);
  420. return normalized_host == "localhost" ||
  421. IsNormalizedLocalhostTLD(normalized_host);
  422. }
  423. } // namespace net