gurl.h 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543
  1. // Copyright 2013 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #ifndef URL_GURL_H_
  5. #define URL_GURL_H_
  6. #include <stddef.h>
  7. #include <iosfwd>
  8. #include <memory>
  9. #include <string>
  10. #include "base/component_export.h"
  11. #include "base/debug/alias.h"
  12. #include "base/debug/crash_logging.h"
  13. #include "base/strings/string_piece.h"
  14. #include "third_party/perfetto/include/perfetto/tracing/traced_value_forward.h"
  15. #include "url/third_party/mozilla/url_parse.h"
  16. #include "url/url_canon.h"
  17. #include "url/url_canon_stdstring.h"
  18. #include "url/url_constants.h"
  19. // Represents a URL. GURL is Google's URL parsing library.
  20. //
  21. // A parsed canonicalized URL is guaranteed to be UTF-8. Any non-ASCII input
  22. // characters are UTF-8 encoded and % escaped to ASCII.
  23. //
  24. // The string representation of a URL is called the spec(). Getting the
  25. // spec will assert if the URL is invalid to help protect against malicious
  26. // URLs. If you want the "best effort" canonicalization of an invalid URL, you
  27. // can use possibly_invalid_spec(). Test validity with is_valid(). Data and
  28. // javascript URLs use GetContent() to extract the data.
  29. //
  30. // This class has existence checkers and getters for the various components of
  31. // a URL. Existence is different than being nonempty. "http://www.google.com/?"
  32. // has a query that just happens to be empty, and has_query() will return true
  33. // while the query getters will return the empty string.
  34. //
  35. // Prefer not to modify a URL using string operations (though sometimes this is
  36. // unavoidable). Instead, use ReplaceComponents which can replace or delete
  37. // multiple parts of a URL in one step, doesn't re-canonicalize unchanged
  38. // sections, and avoids some screw-ups. An example is creating a URL with a
  39. // path that contains a literal '#'. Using string concatenation will generate a
  40. // URL with a truncated path and a reference fragment, while ReplaceComponents
  41. // will know to escape this and produce the desired result.
  42. class COMPONENT_EXPORT(URL) GURL {
  43. public:
  44. typedef url::StringPieceReplacements<char> Replacements;
  45. typedef url::StringPieceReplacements<char16_t> ReplacementsW;
  46. // Creates an empty, invalid URL.
  47. GURL();
  48. // Copy construction is relatively inexpensive, with most of the time going
  49. // to reallocating the string. It does not re-parse.
  50. GURL(const GURL& other);
  51. GURL(GURL&& other) noexcept;
  52. // The strings to this contructor should be UTF-8 / UTF-16.
  53. explicit GURL(base::StringPiece url_string);
  54. explicit GURL(base::StringPiece16 url_string);
  55. // Constructor for URLs that have already been parsed and canonicalized. This
  56. // is used for conversions from KURL, for example. The caller must supply all
  57. // information associated with the URL, which must be correct and consistent.
  58. GURL(const char* canonical_spec,
  59. size_t canonical_spec_len,
  60. const url::Parsed& parsed,
  61. bool is_valid);
  62. // Notice that we take the canonical_spec by value so that we can convert
  63. // from WebURL without copying the string. When we call this constructor
  64. // we pass in a temporary std::string, which lets the compiler skip the
  65. // copy and just move the std::string into the function argument. In the
  66. // implementation, we use std::move to move the data into the GURL itself,
  67. // which means we end up with zero copies.
  68. GURL(std::string canonical_spec, const url::Parsed& parsed, bool is_valid);
  69. ~GURL();
  70. GURL& operator=(const GURL& other);
  71. GURL& operator=(GURL&& other) noexcept;
  72. // Returns true when this object represents a valid parsed URL. When not
  73. // valid, other functions will still succeed, but you will not get canonical
  74. // data out in the format you may be expecting. Instead, we keep something
  75. // "reasonable looking" so that the user can see how it's busted if
  76. // displayed to them.
  77. bool is_valid() const {
  78. return is_valid_;
  79. }
  80. // Returns true if the URL is zero-length. Note that empty URLs are also
  81. // invalid, and is_valid() will return false for them. This is provided
  82. // because some users may want to treat the empty case differently.
  83. bool is_empty() const {
  84. return spec_.empty();
  85. }
  86. // Returns the raw spec, i.e., the full text of the URL, in canonical UTF-8,
  87. // if the URL is valid. If the URL is not valid, this will assert and return
  88. // the empty string (for safety in release builds, to keep them from being
  89. // misused which might be a security problem).
  90. //
  91. // The URL will be ASCII (non-ASCII characters will be %-escaped UTF-8).
  92. //
  93. // The exception is for empty() URLs (which are !is_valid()) but this will
  94. // return the empty string without asserting.
  95. //
  96. // Use invalid_spec() below to get the unusable spec of an invalid URL. This
  97. // separation is designed to prevent errors that may cause security problems
  98. // that could result from the mistaken use of an invalid URL.
  99. const std::string& spec() const;
  100. // Returns the potentially invalid spec for a the URL. This spec MUST NOT be
  101. // modified or sent over the network. It is designed to be displayed in error
  102. // messages to the user, as the appearance of the spec may explain the error.
  103. // If the spec is valid, the valid spec will be returned.
  104. //
  105. // The returned string is guaranteed to be valid UTF-8.
  106. const std::string& possibly_invalid_spec() const {
  107. return spec_;
  108. }
  109. // Getter for the raw parsed structure. This allows callers to locate parts
  110. // of the URL within the spec themselves. Most callers should consider using
  111. // the individual component getters below.
  112. //
  113. // The returned parsed structure will reference into the raw spec, which may
  114. // or may not be valid. If you are using this to index into the spec, BE
  115. // SURE YOU ARE USING possibly_invalid_spec() to get the spec, and that you
  116. // don't do anything "important" with invalid specs.
  117. const url::Parsed& parsed_for_possibly_invalid_spec() const {
  118. return parsed_;
  119. }
  120. // Allows GURL to used as a key in STL (for example, a std::set or std::map).
  121. bool operator<(const GURL& other) const;
  122. bool operator>(const GURL& other) const;
  123. // Resolves a URL that's possibly relative to this object's URL, and returns
  124. // it. Absolute URLs are also handled according to the rules of URLs on web
  125. // pages.
  126. //
  127. // It may be impossible to resolve the URLs properly. If the input is not
  128. // "standard" (IsStandard() == false) and the input looks relative, we can't
  129. // resolve it. In these cases, the result will be an empty, invalid GURL.
  130. //
  131. // The result may also be a nonempty, invalid URL if the input has some kind
  132. // of encoding error. In these cases, we will try to construct a "good" URL
  133. // that may have meaning to the user, but it will be marked invalid.
  134. //
  135. // It is an error to resolve a URL relative to an invalid URL. The result
  136. // will be the empty URL.
  137. GURL Resolve(base::StringPiece relative) const;
  138. GURL Resolve(base::StringPiece16 relative) const;
  139. // Creates a new GURL by replacing the current URL's components with the
  140. // supplied versions. See the Replacements class in url_canon.h for more.
  141. //
  142. // These are not particularly quick, so avoid doing mutations when possible.
  143. // Prefer the 8-bit version when possible.
  144. //
  145. // It is an error to replace components of an invalid URL. The result will
  146. // be the empty URL.
  147. //
  148. // Note that this intentionally disallows direct use of url::Replacements,
  149. // which is harder to use correctly.
  150. GURL ReplaceComponents(const Replacements& replacements) const;
  151. GURL ReplaceComponents(const ReplacementsW& replacements) const;
  152. // A helper function that is equivalent to replacing the path with a slash
  153. // and clearing out everything after that. We sometimes need to know just the
  154. // scheme and the authority. If this URL is not a standard URL (it doesn't
  155. // have the regular authority and path sections), then the result will be
  156. // an empty, invalid GURL. Note that this *does* work for file: URLs, which
  157. // some callers may want to filter out before calling this.
  158. //
  159. // It is an error to get an empty path on an invalid URL. The result
  160. // will be the empty URL.
  161. GURL GetWithEmptyPath() const;
  162. // A helper function to return a GURL without the filename, query values, and
  163. // fragment. For example,
  164. // GURL("https://www.foo.com/index.html?q=test").GetWithoutFilename().spec()
  165. // will return "https://www.foo.com/".
  166. // GURL("https://www.foo.com/bar/").GetWithoutFilename().spec()
  167. // will return "https://www.foo.com/bar/". If the GURL is invalid or missing a
  168. // scheme, authority or path, it will return an empty, invalid GURL.
  169. GURL GetWithoutFilename() const;
  170. // A helper function to return a GURL containing just the scheme, host,
  171. // and port from a URL. Equivalent to clearing any username and password,
  172. // replacing the path with a slash, and clearing everything after that. If
  173. // this URL is not a standard URL, then the result will be an empty,
  174. // invalid GURL. If the URL has neither username nor password, this
  175. // degenerates to GetWithEmptyPath().
  176. //
  177. // It is an error to get the origin of an invalid URL. The result
  178. // will be the empty URL.
  179. //
  180. // WARNING: Please avoid converting urls into origins if at all possible!
  181. // //docs/security/origin-vs-url.md is a list of gotchas that can result. Such
  182. // conversions will likely return a wrong result for about:blank and/or
  183. // in the presence of iframe.sandbox attribute. Prefer to get origins directly
  184. // from the source (e.g. RenderFrameHost::GetLastCommittedOrigin).
  185. GURL DeprecatedGetOriginAsURL() const;
  186. // A helper function to return a GURL stripped from the elements that are not
  187. // supposed to be sent as HTTP referrer: username, password and ref fragment.
  188. // For invalid URLs or URLs that no valid referrers, an empty URL will be
  189. // returned.
  190. GURL GetAsReferrer() const;
  191. // Returns true if the scheme for the current URL is a known "standard-format"
  192. // scheme. A standard-format scheme adheres to what RFC 3986 calls "generic
  193. // URI syntax" (https://tools.ietf.org/html/rfc3986#section-3). This includes
  194. // file: and filesystem:, which some callers may want to filter out explicitly
  195. // by calling SchemeIsFile[System].
  196. bool IsStandard() const;
  197. // Returns true when the url is of the form about:blank, about:blank?foo or
  198. // about:blank/#foo.
  199. bool IsAboutBlank() const;
  200. // Returns true when the url is of the form about:srcdoc, about:srcdoc?foo or
  201. // about:srcdoc/#foo.
  202. bool IsAboutSrcdoc() const;
  203. // Returns true if the given parameter (should be lower-case ASCII to match
  204. // the canonicalized scheme) is the scheme for this URL. Do not include a
  205. // colon.
  206. bool SchemeIs(base::StringPiece lower_ascii_scheme) const;
  207. // Returns true if the scheme is "http" or "https".
  208. bool SchemeIsHTTPOrHTTPS() const;
  209. // Returns true is the scheme is "ws" or "wss".
  210. bool SchemeIsWSOrWSS() const;
  211. // We often need to know if this is a file URL. File URLs are "standard", but
  212. // are often treated separately by some programs.
  213. bool SchemeIsFile() const {
  214. return SchemeIs(url::kFileScheme);
  215. }
  216. // FileSystem URLs need to be treated differently in some cases.
  217. bool SchemeIsFileSystem() const {
  218. return SchemeIs(url::kFileSystemScheme);
  219. }
  220. // Returns true if the scheme indicates a network connection that uses TLS or
  221. // some other cryptographic protocol (e.g. QUIC) for security.
  222. //
  223. // This function is a not a complete test of whether or not an origin's code
  224. // is minimally trustworthy. For that, see Chromium's |IsOriginSecure| for a
  225. // higher-level and more complete semantics. See that function's documentation
  226. // for more detail.
  227. bool SchemeIsCryptographic() const;
  228. // As above, but static. Parameter should be lower-case ASCII.
  229. static bool SchemeIsCryptographic(base::StringPiece lower_ascii_scheme);
  230. // Returns true if the scheme is "blob".
  231. bool SchemeIsBlob() const {
  232. return SchemeIs(url::kBlobScheme);
  233. }
  234. // Returns true if the scheme is a local scheme, as defined in Fetch:
  235. // https://fetch.spec.whatwg.org/#local-scheme
  236. bool SchemeIsLocal() const;
  237. // For most URLs, the "content" is everything after the scheme (skipping the
  238. // scheme delimiting colon) and before the fragment (skipping the fragment
  239. // delimiting octothorpe). For javascript URLs the "content" also includes the
  240. // fragment delimiter and fragment.
  241. //
  242. // It is an error to get the content of an invalid URL: the result will be an
  243. // empty string.
  244. std::string GetContent() const;
  245. base::StringPiece GetContentPiece() const;
  246. // Returns true if the hostname is an IP address. Note: this function isn't
  247. // as cheap as a simple getter because it re-parses the hostname to verify.
  248. bool HostIsIPAddress() const;
  249. // Not including the colon. If you are comparing schemes, prefer SchemeIs.
  250. bool has_scheme() const {
  251. return parsed_.scheme.len >= 0;
  252. }
  253. std::string scheme() const {
  254. return ComponentString(parsed_.scheme);
  255. }
  256. base::StringPiece scheme_piece() const {
  257. return ComponentStringPiece(parsed_.scheme);
  258. }
  259. bool has_username() const {
  260. return parsed_.username.len >= 0;
  261. }
  262. std::string username() const {
  263. return ComponentString(parsed_.username);
  264. }
  265. base::StringPiece username_piece() const {
  266. return ComponentStringPiece(parsed_.username);
  267. }
  268. bool has_password() const {
  269. return parsed_.password.len >= 0;
  270. }
  271. std::string password() const {
  272. return ComponentString(parsed_.password);
  273. }
  274. base::StringPiece password_piece() const {
  275. return ComponentStringPiece(parsed_.password);
  276. }
  277. // The host may be a hostname, an IPv4 address, or an IPv6 literal surrounded
  278. // by square brackets, like "[2001:db8::1]". To exclude these brackets, use
  279. // HostNoBrackets() below.
  280. bool has_host() const {
  281. // Note that hosts are special, absence of host means length 0.
  282. return parsed_.host.len > 0;
  283. }
  284. std::string host() const {
  285. return ComponentString(parsed_.host);
  286. }
  287. base::StringPiece host_piece() const {
  288. return ComponentStringPiece(parsed_.host);
  289. }
  290. // The port if one is explicitly specified. Most callers will want IntPort()
  291. // or EffectiveIntPort() instead of these. The getters will not include the
  292. // ':'.
  293. bool has_port() const {
  294. return parsed_.port.len >= 0;
  295. }
  296. std::string port() const {
  297. return ComponentString(parsed_.port);
  298. }
  299. base::StringPiece port_piece() const {
  300. return ComponentStringPiece(parsed_.port);
  301. }
  302. // Including first slash following host, up to the query. The URL
  303. // "http://www.google.com/" has a path of "/".
  304. bool has_path() const {
  305. return parsed_.path.len >= 0;
  306. }
  307. std::string path() const {
  308. return ComponentString(parsed_.path);
  309. }
  310. base::StringPiece path_piece() const {
  311. return ComponentStringPiece(parsed_.path);
  312. }
  313. // Stuff following '?' up to the ref. The getters will not include the '?'.
  314. bool has_query() const {
  315. return parsed_.query.len >= 0;
  316. }
  317. std::string query() const {
  318. return ComponentString(parsed_.query);
  319. }
  320. base::StringPiece query_piece() const {
  321. return ComponentStringPiece(parsed_.query);
  322. }
  323. // Stuff following '#' to the end of the string. This will be %-escaped UTF-8.
  324. // The getters will not include the '#'.
  325. bool has_ref() const {
  326. return parsed_.ref.len >= 0;
  327. }
  328. std::string ref() const {
  329. return ComponentString(parsed_.ref);
  330. }
  331. base::StringPiece ref_piece() const {
  332. return ComponentStringPiece(parsed_.ref);
  333. }
  334. // Returns a parsed version of the port. Can also be any of the special
  335. // values defined in Parsed for ExtractPort.
  336. int IntPort() const;
  337. // Returns the port number of the URL, or the default port number.
  338. // If the scheme has no concept of port (or unknown default) returns
  339. // PORT_UNSPECIFIED.
  340. int EffectiveIntPort() const;
  341. // Extracts the filename portion of the path and returns it. The filename
  342. // is everything after the last slash in the path. This may be empty.
  343. std::string ExtractFileName() const;
  344. // Returns the path that should be sent to the server. This is the path,
  345. // parameter, and query portions of the URL. It is guaranteed to be ASCII.
  346. std::string PathForRequest() const;
  347. // Returns the same characters as PathForRequest(), avoiding a copy.
  348. base::StringPiece PathForRequestPiece() const;
  349. // Returns the host, excluding the square brackets surrounding IPv6 address
  350. // literals. This can be useful for passing to getaddrinfo().
  351. std::string HostNoBrackets() const;
  352. // Returns the same characters as HostNoBrackets(), avoiding a copy.
  353. base::StringPiece HostNoBracketsPiece() const;
  354. // Returns true if this URL's host matches or is in the same domain as
  355. // the given input string. For example, if the hostname of the URL is
  356. // "www.google.com", this will return true for "com", "google.com", and
  357. // "www.google.com".
  358. //
  359. // The input domain should match host canonicalization rules. i.e. the input
  360. // should be lowercase except for escape chars.
  361. //
  362. // This call is more efficient than getting the host and checking whether the
  363. // host has the specific domain or not because no copies or object
  364. // constructions are done.
  365. bool DomainIs(base::StringPiece canonical_domain) const;
  366. // Checks whether or not two URLs differ only in the ref (the part after
  367. // the # character).
  368. bool EqualsIgnoringRef(const GURL& other) const;
  369. // Swaps the contents of this GURL object with |other|, without doing
  370. // any memory allocations.
  371. void Swap(GURL* other);
  372. // Returns a reference to a singleton empty GURL. This object is for callers
  373. // who return references but don't have anything to return in some cases.
  374. // If you just want an empty URL for normal use, prefer GURL(). This function
  375. // may be called from any thread.
  376. static const GURL& EmptyGURL();
  377. // Returns the inner URL of a nested URL (currently only non-null for
  378. // filesystem URLs).
  379. //
  380. // TODO(mmenke): inner_url().spec() currently returns the same value as
  381. // caling spec() on the GURL itself. This should be fixed.
  382. // See https://crbug.com/619596
  383. const GURL* inner_url() const {
  384. return inner_url_.get();
  385. }
  386. // Estimates dynamic memory usage.
  387. // See base/trace_event/memory_usage_estimator.h for more info.
  388. size_t EstimateMemoryUsage() const;
  389. // Helper used by GURL::IsAboutUrl and KURL::IsAboutURL.
  390. static bool IsAboutPath(base::StringPiece actual_path,
  391. base::StringPiece allowed_path);
  392. void WriteIntoTrace(perfetto::TracedValue context) const;
  393. private:
  394. // Variant of the string parsing constructor that allows the caller to elect
  395. // retain trailing whitespace, if any, on the passed URL spec, but only if
  396. // the scheme is one that allows trailing whitespace. The primary use-case is
  397. // for data: URLs. In most cases, you want to use the single parameter
  398. // constructor above.
  399. enum RetainWhiteSpaceSelector { RETAIN_TRAILING_PATH_WHITEPACE };
  400. GURL(const std::string& url_string, RetainWhiteSpaceSelector);
  401. template <typename T, typename CharT = typename T::value_type>
  402. void InitCanonical(T input_spec, bool trim_path_end);
  403. void InitializeFromCanonicalSpec();
  404. // Helper used by IsAboutBlank and IsAboutSrcdoc.
  405. bool IsAboutUrl(base::StringPiece allowed_path) const;
  406. // Returns the substring of the input identified by the given component.
  407. std::string ComponentString(const url::Component& comp) const {
  408. if (!comp.is_nonempty())
  409. return std::string();
  410. return std::string(spec_, static_cast<size_t>(comp.begin),
  411. static_cast<size_t>(comp.len));
  412. }
  413. base::StringPiece ComponentStringPiece(const url::Component& comp) const {
  414. if (!comp.is_nonempty())
  415. return base::StringPiece();
  416. return base::StringPiece(&spec_[static_cast<size_t>(comp.begin)],
  417. static_cast<size_t>(comp.len));
  418. }
  419. void ProcessFileSystemURLAfterReplaceComponents();
  420. // The actual text of the URL, in canonical ASCII form.
  421. std::string spec_;
  422. // Set when the given URL is valid. Otherwise, we may still have a spec and
  423. // components, but they may not identify valid resources (for example, an
  424. // invalid port number, invalid characters in the scheme, etc.).
  425. bool is_valid_;
  426. // Identified components of the canonical spec.
  427. url::Parsed parsed_;
  428. // Used for nested schemes [currently only filesystem:].
  429. std::unique_ptr<GURL> inner_url_;
  430. };
  431. // Stream operator so GURL can be used in assertion statements.
  432. COMPONENT_EXPORT(URL)
  433. std::ostream& operator<<(std::ostream& out, const GURL& url);
  434. COMPONENT_EXPORT(URL) bool operator==(const GURL& x, const GURL& y);
  435. COMPONENT_EXPORT(URL) bool operator!=(const GURL& x, const GURL& y);
  436. // Equality operator for comparing raw spec_. This should be used in place of
  437. // url == GURL(spec) where |spec| is known (i.e. constants). This is to prevent
  438. // needlessly re-parsing |spec| into a temporary GURL.
  439. COMPONENT_EXPORT(URL)
  440. bool operator==(const GURL& x, const base::StringPiece& spec);
  441. COMPONENT_EXPORT(URL)
  442. bool operator==(const base::StringPiece& spec, const GURL& x);
  443. COMPONENT_EXPORT(URL)
  444. bool operator!=(const GURL& x, const base::StringPiece& spec);
  445. COMPONENT_EXPORT(URL)
  446. bool operator!=(const base::StringPiece& spec, const GURL& x);
  447. // DEBUG_ALIAS_FOR_GURL(var_name, url) copies |url| into a new stack-allocated
  448. // variable named |<var_name>|. This helps ensure that the value of |url| gets
  449. // preserved in crash dumps.
  450. #define DEBUG_ALIAS_FOR_GURL(var_name, url) \
  451. DEBUG_ALIAS_FOR_CSTR(var_name, (url).possibly_invalid_spec().c_str(), 128)
  452. namespace url::debug {
  453. class COMPONENT_EXPORT(URL) ScopedUrlCrashKey {
  454. public:
  455. ScopedUrlCrashKey(base::debug::CrashKeyString* crash_key, const GURL& value);
  456. ~ScopedUrlCrashKey();
  457. ScopedUrlCrashKey(const ScopedUrlCrashKey&) = delete;
  458. ScopedUrlCrashKey& operator=(const ScopedUrlCrashKey&) = delete;
  459. private:
  460. base::debug::ScopedCrashKeyString scoped_string_value_;
  461. };
  462. } // namespace url::debug
  463. #endif // URL_GURL_H_