url_formatter.h 9.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226
  1. // Copyright 2015 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. // url_formatter contains routines for formatting URLs in a way that can be
  5. // safely and securely displayed to users. For example, it is responsible
  6. // for determining when to convert an IDN A-Label (e.g. "xn--[something]")
  7. // into the IDN U-Label.
  8. //
  9. // Note that this formatting is only intended for display purposes; it would
  10. // be insecure and insufficient to make comparisons solely on formatted URLs
  11. // (that is, it should not be used for normalizing URLs for comparison for
  12. // security decisions).
  13. #ifndef COMPONENTS_URL_FORMATTER_URL_FORMATTER_H_
  14. #define COMPONENTS_URL_FORMATTER_URL_FORMATTER_H_
  15. #include <stddef.h>
  16. #include <stdint.h>
  17. #include <string>
  18. #include <vector>
  19. #include "base/containers/flat_set.h"
  20. #include "base/strings/escape.h"
  21. #include "base/strings/string_piece.h"
  22. #include "base/strings/utf_offset_string_conversions.h"
  23. #include "components/url_formatter/spoof_checks/idn_spoof_checker.h"
  24. class GURL;
  25. namespace url {
  26. struct Component;
  27. struct Parsed;
  28. }
  29. namespace url_formatter {
  30. using Skeletons = base::flat_set<std::string>;
  31. // Used by FormatUrl to specify handling of certain parts of the url.
  32. typedef uint32_t FormatUrlType;
  33. typedef uint32_t FormatUrlTypes;
  34. // The result of an IDN to Unicode conversion.
  35. struct IDNConversionResult {
  36. // The result of the conversion. If the input is a safe-to-display IDN encoded
  37. // as punycode, this will be its unicode representation. Otherwise, it'll be
  38. // the same as input.
  39. std::u16string result;
  40. // True if the hostname of the input has an IDN component, even if the result
  41. // wasn't converted.
  42. bool has_idn_component = false;
  43. // The top domain that the hostname of the input is visually similar to. Is
  44. // empty if the input didn't match any top domain.
  45. // E.g. IDNToUnicodeWithDetails("googlé.com") will fill |result| with
  46. // "xn--googl-fsa.com" and |matching_top_domain.domain| with "google.com".
  47. TopDomainEntry matching_top_domain;
  48. // Result of the spoof check. If the domain was converted to unicode, this
  49. // must be kSafe. Otherwise, this will be the failure reason
  50. // for the domain component (i.e. label) that failed the spoof checks. If
  51. // multiple labels fail the checks, this will be the result of the first
  52. // component that failed, counting from the left in the punycode form.
  53. IDNSpoofChecker::Result spoof_check_result = IDNSpoofChecker::Result::kNone;
  54. };
  55. // Nothing is omitted.
  56. extern const FormatUrlType kFormatUrlOmitNothing;
  57. // If set, any username and password are removed.
  58. extern const FormatUrlType kFormatUrlOmitUsernamePassword;
  59. // If the scheme is 'http://', it's removed.
  60. extern const FormatUrlType kFormatUrlOmitHTTP;
  61. // Omits the path if it is just a slash and there is no query or ref. This is
  62. // meaningful for non-file "standard" URLs.
  63. extern const FormatUrlType kFormatUrlOmitTrailingSlashOnBareHostname;
  64. // If the scheme is 'https://', it's removed. Not in kFormatUrlOmitDefaults.
  65. extern const FormatUrlType kFormatUrlOmitHTTPS;
  66. // Omits some trivially informative subdomains such as "www". Not in
  67. // kFormatUrlOmitDefaults.
  68. extern const FormatUrlType kFormatUrlOmitTrivialSubdomains;
  69. // Omits everything after the host: the path, query, ref, username and password
  70. // are all omitted.
  71. extern const FormatUrlType kFormatUrlTrimAfterHost;
  72. // If the scheme is 'file://', it's removed. Not in kFormatUrlOmitDefaults.
  73. extern const FormatUrlType kFormatUrlOmitFileScheme;
  74. // If the scheme is 'mailto:', it's removed. Not in kFormatUrlOmitDefaults.
  75. extern const FormatUrlType kFormatUrlOmitMailToScheme;
  76. // Omits the mobile prefix "m". Not in kFormatUrlOmitDefaults.
  77. extern const FormatUrlType kFormatUrlOmitMobilePrefix;
  78. // Convenience for omitting all unnecessary types. Does not include HTTPS scheme
  79. // removal, or experimental flags.
  80. extern const FormatUrlType kFormatUrlOmitDefaults;
  81. // Creates a string representation of |url|. The IDN host name is turned to
  82. // Unicode if the Unicode representation is deemed safe. |format_type| is a
  83. // bitmask of FormatUrlTypes, see it for details. |unescape_rules| defines how
  84. // to clean the URL for human readability. You will generally want
  85. // |UnescapeRule::SPACES| for display to the user if you can handle spaces, or
  86. // |UnescapeRule::NORMAL| if not. If the path part and the query part seem to
  87. // be encoded in %-encoded UTF-8, decodes %-encoding and UTF-8.
  88. //
  89. // The last three parameters may be NULL.
  90. //
  91. // |new_parsed| will be set to the parsing parameters of the resultant URL.
  92. //
  93. // |prefix_end| will be the length before the hostname of the resultant URL.
  94. //
  95. // |offset[s]_for_adjustment| specifies one or more offsets into the original
  96. // URL, representing insertion or selection points between characters: if the
  97. // input is "http://foo.com/", offset 0 is before the entire URL, offset 7 is
  98. // between the scheme and the host, and offset 15 is after the end of the URL.
  99. // Valid input offsets range from 0 to the length of the input URL string. On
  100. // exit, each offset will have been modified to reflect any changes made to the
  101. // output string. For example, if |url| is "http://a:b@c.com/",
  102. // |omit_username_password| is true, and an offset is 12 (pointing between 'c'
  103. // and '.'), then on return the output string will be "http://c.com/" and the
  104. // offset will be 8. If an offset cannot be successfully adjusted (e.g. because
  105. // it points into the middle of a component that was entirely removed or into
  106. // the middle of an encoding sequence), it will be set to std::u16string::npos.
  107. // For consistency, if an input offset points between the scheme and the
  108. // username/password, and both are removed, on output this offset will be 0
  109. // rather than npos; this means that offsets at the starts and ends of removed
  110. // components are always transformed the same way regardless of what other
  111. // components are adjacent.
  112. std::u16string FormatUrl(const GURL& url,
  113. FormatUrlTypes format_types,
  114. base::UnescapeRule::Type unescape_rules,
  115. url::Parsed* new_parsed,
  116. size_t* prefix_end,
  117. size_t* offset_for_adjustment);
  118. std::u16string FormatUrlWithOffsets(
  119. const GURL& url,
  120. FormatUrlTypes format_types,
  121. base::UnescapeRule::Type unescape_rules,
  122. url::Parsed* new_parsed,
  123. size_t* prefix_end,
  124. std::vector<size_t>* offsets_for_adjustment);
  125. // This function is like those above except it takes |adjustments| rather
  126. // than |offset[s]_for_adjustment|. |adjustments| will be set to reflect all
  127. // the transformations that happened to |url| to convert it into the returned
  128. // value.
  129. std::u16string FormatUrlWithAdjustments(
  130. const GURL& url,
  131. FormatUrlTypes format_types,
  132. base::UnescapeRule::Type unescape_rules,
  133. url::Parsed* new_parsed,
  134. size_t* prefix_end,
  135. base::OffsetAdjuster::Adjustments* adjustments);
  136. // This is a convenience function for FormatUrl() with
  137. // format_types = kFormatUrlOmitDefaults and unescape = SPACES. This is the
  138. // typical set of flags for "URLs to display to the user". You should be
  139. // cautious about using this for URLs which will be parsed or sent to other
  140. // applications.
  141. inline std::u16string FormatUrl(const GURL& url) {
  142. return FormatUrl(url, kFormatUrlOmitDefaults, base::UnescapeRule::SPACES,
  143. nullptr, nullptr, nullptr);
  144. }
  145. // Returns whether FormatUrl() would strip a trailing slash from |url|, given a
  146. // format flag including kFormatUrlOmitTrailingSlashOnBareHostname.
  147. bool CanStripTrailingSlash(const GURL& url);
  148. // Formats the host in |url| and appends it to |output|.
  149. void AppendFormattedHost(const GURL& url, std::u16string* output);
  150. // Converts the given host name to unicode characters. This can be called for
  151. // any host name, if the input is not IDN or is invalid in some way, we'll just
  152. // return the ASCII source so it is still usable.
  153. //
  154. // The input should be the canonicalized ASCII host name from GURL. This
  155. // function does NOT accept UTF-8!
  156. std::u16string IDNToUnicode(base::StringPiece host);
  157. // Same as IDNToUnicode, but disables spoof checks and returns more details.
  158. // In particular, it doesn't fall back to punycode if |host| fails spoof checks
  159. // in IDN spoof checker or is a lookalike of a top domain.
  160. // DO NOT use this for displaying URLs.
  161. IDNConversionResult UnsafeIDNToUnicodeWithDetails(base::StringPiece host);
  162. // Strips a "www." prefix from |host| if present and if |host| is eligible.
  163. // |host| is only eligible for www-stripping if it is not a private or intranet
  164. // hostname, and if "www." is part of the subdomain (not the eTLD+1).
  165. std::string StripWWW(const std::string& host);
  166. // Strips a "m." prefix from |host| if present.
  167. std::string StripMobilePrefix(const std::string& text);
  168. // If the |host| component of |url| begins with a "www." prefix (and meets the
  169. // conditions described for StripWWW), then updates |host| to strip the "www."
  170. // prefix.
  171. void StripWWWFromHostComponent(const std::string& url, url::Component* host);
  172. // Returns skeleton strings computed from |host| for spoof checking.
  173. Skeletons GetSkeletons(const std::u16string& host);
  174. // Returns a domain from the top 10K list matching the given skeleton. Used for
  175. // spoof checking. Different types of skeletons are saved in the skeleton trie.
  176. // Providing |type| makes sure the right type of skeletons are looked up. For
  177. // example if |skeleton|="googlecorn", |type|="kFull", no match would be found
  178. // even though the skeleton is saved in the trie, because the type of this
  179. // skeleton in the trie is "kSeparatorsRemoved".
  180. TopDomainEntry LookupSkeletonInTopDomains(
  181. const std::string& skeleton,
  182. const SkeletonType type = SkeletonType::kFull);
  183. // Removes diacritics from `host` and returns the new string if the input
  184. // only contains Latin-Greek-Cyrillic characters. Otherwise, returns the
  185. // input string.
  186. std::u16string MaybeRemoveDiacritics(const std::u16string& host);
  187. } // namespace url_formatter
  188. #endif // COMPONENTS_URL_FORMATTER_URL_FORMATTER_H_