url_formatter.cc 34 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843
  1. // Copyright 2015 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "components/url_formatter/url_formatter.h"
  5. #include <algorithm>
  6. #include <ostream>
  7. #include <utility>
  8. #include <vector>
  9. #include "base/lazy_instance.h"
  10. #include "base/memory/raw_ptr.h"
  11. #include "base/numerics/safe_conversions.h"
  12. #include "base/strings/strcat.h"
  13. #include "base/strings/string_piece.h"
  14. #include "base/strings/string_util.h"
  15. #include "base/strings/utf_offset_string_conversions.h"
  16. #include "base/strings/utf_string_conversions.h"
  17. #include "base/threading/thread_local_storage.h"
  18. #include "build/build_config.h"
  19. #include "net/base/registry_controlled_domains/registry_controlled_domain.h"
  20. #include "third_party/icu/source/common/unicode/uidna.h"
  21. #include "third_party/icu/source/common/unicode/utypes.h"
  22. #include "url/gurl.h"
  23. #include "url/third_party/mozilla/url_parse.h"
  24. #include "url/url_constants.h"
  25. #include "url/url_util.h"
  26. namespace url_formatter {
  27. namespace {
  28. const char kWww[] = "www.";
  29. constexpr size_t kWwwLength = 4;
  30. const char kMobilePrefix[] = "m.";
  31. constexpr size_t kMobilePrefixLength = 2;
  32. IDNConversionResult IDNToUnicodeWithAdjustments(
  33. base::StringPiece host,
  34. base::OffsetAdjuster::Adjustments* adjustments);
  35. // Result of converting a single IDN component (i.e. label) to unicode.
  36. struct ComponentResult {
  37. // Set to true if the component is converted to unicode.
  38. bool converted = false;
  39. // Set to true if the component is IDN, even if it's not converted to unicode.
  40. bool has_idn_component = false;
  41. // Result of the IDN spoof check.
  42. IDNSpoofChecker::Result spoof_check_result = IDNSpoofChecker::Result::kNone;
  43. };
  44. ComponentResult IDNToUnicodeOneComponent(
  45. base::StringPiece16 comp,
  46. base::StringPiece top_level_domain,
  47. base::StringPiece16 top_level_domain_unicode,
  48. bool ignore_spoof_check_results,
  49. std::u16string* out);
  50. class AppendComponentTransform {
  51. public:
  52. AppendComponentTransform() = default;
  53. virtual ~AppendComponentTransform() = default;
  54. virtual std::u16string Execute(
  55. const std::string& component_text,
  56. base::OffsetAdjuster::Adjustments* adjustments) const = 0;
  57. // NOTE: No DISALLOW_COPY_AND_ASSIGN here, since gcc < 4.3.0 requires an
  58. // accessible copy constructor in order to call AppendFormattedComponent()
  59. // with an inline temporary (see http://gcc.gnu.org/bugs/#cxx%5Frvalbind ).
  60. };
  61. class HostComponentTransform : public AppendComponentTransform {
  62. public:
  63. HostComponentTransform(bool trim_trivial_subdomains, bool trim_mobile_prefix)
  64. : trim_trivial_subdomains_(trim_trivial_subdomains),
  65. trim_mobile_prefix_(trim_mobile_prefix) {}
  66. private:
  67. std::u16string Execute(
  68. const std::string& component_text,
  69. base::OffsetAdjuster::Adjustments* adjustments) const override {
  70. // Nothing to change.
  71. if (!trim_trivial_subdomains_ && !trim_mobile_prefix_)
  72. return IDNToUnicodeWithAdjustments(component_text, adjustments).result;
  73. std::string stripped_component_text = component_text;
  74. if (base::StartsWith(component_text, "www.m.") &&
  75. trim_trivial_subdomains_ && trim_mobile_prefix_) {
  76. stripped_component_text = StripWWW(stripped_component_text);
  77. stripped_component_text = StripMobilePrefix(stripped_component_text);
  78. } else if (base::StartsWith(component_text, "m.www.") &&
  79. trim_mobile_prefix_ && trim_trivial_subdomains_) {
  80. stripped_component_text = StripMobilePrefix(stripped_component_text);
  81. stripped_component_text = StripWWW(stripped_component_text);
  82. } else {
  83. if (trim_trivial_subdomains_) {
  84. stripped_component_text = StripWWW(component_text);
  85. }
  86. if (trim_mobile_prefix_) {
  87. stripped_component_text = StripMobilePrefix(stripped_component_text);
  88. }
  89. }
  90. // If StripWWW() and StripMobilePrefix() did nothing, then "www." and "m."
  91. // weren't a prefix, or it otherwise didn't meet conditions for stripping
  92. // "www." (such as intranet hostnames). In this case, no adjustments for
  93. // trivial subdomains are needed.
  94. if (stripped_component_text == component_text)
  95. return IDNToUnicodeWithAdjustments(component_text, adjustments).result;
  96. base::OffsetAdjuster::Adjustments offset_adjustments;
  97. if (component_text.length() ==
  98. stripped_component_text.length() + kMobilePrefixLength + kWwwLength) {
  99. // Add www. and m. offsets.
  100. offset_adjustments.push_back(
  101. base::OffsetAdjuster::Adjustment(0, kWwwLength, 0));
  102. offset_adjustments.push_back(
  103. base::OffsetAdjuster::Adjustment(0, kMobilePrefixLength, 0));
  104. } else if (component_text.length() ==
  105. stripped_component_text.length() + kWwwLength) {
  106. // Add www. offset.
  107. offset_adjustments.push_back(
  108. base::OffsetAdjuster::Adjustment(0, kWwwLength, 0));
  109. } else if (component_text.length() ==
  110. stripped_component_text.length() + kMobilePrefixLength) {
  111. // Add m. offset
  112. offset_adjustments.push_back(
  113. base::OffsetAdjuster::Adjustment(0, kMobilePrefixLength, 0));
  114. }
  115. std::u16string unicode_result =
  116. IDNToUnicodeWithAdjustments(stripped_component_text, adjustments)
  117. .result;
  118. base::OffsetAdjuster::MergeSequentialAdjustments(offset_adjustments,
  119. adjustments);
  120. return unicode_result;
  121. }
  122. bool trim_trivial_subdomains_;
  123. bool trim_mobile_prefix_;
  124. };
  125. class NonHostComponentTransform : public AppendComponentTransform {
  126. public:
  127. explicit NonHostComponentTransform(base::UnescapeRule::Type unescape_rules)
  128. : unescape_rules_(unescape_rules) {}
  129. private:
  130. std::u16string Execute(
  131. const std::string& component_text,
  132. base::OffsetAdjuster::Adjustments* adjustments) const override {
  133. return (unescape_rules_ == base::UnescapeRule::NONE)
  134. ? base::UTF8ToUTF16WithAdjustments(component_text, adjustments)
  135. : base::UnescapeAndDecodeUTF8URLComponentWithAdjustments(
  136. component_text, unescape_rules_, adjustments);
  137. }
  138. const base::UnescapeRule::Type unescape_rules_;
  139. };
  140. // Transforms the portion of |spec| covered by |original_component| according to
  141. // |transform|. Appends the result to |output|. If |output_component| is
  142. // non-NULL, its start and length are set to the transformed component's new
  143. // start and length. If |adjustments| is non-NULL, appends adjustments (if
  144. // any) that reflect the transformation the original component underwent to
  145. // become the transformed value appended to |output|.
  146. void AppendFormattedComponent(const std::string& spec,
  147. const url::Component& original_component,
  148. const AppendComponentTransform& transform,
  149. std::u16string* output,
  150. url::Component* output_component,
  151. base::OffsetAdjuster::Adjustments* adjustments) {
  152. DCHECK(output);
  153. if (original_component.is_nonempty()) {
  154. size_t original_component_begin =
  155. static_cast<size_t>(original_component.begin);
  156. size_t output_component_begin = output->length();
  157. std::string component_str(spec, original_component_begin,
  158. static_cast<size_t>(original_component.len));
  159. // Transform |component_str| and modify |adjustments| appropriately.
  160. base::OffsetAdjuster::Adjustments component_transform_adjustments;
  161. output->append(
  162. transform.Execute(component_str, &component_transform_adjustments));
  163. // Shift all the adjustments made for this component so the offsets are
  164. // valid for the original string and add them to |adjustments|.
  165. for (auto& component_transform_adjustment :
  166. component_transform_adjustments) {
  167. component_transform_adjustment.original_offset +=
  168. original_component_begin;
  169. }
  170. if (adjustments) {
  171. adjustments->insert(adjustments->end(),
  172. component_transform_adjustments.begin(),
  173. component_transform_adjustments.end());
  174. }
  175. // Set positions of the parsed component.
  176. if (output_component) {
  177. output_component->begin = static_cast<int>(output_component_begin);
  178. output_component->len =
  179. static_cast<int>(output->length() - output_component_begin);
  180. }
  181. } else if (output_component) {
  182. output_component->reset();
  183. }
  184. }
  185. // If |component| is valid, its begin is incremented by |delta|.
  186. void AdjustComponent(int delta, url::Component* component) {
  187. if (!component->is_valid())
  188. return;
  189. DCHECK(delta >= 0 || component->begin >= -delta);
  190. component->begin += delta;
  191. }
  192. // Adjusts all the components of |parsed| by |delta|, except for the scheme.
  193. void AdjustAllComponentsButScheme(int delta, url::Parsed* parsed) {
  194. AdjustComponent(delta, &(parsed->username));
  195. AdjustComponent(delta, &(parsed->password));
  196. AdjustComponent(delta, &(parsed->host));
  197. AdjustComponent(delta, &(parsed->port));
  198. AdjustComponent(delta, &(parsed->path));
  199. AdjustComponent(delta, &(parsed->query));
  200. AdjustComponent(delta, &(parsed->ref));
  201. }
  202. // Helper for FormatUrlWithOffsets().
  203. std::u16string FormatViewSourceUrl(
  204. const GURL& url,
  205. FormatUrlTypes format_types,
  206. base::UnescapeRule::Type unescape_rules,
  207. url::Parsed* new_parsed,
  208. size_t* prefix_end,
  209. base::OffsetAdjuster::Adjustments* adjustments) {
  210. DCHECK(new_parsed);
  211. static constexpr base::StringPiece16 kViewSource = u"view-source:";
  212. // The URL embedded within view-source should never have destructive elisions
  213. // applied to it. Users of view-source likely want to see the full URL.
  214. format_types &= ~kFormatUrlOmitHTTPS;
  215. format_types &= ~kFormatUrlOmitTrivialSubdomains;
  216. format_types &= ~kFormatUrlTrimAfterHost;
  217. format_types &= ~kFormatUrlOmitFileScheme;
  218. // Format the underlying URL and record adjustments.
  219. const std::string& url_str(url.possibly_invalid_spec());
  220. adjustments->clear();
  221. std::u16string result = base::StrCat(
  222. {kViewSource, FormatUrlWithAdjustments(
  223. GURL(url_str.substr(kViewSource.size())), format_types,
  224. unescape_rules, new_parsed, prefix_end, adjustments)});
  225. // Revise |adjustments| by shifting to the offsets to prefix that the above
  226. // call to FormatUrl didn't get to see.
  227. for (auto& adjustment : *adjustments)
  228. adjustment.original_offset += kViewSource.size();
  229. // Adjust positions of the parsed components.
  230. if (new_parsed->scheme.is_nonempty()) {
  231. // Assume "view-source:real-scheme" as a scheme.
  232. new_parsed->scheme.len += kViewSource.size();
  233. } else {
  234. new_parsed->scheme.begin = 0;
  235. new_parsed->scheme.len = kViewSource.size() - 1;
  236. }
  237. AdjustAllComponentsButScheme(kViewSource.size(), new_parsed);
  238. if (prefix_end)
  239. *prefix_end += kViewSource.size();
  240. return result;
  241. }
  242. base::LazyInstance<IDNSpoofChecker>::Leaky g_idn_spoof_checker =
  243. LAZY_INSTANCE_INITIALIZER;
  244. // Computes the top level domain from |host|. top_level_domain_unicode will
  245. // contain the unicode version of top_level_domain. top_level_domain_unicode can
  246. // remain empty if the TLD is not well formed punycode.
  247. void GetTopLevelDomain(base::StringPiece host,
  248. base::StringPiece* top_level_domain,
  249. std::u16string* top_level_domain_unicode) {
  250. size_t last_dot = host.rfind('.');
  251. if (last_dot == base::StringPiece::npos)
  252. return;
  253. *top_level_domain = host.substr(last_dot + 1);
  254. std::u16string tld16;
  255. tld16.reserve(top_level_domain->length());
  256. tld16.insert(tld16.end(), top_level_domain->begin(), top_level_domain->end());
  257. // Convert the TLD to unicode, ignoring the spoof check results. This will
  258. // always decode the input to unicode as long as it's valid punycode.
  259. IDNToUnicodeOneComponent(tld16, std::string(), std::u16string(),
  260. /*ignore_spoof_check_results=*/true,
  261. top_level_domain_unicode);
  262. }
  263. IDNConversionResult IDNToUnicodeWithAdjustmentsImpl(
  264. base::StringPiece host,
  265. base::OffsetAdjuster::Adjustments* adjustments,
  266. bool ignore_spoof_check_results) {
  267. if (adjustments)
  268. adjustments->clear();
  269. // Convert the ASCII input to a std::u16string for ICU.
  270. std::u16string host16;
  271. host16.reserve(host.length());
  272. host16.insert(host16.end(), host.begin(), host.end());
  273. // Compute the top level domain to be used in spoof checks later.
  274. base::StringPiece top_level_domain;
  275. std::u16string top_level_domain_unicode;
  276. GetTopLevelDomain(host, &top_level_domain, &top_level_domain_unicode);
  277. IDNConversionResult result;
  278. // Do each component of the host separately, since we enforce script matching
  279. // on a per-component basis.
  280. std::u16string out16;
  281. for (size_t component_start = 0, component_end;
  282. component_start < host16.length(); component_start = component_end + 1) {
  283. // Find the end of the component.
  284. component_end = host16.find('.', component_start);
  285. if (component_end == std::u16string::npos)
  286. component_end = host16.length(); // For getting the last component.
  287. size_t component_length = component_end - component_start;
  288. size_t new_component_start = out16.length();
  289. ComponentResult component_result;
  290. if (component_end > component_start) {
  291. // Add the substring that we just found.
  292. component_result = IDNToUnicodeOneComponent(
  293. {host16.data() + component_start, component_length}, top_level_domain,
  294. top_level_domain_unicode, ignore_spoof_check_results, &out16);
  295. result.has_idn_component |= component_result.has_idn_component;
  296. if (component_result.spoof_check_result !=
  297. IDNSpoofChecker::Result::kNone &&
  298. (result.spoof_check_result == IDNSpoofChecker::Result::kNone ||
  299. result.spoof_check_result == IDNSpoofChecker::Result::kSafe)) {
  300. result.spoof_check_result = component_result.spoof_check_result;
  301. }
  302. }
  303. size_t new_component_length = out16.length() - new_component_start;
  304. if (component_result.converted && adjustments) {
  305. adjustments->push_back(base::OffsetAdjuster::Adjustment(
  306. component_start, component_length, new_component_length));
  307. }
  308. // Need to add the dot we just found (if we found one).
  309. if (component_end < host16.length())
  310. out16.push_back('.');
  311. }
  312. result.result = out16;
  313. // Leave as punycode any inputs that spoof top domains.
  314. if (result.has_idn_component) {
  315. result.matching_top_domain =
  316. g_idn_spoof_checker.Get().GetSimilarTopDomain(out16);
  317. if (!ignore_spoof_check_results &&
  318. !result.matching_top_domain.domain.empty()) {
  319. if (adjustments)
  320. adjustments->clear();
  321. result.result = host16;
  322. }
  323. }
  324. return result;
  325. }
  326. // TODO(brettw): We may want to skip this step in the case of file URLs to
  327. // allow unicode UNC hostnames regardless of encodings.
  328. IDNConversionResult IDNToUnicodeWithAdjustments(
  329. base::StringPiece host,
  330. base::OffsetAdjuster::Adjustments* adjustments) {
  331. return IDNToUnicodeWithAdjustmentsImpl(host, adjustments,
  332. /*ignore_spoof_check_results=*/false);
  333. }
  334. IDNConversionResult UnsafeIDNToUnicodeWithAdjustments(
  335. base::StringPiece host,
  336. base::OffsetAdjuster::Adjustments* adjustments) {
  337. return IDNToUnicodeWithAdjustmentsImpl(host, adjustments,
  338. /*ignore_spoof_check_results=*/true);
  339. }
  340. // Returns true if the given Unicode host component is safe to display to the
  341. // user. Note that this function does not deal with pure ASCII domain labels at
  342. // all even though it's possible to make up look-alike labels with ASCII
  343. // characters alone.
  344. IDNSpoofChecker::Result SpoofCheckIDNComponent(
  345. base::StringPiece16 label,
  346. base::StringPiece top_level_domain,
  347. base::StringPiece16 top_level_domain_unicode) {
  348. return g_idn_spoof_checker.Get().SafeToDisplayAsUnicode(
  349. label, top_level_domain, top_level_domain_unicode);
  350. }
  351. // A wrapper to use LazyInstance<>::Leaky with ICU's UIDNA, a C pointer to
  352. // a UTS46/IDNA 2008 handling object opened with uidna_openUTS46().
  353. //
  354. // We use UTS46 with BiDiCheck to migrate from IDNA 2003 to IDNA 2008 with the
  355. // backward compatibility in mind. What it does:
  356. //
  357. // 1. Use the up-to-date Unicode data.
  358. // 2. Define a case folding/mapping with the up-to-date Unicode data as in
  359. // IDNA 2003.
  360. // 3. Use transitional mechanism for 4 deviation characters (sharp-s,
  361. // final sigma, ZWJ and ZWNJ) for now.
  362. // 4. Continue to allow symbols and punctuations.
  363. // 5. Apply new BiDi check rules more permissive than the IDNA 2003 BiDI rules.
  364. // 6. Do not apply STD3 rules
  365. // 7. Do not allow unassigned code points.
  366. //
  367. // It also closely matches what IE 10 does except for the BiDi check (
  368. // http://goo.gl/3XBhqw ).
  369. // See http://http://unicode.org/reports/tr46/ and references therein/ for more
  370. // details.
  371. struct UIDNAWrapper {
  372. UIDNAWrapper() {
  373. UErrorCode err = U_ZERO_ERROR;
  374. // TODO(jungshik): Change options as different parties (browsers,
  375. // registrars, search engines) converge toward a consensus.
  376. value = uidna_openUTS46(UIDNA_CHECK_BIDI, &err);
  377. CHECK(U_SUCCESS(err)) << "failed to open UTS46 data with error: "
  378. << u_errorName(err)
  379. << ". If you see this error message in a test "
  380. << "environment your test environment likely lacks "
  381. << "the required data tables for libicu. See "
  382. << "https://crbug.com/778929.";
  383. }
  384. raw_ptr<UIDNA> value;
  385. };
  386. base::LazyInstance<UIDNAWrapper>::Leaky g_uidna = LAZY_INSTANCE_INITIALIZER;
  387. // Converts one component (label) of a host (between dots) to Unicode if safe.
  388. // If |ignore_spoof_check_results| is true and input is valid unicode, ignores
  389. // spoof check results and always converts the input to unicode. The result will
  390. // be APPENDED to the given output string and will be the same as the input if
  391. // it is not IDN in ACE/punycode or the IDN is unsafe to display. Returns true
  392. // if conversion was made. Sets |has_idn_component| to true if the input has
  393. // IDN, regardless of whether it was converted to unicode or not.
  394. ComponentResult IDNToUnicodeOneComponent(
  395. base::StringPiece16 comp,
  396. base::StringPiece top_level_domain,
  397. base::StringPiece16 top_level_domain_unicode,
  398. bool ignore_spoof_check_results,
  399. std::u16string* out) {
  400. DCHECK(out);
  401. ComponentResult result;
  402. if (comp.empty())
  403. return result;
  404. // Early return if the input cannot be an IDN component.
  405. // Valid punycode must not end with a dash.
  406. static constexpr char16_t kIdnPrefix[] = u"xn--";
  407. if (!base::StartsWith(comp, kIdnPrefix) || comp.back() == '-') {
  408. out->append(comp.data(), comp.size());
  409. return result;
  410. }
  411. UIDNA* uidna = g_uidna.Get().value;
  412. DCHECK(uidna != nullptr);
  413. size_t original_length = out->length();
  414. int32_t output_length = 64;
  415. UIDNAInfo info = UIDNA_INFO_INITIALIZER;
  416. UErrorCode status;
  417. do {
  418. out->resize(original_length + output_length);
  419. status = U_ZERO_ERROR;
  420. // This returns the actual length required. If this is more than 64
  421. // code units, |status| will be U_BUFFER_OVERFLOW_ERROR and we'll try
  422. // the conversion again, but with a sufficiently large buffer.
  423. output_length = uidna_labelToUnicode(
  424. uidna, comp.data(), static_cast<int32_t>(comp.size()),
  425. &(*out)[original_length], output_length, &info, &status);
  426. } while ((status == U_BUFFER_OVERFLOW_ERROR && info.errors == 0));
  427. if (U_SUCCESS(status) && info.errors == 0) {
  428. result.has_idn_component = true;
  429. // Converted successfully. At this point the length of the output string
  430. // is original_length + output_length which may be shorter than the current
  431. // length of |out|. Trim |out| and ensure that the converted component can
  432. // be safely displayed to the user.
  433. out->resize(original_length + output_length);
  434. result.spoof_check_result = SpoofCheckIDNComponent(
  435. base::StringPiece16(out->data() + original_length,
  436. base::checked_cast<size_t>(output_length)),
  437. top_level_domain, top_level_domain_unicode);
  438. DCHECK_NE(IDNSpoofChecker::Result::kNone, result.spoof_check_result);
  439. if (ignore_spoof_check_results ||
  440. result.spoof_check_result == IDNSpoofChecker::Result::kSafe) {
  441. result.converted = true;
  442. return result;
  443. }
  444. }
  445. // We get here with no IDN or on error, in which case we just revert to
  446. // original string and append the literal input.
  447. out->resize(original_length);
  448. out->append(comp.data(), comp.size());
  449. return result;
  450. }
  451. // Returns true iff URL-parsing `spec` would reveal that it has the
  452. // "view-source" scheme, and that parsing the spec minus that scheme also has
  453. // the "view-source" scheme.
  454. bool HasTwoViewSourceSchemes(base::StringPiece spec) {
  455. static constexpr char kViewSource[] = "view-source";
  456. url::Component scheme;
  457. if (!url::FindAndCompareScheme(spec.data(),
  458. base::checked_cast<int>(spec.size()),
  459. kViewSource, &scheme)) {
  460. return false;
  461. }
  462. // Consume the scheme.
  463. spec.remove_prefix(scheme.begin + scheme.len);
  464. // Consume the trailing colon. If it's not there, then `spec` didn't really
  465. // have the first view-source scheme.
  466. if (spec.empty() || spec[0] != ':')
  467. return false;
  468. spec.remove_prefix(1);
  469. return url::FindAndCompareScheme(
  470. spec.data(), base::checked_cast<int>(spec.size()), kViewSource, &scheme);
  471. }
  472. } // namespace
  473. const FormatUrlType kFormatUrlOmitNothing = 0;
  474. const FormatUrlType kFormatUrlOmitUsernamePassword = 1 << 0;
  475. const FormatUrlType kFormatUrlOmitHTTP = 1 << 1;
  476. const FormatUrlType kFormatUrlOmitTrailingSlashOnBareHostname = 1 << 2;
  477. const FormatUrlType kFormatUrlOmitHTTPS = 1 << 3;
  478. const FormatUrlType kFormatUrlOmitTrivialSubdomains = 1 << 5;
  479. const FormatUrlType kFormatUrlTrimAfterHost = 1 << 6;
  480. const FormatUrlType kFormatUrlOmitFileScheme = 1 << 7;
  481. const FormatUrlType kFormatUrlOmitMailToScheme = 1 << 8;
  482. const FormatUrlType kFormatUrlOmitMobilePrefix = 1 << 9;
  483. const FormatUrlType kFormatUrlOmitDefaults =
  484. kFormatUrlOmitUsernamePassword | kFormatUrlOmitHTTP |
  485. kFormatUrlOmitTrailingSlashOnBareHostname;
  486. std::u16string FormatUrl(const GURL& url,
  487. FormatUrlTypes format_types,
  488. base::UnescapeRule::Type unescape_rules,
  489. url::Parsed* new_parsed,
  490. size_t* prefix_end,
  491. size_t* offset_for_adjustment) {
  492. base::OffsetAdjuster::Adjustments adjustments;
  493. std::u16string result = FormatUrlWithAdjustments(
  494. url, format_types, unescape_rules, new_parsed, prefix_end, &adjustments);
  495. if (offset_for_adjustment) {
  496. base::OffsetAdjuster::AdjustOffset(adjustments, offset_for_adjustment,
  497. result.length());
  498. }
  499. return result;
  500. }
  501. std::u16string FormatUrlWithOffsets(
  502. const GURL& url,
  503. FormatUrlTypes format_types,
  504. base::UnescapeRule::Type unescape_rules,
  505. url::Parsed* new_parsed,
  506. size_t* prefix_end,
  507. std::vector<size_t>* offsets_for_adjustment) {
  508. base::OffsetAdjuster::Adjustments adjustments;
  509. const std::u16string& result = FormatUrlWithAdjustments(
  510. url, format_types, unescape_rules, new_parsed, prefix_end, &adjustments);
  511. base::OffsetAdjuster::AdjustOffsets(adjustments, offsets_for_adjustment,
  512. result.length());
  513. return result;
  514. }
  515. std::u16string FormatUrlWithAdjustments(
  516. const GURL& url,
  517. FormatUrlTypes format_types,
  518. base::UnescapeRule::Type unescape_rules,
  519. url::Parsed* new_parsed,
  520. size_t* prefix_end,
  521. base::OffsetAdjuster::Adjustments* adjustments) {
  522. DCHECK(adjustments);
  523. adjustments->clear();
  524. url::Parsed parsed_temp;
  525. if (!new_parsed)
  526. new_parsed = &parsed_temp;
  527. else
  528. *new_parsed = url::Parsed();
  529. // Special handling for view-source:. Don't use content::kViewSourceScheme
  530. // because this library shouldn't depend on chrome. Reject repeated
  531. // view-source schemes to avoid recursion.
  532. static constexpr base::StringPiece kViewSource = "view-source";
  533. if (url.SchemeIs(kViewSource) &&
  534. !HasTwoViewSourceSchemes(url.possibly_invalid_spec())) {
  535. return FormatViewSourceUrl(url, format_types, unescape_rules, new_parsed,
  536. prefix_end, adjustments);
  537. }
  538. // We handle both valid and invalid URLs (this will give us the spec
  539. // regardless of validity).
  540. const std::string& spec = url.possibly_invalid_spec();
  541. const url::Parsed& parsed = url.parsed_for_possibly_invalid_spec();
  542. // Scheme & separators. These are ASCII.
  543. size_t scheme_size = static_cast<size_t>(parsed.CountCharactersBefore(
  544. url::Parsed::USERNAME, true /* include_delimiter */));
  545. std::u16string url_string;
  546. url_string.insert(url_string.end(), spec.begin(), spec.begin() + scheme_size);
  547. new_parsed->scheme = parsed.scheme;
  548. // Username & password.
  549. if (((format_types & kFormatUrlOmitUsernamePassword) != 0) ||
  550. ((format_types & kFormatUrlTrimAfterHost) != 0)) {
  551. // Remove the username and password fields. We don't want to display those
  552. // to the user since they can be used for attacks,
  553. // e.g. "http://google.com:search@evil.ru/"
  554. new_parsed->username.reset();
  555. new_parsed->password.reset();
  556. // Update the adjustments based on removed username and/or password.
  557. if (parsed.username.is_nonempty() || parsed.password.is_nonempty()) {
  558. if (parsed.username.is_nonempty() && parsed.password.is_nonempty()) {
  559. // The seeming off-by-two is to account for the ':' after the username
  560. // and '@' after the password.
  561. adjustments->push_back(base::OffsetAdjuster::Adjustment(
  562. static_cast<size_t>(parsed.username.begin),
  563. static_cast<size_t>(parsed.username.len + parsed.password.len + 2),
  564. 0));
  565. } else {
  566. const url::Component* nonempty_component =
  567. parsed.username.is_nonempty() ? &parsed.username : &parsed.password;
  568. // The seeming off-by-one is to account for the '@' after the
  569. // username/password.
  570. adjustments->push_back(base::OffsetAdjuster::Adjustment(
  571. static_cast<size_t>(nonempty_component->begin),
  572. static_cast<size_t>(nonempty_component->len + 1), 0));
  573. }
  574. }
  575. } else {
  576. AppendFormattedComponent(spec, parsed.username,
  577. NonHostComponentTransform(unescape_rules),
  578. &url_string, &new_parsed->username, adjustments);
  579. if (parsed.password.is_valid())
  580. url_string.push_back(':');
  581. AppendFormattedComponent(spec, parsed.password,
  582. NonHostComponentTransform(unescape_rules),
  583. &url_string, &new_parsed->password, adjustments);
  584. if (parsed.username.is_valid() || parsed.password.is_valid())
  585. url_string.push_back('@');
  586. }
  587. if (prefix_end)
  588. *prefix_end = static_cast<size_t>(url_string.length());
  589. // Host.
  590. bool trim_trivial_subdomains =
  591. (format_types & kFormatUrlOmitTrivialSubdomains) != 0;
  592. bool trim_mobile_prefix = (format_types & kFormatUrlOmitMobilePrefix) != 0;
  593. AppendFormattedComponent(
  594. spec, parsed.host,
  595. HostComponentTransform(trim_trivial_subdomains, trim_mobile_prefix),
  596. &url_string, &new_parsed->host, adjustments);
  597. // Port.
  598. if (parsed.port.is_nonempty()) {
  599. url_string.push_back(':');
  600. new_parsed->port.begin = url_string.length();
  601. url_string.insert(url_string.end(), spec.begin() + parsed.port.begin,
  602. spec.begin() + parsed.port.end());
  603. new_parsed->port.len = url_string.length() - new_parsed->port.begin;
  604. } else {
  605. new_parsed->port.reset();
  606. }
  607. // Path & query. Both get the same general unescape & convert treatment.
  608. if ((format_types & kFormatUrlTrimAfterHost) && url.IsStandard() &&
  609. !url.SchemeIsFile() && !url.SchemeIsFileSystem()) {
  610. size_t trimmed_length = parsed.path.len;
  611. // Remove query and the '?' delimeter.
  612. if (parsed.query.is_valid())
  613. trimmed_length += parsed.query.len + 1;
  614. // Remove ref and the '#" delimiter.
  615. if (parsed.ref.is_valid())
  616. trimmed_length += parsed.ref.len + 1;
  617. adjustments->push_back(
  618. base::OffsetAdjuster::Adjustment(parsed.path.begin, trimmed_length, 0));
  619. } else if ((format_types & kFormatUrlOmitTrailingSlashOnBareHostname) &&
  620. CanStripTrailingSlash(url)) {
  621. // Omit the path, which is a single trailing slash. There's no query or ref.
  622. if (parsed.path.len > 0) {
  623. adjustments->push_back(base::OffsetAdjuster::Adjustment(
  624. parsed.path.begin, parsed.path.len, 0));
  625. }
  626. } else {
  627. // Append the formatted path, query, and ref.
  628. AppendFormattedComponent(spec, parsed.path,
  629. NonHostComponentTransform(unescape_rules),
  630. &url_string, &new_parsed->path, adjustments);
  631. if (parsed.query.is_valid())
  632. url_string.push_back('?');
  633. AppendFormattedComponent(spec, parsed.query,
  634. NonHostComponentTransform(unescape_rules),
  635. &url_string, &new_parsed->query, adjustments);
  636. if (parsed.ref.is_valid())
  637. url_string.push_back('#');
  638. AppendFormattedComponent(spec, parsed.ref,
  639. NonHostComponentTransform(unescape_rules),
  640. &url_string, &new_parsed->ref, adjustments);
  641. }
  642. // url_formatter::FixupURL() treats "ftp.foo.com" as ftp://ftp.foo.com. This
  643. // means that if we trim the scheme off a URL whose host starts with "ftp."
  644. // and the user inputs this into any field subject to fixup (which is
  645. // basically all input fields), the meaning would be changed. (In fact, often
  646. // the formatted URL is directly pre-filled into an input field.) For this
  647. // reason we avoid stripping schemes in this case.
  648. const char kFTP[] = "ftp.";
  649. bool strip_scheme =
  650. !base::StartsWith(url.host(), kFTP, base::CompareCase::SENSITIVE) &&
  651. (((format_types & kFormatUrlOmitHTTP) &&
  652. url.SchemeIs(url::kHttpScheme)) ||
  653. ((format_types & kFormatUrlOmitHTTPS) &&
  654. url.SchemeIs(url::kHttpsScheme)) ||
  655. ((format_types & kFormatUrlOmitFileScheme) &&
  656. url.SchemeIs(url::kFileScheme)) ||
  657. ((format_types & kFormatUrlOmitMailToScheme) &&
  658. url.SchemeIs(url::kMailToScheme)));
  659. // If we need to strip out schemes do it after the fact.
  660. if (strip_scheme) {
  661. DCHECK(new_parsed->scheme.is_valid());
  662. size_t scheme_and_separator_len =
  663. url.SchemeIs(url::kMailToScheme)
  664. ? new_parsed->scheme.len + 1 // +1 for :.
  665. : new_parsed->scheme.len + 3; // +3 for ://.
  666. #if BUILDFLAG(IS_WIN)
  667. // Because there's an additional leading slash after the scheme for local
  668. // files on Windows, we should remove it for URL display when eliding
  669. // the scheme by offsetting by an additional character.
  670. if (url.SchemeIs(url::kFileScheme) &&
  671. base::StartsWith(url_string, u"file:///",
  672. base::CompareCase::INSENSITIVE_ASCII)) {
  673. ++new_parsed->path.begin;
  674. ++scheme_size;
  675. ++scheme_and_separator_len;
  676. }
  677. #endif
  678. url_string.erase(0, scheme_size);
  679. // Because offsets in the |adjustments| are already calculated with respect
  680. // to the string with the http:// prefix in it, those offsets remain correct
  681. // after stripping the prefix. The only thing necessary is to add an
  682. // adjustment to reflect the stripped prefix.
  683. adjustments->insert(adjustments->begin(),
  684. base::OffsetAdjuster::Adjustment(0, scheme_size, 0));
  685. if (prefix_end)
  686. *prefix_end -= scheme_size;
  687. // Adjust new_parsed.
  688. new_parsed->scheme.reset();
  689. AdjustAllComponentsButScheme(-scheme_and_separator_len, new_parsed);
  690. }
  691. return url_string;
  692. }
  693. bool CanStripTrailingSlash(const GURL& url) {
  694. // Omit the path only for standard, non-file URLs with nothing but "/" after
  695. // the hostname.
  696. return url.IsStandard() && !url.SchemeIsFile() && !url.SchemeIsFileSystem() &&
  697. !url.has_query() && !url.has_ref() && url.path_piece() == "/";
  698. }
  699. void AppendFormattedHost(const GURL& url, std::u16string* output) {
  700. AppendFormattedComponent(
  701. url.possibly_invalid_spec(), url.parsed_for_possibly_invalid_spec().host,
  702. HostComponentTransform(false, false), output, nullptr, nullptr);
  703. }
  704. IDNConversionResult UnsafeIDNToUnicodeWithDetails(base::StringPiece host) {
  705. return UnsafeIDNToUnicodeWithAdjustments(host, nullptr);
  706. }
  707. std::u16string IDNToUnicode(base::StringPiece host) {
  708. return IDNToUnicodeWithAdjustments(host, nullptr).result;
  709. }
  710. std::string StripWWW(const std::string& text) {
  711. // Exclude the registry and domain from trivial subdomain stripping.
  712. std::string domain_and_registry =
  713. net::registry_controlled_domains::GetDomainAndRegistry(
  714. text, net::registry_controlled_domains::INCLUDE_PRIVATE_REGISTRIES);
  715. // If there is no domain and registry, we may be looking at an intranet
  716. // or otherwise non-standard host. Leave those alone.
  717. if (domain_and_registry.empty())
  718. return text;
  719. return text.size() - domain_and_registry.length() >= kWwwLength &&
  720. base::StartsWith(text, kWww, base::CompareCase::SENSITIVE)
  721. ? text.substr(kWwwLength)
  722. : text;
  723. }
  724. void StripWWWFromHostComponent(const std::string& url, url::Component* host) {
  725. std::string host_str = url.substr(host->begin, host->len);
  726. if (StripWWW(host_str) == host_str)
  727. return;
  728. host->begin += kWwwLength;
  729. host->len -= kWwwLength;
  730. }
  731. std::string StripMobilePrefix(const std::string& text) {
  732. return text.size() >= kMobilePrefixLength &&
  733. base::StartsWith(text, kMobilePrefix,
  734. base::CompareCase::SENSITIVE)
  735. ? text.substr(kMobilePrefixLength)
  736. : text;
  737. }
  738. Skeletons GetSkeletons(const std::u16string& host) {
  739. return g_idn_spoof_checker.Get().GetSkeletons(host);
  740. }
  741. TopDomainEntry LookupSkeletonInTopDomains(const std::string& skeleton,
  742. const SkeletonType type) {
  743. return g_idn_spoof_checker.Get().LookupSkeletonInTopDomains(skeleton, type);
  744. }
  745. std::u16string MaybeRemoveDiacritics(const std::u16string& host) {
  746. return g_idn_spoof_checker.Get().MaybeRemoveDiacritics(host);
  747. }
  748. } // namespace url_formatter