url_canon_internal.h 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469
  1. // Copyright 2013 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #ifndef URL_URL_CANON_INTERNAL_H_
  5. #define URL_URL_CANON_INTERNAL_H_
  6. // This file is intended to be included in another C++ file where the character
  7. // types are defined. This allows us to write mostly generic code, but not have
  8. // template bloat because everything is inlined when anybody calls any of our
  9. // functions.
  10. #include <stddef.h>
  11. #include <stdlib.h>
  12. #include "base/component_export.h"
  13. #include "base/notreached.h"
  14. #include "base/third_party/icu/icu_utf.h"
  15. #include "url/url_canon.h"
  16. namespace url {
  17. // Character type handling -----------------------------------------------------
  18. // Bits that identify different character types. These types identify different
  19. // bits that are set for each 8-bit character in the kSharedCharTypeTable.
  20. enum SharedCharTypes {
  21. // Characters that do not require escaping in queries. Characters that do
  22. // not have this flag will be escaped; see url_canon_query.cc
  23. CHAR_QUERY = 1,
  24. // Valid in the username/password field.
  25. CHAR_USERINFO = 2,
  26. // Valid in a IPv4 address (digits plus dot and 'x' for hex).
  27. CHAR_IPV4 = 4,
  28. // Valid in an ASCII-representation of a hex digit (as in %-escaped).
  29. CHAR_HEX = 8,
  30. // Valid in an ASCII-representation of a decimal digit.
  31. CHAR_DEC = 16,
  32. // Valid in an ASCII-representation of an octal digit.
  33. CHAR_OCT = 32,
  34. // Characters that do not require escaping in encodeURIComponent. Characters
  35. // that do not have this flag will be escaped; see url_util.cc.
  36. CHAR_COMPONENT = 64,
  37. };
  38. // This table contains the flags in SharedCharTypes for each 8-bit character.
  39. // Some canonicalization functions have their own specialized lookup table.
  40. // For those with simple requirements, we have collected the flags in one
  41. // place so there are fewer lookup tables to load into the CPU cache.
  42. //
  43. // Using an unsigned char type has a small but measurable performance benefit
  44. // over using a 32-bit number.
  45. extern const unsigned char kSharedCharTypeTable[0x100];
  46. // More readable wrappers around the character type lookup table.
  47. inline bool IsCharOfType(unsigned char c, SharedCharTypes type) {
  48. return !!(kSharedCharTypeTable[c] & type);
  49. }
  50. inline bool IsQueryChar(unsigned char c) {
  51. return IsCharOfType(c, CHAR_QUERY);
  52. }
  53. inline bool IsIPv4Char(unsigned char c) {
  54. return IsCharOfType(c, CHAR_IPV4);
  55. }
  56. inline bool IsHexChar(unsigned char c) {
  57. return IsCharOfType(c, CHAR_HEX);
  58. }
  59. inline bool IsComponentChar(unsigned char c) {
  60. return IsCharOfType(c, CHAR_COMPONENT);
  61. }
  62. // Appends the given string to the output, escaping characters that do not
  63. // match the given |type| in SharedCharTypes.
  64. void AppendStringOfType(const char* source,
  65. size_t length,
  66. SharedCharTypes type,
  67. CanonOutput* output);
  68. void AppendStringOfType(const char16_t* source,
  69. size_t length,
  70. SharedCharTypes type,
  71. CanonOutput* output);
  72. // Maps the hex numerical values 0x0 to 0xf to the corresponding ASCII digit
  73. // that will be used to represent it.
  74. COMPONENT_EXPORT(URL) extern const char kHexCharLookup[0x10];
  75. // This lookup table allows fast conversion between ASCII hex letters and their
  76. // corresponding numerical value. The 8-bit range is divided up into 8
  77. // regions of 0x20 characters each. Each of the three character types (numbers,
  78. // uppercase, lowercase) falls into different regions of this range. The table
  79. // contains the amount to subtract from characters in that range to get at
  80. // the corresponding numerical value.
  81. //
  82. // See HexDigitToValue for the lookup.
  83. extern const char kCharToHexLookup[8];
  84. // Assumes the input is a valid hex digit! Call IsHexChar before using this.
  85. inline int HexCharToValue(unsigned char c) {
  86. return c - kCharToHexLookup[c / 0x20];
  87. }
  88. // Indicates if the given character is a dot or dot equivalent, returning the
  89. // number of characters taken by it. This will be one for a literal dot, 3 for
  90. // an escaped dot. If the character is not a dot, this will return 0.
  91. template <typename CHAR>
  92. inline size_t IsDot(const CHAR* spec, size_t offset, size_t end) {
  93. if (spec[offset] == '.') {
  94. return 1;
  95. } else if (spec[offset] == '%' && offset + 3 <= end &&
  96. spec[offset + 1] == '2' &&
  97. (spec[offset + 2] == 'e' || spec[offset + 2] == 'E')) {
  98. // Found "%2e"
  99. return 3;
  100. }
  101. return 0;
  102. }
  103. // Returns the canonicalized version of the input character according to scheme
  104. // rules. This is implemented alongside the scheme canonicalizer, and is
  105. // required for relative URL resolving to test for scheme equality.
  106. //
  107. // Returns 0 if the input character is not a valid scheme character.
  108. char CanonicalSchemeChar(char16_t ch);
  109. // Write a single character, escaped, to the output. This always escapes: it
  110. // does no checking that thee character requires escaping.
  111. // Escaping makes sense only 8 bit chars, so code works in all cases of
  112. // input parameters (8/16bit).
  113. template<typename UINCHAR, typename OUTCHAR>
  114. inline void AppendEscapedChar(UINCHAR ch,
  115. CanonOutputT<OUTCHAR>* output) {
  116. output->push_back('%');
  117. output->push_back(static_cast<OUTCHAR>(kHexCharLookup[(ch >> 4) & 0xf]));
  118. output->push_back(static_cast<OUTCHAR>(kHexCharLookup[ch & 0xf]));
  119. }
  120. // The character we'll substitute for undecodable or invalid characters.
  121. extern const base_icu::UChar32 kUnicodeReplacementCharacter;
  122. // UTF-8 functions ------------------------------------------------------------
  123. // Reads one character in UTF-8 starting at |*begin| in |str| and places
  124. // the decoded value into |*code_point|. If the character is valid, we will
  125. // return true. If invalid, we'll return false and put the
  126. // kUnicodeReplacementCharacter into |*code_point|.
  127. //
  128. // |*begin| will be updated to point to the last character consumed so it
  129. // can be incremented in a loop and will be ready for the next character.
  130. // (for a single-byte ASCII character, it will not be changed).
  131. COMPONENT_EXPORT(URL)
  132. bool ReadUTFChar(const char* str,
  133. size_t* begin,
  134. size_t length,
  135. base_icu::UChar32* code_point_out);
  136. // Generic To-UTF-8 converter. This will call the given append method for each
  137. // character that should be appended, with the given output method. Wrappers
  138. // are provided below for escaped and non-escaped versions of this.
  139. //
  140. // The char_value must have already been checked that it's a valid Unicode
  141. // character.
  142. template <class Output, void Appender(unsigned char, Output*)>
  143. inline void DoAppendUTF8(base_icu::UChar32 char_value, Output* output) {
  144. DCHECK(char_value >= 0);
  145. DCHECK(char_value <= 0x10FFFF);
  146. if (char_value <= 0x7f) {
  147. Appender(static_cast<unsigned char>(char_value), output);
  148. } else if (char_value <= 0x7ff) {
  149. // 110xxxxx 10xxxxxx
  150. Appender(static_cast<unsigned char>(0xC0 | (char_value >> 6)),
  151. output);
  152. Appender(static_cast<unsigned char>(0x80 | (char_value & 0x3f)),
  153. output);
  154. } else if (char_value <= 0xffff) {
  155. // 1110xxxx 10xxxxxx 10xxxxxx
  156. Appender(static_cast<unsigned char>(0xe0 | (char_value >> 12)),
  157. output);
  158. Appender(static_cast<unsigned char>(0x80 | ((char_value >> 6) & 0x3f)),
  159. output);
  160. Appender(static_cast<unsigned char>(0x80 | (char_value & 0x3f)),
  161. output);
  162. } else {
  163. // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
  164. Appender(static_cast<unsigned char>(0xf0 | (char_value >> 18)),
  165. output);
  166. Appender(static_cast<unsigned char>(0x80 | ((char_value >> 12) & 0x3f)),
  167. output);
  168. Appender(static_cast<unsigned char>(0x80 | ((char_value >> 6) & 0x3f)),
  169. output);
  170. Appender(static_cast<unsigned char>(0x80 | (char_value & 0x3f)), output);
  171. }
  172. }
  173. // Helper used by AppendUTF8Value below. We use an unsigned parameter so there
  174. // are no funny sign problems with the input, but then have to convert it to
  175. // a regular char for appending.
  176. inline void AppendCharToOutput(unsigned char ch, CanonOutput* output) {
  177. output->push_back(static_cast<char>(ch));
  178. }
  179. // Writes the given character to the output as UTF-8. This does NO checking
  180. // of the validity of the Unicode characters; the caller should ensure that
  181. // the value it is appending is valid to append.
  182. inline void AppendUTF8Value(base_icu::UChar32 char_value, CanonOutput* output) {
  183. DoAppendUTF8<CanonOutput, AppendCharToOutput>(char_value, output);
  184. }
  185. // Writes the given character to the output as UTF-8, escaping ALL
  186. // characters (even when they are ASCII). This does NO checking of the
  187. // validity of the Unicode characters; the caller should ensure that the value
  188. // it is appending is valid to append.
  189. inline void AppendUTF8EscapedValue(base_icu::UChar32 char_value,
  190. CanonOutput* output) {
  191. DoAppendUTF8<CanonOutput, AppendEscapedChar>(char_value, output);
  192. }
  193. // UTF-16 functions -----------------------------------------------------------
  194. // Reads one character in UTF-16 starting at |*begin| in |str| and places
  195. // the decoded value into |*code_point|. If the character is valid, we will
  196. // return true. If invalid, we'll return false and put the
  197. // kUnicodeReplacementCharacter into |*code_point|.
  198. //
  199. // |*begin| will be updated to point to the last character consumed so it
  200. // can be incremented in a loop and will be ready for the next character.
  201. // (for a single-16-bit-word character, it will not be changed).
  202. COMPONENT_EXPORT(URL)
  203. bool ReadUTFChar(const char16_t* str,
  204. size_t* begin,
  205. size_t length,
  206. base_icu::UChar32* code_point_out);
  207. // Equivalent to U16_APPEND_UNSAFE in ICU but uses our output method.
  208. inline void AppendUTF16Value(base_icu::UChar32 code_point,
  209. CanonOutputT<char16_t>* output) {
  210. if (code_point > 0xffff) {
  211. output->push_back(static_cast<char16_t>((code_point >> 10) + 0xd7c0));
  212. output->push_back(static_cast<char16_t>((code_point & 0x3ff) | 0xdc00));
  213. } else {
  214. output->push_back(static_cast<char16_t>(code_point));
  215. }
  216. }
  217. // Escaping functions ---------------------------------------------------------
  218. // Writes the given character to the output as UTF-8, escaped. Call this
  219. // function only when the input is wide. Returns true on success. Failure
  220. // means there was some problem with the encoding, we'll still try to
  221. // update the |*begin| pointer and add a placeholder character to the
  222. // output so processing can continue.
  223. //
  224. // We will append the character starting at ch[begin] with the buffer ch
  225. // being |length|. |*begin| will be updated to point to the last character
  226. // consumed (we may consume more than one for UTF-16) so that if called in
  227. // a loop, incrementing the pointer will move to the next character.
  228. //
  229. // Every single output character will be escaped. This means that if you
  230. // give it an ASCII character as input, it will be escaped. Some code uses
  231. // this when it knows that a character is invalid according to its rules
  232. // for validity. If you don't want escaping for ASCII characters, you will
  233. // have to filter them out prior to calling this function.
  234. //
  235. // Assumes that ch[begin] is within range in the array, but does not assume
  236. // that any following characters are.
  237. inline bool AppendUTF8EscapedChar(const char16_t* str,
  238. size_t* begin,
  239. size_t length,
  240. CanonOutput* output) {
  241. // UTF-16 input. ReadUTFChar will handle invalid characters for us and give
  242. // us the kUnicodeReplacementCharacter, so we don't have to do special
  243. // checking after failure, just pass through the failure to the caller.
  244. base_icu::UChar32 char_value;
  245. bool success = ReadUTFChar(str, begin, length, &char_value);
  246. AppendUTF8EscapedValue(char_value, output);
  247. return success;
  248. }
  249. // Handles UTF-8 input. See the wide version above for usage.
  250. inline bool AppendUTF8EscapedChar(const char* str,
  251. size_t* begin,
  252. size_t length,
  253. CanonOutput* output) {
  254. // ReadUTF8Char will handle invalid characters for us and give us the
  255. // kUnicodeReplacementCharacter, so we don't have to do special checking
  256. // after failure, just pass through the failure to the caller.
  257. base_icu::UChar32 ch;
  258. bool success = ReadUTFChar(str, begin, length, &ch);
  259. AppendUTF8EscapedValue(ch, output);
  260. return success;
  261. }
  262. // Given a '%' character at |*begin| in the string |spec|, this will decode
  263. // the escaped value and put it into |*unescaped_value| on success (returns
  264. // true). On failure, this will return false, and will not write into
  265. // |*unescaped_value|.
  266. //
  267. // |*begin| will be updated to point to the last character of the escape
  268. // sequence so that when called with the index of a for loop, the next time
  269. // through it will point to the next character to be considered. On failure,
  270. // |*begin| will be unchanged.
  271. inline bool Is8BitChar(char c) {
  272. return true; // this case is specialized to avoid a warning
  273. }
  274. inline bool Is8BitChar(char16_t c) {
  275. return c <= 255;
  276. }
  277. template <typename CHAR>
  278. inline bool DecodeEscaped(const CHAR* spec,
  279. size_t* begin,
  280. size_t end,
  281. unsigned char* unescaped_value) {
  282. if (*begin + 3 > end ||
  283. !Is8BitChar(spec[*begin + 1]) || !Is8BitChar(spec[*begin + 2])) {
  284. // Invalid escape sequence because there's not enough room, or the
  285. // digits are not ASCII.
  286. return false;
  287. }
  288. unsigned char first = static_cast<unsigned char>(spec[*begin + 1]);
  289. unsigned char second = static_cast<unsigned char>(spec[*begin + 2]);
  290. if (!IsHexChar(first) || !IsHexChar(second)) {
  291. // Invalid hex digits, fail.
  292. return false;
  293. }
  294. // Valid escape sequence.
  295. *unescaped_value = static_cast<unsigned char>((HexCharToValue(first) << 4) +
  296. HexCharToValue(second));
  297. *begin += 2;
  298. return true;
  299. }
  300. // Appends the given substring to the output, escaping "some" characters that
  301. // it feels may not be safe. It assumes the input values are all contained in
  302. // 8-bit although it allows any type.
  303. //
  304. // This is used in error cases to append invalid output so that it looks
  305. // approximately correct. Non-error cases should not call this function since
  306. // the escaping rules are not guaranteed!
  307. void AppendInvalidNarrowString(const char* spec,
  308. size_t begin,
  309. size_t end,
  310. CanonOutput* output);
  311. void AppendInvalidNarrowString(const char16_t* spec,
  312. size_t begin,
  313. size_t end,
  314. CanonOutput* output);
  315. // Misc canonicalization helpers ----------------------------------------------
  316. // Converts between UTF-8 and UTF-16, returning true on successful conversion.
  317. // The output will be appended to the given canonicalizer output (so make sure
  318. // it's empty if you want to replace).
  319. //
  320. // On invalid input, this will still write as much output as possible,
  321. // replacing the invalid characters with the "invalid character". It will
  322. // return false in the failure case, and the caller should not continue as
  323. // normal.
  324. COMPONENT_EXPORT(URL)
  325. bool ConvertUTF16ToUTF8(const char16_t* input,
  326. size_t input_len,
  327. CanonOutput* output);
  328. COMPONENT_EXPORT(URL)
  329. bool ConvertUTF8ToUTF16(const char* input,
  330. size_t input_len,
  331. CanonOutputT<char16_t>* output);
  332. // Converts from UTF-16 to 8-bit using the character set converter. If the
  333. // converter is NULL, this will use UTF-8.
  334. void ConvertUTF16ToQueryEncoding(const char16_t* input,
  335. const Component& query,
  336. CharsetConverter* converter,
  337. CanonOutput* output);
  338. // Applies the replacements to the given component source. The component source
  339. // should be pre-initialized to the "old" base. That is, all pointers will
  340. // point to the spec of the old URL, and all of the Parsed components will
  341. // be indices into that string.
  342. //
  343. // The pointers and components in the |source| for all non-NULL strings in the
  344. // |repl| (replacements) will be updated to reference those strings.
  345. // Canonicalizing with the new |source| and |parsed| can then combine URL
  346. // components from many different strings.
  347. void SetupOverrideComponents(const char* base,
  348. const Replacements<char>& repl,
  349. URLComponentSource<char>* source,
  350. Parsed* parsed);
  351. // Like the above 8-bit version, except that it additionally converts the
  352. // UTF-16 input to UTF-8 before doing the overrides.
  353. //
  354. // The given utf8_buffer is used to store the converted components. They will
  355. // be appended one after another, with the parsed structure identifying the
  356. // appropriate substrings. This buffer is a parameter because the source has
  357. // no storage, so the buffer must have the same lifetime as the source
  358. // parameter owned by the caller.
  359. //
  360. // THE CALLER MUST NOT ADD TO THE |utf8_buffer| AFTER THIS CALL. Members of
  361. // |source| will point into this buffer, which could be invalidated if
  362. // additional data is added and the CanonOutput resizes its buffer.
  363. //
  364. // Returns true on success. False means that the input was not valid UTF-16,
  365. // although we will have still done the override with "invalid characters" in
  366. // place of errors.
  367. bool SetupUTF16OverrideComponents(const char* base,
  368. const Replacements<char16_t>& repl,
  369. CanonOutput* utf8_buffer,
  370. URLComponentSource<char>* source,
  371. Parsed* parsed);
  372. // Implemented in url_canon_path.cc, these are required by the relative URL
  373. // resolver as well, so we declare them here.
  374. bool CanonicalizePartialPathInternal(const char* spec,
  375. const Component& path,
  376. size_t path_begin_in_output,
  377. CanonOutput* output);
  378. bool CanonicalizePartialPathInternal(const char16_t* spec,
  379. const Component& path,
  380. size_t path_begin_in_output,
  381. CanonOutput* output);
  382. // Find the position of a bona fide Windows drive letter in the given path. If
  383. // no leading drive letter is found, -1 is returned. This function correctly
  384. // treats /c:/foo and /./c:/foo as having drive letters, and /def/c:/foo as not
  385. // having a drive letter.
  386. //
  387. // Exported for tests.
  388. COMPONENT_EXPORT(URL)
  389. int FindWindowsDriveLetter(const char* spec, int begin, int end);
  390. COMPONENT_EXPORT(URL)
  391. int FindWindowsDriveLetter(const char16_t* spec, int begin, int end);
  392. #ifndef WIN32
  393. // Implementations of Windows' int-to-string conversions
  394. COMPONENT_EXPORT(URL)
  395. int _itoa_s(int value, char* buffer, size_t size_in_chars, int radix);
  396. COMPONENT_EXPORT(URL)
  397. int _itow_s(int value, char16_t* buffer, size_t size_in_chars, int radix);
  398. // Secure template overloads for these functions
  399. template<size_t N>
  400. inline int _itoa_s(int value, char (&buffer)[N], int radix) {
  401. return _itoa_s(value, buffer, N, radix);
  402. }
  403. template <size_t N>
  404. inline int _itow_s(int value, char16_t (&buffer)[N], int radix) {
  405. return _itow_s(value, buffer, N, radix);
  406. }
  407. // _strtoui64 and strtoull behave the same
  408. inline unsigned long long _strtoui64(const char* nptr,
  409. char** endptr, int base) {
  410. return strtoull(nptr, endptr, base);
  411. }
  412. #endif // WIN32
  413. } // namespace url
  414. #endif // URL_URL_CANON_INTERNAL_H_