123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442 |
- // Copyright 2013 The Chromium Authors. All rights reserved.
- // Use of this source code is governed by a BSD-style license that can be
- // found in the LICENSE file.
- #include "base/check.h"
- #include "base/cpu_reduction_experiment.h"
- #include "url/url_canon.h"
- #include "url/url_canon_internal.h"
- namespace url {
- namespace {
- // For reference, here's what IE supports:
- // Key: 0 (disallowed: failure if present in the input)
- // + (allowed either escaped or unescaped, and unmodified)
- // U (allowed escaped or unescaped but always unescaped if present in
- // escaped form)
- // E (allowed escaped or unescaped but always escaped if present in
- // unescaped form)
- // % (only allowed escaped in the input, will be unmodified).
- // I left blank alpha numeric characters.
- //
- // 00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f
- // -----------------------------------------------
- // 0 0 E E E E E E E E E E E E E E E
- // 1 E E E E E E E E E E E E E E E E
- // 2 E + E E + E + + + + + + + U U 0
- // 3 % % E + E 0 <-- Those are : ; < = > ?
- // 4 %
- // 5 U 0 U U U <-- Those are [ \ ] ^ _
- // 6 E <-- That's `
- // 7 E E E U E <-- Those are { | } ~ (UNPRINTABLE)
- //
- // NOTE: I didn't actually test all the control characters. Some may be
- // disallowed in the input, but they are all accepted escaped except for 0.
- // I also didn't test if characters affecting HTML parsing are allowed
- // unescaped, e.g. (") or (#), which would indicate the beginning of the path.
- // Surprisingly, space is accepted in the input and always escaped.
- // This table lists the canonical version of all characters we allow in the
- // input, with 0 indicating it is disallowed. We use the magic kEscapedHostChar
- // value to indicate that this character should be escaped. We are a little more
- // restrictive than IE, but less restrictive than Firefox.
- //
- // Note that we disallow the % character. We will allow it when part of an
- // escape sequence, of course, but this disallows "%25". Even though IE allows
- // it, allowing it would put us in a funny state. If there was an invalid
- // escape sequence like "%zz", we'll add "%25zz" to the output and fail.
- // Allowing percents means we'll succeed a second time, so validity would change
- // based on how many times you run the canonicalizer. We prefer to always report
- // the same vailidity, so reject this.
- const unsigned char kEsc = 0xff;
- const unsigned char kHostCharLookup[0x80] = {
- // 00-1f: all are invalid
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- // ' ' ! " # $ % & ' ( ) * + , - . /
- kEsc,kEsc,kEsc,kEsc,kEsc, 0, kEsc,kEsc,kEsc,kEsc,kEsc, '+',kEsc, '-', '.', 0,
- // 0 1 2 3 4 5 6 7 8 9 : ; < = > ?
- '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ':', 0 ,kEsc,kEsc,kEsc, 0 ,
- // @ A B C D E F G H I J K L M N O
- kEsc, 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o',
- // P Q R S T U V W X Y Z [ \ ] ^ _
- 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '[', 0 , ']', 0 , '_',
- // ` a b c d e f g h i j k l m n o
- kEsc, 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o',
- // p q r s t u v w x y z { | } ~
- 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',kEsc,kEsc,kEsc, 0 , 0 };
- // RFC1034 maximum FQDN length.
- constexpr size_t kMaxHostLength = 253;
- // Generous padding to account for the fact that UTS#46 normalization can cause
- // a long string to actually shrink and fit within the 253 character RFC1034
- // FQDN length limit. Note that this can still be too short for pathological
- // cases: An arbitrary number of characters (e.g. U+00AD SOFT HYPHEN) can be
- // removed from the input by UTS#46 processing. However, this should be
- // sufficient for all normally-encountered, non-abusive hostname strings.
- constexpr size_t kMaxHostBufferLength = kMaxHostLength * 5;
- constexpr size_t kTempHostBufferLen = 1024;
- using StackBuffer = RawCanonOutputT<char, kTempHostBufferLen>;
- using StackBufferW = RawCanonOutputT<char16_t, kTempHostBufferLen>;
- // Scans a host name and fills in the output flags according to what we find.
- // |has_non_ascii| will be true if there are any non-7-bit characters, and
- // |has_escaped| will be true if there is a percent sign.
- template<typename CHAR, typename UCHAR>
- void ScanHostname(const CHAR* spec,
- const Component& host,
- bool* has_non_ascii,
- bool* has_escaped) {
- int end = host.end();
- *has_non_ascii = false;
- *has_escaped = false;
- for (int i = host.begin; i < end; i++) {
- if (static_cast<UCHAR>(spec[i]) >= 0x80)
- *has_non_ascii = true;
- else if (spec[i] == '%')
- *has_escaped = true;
- }
- }
- // Canonicalizes a host name that is entirely 8-bit characters (even though
- // the type holding them may be 16 bits. Escaped characters will be unescaped.
- // Non-7-bit characters (for example, UTF-8) will be passed unchanged.
- //
- // The |*has_non_ascii| flag will be true if there are non-7-bit characters in
- // the output.
- //
- // This function is used in two situations:
- //
- // * When the caller knows there is no non-ASCII or percent escaped
- // characters. This is what DoHost does. The result will be a completely
- // canonicalized host since we know nothing weird can happen (escaped
- // characters could be unescaped to non-7-bit, so they have to be treated
- // with suspicion at this point). It does not use the |has_non_ascii| flag.
- //
- // * When the caller has an 8-bit string that may need unescaping.
- // DoComplexHost calls us this situation to do unescaping and validation.
- // After this, it may do other IDN operations depending on the value of the
- // |*has_non_ascii| flag.
- //
- // The return value indicates if the output is a potentially valid host name.
- template <typename INCHAR, typename OUTCHAR>
- bool DoSimpleHost(const INCHAR* host,
- size_t host_len,
- CanonOutputT<OUTCHAR>* output,
- bool* has_non_ascii) {
- *has_non_ascii = false;
- bool success = true;
- for (size_t i = 0; i < host_len; ++i) {
- unsigned int source = host[i];
- if (source == '%') {
- // Unescape first, if possible.
- // Source will be used only if decode operation was successful.
- if (!DecodeEscaped(host, &i, host_len,
- reinterpret_cast<unsigned char*>(&source))) {
- // Invalid escaped character. There is nothing that can make this
- // host valid. We append an escaped percent so the URL looks reasonable
- // and mark as failed.
- AppendEscapedChar('%', output);
- success = false;
- continue;
- }
- }
- if (source < 0x80) {
- // We have ASCII input, we can use our lookup table.
- unsigned char replacement = kHostCharLookup[source];
- if (!replacement) {
- // Invalid character, add it as percent-escaped and mark as failed.
- AppendEscapedChar(source, output);
- success = false;
- } else if (replacement == kEsc) {
- // This character is valid but should be escaped.
- AppendEscapedChar(source, output);
- } else {
- // Common case, the given character is valid in a hostname, the lookup
- // table tells us the canonical representation of that character (lower
- // cased).
- output->push_back(replacement);
- }
- } else {
- // It's a non-ascii char. Just push it to the output.
- // In case where we have char16 input, and char output it's safe to
- // cast char16->char only if input string was converted to ASCII.
- output->push_back(static_cast<OUTCHAR>(source));
- *has_non_ascii = true;
- }
- }
- return success;
- }
- // Canonicalizes a host that requires IDN conversion. Returns true on success
- bool DoIDNHost(const char16_t* src, size_t src_len, CanonOutput* output) {
- int original_output_len = output->length(); // So we can rewind below.
- // We need to escape URL before doing IDN conversion, since punicode strings
- // cannot be escaped after they are created.
- RawCanonOutputW<kTempHostBufferLen> url_escaped_host;
- bool has_non_ascii;
- DoSimpleHost(src, src_len, &url_escaped_host, &has_non_ascii);
- if (url_escaped_host.length() > kMaxHostBufferLength) {
- AppendInvalidNarrowString(src, 0, src_len, output);
- return false;
- }
- StackBufferW wide_output;
- if (!IDNToASCII(url_escaped_host.data(),
- url_escaped_host.length(),
- &wide_output)) {
- // Some error, give up. This will write some reasonable looking
- // representation of the string to the output.
- AppendInvalidNarrowString(src, 0, src_len, output);
- return false;
- }
- // Now we check the ASCII output like a normal host. It will also handle
- // unescaping. Although we unescaped everything before this function call, if
- // somebody does %00 as fullwidth, ICU will convert this to ASCII.
- bool success = DoSimpleHost(wide_output.data(), wide_output.length(), output,
- &has_non_ascii);
- if (has_non_ascii) {
- // ICU generated something that DoSimpleHost didn't think looked like
- // ASCII. This is quite rare, but ICU might convert some characters to
- // percent signs which might generate new escape sequences which might in
- // turn be invalid. An example is U+FE6A "small percent" which ICU will
- // name prep into an ASCII percent and then we can interpret the following
- // characters as escaped characters.
- //
- // If DoSimpleHost didn't think the output was ASCII, just escape the
- // thing we gave ICU and give up. DoSimpleHost will have handled a further
- // level of escaping from ICU for simple ASCII cases (i.e. if ICU generates
- // a new escaped ASCII sequence like "%41" we'll unescape it) but it won't
- // do more (like handle escaped non-ASCII sequences). Handling the escaped
- // ASCII isn't strictly necessary, but DoSimpleHost handles this case
- // anyway so we handle it/
- output->set_length(original_output_len);
- AppendInvalidNarrowString(wide_output.data(), 0, wide_output.length(),
- output);
- return false;
- }
- return success;
- }
- // 8-bit convert host to its ASCII version: this converts the UTF-8 input to
- // UTF-16. The has_escaped flag should be set if the input string requires
- // unescaping.
- bool DoComplexHost(const char* host,
- size_t host_len,
- bool has_non_ascii,
- bool has_escaped,
- CanonOutput* output) {
- // Save the current position in the output. We may write stuff and rewind it
- // below, so we need to know where to rewind to.
- size_t begin_length = output->length();
- // Points to the UTF-8 data we want to convert. This will either be the
- // input or the unescaped version written to |*output| if necessary.
- const char* utf8_source;
- size_t utf8_source_len;
- bool are_all_escaped_valid = true;
- if (has_escaped) {
- // Unescape before converting to UTF-16 for IDN. We write this into the
- // output because it most likely does not require IDNization, and we can
- // save another huge stack buffer. It will be replaced below if it requires
- // IDN. This will also update our non-ASCII flag so we know whether the
- // unescaped input requires IDN.
- if (!DoSimpleHost(host, host_len, output, &has_non_ascii)) {
- // Error with some escape sequence. We'll call the current output
- // complete. DoSimpleHost will have written some "reasonable" output
- // for the invalid escapes, but the output could be non-ASCII and
- // needs to go through re-encoding below.
- are_all_escaped_valid = false;
- }
- // Unescaping may have left us with ASCII input, in which case the
- // unescaped version we wrote to output is complete.
- if (!has_non_ascii) {
- return are_all_escaped_valid;
- }
- // Save the pointer into the data was just converted (it may be appended to
- // other data in the output buffer).
- utf8_source = &output->data()[begin_length];
- utf8_source_len = output->length() - begin_length;
- } else {
- // We don't need to unescape, use input for IDNization later. (We know the
- // input has non-ASCII, or the simple version would have been called
- // instead of us.)
- utf8_source = host;
- utf8_source_len = host_len;
- }
- // Non-ASCII input requires IDN, convert to UTF-16 and do the IDN conversion.
- // Above, we may have used the output to write the unescaped values to, so
- // we have to rewind it to where we started after we convert it to UTF-16.
- StackBufferW utf16;
- if (!ConvertUTF8ToUTF16(utf8_source, utf8_source_len, &utf16)) {
- // In this error case, the input may or may not be the output.
- StackBuffer utf8;
- for (size_t i = 0; i < utf8_source_len; i++)
- utf8.push_back(utf8_source[i]);
- output->set_length(begin_length);
- AppendInvalidNarrowString(utf8.data(), 0, utf8.length(), output);
- return false;
- }
- output->set_length(begin_length);
- // This will call DoSimpleHost which will do normal ASCII canonicalization
- // and also check for IP addresses in the outpt.
- return DoIDNHost(utf16.data(), utf16.length(), output) &&
- are_all_escaped_valid;
- }
- // UTF-16 convert host to its ASCII version. The set up is already ready for
- // the backend, so we just pass through. The has_escaped flag should be set if
- // the input string requires unescaping.
- bool DoComplexHost(const char16_t* host,
- size_t host_len,
- bool has_non_ascii,
- bool has_escaped,
- CanonOutput* output) {
- if (has_escaped) {
- // Yikes, we have escaped characters with wide input. The escaped
- // characters should be interpreted as UTF-8. To solve this problem,
- // we convert to UTF-8, unescape, then convert back to UTF-16 for IDN.
- //
- // We don't bother to optimize the conversion in the ASCII case (which
- // *could* just be a copy) and use the UTF-8 path, because it should be
- // very rare that host names have escaped characters, and it is relatively
- // fast to do the conversion anyway.
- StackBuffer utf8;
- if (!ConvertUTF16ToUTF8(host, host_len, &utf8)) {
- AppendInvalidNarrowString(host, 0, host_len, output);
- return false;
- }
- // Once we convert to UTF-8, we can use the 8-bit version of the complex
- // host handling code above.
- return DoComplexHost(utf8.data(), utf8.length(), has_non_ascii, has_escaped,
- output);
- }
- // No unescaping necessary, we can safely pass the input to ICU. This
- // function will only get called if we either have escaped or non-ascii
- // input, so it's safe to just use ICU now. Even if the input is ASCII,
- // this function will do the right thing (just slower than we could).
- return DoIDNHost(host, host_len, output);
- }
- template <typename CHAR, typename UCHAR>
- bool DoHostSubstring(const CHAR* spec,
- const Component& host,
- CanonOutput* output) {
- DCHECK(host.is_valid());
- bool has_non_ascii, has_escaped;
- ScanHostname<CHAR, UCHAR>(spec, host, &has_non_ascii, &has_escaped);
- if (has_non_ascii || has_escaped) {
- return DoComplexHost(&spec[host.begin], static_cast<size_t>(host.len),
- has_non_ascii, has_escaped, output);
- }
- const bool success = DoSimpleHost(
- &spec[host.begin], static_cast<size_t>(host.len), output, &has_non_ascii);
- DCHECK(!has_non_ascii);
- return success;
- }
- template <typename CHAR, typename UCHAR>
- void DoHost(const CHAR* spec,
- const Component& host,
- CanonOutput* output,
- CanonHostInfo* host_info) {
- if (!host.is_nonempty()) {
- // Empty hosts don't need anything.
- host_info->family = CanonHostInfo::NEUTRAL;
- host_info->out_host = Component();
- return;
- }
- // Keep track of output's initial length, so we can rewind later.
- const int output_begin = output->length();
- if (DoHostSubstring<CHAR, UCHAR>(spec, host, output)) {
- // After all the other canonicalization, check if we ended up with an IP
- // address. IP addresses are small, so writing into this temporary buffer
- // should not cause an allocation.
- RawCanonOutput<64> canon_ip;
- CanonicalizeIPAddress(output->data(),
- MakeRange(output_begin, output->length()),
- &canon_ip, host_info);
- // If we got an IPv4/IPv6 address, copy the canonical form back to the
- // real buffer. Otherwise, it's a hostname or broken IP, in which case
- // we just leave it in place.
- if (host_info->IsIPAddress()) {
- output->set_length(output_begin);
- output->Append(canon_ip.data(), canon_ip.length());
- }
- } else {
- // Canonicalization failed. Set BROKEN to notify the caller.
- host_info->family = CanonHostInfo::BROKEN;
- }
- host_info->out_host = MakeRange(output_begin, output->length());
- }
- } // namespace
- bool CanonicalizeHost(const char* spec,
- const Component& host,
- CanonOutput* output,
- Component* out_host) {
- CanonHostInfo host_info;
- DoHost<char, unsigned char>(spec, host, output, &host_info);
- *out_host = host_info.out_host;
- return (host_info.family != CanonHostInfo::BROKEN);
- }
- bool CanonicalizeHost(const char16_t* spec,
- const Component& host,
- CanonOutput* output,
- Component* out_host) {
- CanonHostInfo host_info;
- DoHost<char16_t, char16_t>(spec, host, output, &host_info);
- *out_host = host_info.out_host;
- return (host_info.family != CanonHostInfo::BROKEN);
- }
- void CanonicalizeHostVerbose(const char* spec,
- const Component& host,
- CanonOutput* output,
- CanonHostInfo* host_info) {
- DoHost<char, unsigned char>(spec, host, output, host_info);
- }
- void CanonicalizeHostVerbose(const char16_t* spec,
- const Component& host,
- CanonOutput* output,
- CanonHostInfo* host_info) {
- DoHost<char16_t, char16_t>(spec, host, output, host_info);
- }
- bool CanonicalizeHostSubstring(const char* spec,
- const Component& host,
- CanonOutput* output) {
- return DoHostSubstring<char, unsigned char>(spec, host, output);
- }
- bool CanonicalizeHostSubstring(const char16_t* spec,
- const Component& host,
- CanonOutput* output) {
- return DoHostSubstring<char16_t, char16_t>(spec, host, output);
- }
- } // namespace url
|