mime_sniffer.cc 33 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839
  1. // Copyright (c) 2012 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. // Detecting mime types is a tricky business because we need to balance
  5. // compatibility concerns with security issues. Here is a survey of how other
  6. // browsers behave and then a description of how we intend to behave.
  7. //
  8. // HTML payload, no Content-Type header:
  9. // * IE 7: Render as HTML
  10. // * Firefox 2: Render as HTML
  11. // * Safari 3: Render as HTML
  12. // * Opera 9: Render as HTML
  13. //
  14. // Here the choice seems clear:
  15. // => Chrome: Render as HTML
  16. //
  17. // HTML payload, Content-Type: "text/plain":
  18. // * IE 7: Render as HTML
  19. // * Firefox 2: Render as text
  20. // * Safari 3: Render as text (Note: Safari will Render as HTML if the URL
  21. // has an HTML extension)
  22. // * Opera 9: Render as text
  23. //
  24. // Here we choose to follow the majority (and break some compatibility with IE).
  25. // Many folks dislike IE's behavior here.
  26. // => Chrome: Render as text
  27. // We generalize this as follows. If the Content-Type header is text/plain
  28. // we won't detect dangerous mime types (those that can execute script).
  29. //
  30. // HTML payload, Content-Type: "application/octet-stream":
  31. // * IE 7: Render as HTML
  32. // * Firefox 2: Download as application/octet-stream
  33. // * Safari 3: Render as HTML
  34. // * Opera 9: Render as HTML
  35. //
  36. // We follow Firefox.
  37. // => Chrome: Download as application/octet-stream
  38. // One factor in this decision is that IIS 4 and 5 will send
  39. // application/octet-stream for .xhtml files (because they don't recognize
  40. // the extension). We did some experiments and it looks like this doesn't occur
  41. // very often on the web. We choose the more secure option.
  42. //
  43. // GIF payload, no Content-Type header:
  44. // * IE 7: Render as GIF
  45. // * Firefox 2: Render as GIF
  46. // * Safari 3: Download as Unknown (Note: Safari will Render as GIF if the
  47. // URL has an GIF extension)
  48. // * Opera 9: Render as GIF
  49. //
  50. // The choice is clear.
  51. // => Chrome: Render as GIF
  52. // Once we decide to render HTML without a Content-Type header, there isn't much
  53. // reason not to render GIFs.
  54. //
  55. // GIF payload, Content-Type: "text/plain":
  56. // * IE 7: Render as GIF
  57. // * Firefox 2: Download as application/octet-stream (Note: Firefox will
  58. // Download as GIF if the URL has an GIF extension)
  59. // * Safari 3: Download as Unknown (Note: Safari will Render as GIF if the
  60. // URL has an GIF extension)
  61. // * Opera 9: Render as GIF
  62. //
  63. // Displaying as text/plain makes little sense as the content will look like
  64. // gibberish. Here, we could change our minds and download.
  65. // => Chrome: Render as GIF
  66. //
  67. // GIF payload, Content-Type: "application/octet-stream":
  68. // * IE 7: Render as GIF
  69. // * Firefox 2: Download as application/octet-stream (Note: Firefox will
  70. // Download as GIF if the URL has an GIF extension)
  71. // * Safari 3: Download as Unknown (Note: Safari will Render as GIF if the
  72. // URL has an GIF extension)
  73. // * Opera 9: Render as GIF
  74. //
  75. // We used to render as GIF here, but the problem is that some sites want to
  76. // trigger downloads by sending application/octet-stream (even though they
  77. // should be sending Content-Disposition: attachment). Although it is safe
  78. // to render as GIF from a security perspective, we actually get better
  79. // compatibility if we don't sniff from application/octet stream at all.
  80. // => Chrome: Download as application/octet-stream
  81. //
  82. // Note that our definition of HTML payload is much stricter than IE's
  83. // definition and roughly the same as Firefox's definition.
  84. #include <stdint.h>
  85. #include <string>
  86. #include "net/base/mime_sniffer.h"
  87. #include "base/check_op.h"
  88. #include "base/containers/span.h"
  89. #include "base/notreached.h"
  90. #include "base/strings/string_util.h"
  91. #include "build/build_config.h"
  92. #include "url/gurl.h"
  93. namespace net {
  94. // The number of content bytes we need to use all our magic numbers. Feel free
  95. // to increase this number if you add a longer magic number.
  96. static const size_t kBytesRequiredForMagic = 42;
  97. struct MagicNumber {
  98. const char* const mime_type;
  99. const base::StringPiece magic;
  100. bool is_string;
  101. const char* const mask; // if set, must have same length as |magic|
  102. };
  103. #define MAGIC_NUMBER(mime_type, magic) \
  104. { (mime_type), base::StringPiece((magic), sizeof(magic) - 1), false, nullptr }
  105. template <int MagicSize, int MaskSize>
  106. class VerifySizes {
  107. static_assert(MagicSize == MaskSize, "sizes must be equal");
  108. public:
  109. enum { SIZES = MagicSize };
  110. };
  111. #define verified_sizeof(magic, mask) \
  112. VerifySizes<sizeof(magic), sizeof(mask)>::SIZES
  113. #define MAGIC_MASK(mime_type, magic, mask) \
  114. { \
  115. (mime_type), base::StringPiece((magic), verified_sizeof(magic, mask) - 1), \
  116. false, (mask) \
  117. }
  118. // Magic strings are case insensitive and must not include '\0' characters
  119. #define MAGIC_STRING(mime_type, magic) \
  120. { (mime_type), base::StringPiece((magic), sizeof(magic) - 1), true, nullptr }
  121. static const MagicNumber kMagicNumbers[] = {
  122. // Source: HTML 5 specification
  123. MAGIC_NUMBER("application/pdf", "%PDF-"),
  124. MAGIC_NUMBER("application/postscript", "%!PS-Adobe-"),
  125. MAGIC_NUMBER("image/gif", "GIF87a"),
  126. MAGIC_NUMBER("image/gif", "GIF89a"),
  127. MAGIC_NUMBER("image/png", "\x89" "PNG\x0D\x0A\x1A\x0A"),
  128. MAGIC_NUMBER("image/jpeg", "\xFF\xD8\xFF"),
  129. MAGIC_NUMBER("image/bmp", "BM"),
  130. // Source: Mozilla
  131. MAGIC_NUMBER("text/plain", "#!"), // Script
  132. MAGIC_NUMBER("text/plain", "%!"), // Script, similar to PS
  133. MAGIC_NUMBER("text/plain", "From"),
  134. MAGIC_NUMBER("text/plain", ">From"),
  135. // Chrome specific
  136. MAGIC_NUMBER("application/x-gzip", "\x1F\x8B\x08"),
  137. MAGIC_NUMBER("audio/x-pn-realaudio", "\x2E\x52\x4D\x46"),
  138. MAGIC_NUMBER("video/x-ms-asf",
  139. "\x30\x26\xB2\x75\x8E\x66\xCF\x11\xA6\xD9\x00\xAA\x00\x62\xCE\x6C"),
  140. MAGIC_NUMBER("image/tiff", "I I"),
  141. MAGIC_NUMBER("image/tiff", "II*"),
  142. MAGIC_NUMBER("image/tiff", "MM\x00*"),
  143. MAGIC_NUMBER("audio/mpeg", "ID3"),
  144. MAGIC_NUMBER("image/webp", "RIFF....WEBPVP"),
  145. MAGIC_NUMBER("video/webm", "\x1A\x45\xDF\xA3"),
  146. MAGIC_NUMBER("application/zip", "PK\x03\x04"),
  147. MAGIC_NUMBER("application/x-rar-compressed", "Rar!\x1A\x07\x00"),
  148. MAGIC_NUMBER("application/x-msmetafile", "\xD7\xCD\xC6\x9A"),
  149. MAGIC_NUMBER("application/octet-stream", "MZ"), // EXE
  150. // Sniffing for Flash:
  151. //
  152. // MAGIC_NUMBER("application/x-shockwave-flash", "CWS"),
  153. // MAGIC_NUMBER("application/x-shockwave-flash", "FLV"),
  154. // MAGIC_NUMBER("application/x-shockwave-flash", "FWS"),
  155. //
  156. // Including these magic number for Flash is a trade off.
  157. //
  158. // Pros:
  159. // * Flash is an important and popular file format
  160. //
  161. // Cons:
  162. // * These patterns are fairly weak
  163. // * If we mistakenly decide something is Flash, we will execute it
  164. // in the origin of an unsuspecting site. This could be a security
  165. // vulnerability if the site allows users to upload content.
  166. //
  167. // On balance, we do not include these patterns.
  168. };
  169. // The number of content bytes we need to use all our Microsoft Office magic
  170. // numbers.
  171. static const size_t kBytesRequiredForOfficeMagic = 8;
  172. static const MagicNumber kOfficeMagicNumbers[] = {
  173. MAGIC_NUMBER("CFB", "\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1"),
  174. MAGIC_NUMBER("OOXML", "PK\x03\x04"),
  175. };
  176. enum OfficeDocType {
  177. DOC_TYPE_WORD,
  178. DOC_TYPE_EXCEL,
  179. DOC_TYPE_POWERPOINT,
  180. DOC_TYPE_NONE
  181. };
  182. struct OfficeExtensionType {
  183. OfficeDocType doc_type;
  184. const base::StringPiece extension;
  185. };
  186. #define OFFICE_EXTENSION(type, extension) \
  187. { (type), base::StringPiece((extension), sizeof(extension) - 1) }
  188. static const OfficeExtensionType kOfficeExtensionTypes[] = {
  189. OFFICE_EXTENSION(DOC_TYPE_WORD, ".doc"),
  190. OFFICE_EXTENSION(DOC_TYPE_EXCEL, ".xls"),
  191. OFFICE_EXTENSION(DOC_TYPE_POWERPOINT, ".ppt"),
  192. OFFICE_EXTENSION(DOC_TYPE_WORD, ".docx"),
  193. OFFICE_EXTENSION(DOC_TYPE_EXCEL, ".xlsx"),
  194. OFFICE_EXTENSION(DOC_TYPE_POWERPOINT, ".pptx"),
  195. };
  196. static const MagicNumber kExtraMagicNumbers[] = {
  197. MAGIC_NUMBER("image/x-xbitmap", "#define"),
  198. MAGIC_NUMBER("image/x-icon", "\x00\x00\x01\x00"),
  199. MAGIC_NUMBER("audio/wav", "RIFF....WAVEfmt "),
  200. MAGIC_NUMBER("video/avi", "RIFF....AVI LIST"),
  201. MAGIC_NUMBER("audio/ogg", "OggS\0"),
  202. MAGIC_MASK("video/mpeg", "\x00\x00\x01\xB0", "\xFF\xFF\xFF\xF0"),
  203. MAGIC_MASK("audio/mpeg", "\xFF\xE0", "\xFF\xE0"),
  204. MAGIC_NUMBER("video/3gpp", "....ftyp3g"),
  205. MAGIC_NUMBER("video/3gpp", "....ftypavcl"),
  206. MAGIC_NUMBER("video/mp4", "....ftyp"),
  207. MAGIC_NUMBER("video/quicktime", "....moov"),
  208. MAGIC_NUMBER("application/x-shockwave-flash", "CWS"),
  209. MAGIC_NUMBER("application/x-shockwave-flash", "FWS"),
  210. MAGIC_NUMBER("video/x-flv", "FLV"),
  211. MAGIC_NUMBER("audio/x-flac", "fLaC"),
  212. // Per https://tools.ietf.org/html/rfc3267#section-8.1
  213. MAGIC_NUMBER("audio/amr", "#!AMR\n"),
  214. // RAW image types.
  215. MAGIC_NUMBER("image/x-canon-cr2", "II\x2a\x00\x10\x00\x00\x00CR"),
  216. MAGIC_NUMBER("image/x-canon-crw", "II\x1a\x00\x00\x00HEAPCCDR"),
  217. MAGIC_NUMBER("image/x-minolta-mrw", "\x00MRM"),
  218. MAGIC_NUMBER("image/x-olympus-orf", "MMOR"), // big-endian
  219. MAGIC_NUMBER("image/x-olympus-orf", "IIRO"), // little-endian
  220. MAGIC_NUMBER("image/x-olympus-orf", "IIRS"), // little-endian
  221. MAGIC_NUMBER("image/x-fuji-raf", "FUJIFILMCCD-RAW "),
  222. MAGIC_NUMBER("image/x-panasonic-raw",
  223. "IIU\x00\x08\x00\x00\x00"), // Panasonic .raw
  224. MAGIC_NUMBER("image/x-panasonic-raw",
  225. "IIU\x00\x18\x00\x00\x00"), // Panasonic .rw2
  226. MAGIC_NUMBER("image/x-phaseone-raw", "MMMMRaw"),
  227. MAGIC_NUMBER("image/x-x3f", "FOVb"),
  228. };
  229. // Our HTML sniffer differs slightly from Mozilla. For example, Mozilla will
  230. // decide that a document that begins "<!DOCTYPE SOAP-ENV:Envelope PUBLIC " is
  231. // HTML, but we will not.
  232. #define MAGIC_HTML_TAG(tag) \
  233. MAGIC_STRING("text/html", "<" tag)
  234. static const MagicNumber kSniffableTags[] = {
  235. // XML processing directive. Although this is not an HTML mime type, we sniff
  236. // for this in the HTML phase because text/xml is just as powerful as HTML and
  237. // we want to leverage our white space skipping technology.
  238. MAGIC_NUMBER("text/xml", "<?xml"), // Mozilla
  239. // DOCTYPEs
  240. MAGIC_HTML_TAG("!DOCTYPE html"), // HTML5 spec
  241. // Sniffable tags, ordered by how often they occur in sniffable documents.
  242. MAGIC_HTML_TAG("script"), // HTML5 spec, Mozilla
  243. MAGIC_HTML_TAG("html"), // HTML5 spec, Mozilla
  244. MAGIC_HTML_TAG("!--"),
  245. MAGIC_HTML_TAG("head"), // HTML5 spec, Mozilla
  246. MAGIC_HTML_TAG("iframe"), // Mozilla
  247. MAGIC_HTML_TAG("h1"), // Mozilla
  248. MAGIC_HTML_TAG("div"), // Mozilla
  249. MAGIC_HTML_TAG("font"), // Mozilla
  250. MAGIC_HTML_TAG("table"), // Mozilla
  251. MAGIC_HTML_TAG("a"), // Mozilla
  252. MAGIC_HTML_TAG("style"), // Mozilla
  253. MAGIC_HTML_TAG("title"), // Mozilla
  254. MAGIC_HTML_TAG("b"), // Mozilla
  255. MAGIC_HTML_TAG("body"), // Mozilla
  256. MAGIC_HTML_TAG("br"),
  257. MAGIC_HTML_TAG("p"), // Mozilla
  258. };
  259. // Compare content header to a magic number where magic_entry can contain '.'
  260. // for single character of anything, allowing some bytes to be skipped.
  261. static bool MagicCmp(base::StringPiece content, base::StringPiece magic_entry) {
  262. DCHECK_GE(content.length(), magic_entry.length());
  263. for (size_t i = 0; i < magic_entry.length(); ++i) {
  264. if (magic_entry[i] != '.' && magic_entry[i] != content[i])
  265. return false;
  266. }
  267. return true;
  268. }
  269. // Like MagicCmp() except that it ANDs each byte with a mask before
  270. // the comparison, because there are some bits we don't care about.
  271. static bool MagicMaskCmp(base::StringPiece content,
  272. base::StringPiece magic_entry,
  273. base::StringPiece magic_mask) {
  274. DCHECK_GE(content.length(), magic_entry.length());
  275. for (size_t i = 0; i < magic_entry.length(); ++i) {
  276. if (magic_entry[i] != '.' && magic_entry[i] != (magic_mask[i] & content[i]))
  277. return false;
  278. }
  279. return true;
  280. }
  281. static bool MatchMagicNumber(base::StringPiece content,
  282. const MagicNumber& magic_entry,
  283. std::string* result) {
  284. // Keep kBytesRequiredForMagic honest.
  285. DCHECK_LE(magic_entry.magic.length(), kBytesRequiredForMagic);
  286. bool match = false;
  287. if (content.length() >= magic_entry.magic.length()) {
  288. if (magic_entry.is_string) {
  289. // Consistency check - string entries should have no embedded nulls.
  290. DCHECK_EQ(base::StringPiece::npos, magic_entry.magic.find('\0'));
  291. // Do a case-insensitive prefix comparison.
  292. match = base::StartsWith(content, magic_entry.magic,
  293. base::CompareCase::INSENSITIVE_ASCII);
  294. } else if (!magic_entry.mask) {
  295. match = MagicCmp(content, magic_entry.magic);
  296. } else {
  297. base::StringPiece magic_mask(magic_entry.mask,
  298. magic_entry.magic.length());
  299. match = MagicMaskCmp(content, magic_entry.magic, magic_mask);
  300. }
  301. }
  302. if (match) {
  303. result->assign(magic_entry.mime_type);
  304. return true;
  305. }
  306. return false;
  307. }
  308. static bool CheckForMagicNumbers(base::StringPiece content,
  309. base::span<const MagicNumber> magic_numbers,
  310. std::string* result) {
  311. for (const MagicNumber& magic : magic_numbers) {
  312. if (MatchMagicNumber(content, magic, result))
  313. return true;
  314. }
  315. return false;
  316. }
  317. // Truncates |string_piece| to length |max_size| and returns true if
  318. // |string_piece| is now exactly |max_size|.
  319. static bool TruncateStringPiece(const size_t max_size,
  320. base::StringPiece* string_piece) {
  321. // Keep kMaxBytesToSniff honest.
  322. DCHECK_LE(static_cast<int>(max_size), kMaxBytesToSniff);
  323. *string_piece = string_piece->substr(0, max_size);
  324. return string_piece->length() == max_size;
  325. }
  326. // Returns true and sets result if the content appears to be HTML.
  327. // Clears have_enough_content if more data could possibly change the result.
  328. static bool SniffForHTML(base::StringPiece content,
  329. bool* have_enough_content,
  330. std::string* result) {
  331. // For HTML, we are willing to consider up to 512 bytes. This may be overly
  332. // conservative as IE only considers 256.
  333. *have_enough_content &= TruncateStringPiece(512, &content);
  334. // We adopt a strategy similar to that used by Mozilla to sniff HTML tags,
  335. // but with some modifications to better match the HTML5 spec.
  336. base::StringPiece trimmed =
  337. base::TrimWhitespaceASCII(content, base::TRIM_LEADING);
  338. // |trimmed| now starts at first non-whitespace character (or is empty).
  339. return CheckForMagicNumbers(trimmed, kSniffableTags, result);
  340. }
  341. // Returns true and sets result if the content matches any of kMagicNumbers.
  342. // Clears have_enough_content if more data could possibly change the result.
  343. static bool SniffForMagicNumbers(base::StringPiece content,
  344. bool* have_enough_content,
  345. std::string* result) {
  346. *have_enough_content &= TruncateStringPiece(kBytesRequiredForMagic, &content);
  347. // Check our big table of Magic Numbers
  348. return CheckForMagicNumbers(content, kMagicNumbers, result);
  349. }
  350. // Returns true and sets result if the content matches any of
  351. // kOfficeMagicNumbers, and the URL has the proper extension.
  352. // Clears |have_enough_content| if more data could possibly change the result.
  353. static bool SniffForOfficeDocs(base::StringPiece content,
  354. const GURL& url,
  355. bool* have_enough_content,
  356. std::string* result) {
  357. *have_enough_content &=
  358. TruncateStringPiece(kBytesRequiredForOfficeMagic, &content);
  359. // Check our table of magic numbers for Office file types.
  360. std::string office_version;
  361. if (!CheckForMagicNumbers(content, kOfficeMagicNumbers, &office_version))
  362. return false;
  363. OfficeDocType type = DOC_TYPE_NONE;
  364. base::StringPiece url_path = url.path_piece();
  365. for (const auto& office_extension : kOfficeExtensionTypes) {
  366. if (base::EndsWith(url_path, office_extension.extension,
  367. base::CompareCase::INSENSITIVE_ASCII)) {
  368. type = office_extension.doc_type;
  369. break;
  370. }
  371. }
  372. if (type == DOC_TYPE_NONE)
  373. return false;
  374. if (office_version == "CFB") {
  375. switch (type) {
  376. case DOC_TYPE_WORD:
  377. *result = "application/msword";
  378. return true;
  379. case DOC_TYPE_EXCEL:
  380. *result = "application/vnd.ms-excel";
  381. return true;
  382. case DOC_TYPE_POWERPOINT:
  383. *result = "application/vnd.ms-powerpoint";
  384. return true;
  385. case DOC_TYPE_NONE:
  386. NOTREACHED();
  387. return false;
  388. }
  389. } else if (office_version == "OOXML") {
  390. switch (type) {
  391. case DOC_TYPE_WORD:
  392. *result = "application/vnd.openxmlformats-officedocument."
  393. "wordprocessingml.document";
  394. return true;
  395. case DOC_TYPE_EXCEL:
  396. *result = "application/vnd.openxmlformats-officedocument."
  397. "spreadsheetml.sheet";
  398. return true;
  399. case DOC_TYPE_POWERPOINT:
  400. *result = "application/vnd.openxmlformats-officedocument."
  401. "presentationml.presentation";
  402. return true;
  403. case DOC_TYPE_NONE:
  404. NOTREACHED();
  405. return false;
  406. }
  407. }
  408. NOTREACHED();
  409. return false;
  410. }
  411. static bool IsOfficeType(const std::string& type_hint) {
  412. return (type_hint == "application/msword" ||
  413. type_hint == "application/vnd.ms-excel" ||
  414. type_hint == "application/vnd.ms-powerpoint" ||
  415. type_hint == "application/vnd.openxmlformats-officedocument."
  416. "wordprocessingml.document" ||
  417. type_hint == "application/vnd.openxmlformats-officedocument."
  418. "spreadsheetml.sheet" ||
  419. type_hint == "application/vnd.openxmlformats-officedocument."
  420. "presentationml.presentation" ||
  421. type_hint == "application/vnd.ms-excel.sheet.macroenabled.12" ||
  422. type_hint == "application/vnd.ms-word.document.macroenabled.12" ||
  423. type_hint == "application/vnd.ms-powerpoint.presentation."
  424. "macroenabled.12" ||
  425. type_hint == "application/mspowerpoint" ||
  426. type_hint == "application/msexcel" ||
  427. type_hint == "application/vnd.ms-word" ||
  428. type_hint == "application/vnd.ms-word.document.12" ||
  429. type_hint == "application/vnd.msword");
  430. }
  431. // This function checks for files that have a Microsoft Office MIME type
  432. // set, but are not actually Office files.
  433. //
  434. // If this is not actually an Office file, |*result| is set to
  435. // "application/octet-stream", otherwise it is not modified.
  436. //
  437. // Returns false if additional data is required to determine the file type, or
  438. // true if there is enough data to make a decision.
  439. static bool SniffForInvalidOfficeDocs(base::StringPiece content,
  440. const GURL& url,
  441. std::string* result) {
  442. if (!TruncateStringPiece(kBytesRequiredForOfficeMagic, &content))
  443. return false;
  444. // Check our table of magic numbers for Office file types. If it does not
  445. // match one, the MIME type was invalid. Set it instead to a safe value.
  446. std::string office_version;
  447. if (!CheckForMagicNumbers(content, kOfficeMagicNumbers, &office_version)) {
  448. *result = "application/octet-stream";
  449. }
  450. // We have enough information to determine if this was a Microsoft Office
  451. // document or not, so sniffing is completed.
  452. return true;
  453. }
  454. // Tags that indicate the content is likely XML.
  455. static const MagicNumber kMagicXML[] = {
  456. MAGIC_STRING("application/atom+xml", "<feed"),
  457. MAGIC_STRING("application/rss+xml", "<rss"),
  458. };
  459. // Returns true and sets result if the content appears to contain XHTML or a
  460. // feed.
  461. // Clears have_enough_content if more data could possibly change the result.
  462. //
  463. // TODO(evanm): this is similar but more conservative than what Safari does,
  464. // while HTML5 has a different recommendation -- what should we do?
  465. // TODO(evanm): this is incorrect for documents whose encoding isn't a superset
  466. // of ASCII -- do we care?
  467. static bool SniffXML(base::StringPiece content,
  468. bool* have_enough_content,
  469. std::string* result) {
  470. // We allow at most 300 bytes of content before we expect the opening tag.
  471. *have_enough_content &= TruncateStringPiece(300, &content);
  472. // This loop iterates through tag-looking offsets in the file.
  473. // We want to skip XML processing instructions (of the form "<?xml ...")
  474. // and stop at the first "plain" tag, then make a decision on the mime-type
  475. // based on the name (or possibly attributes) of that tag.
  476. const int kMaxTagIterations = 5;
  477. size_t pos = 0;
  478. for (size_t i = 0; i < kMaxTagIterations && pos < content.length(); ++i) {
  479. pos = content.find('<', pos);
  480. if (pos == base::StringPiece::npos)
  481. return false;
  482. base::StringPiece current = content.substr(pos);
  483. // Skip XML and DOCTYPE declarations.
  484. static constexpr base::StringPiece kXmlPrefix("<?xml");
  485. static constexpr base::StringPiece kDocTypePrefix("<!DOCTYPE");
  486. if (base::StartsWith(current, kXmlPrefix,
  487. base::CompareCase::INSENSITIVE_ASCII) ||
  488. base::StartsWith(current, kDocTypePrefix,
  489. base::CompareCase::INSENSITIVE_ASCII)) {
  490. ++pos;
  491. continue;
  492. }
  493. if (CheckForMagicNumbers(current, kMagicXML, result))
  494. return true;
  495. // TODO(evanm): handle RSS 1.0, which is an RDF format and more difficult
  496. // to identify.
  497. // If we get here, we've hit an initial tag that hasn't matched one of the
  498. // above tests. Abort.
  499. return true;
  500. }
  501. // We iterated too far without finding a start tag.
  502. // If we have more content to look at, we aren't going to change our mind by
  503. // seeing more bytes from the network.
  504. return pos < content.length();
  505. }
  506. // Byte order marks
  507. static const MagicNumber kByteOrderMark[] = {
  508. MAGIC_NUMBER("text/plain", "\xFE\xFF"), // UTF-16BE
  509. MAGIC_NUMBER("text/plain", "\xFF\xFE"), // UTF-16LE
  510. MAGIC_NUMBER("text/plain", "\xEF\xBB\xBF"), // UTF-8
  511. };
  512. // Returns true and sets result to "application/octet-stream" if the content
  513. // appears to be binary data. Otherwise, returns false and sets "text/plain".
  514. // Clears have_enough_content if more data could possibly change the result.
  515. static bool SniffBinary(base::StringPiece content,
  516. bool* have_enough_content,
  517. std::string* result) {
  518. // There is no concensus about exactly how to sniff for binary content.
  519. // * IE 7: Don't sniff for binary looking bytes, but trust the file extension.
  520. // * Firefox 3.5: Sniff first 4096 bytes for a binary looking byte.
  521. // Here, we side with FF, but with a smaller buffer. This size was chosen
  522. // because it is small enough to comfortably fit into a single packet (after
  523. // allowing for headers) and yet large enough to account for binary formats
  524. // that have a significant amount of ASCII at the beginning (crbug.com/15314).
  525. const bool is_truncated = TruncateStringPiece(kMaxBytesToSniff, &content);
  526. // First, we look for a BOM.
  527. std::string unused;
  528. if (CheckForMagicNumbers(content, kByteOrderMark, &unused)) {
  529. // If there is BOM, we think the buffer is not binary.
  530. result->assign("text/plain");
  531. return false;
  532. }
  533. // Next we look to see if any of the bytes "look binary."
  534. if (LooksLikeBinary(content)) {
  535. result->assign("application/octet-stream");
  536. return true;
  537. }
  538. // No evidence either way. Default to non-binary and, if truncated, clear
  539. // have_enough_content because there could be a binary looking byte in the
  540. // truncated data.
  541. *have_enough_content &= is_truncated;
  542. result->assign("text/plain");
  543. return false;
  544. }
  545. static bool IsUnknownMimeType(base::StringPiece mime_type) {
  546. // TODO(tc): Maybe reuse some code in net/http/http_response_headers.* here.
  547. // If we do, please be careful not to alter the semantics at all.
  548. static const char* const kUnknownMimeTypes[] = {
  549. // Empty mime types are as unknown as they get.
  550. "",
  551. // The unknown/unknown type is popular and uninformative
  552. "unknown/unknown",
  553. // The second most popular unknown mime type is application/unknown
  554. "application/unknown",
  555. // Firefox rejects a mime type if it is exactly */*
  556. "*/*",
  557. };
  558. for (const char* const unknown_mime_type : kUnknownMimeTypes) {
  559. if (mime_type == unknown_mime_type)
  560. return true;
  561. }
  562. if (mime_type.find('/') == base::StringPiece::npos) {
  563. // Firefox rejects a mime type if it does not contain a slash
  564. return true;
  565. }
  566. return false;
  567. }
  568. // Returns true and sets result if the content appears to be a crx (Chrome
  569. // extension) file.
  570. // Clears have_enough_content if more data could possibly change the result.
  571. static bool SniffCRX(base::StringPiece content,
  572. const GURL& url,
  573. bool* have_enough_content,
  574. std::string* result) {
  575. // Technically, the crx magic number is just Cr24, but the bytes after that
  576. // are a version number which changes infrequently. Including it in the
  577. // sniffing gives us less room for error. If the version number ever changes,
  578. // we can just add an entry to this list.
  579. static const struct MagicNumber kCRXMagicNumbers[] = {
  580. MAGIC_NUMBER("application/x-chrome-extension", "Cr24\x02\x00\x00\x00"),
  581. MAGIC_NUMBER("application/x-chrome-extension", "Cr24\x03\x00\x00\x00")};
  582. // Only consider files that have the extension ".crx".
  583. if (!base::EndsWith(url.path_piece(), ".crx", base::CompareCase::SENSITIVE))
  584. return false;
  585. *have_enough_content &= TruncateStringPiece(kBytesRequiredForMagic, &content);
  586. return CheckForMagicNumbers(content, kCRXMagicNumbers, result);
  587. }
  588. bool ShouldSniffMimeType(const GURL& url, base::StringPiece mime_type) {
  589. bool sniffable_scheme = url.is_empty() || url.SchemeIsHTTPOrHTTPS() ||
  590. #if BUILDFLAG(IS_ANDROID)
  591. url.SchemeIs("content") ||
  592. #endif
  593. url.SchemeIsFile() || url.SchemeIsFileSystem();
  594. if (!sniffable_scheme)
  595. return false;
  596. static const char* const kSniffableTypes[] = {
  597. // Many web servers are misconfigured to send text/plain for many
  598. // different types of content.
  599. "text/plain",
  600. // We want to sniff application/octet-stream for
  601. // application/x-chrome-extension, but nothing else.
  602. "application/octet-stream",
  603. // XHTML and Atom/RSS feeds are often served as plain xml instead of
  604. // their more specific mime types.
  605. "text/xml",
  606. "application/xml",
  607. // Check for false Microsoft Office MIME types.
  608. "application/msword",
  609. "application/vnd.ms-excel",
  610. "application/vnd.ms-powerpoint",
  611. "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
  612. "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
  613. "application/vnd.openxmlformats-officedocument.presentationml.presentation",
  614. "application/vnd.ms-excel.sheet.macroenabled.12",
  615. "application/vnd.ms-word.document.macroenabled.12",
  616. "application/vnd.ms-powerpoint.presentation.macroenabled.12",
  617. "application/mspowerpoint",
  618. "application/msexcel",
  619. "application/vnd.ms-word",
  620. "application/vnd.ms-word.document.12",
  621. "application/vnd.msword",
  622. };
  623. for (const char* const sniffable_type : kSniffableTypes) {
  624. if (mime_type == sniffable_type)
  625. return true;
  626. }
  627. if (IsUnknownMimeType(mime_type)) {
  628. // The web server didn't specify a content type or specified a mime
  629. // type that we ignore.
  630. return true;
  631. }
  632. return false;
  633. }
  634. bool SniffMimeType(base::StringPiece content,
  635. const GURL& url,
  636. const std::string& type_hint,
  637. ForceSniffFileUrlsForHtml force_sniff_file_url_for_html,
  638. std::string* result) {
  639. // Sanity check.
  640. DCHECK_LT(content.length(), 1000000U);
  641. DCHECK(result);
  642. // By default, we assume we have enough content.
  643. // Each sniff routine may unset this if it wasn't provided enough content.
  644. bool have_enough_content = true;
  645. // By default, we'll return the type hint.
  646. // Each sniff routine may modify this if it has a better guess..
  647. result->assign(type_hint);
  648. // If the file has a Microsoft Office MIME type, we should only check that it
  649. // is a valid Office file. Because this is the only reason we sniff files
  650. // with a Microsoft Office MIME type, we can return early.
  651. if (IsOfficeType(type_hint))
  652. return SniffForInvalidOfficeDocs(content, url, result);
  653. // Cache information about the type_hint
  654. bool hint_is_unknown_mime_type = IsUnknownMimeType(type_hint);
  655. // First check for HTML, unless it's a file URL and
  656. // |allow_sniffing_files_urls_as_html| is false.
  657. if (hint_is_unknown_mime_type &&
  658. (!url.SchemeIsFile() ||
  659. force_sniff_file_url_for_html == ForceSniffFileUrlsForHtml::kEnabled)) {
  660. // We're only willing to sniff HTML if the server has not supplied a mime
  661. // type, or if the type it did supply indicates that it doesn't know what
  662. // the type should be.
  663. if (SniffForHTML(content, &have_enough_content, result))
  664. return true; // We succeeded in sniffing HTML. No more content needed.
  665. }
  666. // We're only willing to sniff for binary in 3 cases:
  667. // 1. The server has not supplied a mime type.
  668. // 2. The type it did supply indicates that it doesn't know what the type
  669. // should be.
  670. // 3. The type is "text/plain" which is the default on some web servers and
  671. // could be indicative of a mis-configuration that we shield the user from.
  672. const bool hint_is_text_plain = (type_hint == "text/plain");
  673. if (hint_is_unknown_mime_type || hint_is_text_plain) {
  674. if (!SniffBinary(content, &have_enough_content, result)) {
  675. // If the server said the content was text/plain and it doesn't appear
  676. // to be binary, then we trust it.
  677. if (hint_is_text_plain) {
  678. return have_enough_content;
  679. }
  680. }
  681. }
  682. // If we have plain XML, sniff XML subtypes.
  683. if (type_hint == "text/xml" || type_hint == "application/xml") {
  684. // We're not interested in sniffing these types for images and the like.
  685. // Instead, we're looking explicitly for a feed. If we don't find one
  686. // we're done and return early.
  687. if (SniffXML(content, &have_enough_content, result))
  688. return true;
  689. return have_enough_content;
  690. }
  691. // CRX files (Chrome extensions) have a special sniffing algorithm. It is
  692. // tighter than the others because we don't have to match legacy behavior.
  693. if (SniffCRX(content, url, &have_enough_content, result))
  694. return true;
  695. // Check the file extension and magic numbers to see if this is an Office
  696. // document. This needs to be checked before the general magic numbers
  697. // because zip files and Office documents (OOXML) have the same magic number.
  698. if (SniffForOfficeDocs(content, url, &have_enough_content, result)) {
  699. return true; // We've matched a magic number. No more content needed.
  700. }
  701. // We're not interested in sniffing for magic numbers when the type_hint
  702. // is application/octet-stream. Time to bail out.
  703. if (type_hint == "application/octet-stream")
  704. return have_enough_content;
  705. // Now we look in our large table of magic numbers to see if we can find
  706. // anything that matches the content.
  707. if (SniffForMagicNumbers(content, &have_enough_content, result))
  708. return true; // We've matched a magic number. No more content needed.
  709. return have_enough_content;
  710. }
  711. bool SniffMimeType(const char* content,
  712. size_t content_size,
  713. const GURL& url,
  714. const std::string& type_hint,
  715. ForceSniffFileUrlsForHtml force_sniff_file_url_for_html,
  716. std::string* result) {
  717. return SniffMimeType(base::StringPiece(content, content_size), url, type_hint,
  718. force_sniff_file_url_for_html, result);
  719. }
  720. NET_EXPORT bool SniffMimeTypeFromLocalData(base::StringPiece content,
  721. std::string* result) {
  722. // First check the extra table.
  723. if (CheckForMagicNumbers(content, kExtraMagicNumbers, result))
  724. return true;
  725. // Finally check the original table.
  726. return CheckForMagicNumbers(content, kMagicNumbers, result);
  727. }
  728. bool SniffMimeTypeFromLocalData(const char* content,
  729. size_t size,
  730. std::string* result) {
  731. return SniffMimeTypeFromLocalData(base::StringPiece(content, size), result);
  732. }
  733. bool LooksLikeBinary(base::StringPiece content) {
  734. // The definition of "binary bytes" is from the spec at
  735. // https://mimesniff.spec.whatwg.org/#binary-data-byte
  736. //
  737. // The bytes which are considered to be "binary" are all < 0x20. Encode them
  738. // one bit per byte, with 1 for a "binary" bit, and 0 for a "text" bit. The
  739. // least-significant bit represents byte 0x00, the most-significant bit
  740. // represents byte 0x1F.
  741. const uint32_t kBinaryBits =
  742. ~(1u << '\t' | 1u << '\n' | 1u << '\r' | 1u << '\f' | 1u << '\x1b');
  743. for (char c : content) {
  744. uint8_t byte = static_cast<uint8_t>(c);
  745. if (byte < 0x20 && (kBinaryBits & (1u << byte)))
  746. return true;
  747. }
  748. return false;
  749. }
  750. } // namespace net