network_fetcher.cc 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553
  1. // Copyright 2019 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "components/winhttp/network_fetcher.h"
  5. #include <limits>
  6. #include <string>
  7. #include <utility>
  8. #include "base/bind.h"
  9. #include "base/callback.h"
  10. #include "base/files/file.h"
  11. #include "base/files/file_util.h"
  12. #include "base/logging.h"
  13. #include "base/memory/scoped_refptr.h"
  14. #include "base/numerics/safe_math.h"
  15. #include "base/strings/strcat.h"
  16. #include "base/strings/string_number_conversions.h"
  17. #include "base/strings/string_piece.h"
  18. #include "base/strings/stringprintf.h"
  19. #include "base/strings/sys_string_conversions.h"
  20. #include "base/task/task_traits.h"
  21. #include "base/task/thread_pool.h"
  22. #include "base/threading/thread_task_runner_handle.h"
  23. #include "base/win/windows_version.h"
  24. #include "components/winhttp/net_util.h"
  25. #include "components/winhttp/proxy_info.h"
  26. #include "components/winhttp/scoped_hinternet.h"
  27. #include "components/winhttp/scoped_winttp_proxy_info.h"
  28. #include "url/url_constants.h"
  29. namespace winhttp {
  30. namespace {
  31. constexpr base::TaskTraits kTaskTraits = {
  32. base::MayBlock(), base::TaskPriority::BEST_EFFORT,
  33. base::TaskShutdownBehavior::SKIP_ON_SHUTDOWN};
  34. void CrackUrl(const GURL& url,
  35. bool* is_https,
  36. std::string* host,
  37. int* port,
  38. std::string* path_for_request) {
  39. if (is_https)
  40. *is_https = url.SchemeIs(url::kHttpsScheme);
  41. if (host)
  42. *host = url.host();
  43. if (port)
  44. *port = url.EffectiveIntPort();
  45. if (path_for_request)
  46. *path_for_request = url.PathForRequest();
  47. }
  48. } // namespace
  49. NetworkFetcher::NetworkFetcher(
  50. const HINTERNET& session_handle,
  51. scoped_refptr<ProxyConfiguration> proxy_configuration)
  52. : main_thread_task_runner_(base::ThreadTaskRunnerHandle::Get()),
  53. session_handle_(session_handle),
  54. proxy_configuration_(proxy_configuration) {}
  55. NetworkFetcher::~NetworkFetcher() {
  56. DVLOG(3) << "~NetworkFetcher";
  57. }
  58. void NetworkFetcher::Close() {
  59. // |write_data_callback_| maintains an outstanding reference to this object
  60. // and the reference must be released to avoid leaking the object.
  61. write_data_callback_.Reset();
  62. request_handle_.reset();
  63. }
  64. void NetworkFetcher::CompleteFetch() {
  65. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  66. if (!file_.IsValid()) {
  67. std::move(fetch_complete_callback_).Run(response_code_);
  68. return;
  69. }
  70. base::ThreadPool::PostTaskAndReply(
  71. FROM_HERE, kTaskTraits,
  72. base::BindOnce([](base::File& file) { file.Close(); }, std::ref(file_)),
  73. base::BindOnce(&NetworkFetcher::CompleteFetch, this));
  74. }
  75. HRESULT NetworkFetcher::QueryHeaderString(const std::wstring& name,
  76. std::wstring* value) const {
  77. return QueryHeadersString(request_handle_.get(), WINHTTP_QUERY_CUSTOM,
  78. name.c_str(), value);
  79. }
  80. HRESULT NetworkFetcher::QueryHeaderInt(const std::wstring& name,
  81. int* value) const {
  82. return QueryHeadersInt(request_handle_.get(), WINHTTP_QUERY_CUSTOM,
  83. name.c_str(), value);
  84. }
  85. std::string NetworkFetcher::GetResponseBody() const {
  86. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  87. return post_response_body_;
  88. }
  89. HRESULT NetworkFetcher::GetNetError() const {
  90. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  91. return net_error_;
  92. }
  93. base::FilePath NetworkFetcher::GetFilePath() const {
  94. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  95. return file_path_;
  96. }
  97. int64_t NetworkFetcher::GetContentSize() const {
  98. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  99. return content_size_;
  100. }
  101. void NetworkFetcher::PostRequest(
  102. const GURL& url,
  103. const std::string& post_data,
  104. const std::string& content_type,
  105. const base::flat_map<std::string, std::string>& post_additional_headers,
  106. FetchStartedCallback fetch_started_callback,
  107. FetchProgressCallback fetch_progress_callback,
  108. FetchCompleteCallback fetch_complete_callback) {
  109. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  110. url_ = url;
  111. fetch_started_callback_ = std::move(fetch_started_callback);
  112. fetch_progress_callback_ = std::move(fetch_progress_callback);
  113. fetch_complete_callback_ = std::move(fetch_complete_callback);
  114. CrackUrl(url_, &is_https_, &host_, &port_, &path_for_request_);
  115. verb_ = L"POST";
  116. content_type_ = content_type;
  117. write_data_callback_ =
  118. base::BindRepeating(&NetworkFetcher::WriteDataToMemory, this);
  119. net_error_ = BeginFetch(post_data, post_additional_headers);
  120. if (FAILED(net_error_))
  121. CompleteFetch();
  122. }
  123. void NetworkFetcher::DownloadToFile(
  124. const GURL& url,
  125. const base::FilePath& file_path,
  126. FetchStartedCallback fetch_started_callback,
  127. FetchProgressCallback fetch_progress_callback,
  128. FetchCompleteCallback fetch_complete_callback) {
  129. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  130. url_ = url;
  131. file_path_ = file_path;
  132. fetch_started_callback_ = std::move(fetch_started_callback);
  133. fetch_progress_callback_ = std::move(fetch_progress_callback);
  134. fetch_complete_callback_ = std::move(fetch_complete_callback);
  135. CrackUrl(url, &is_https_, &host_, &port_, &path_for_request_);
  136. verb_ = L"GET";
  137. write_data_callback_ =
  138. base::BindRepeating(&NetworkFetcher::WriteDataToFile, this);
  139. net_error_ = BeginFetch({}, {});
  140. if (FAILED(net_error_))
  141. CompleteFetch();
  142. }
  143. HRESULT NetworkFetcher::BeginFetch(
  144. const std::string& data,
  145. base::flat_map<std::string, std::string> additional_headers) {
  146. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  147. if (!url_.SchemeIsHTTPOrHTTPS())
  148. return E_INVALIDARG;
  149. connect_handle_ = Connect();
  150. if (!connect_handle_.get())
  151. return HRESULTFromLastError();
  152. absl::optional<ScopedWinHttpProxyInfo> winhttp_proxy_info =
  153. proxy_configuration_->GetProxyForUrl(session_handle_, url_);
  154. request_handle_ = OpenRequest();
  155. if (!request_handle_.get())
  156. return HRESULTFromLastError();
  157. SetProxyForRequest(request_handle_.get(), winhttp_proxy_info);
  158. const auto winhttp_callback = ::WinHttpSetStatusCallback(
  159. request_handle_.get(), &NetworkFetcher::WinHttpStatusCallback,
  160. WINHTTP_CALLBACK_FLAG_ALL_NOTIFICATIONS, 0);
  161. if (winhttp_callback == WINHTTP_INVALID_STATUS_CALLBACK)
  162. return HRESULTFromLastError();
  163. auto hr =
  164. SetOption(request_handle_.get(), WINHTTP_OPTION_CONTEXT_VALUE, context());
  165. if (FAILED(hr))
  166. return hr;
  167. self_ = this;
  168. // Disables both saving and sending cookies.
  169. hr = SetOption(request_handle_.get(), WINHTTP_OPTION_DISABLE_FEATURE,
  170. WINHTTP_DISABLE_COOKIES);
  171. if (FAILED(hr))
  172. return hr;
  173. if (!content_type_.empty())
  174. additional_headers.insert({"Content-Type", content_type_});
  175. for (const auto& header : additional_headers) {
  176. const auto raw_header = base::SysUTF8ToWide(
  177. base::StrCat({header.first, ": ", header.second, "\r\n"}));
  178. if (!::WinHttpAddRequestHeaders(
  179. request_handle_.get(), raw_header.c_str(), raw_header.size(),
  180. WINHTTP_ADDREQ_FLAG_ADD | WINHTTP_ADDREQ_FLAG_REPLACE)) {
  181. PLOG(ERROR) << "Failed to set the request header: " << raw_header;
  182. }
  183. }
  184. hr = SendRequest(data);
  185. if (FAILED(hr))
  186. return hr;
  187. return S_OK;
  188. }
  189. ScopedHInternet NetworkFetcher::Connect() {
  190. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  191. return ScopedHInternet(::WinHttpConnect(
  192. session_handle_, base::SysUTF8ToWide(host_).c_str(), port_, 0));
  193. }
  194. ScopedHInternet NetworkFetcher::OpenRequest() {
  195. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  196. uint32_t flags = WINHTTP_FLAG_REFRESH;
  197. if (is_https_)
  198. flags |= WINHTTP_FLAG_SECURE;
  199. return ScopedHInternet(::WinHttpOpenRequest(
  200. connect_handle_.get(), verb_.data(),
  201. base::SysUTF8ToWide(path_for_request_).c_str(), nullptr,
  202. WINHTTP_NO_REFERER, WINHTTP_DEFAULT_ACCEPT_TYPES, flags));
  203. }
  204. HRESULT NetworkFetcher::SendRequest(const std::string& data) {
  205. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  206. VLOG(2) << data;
  207. const uint32_t bytes_to_send = base::saturated_cast<uint32_t>(data.size());
  208. void* request_body =
  209. bytes_to_send ? const_cast<char*>(data.c_str()) : WINHTTP_NO_REQUEST_DATA;
  210. if (!::WinHttpSendRequest(request_handle_.get(),
  211. WINHTTP_NO_ADDITIONAL_HEADERS, 0, request_body,
  212. bytes_to_send, bytes_to_send, context())) {
  213. return HRESULTFromLastError();
  214. }
  215. return S_OK;
  216. }
  217. void NetworkFetcher::SendRequestComplete() {
  218. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  219. std::wstring all;
  220. QueryHeadersString(
  221. request_handle_.get(),
  222. WINHTTP_QUERY_RAW_HEADERS_CRLF | WINHTTP_QUERY_FLAG_REQUEST_HEADERS,
  223. WINHTTP_HEADER_NAME_BY_INDEX, &all);
  224. VLOG(3) << "request headers: " << all;
  225. net_error_ = ReceiveResponse();
  226. if (FAILED(net_error_))
  227. CompleteFetch();
  228. }
  229. HRESULT NetworkFetcher::ReceiveResponse() {
  230. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  231. if (!::WinHttpReceiveResponse(request_handle_.get(), nullptr))
  232. return HRESULTFromLastError();
  233. return S_OK;
  234. }
  235. void NetworkFetcher::HeadersAvailable() {
  236. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  237. std::wstring all;
  238. QueryHeadersString(request_handle_.get(), WINHTTP_QUERY_RAW_HEADERS_CRLF,
  239. WINHTTP_HEADER_NAME_BY_INDEX, &all);
  240. VLOG(3) << "response headers: " << all;
  241. net_error_ = QueryHeadersInt(request_handle_.get(), WINHTTP_QUERY_STATUS_CODE,
  242. WINHTTP_HEADER_NAME_BY_INDEX, &response_code_);
  243. if (FAILED(net_error_)) {
  244. CompleteFetch();
  245. return;
  246. }
  247. int content_length = 0;
  248. net_error_ =
  249. QueryHeadersInt(request_handle_.get(), WINHTTP_QUERY_CONTENT_LENGTH,
  250. WINHTTP_HEADER_NAME_BY_INDEX, &content_length);
  251. if (FAILED(net_error_)) {
  252. CompleteFetch();
  253. return;
  254. }
  255. std::move(fetch_started_callback_).Run(response_code_, content_length);
  256. // Start reading the body of response.
  257. net_error_ = ReadData();
  258. if (FAILED(net_error_))
  259. CompleteFetch();
  260. }
  261. HRESULT NetworkFetcher::ReadData() {
  262. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  263. // Use a fixed buffer size, larger than the internal WinHTTP buffer size (8K),
  264. // according to the documentation for WinHttpReadData.
  265. constexpr size_t kNumBytesToRead = 0x4000; // 16KiB.
  266. read_buffer_.resize(kNumBytesToRead);
  267. if (!::WinHttpReadData(request_handle_.get(), &read_buffer_.front(),
  268. read_buffer_.size(), nullptr)) {
  269. return HRESULTFromLastError();
  270. }
  271. DVLOG(3) << "reading data...";
  272. return S_OK;
  273. }
  274. void NetworkFetcher::ReadDataComplete(size_t num_bytes_read) {
  275. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  276. read_buffer_.resize(num_bytes_read);
  277. write_data_callback_.Run();
  278. }
  279. void NetworkFetcher::RequestError(const WINHTTP_ASYNC_RESULT* result) {
  280. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  281. net_error_ = HRESULT_FROM_WIN32(result->dwError);
  282. CompleteFetch();
  283. }
  284. void NetworkFetcher::WriteDataToFile() {
  285. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  286. base::ThreadPool::PostTaskAndReplyWithResult(
  287. FROM_HERE, kTaskTraits,
  288. base::BindOnce(&NetworkFetcher::WriteDataToFileBlocking, this),
  289. base::BindOnce(&NetworkFetcher::WriteDataToFileComplete, this));
  290. }
  291. // Returns true if EOF is reached.
  292. bool NetworkFetcher::WriteDataToFileBlocking() {
  293. if (read_buffer_.empty()) {
  294. file_.Close();
  295. net_error_ = S_OK;
  296. return true;
  297. }
  298. if (!file_.IsValid()) {
  299. file_.Initialize(file_path_,
  300. base::File::Flags::FLAG_CREATE_ALWAYS |
  301. base::File::Flags::FLAG_WRITE |
  302. base::File::Flags::FLAG_WIN_SEQUENTIAL_SCAN);
  303. if (!file_.IsValid()) {
  304. net_error_ = HRESULTFromLastError();
  305. return false;
  306. }
  307. }
  308. DCHECK(file_.IsValid());
  309. if (file_.WriteAtCurrentPos(&read_buffer_.front(), read_buffer_.size()) ==
  310. -1) {
  311. net_error_ = HRESULTFromLastError();
  312. file_.Close();
  313. base::DeleteFile(file_path_);
  314. return false;
  315. }
  316. content_size_ += read_buffer_.size();
  317. return false;
  318. }
  319. void NetworkFetcher::WriteDataToFileComplete(bool is_eof) {
  320. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  321. fetch_progress_callback_.Run(base::saturated_cast<int64_t>(content_size_));
  322. if (is_eof || FAILED(net_error_)) {
  323. CompleteFetch();
  324. return;
  325. }
  326. net_error_ = ReadData();
  327. if (FAILED(net_error_))
  328. CompleteFetch();
  329. }
  330. void NetworkFetcher::WriteDataToMemory() {
  331. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  332. if (read_buffer_.empty()) {
  333. VLOG(2) << post_response_body_;
  334. net_error_ = S_OK;
  335. CompleteFetch();
  336. return;
  337. }
  338. post_response_body_.append(read_buffer_.begin(), read_buffer_.end());
  339. content_size_ += read_buffer_.size();
  340. fetch_progress_callback_.Run(base::saturated_cast<int64_t>(content_size_));
  341. net_error_ = ReadData();
  342. if (FAILED(net_error_))
  343. CompleteFetch();
  344. }
  345. void __stdcall NetworkFetcher::WinHttpStatusCallback(HINTERNET handle,
  346. DWORD_PTR context,
  347. DWORD status,
  348. void* info,
  349. DWORD info_len) {
  350. DCHECK(handle);
  351. DCHECK(context);
  352. NetworkFetcher* network_fetcher = reinterpret_cast<NetworkFetcher*>(context);
  353. network_fetcher->main_thread_task_runner_->PostTask(
  354. FROM_HERE,
  355. base::BindOnce(&NetworkFetcher::StatusCallback, network_fetcher, handle,
  356. status, info, info_len));
  357. }
  358. void NetworkFetcher::StatusCallback(HINTERNET handle,
  359. uint32_t status,
  360. void* info,
  361. uint32_t info_len) {
  362. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  363. base::StringPiece status_string;
  364. std::wstring info_string;
  365. switch (status) {
  366. case WINHTTP_CALLBACK_STATUS_HANDLE_CREATED:
  367. status_string = "handle created";
  368. break;
  369. case WINHTTP_CALLBACK_STATUS_HANDLE_CLOSING:
  370. status_string = "handle closing";
  371. break;
  372. case WINHTTP_CALLBACK_STATUS_RESOLVING_NAME:
  373. status_string = "resolving";
  374. info_string.assign(static_cast<wchar_t*>(info), info_len); // host.
  375. break;
  376. case WINHTTP_CALLBACK_STATUS_NAME_RESOLVED:
  377. status_string = "resolved";
  378. break;
  379. case WINHTTP_CALLBACK_STATUS_CONNECTING_TO_SERVER:
  380. status_string = "connecting";
  381. info_string.assign(static_cast<wchar_t*>(info), info_len); // IP.
  382. break;
  383. case WINHTTP_CALLBACK_STATUS_CONNECTED_TO_SERVER:
  384. status_string = "connected";
  385. break;
  386. case WINHTTP_CALLBACK_STATUS_SENDING_REQUEST:
  387. status_string = "sending";
  388. break;
  389. case WINHTTP_CALLBACK_STATUS_REQUEST_SENT:
  390. status_string = "sent";
  391. break;
  392. case WINHTTP_CALLBACK_STATUS_RECEIVING_RESPONSE:
  393. status_string = "receiving response";
  394. break;
  395. case WINHTTP_CALLBACK_STATUS_RESPONSE_RECEIVED:
  396. status_string = "response received";
  397. break;
  398. case WINHTTP_CALLBACK_STATUS_CLOSING_CONNECTION:
  399. status_string = "connection closing";
  400. break;
  401. case WINHTTP_CALLBACK_STATUS_CONNECTION_CLOSED:
  402. status_string = "connection closed";
  403. break;
  404. case WINHTTP_CALLBACK_STATUS_REDIRECT:
  405. // |info| may contain invalid URL data and not safe to reference always.
  406. status_string = "redirect";
  407. break;
  408. case WINHTTP_CALLBACK_STATUS_DATA_AVAILABLE:
  409. status_string = "data available";
  410. DCHECK_EQ(info_len, sizeof(uint32_t));
  411. info_string = base::StringPrintf(L"%lu", *static_cast<uint32_t*>(info));
  412. break;
  413. case WINHTTP_CALLBACK_STATUS_HEADERS_AVAILABLE:
  414. status_string = "headers available";
  415. break;
  416. case WINHTTP_CALLBACK_STATUS_READ_COMPLETE:
  417. status_string = "read complete";
  418. info_string = base::StringPrintf(L"%lu", info_len);
  419. break;
  420. case WINHTTP_CALLBACK_STATUS_SENDREQUEST_COMPLETE:
  421. status_string = "send request complete";
  422. break;
  423. case WINHTTP_CALLBACK_STATUS_WRITE_COMPLETE:
  424. status_string = "write complete";
  425. break;
  426. case WINHTTP_CALLBACK_STATUS_REQUEST_ERROR:
  427. status_string = "request error";
  428. break;
  429. case WINHTTP_CALLBACK_STATUS_SECURE_FAILURE:
  430. status_string = "https failure";
  431. DCHECK(info);
  432. DCHECK_EQ(info_len, sizeof(uint32_t));
  433. info_string = base::StringPrintf(L"%#x", *static_cast<uint32_t*>(info));
  434. break;
  435. default:
  436. status_string = "unknown callback";
  437. break;
  438. }
  439. std::string msg;
  440. if (!status_string.empty())
  441. base::StringAppendF(&msg, "status=%s", status_string.data());
  442. else
  443. base::StringAppendF(&msg, "status=%#x", status);
  444. if (!info_string.empty())
  445. base::StringAppendF(&msg, ", info=%s",
  446. base::SysWideToUTF8(info_string).c_str());
  447. VLOG(3) << "WinHttp status callback:"
  448. << " handle=" << handle << ", " << msg;
  449. switch (status) {
  450. case WINHTTP_CALLBACK_STATUS_HANDLE_CLOSING:
  451. self_ = nullptr;
  452. break;
  453. case WINHTTP_CALLBACK_STATUS_SENDREQUEST_COMPLETE:
  454. SendRequestComplete();
  455. break;
  456. case WINHTTP_CALLBACK_STATUS_HEADERS_AVAILABLE:
  457. HeadersAvailable();
  458. break;
  459. case WINHTTP_CALLBACK_STATUS_READ_COMPLETE:
  460. DCHECK_EQ(info, &read_buffer_.front());
  461. ReadDataComplete(info_len);
  462. break;
  463. case WINHTTP_CALLBACK_STATUS_REQUEST_ERROR:
  464. RequestError(static_cast<const WINHTTP_ASYNC_RESULT*>(info));
  465. break;
  466. }
  467. }
  468. } // namespace winhttp