simple_synchronous_entry.h 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547
  1. // Copyright (c) 2013 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #ifndef NET_DISK_CACHE_SIMPLE_SIMPLE_SYNCHRONOUS_ENTRY_H_
  5. #define NET_DISK_CACHE_SIMPLE_SIMPLE_SYNCHRONOUS_ENTRY_H_
  6. #include <stdint.h>
  7. #include <algorithm>
  8. #include <map>
  9. #include <memory>
  10. #include <string>
  11. #include <utility>
  12. #include <vector>
  13. #include "base/feature_list.h"
  14. #include "base/files/file.h"
  15. #include "base/files/file_path.h"
  16. #include "base/gtest_prod_util.h"
  17. #include "base/memory/raw_ptr.h"
  18. #include "base/memory/ref_counted.h"
  19. #include "base/strings/string_piece_forward.h"
  20. #include "base/time/time.h"
  21. #include "net/base/cache_type.h"
  22. #include "net/base/net_errors.h"
  23. #include "net/base/net_export.h"
  24. #include "net/disk_cache/simple/simple_entry_format.h"
  25. #include "net/disk_cache/simple/simple_file_tracker.h"
  26. #include "net/disk_cache/simple/simple_histogram_enums.h"
  27. namespace net {
  28. class GrowableIOBuffer;
  29. class IOBuffer;
  30. }
  31. FORWARD_DECLARE_TEST(DiskCacheBackendTest, SimpleCacheEnumerationLongKeys);
  32. namespace disk_cache {
  33. class BackendFileOperations;
  34. class UnboundBackendFileOperations;
  35. NET_EXPORT_PRIVATE extern const base::Feature kSimpleCachePrefetchExperiment;
  36. NET_EXPORT_PRIVATE extern const char kSimpleCacheFullPrefetchBytesParam[];
  37. NET_EXPORT_PRIVATE extern const char
  38. kSimpleCacheTrailerPrefetchSpeculativeBytesParam[];
  39. // Returns how large a file would get prefetched on reading the entry.
  40. // If the experiment is disabled, returns 0.
  41. NET_EXPORT_PRIVATE int GetSimpleCachePrefetchSize();
  42. class SimpleSynchronousEntry;
  43. struct RangeResult;
  44. // This class handles the passing of data about the entry between
  45. // SimpleEntryImplementation and SimpleSynchronousEntry and the computation of
  46. // file offsets based on the data size for all streams.
  47. class NET_EXPORT_PRIVATE SimpleEntryStat {
  48. public:
  49. SimpleEntryStat(base::Time last_used,
  50. base::Time last_modified,
  51. const int32_t data_size[],
  52. const int32_t sparse_data_size);
  53. int GetOffsetInFile(size_t key_length, int offset, int stream_index) const;
  54. int GetEOFOffsetInFile(size_t key_length, int stream_index) const;
  55. int GetLastEOFOffsetInFile(size_t key_length, int file_index) const;
  56. int64_t GetFileSize(size_t key_length, int file_index) const;
  57. base::Time last_used() const { return last_used_; }
  58. base::Time last_modified() const { return last_modified_; }
  59. void set_last_used(base::Time last_used) { last_used_ = last_used; }
  60. void set_last_modified(base::Time last_modified) {
  61. last_modified_ = last_modified;
  62. }
  63. int32_t data_size(int stream_index) const { return data_size_[stream_index]; }
  64. void set_data_size(int stream_index, int data_size) {
  65. data_size_[stream_index] = data_size;
  66. }
  67. int32_t sparse_data_size() const { return sparse_data_size_; }
  68. void set_sparse_data_size(int32_t sparse_data_size) {
  69. sparse_data_size_ = sparse_data_size;
  70. }
  71. private:
  72. base::Time last_used_;
  73. base::Time last_modified_;
  74. int32_t data_size_[kSimpleEntryStreamCount];
  75. int32_t sparse_data_size_;
  76. };
  77. struct SimpleStreamPrefetchData {
  78. SimpleStreamPrefetchData();
  79. ~SimpleStreamPrefetchData();
  80. scoped_refptr<net::GrowableIOBuffer> data;
  81. uint32_t stream_crc32;
  82. };
  83. struct SimpleEntryCreationResults {
  84. explicit SimpleEntryCreationResults(SimpleEntryStat entry_stat);
  85. ~SimpleEntryCreationResults();
  86. raw_ptr<SimpleSynchronousEntry> sync_entry;
  87. // This is set when `sync_entry` is null.
  88. std::unique_ptr<UnboundBackendFileOperations> unbound_file_operations;
  89. // Expectation is that [0] will always be filled in, but [1] might not be.
  90. SimpleStreamPrefetchData stream_prefetch_data[2];
  91. SimpleEntryStat entry_stat;
  92. int32_t computed_trailer_prefetch_size = -1;
  93. int result = net::OK;
  94. bool created = false;
  95. };
  96. struct SimpleEntryCloseResults {
  97. int32_t estimated_trailer_prefetch_size = -1;
  98. };
  99. // Worker thread interface to the very simple cache. This interface is not
  100. // thread safe, and callers must ensure that it is only ever accessed from
  101. // a single thread between synchronization points.
  102. class SimpleSynchronousEntry {
  103. public:
  104. struct CRCRecord {
  105. CRCRecord();
  106. CRCRecord(int index_p, bool has_crc32_p, uint32_t data_crc32_p);
  107. int index;
  108. bool has_crc32;
  109. uint32_t data_crc32;
  110. };
  111. struct ReadRequest {
  112. // Also sets request_update_crc to false.
  113. ReadRequest(int index_p, int offset_p, int buf_len_p);
  114. int index;
  115. int offset;
  116. int buf_len;
  117. // Partial CRC of data immediately preceeding this read. Only relevant if
  118. // request_update_crc is set.
  119. uint32_t previous_crc32;
  120. bool request_update_crc = false;
  121. bool request_verify_crc; // only relevant if request_update_crc is set
  122. };
  123. struct ReadResult {
  124. ReadResult() = default;
  125. int result;
  126. uint32_t updated_crc32; // only relevant if crc_updated set
  127. bool crc_updated = false;
  128. };
  129. struct WriteRequest {
  130. WriteRequest(int index_p,
  131. int offset_p,
  132. int buf_len_p,
  133. uint32_t previous_crc32_p,
  134. bool truncate_p,
  135. bool doomed_p,
  136. bool request_update_crc_p);
  137. int index;
  138. int offset;
  139. int buf_len;
  140. uint32_t previous_crc32;
  141. bool truncate;
  142. bool doomed;
  143. bool request_update_crc;
  144. };
  145. struct WriteResult {
  146. WriteResult() = default;
  147. int result;
  148. uint32_t updated_crc32; // only relevant if crc_updated set
  149. bool crc_updated = false;
  150. };
  151. struct SparseRequest {
  152. SparseRequest(int64_t sparse_offset_p, int buf_len_p);
  153. int64_t sparse_offset;
  154. int buf_len;
  155. };
  156. NET_EXPORT_PRIVATE SimpleSynchronousEntry(
  157. net::CacheType cache_type,
  158. const base::FilePath& path,
  159. const std::string& key,
  160. uint64_t entry_hash,
  161. SimpleFileTracker* simple_file_tracker,
  162. std::unique_ptr<UnboundBackendFileOperations> file_operations,
  163. int32_t stream_0_size);
  164. // Like Entry, the SimpleSynchronousEntry self releases when Close() is
  165. // called, but sometimes temporary ones are kept in unique_ptr.
  166. NET_EXPORT_PRIVATE ~SimpleSynchronousEntry();
  167. // Opens a disk cache entry on disk. The |key| parameter is optional, if empty
  168. // the operation may be slower. The |entry_hash| parameter is required.
  169. static void OpenEntry(
  170. net::CacheType cache_type,
  171. const base::FilePath& path,
  172. const std::string& key,
  173. uint64_t entry_hash,
  174. SimpleFileTracker* file_tracker,
  175. std::unique_ptr<UnboundBackendFileOperations> file_operations,
  176. int32_t trailer_prefetch_size,
  177. SimpleEntryCreationResults* out_results);
  178. static void CreateEntry(
  179. net::CacheType cache_type,
  180. const base::FilePath& path,
  181. const std::string& key,
  182. uint64_t entry_hash,
  183. SimpleFileTracker* file_tracker,
  184. std::unique_ptr<UnboundBackendFileOperations> file_operations,
  185. SimpleEntryCreationResults* out_results);
  186. static void OpenOrCreateEntry(
  187. net::CacheType cache_type,
  188. const base::FilePath& path,
  189. const std::string& key,
  190. uint64_t entry_hash,
  191. OpenEntryIndexEnum index_state,
  192. bool optimistic_create,
  193. SimpleFileTracker* file_tracker,
  194. std::unique_ptr<UnboundBackendFileOperations> file_operations,
  195. int32_t trailer_prefetch_size,
  196. SimpleEntryCreationResults* out_results);
  197. // Renames the entry on the file system, making it no longer possible to open
  198. // it again, but allowing operations to continue to be executed through that
  199. // instance. The renamed file will be removed once the entry is closed.
  200. // Returns a net error code.
  201. int Doom();
  202. // Deletes an entry from the file system. This variant should only be used
  203. // if there is no actual open instance around, as it doesn't account for
  204. // possibility of it having been renamed to a non-standard name.
  205. static int DeleteEntryFiles(
  206. const base::FilePath& path,
  207. net::CacheType cache_type,
  208. uint64_t entry_hash,
  209. std::unique_ptr<UnboundBackendFileOperations> unbound_file_operations);
  210. // Like |DeleteEntryFiles()| above, except that it truncates the entry files
  211. // rather than deleting them. Used when dooming entries after the backend has
  212. // shutdown. See implementation of |SimpleEntryImpl::DoomEntryInternal()| for
  213. // more.
  214. static int TruncateEntryFiles(
  215. const base::FilePath& path,
  216. uint64_t entry_hash,
  217. std::unique_ptr<UnboundBackendFileOperations> file_operations);
  218. // Like |DeleteEntryFiles()| above. Deletes all entries corresponding to the
  219. // |key_hashes|. Succeeds only when all entries are deleted. Returns a net
  220. // error code.
  221. static int DeleteEntrySetFiles(
  222. const std::vector<uint64_t>* key_hashes,
  223. const base::FilePath& path,
  224. std::unique_ptr<UnboundBackendFileOperations> unbound_file_operations);
  225. // N.B. ReadData(), WriteData(), CheckEOFRecord(), ReadSparseData(),
  226. // WriteSparseData() and Close() may block on IO.
  227. //
  228. // All of these methods will put the //net return value into |*out_result|.
  229. void ReadData(const ReadRequest& in_entry_op,
  230. SimpleEntryStat* entry_stat,
  231. net::IOBuffer* out_buf,
  232. ReadResult* out_result);
  233. void WriteData(const WriteRequest& in_entry_op,
  234. net::IOBuffer* in_buf,
  235. SimpleEntryStat* out_entry_stat,
  236. WriteResult* out_write_result);
  237. int CheckEOFRecord(BackendFileOperations* file_operations,
  238. base::File* file,
  239. int stream_index,
  240. const SimpleEntryStat& entry_stat,
  241. uint32_t expected_crc32);
  242. void ReadSparseData(const SparseRequest& in_entry_op,
  243. net::IOBuffer* out_buf,
  244. base::Time* out_last_used,
  245. int* out_result);
  246. void WriteSparseData(const SparseRequest& in_entry_op,
  247. net::IOBuffer* in_buf,
  248. uint64_t max_sparse_data_size,
  249. SimpleEntryStat* out_entry_stat,
  250. int* out_result);
  251. void GetAvailableRange(const SparseRequest& in_entry_op,
  252. RangeResult* out_result);
  253. // Close all streams, and add write EOF records to streams indicated by the
  254. // CRCRecord entries in |crc32s_to_write|.
  255. void Close(const SimpleEntryStat& entry_stat,
  256. std::unique_ptr<std::vector<CRCRecord>> crc32s_to_write,
  257. net::GrowableIOBuffer* stream_0_data,
  258. SimpleEntryCloseResults* out_results);
  259. const base::FilePath& path() const { return path_; }
  260. std::string key() const { return key_; }
  261. const SimpleFileTracker::EntryFileKey& entry_file_key() const {
  262. return entry_file_key_;
  263. }
  264. NET_EXPORT_PRIVATE base::FilePath GetFilenameForSubfile(
  265. SimpleFileTracker::SubFile sub_file) const;
  266. int32_t computed_trailer_prefetch_size() const {
  267. return computed_trailer_prefetch_size_;
  268. }
  269. private:
  270. FRIEND_TEST_ALL_PREFIXES(::DiskCacheBackendTest,
  271. SimpleCacheEnumerationLongKeys);
  272. friend class SimpleFileTrackerTest;
  273. class PrefetchData;
  274. class ScopedFileOperationsBinding;
  275. enum FileRequired {
  276. FILE_NOT_REQUIRED,
  277. FILE_REQUIRED
  278. };
  279. struct SparseRange {
  280. int64_t offset;
  281. int64_t length;
  282. uint32_t data_crc32;
  283. int64_t file_offset;
  284. bool operator<(const SparseRange& other) const {
  285. return offset < other.offset;
  286. }
  287. };
  288. // When opening an entry without knowing the key, the header must be read
  289. // without knowing the size of the key. This is how much to read initially, to
  290. // make it likely the entire key is read.
  291. static const size_t kInitialHeaderRead = 64 * 1024;
  292. // Tries to open one of the cache entry files. Succeeds if the open succeeds
  293. // or if the file was not found and is allowed to be omitted if the
  294. // corresponding stream is empty.
  295. bool MaybeOpenFile(BackendFileOperations* file_operations,
  296. int file_index,
  297. base::File::Error* out_error);
  298. // Creates one of the cache entry files if necessary. If the file is allowed
  299. // to be omitted if the corresponding stream is empty, and if |file_required|
  300. // is FILE_NOT_REQUIRED, then the file is not created; otherwise, it is.
  301. bool MaybeCreateFile(BackendFileOperations* file_operations,
  302. int file_index,
  303. FileRequired file_required,
  304. base::File::Error* out_error);
  305. bool OpenFiles(BackendFileOperations* file_operations,
  306. SimpleEntryStat* out_entry_stat);
  307. bool CreateFiles(BackendFileOperations* file_operations,
  308. SimpleEntryStat* out_entry_stat);
  309. void CloseFile(BackendFileOperations* file_operations, int index);
  310. void CloseFiles();
  311. // Read the header and key at the beginning of the file, and validate that
  312. // they are correct. If this entry was opened with a key, the key is checked
  313. // for a match. If not, then the |key_| member is set based on the value in
  314. // this header. Records histograms if any check is failed.
  315. bool CheckHeaderAndKey(base::File* file, int file_index);
  316. // Returns a net error, i.e. net::OK on success.
  317. int InitializeForOpen(BackendFileOperations* file_operations,
  318. SimpleEntryStat* out_entry_stat,
  319. SimpleStreamPrefetchData stream_prefetch_data[2]);
  320. // Writes the header and key to a newly-created stream file. |index| is the
  321. // index of the stream. Returns true on success; returns false and failure.
  322. bool InitializeCreatedFile(BackendFileOperations* file_operations, int index);
  323. // Returns a net error, including net::OK on success and net::FILE_EXISTS
  324. // when the entry already exists.
  325. int InitializeForCreate(BackendFileOperations* file_operations,
  326. SimpleEntryStat* out_entry_stat);
  327. // Allocates and fills a buffer with stream 0 data in |stream_0_data|, then
  328. // checks its crc32. May also optionally read in |stream_1_data| and its
  329. // crc, but might decide not to.
  330. int ReadAndValidateStream0AndMaybe1(
  331. BackendFileOperations* file_operations,
  332. int file_size,
  333. SimpleEntryStat* out_entry_stat,
  334. SimpleStreamPrefetchData stream_prefetch_data[2]);
  335. // Reads the EOF record located at |file_offset| in file |file_index|,
  336. // with |file_0_prefetch| potentially having prefetched file 0 content.
  337. // Puts the result into |*eof_record| and sanity-checks it.
  338. // Returns net status, and records any failures to UMA.
  339. int GetEOFRecordData(base::File* file,
  340. PrefetchData* prefetch_data,
  341. int file_index,
  342. int file_offset,
  343. SimpleFileEOF* eof_record);
  344. // Reads either from |file_0_prefetch| or |file|.
  345. // Range-checks all the in-memory reads.
  346. bool ReadFromFileOrPrefetched(base::File* file,
  347. PrefetchData* prefetch_data,
  348. int file_index,
  349. int offset,
  350. int size,
  351. char* dest);
  352. // Extracts out the payload of stream |stream_index|, reading either from
  353. // |file_0_prefetch|, if available, or |file|. |entry_stat| will be used to
  354. // determine file layout, though |extra_size| additional bytes will be read
  355. // past the stream payload end.
  356. //
  357. // |*stream_data| will be pointed to a fresh buffer with the results,
  358. // and |*out_crc32| will get the checksum, which will be verified against
  359. // |eof_record|.
  360. int PreReadStreamPayload(base::File* file,
  361. PrefetchData* prefetch_data,
  362. int stream_index,
  363. int extra_size,
  364. const SimpleEntryStat& entry_stat,
  365. const SimpleFileEOF& eof_record,
  366. SimpleStreamPrefetchData* out);
  367. // Opens the sparse data file and scans it if it exists.
  368. bool OpenSparseFileIfExists(BackendFileOperations* file_operations,
  369. int32_t* out_sparse_data_size);
  370. // Creates and initializes the sparse data file.
  371. bool CreateSparseFile(BackendFileOperations* file_operations);
  372. // Closes the sparse data file.
  373. void CloseSparseFile(BackendFileOperations* file_operations);
  374. // Writes the header to the (newly-created) sparse file.
  375. bool InitializeSparseFile(base::File* file);
  376. // Removes all but the header of the sparse file.
  377. bool TruncateSparseFile(base::File* sparse_file);
  378. // Scans the existing ranges in the sparse file. Populates |sparse_ranges_|
  379. // and sets |*out_sparse_data_size| to the total size of all the ranges (not
  380. // including headers).
  381. bool ScanSparseFile(base::File* sparse_file, int32_t* out_sparse_data_size);
  382. // Reads from a single sparse range. If asked to read the entire range, also
  383. // verifies the CRC32.
  384. bool ReadSparseRange(base::File* sparse_file,
  385. const SparseRange* range,
  386. int offset,
  387. int len,
  388. char* buf);
  389. // Writes to a single (existing) sparse range. If asked to write the entire
  390. // range, also updates the CRC32; otherwise, invalidates it.
  391. bool WriteSparseRange(base::File* sparse_file,
  392. SparseRange* range,
  393. int offset,
  394. int len,
  395. const char* buf);
  396. // Appends a new sparse range to the sparse data file.
  397. bool AppendSparseRange(base::File* sparse_file,
  398. int64_t offset,
  399. int len,
  400. const char* buf);
  401. static int DeleteEntryFilesInternal(const base::FilePath& path,
  402. net::CacheType cache_type,
  403. uint64_t entry_hash,
  404. BackendFileOperations* file_operations);
  405. static bool DeleteFileForEntryHash(const base::FilePath& path,
  406. uint64_t entry_hash,
  407. int file_index,
  408. BackendFileOperations* file_operations);
  409. static bool DeleteFilesForEntryHash(const base::FilePath& path,
  410. uint64_t entry_hash,
  411. BackendFileOperations* file_operations);
  412. static bool TruncateFilesForEntryHash(const base::FilePath& path,
  413. uint64_t entry_hash,
  414. BackendFileOperations* file_operations);
  415. int DoomInternal(BackendFileOperations* file_operations);
  416. base::FilePath GetFilenameFromFileIndex(int file_index) const;
  417. bool sparse_file_open() const { return sparse_file_open_; }
  418. const net::CacheType cache_type_;
  419. const base::FilePath path_;
  420. SimpleFileTracker::EntryFileKey entry_file_key_;
  421. std::string key_;
  422. bool have_open_files_ = false;
  423. bool initialized_ = false;
  424. // Normally false. This is set to true when an entry is opened without
  425. // checking the file headers. Any subsequent read will perform the check
  426. // before completing.
  427. bool header_and_key_check_needed_[kSimpleEntryNormalFileCount] = {
  428. false,
  429. };
  430. raw_ptr<SimpleFileTracker> file_tracker_;
  431. // An interface to allow file operations. This is in an "unbound" state
  432. // because each operation can run on different sequence from each other.
  433. // Each operation can convert this to a BackendFileOperations with calling
  434. // Bind(), relying on that at most one operation runs at a time.
  435. std::unique_ptr<UnboundBackendFileOperations> unbound_file_operations_;
  436. // The number of trailing bytes in file 0 that we believe should be
  437. // prefetched in order to read the EOF record and stream 0. This is
  438. // a hint from the index and may not be exactly right. -1 if we
  439. // don't have a hinted value.
  440. int32_t trailer_prefetch_size_;
  441. // The exact number of trailing bytes that were needed to read the
  442. // EOF record and stream 0 when the entry was actually opened. This
  443. // may be different from the trailer_prefetch_size_ hint and is
  444. // propagated back to the index in order to optimize the next open.
  445. int32_t computed_trailer_prefetch_size_ = -1;
  446. // True if the corresponding stream is empty and therefore no on-disk file
  447. // was created to store it.
  448. bool empty_file_omitted_[kSimpleEntryNormalFileCount];
  449. typedef std::map<int64_t, SparseRange> SparseRangeOffsetMap;
  450. typedef SparseRangeOffsetMap::iterator SparseRangeIterator;
  451. SparseRangeOffsetMap sparse_ranges_;
  452. bool sparse_file_open_ = false;
  453. // Offset of the end of the sparse file (where the next sparse range will be
  454. // written).
  455. int64_t sparse_tail_offset_;
  456. };
  457. } // namespace disk_cache
  458. #endif // NET_DISK_CACHE_SIMPLE_SIMPLE_SYNCHRONOUS_ENTRY_H_