resource_metadata_storage.cc 43 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247
  1. // Copyright 2013 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "components/drive/resource_metadata_storage.h"
  5. #include <stddef.h>
  6. #include <map>
  7. #include <set>
  8. #include <unordered_map>
  9. #include <utility>
  10. #include "base/bind.h"
  11. #include "base/files/file_util.h"
  12. #include "base/location.h"
  13. #include "base/logging.h"
  14. #include "base/metrics/histogram.h"
  15. #include "base/metrics/histogram_functions.h"
  16. #include "base/metrics/histogram_macros.h"
  17. #include "base/task/sequenced_task_runner.h"
  18. #include "base/threading/scoped_blocking_call.h"
  19. #include "components/drive/drive.pb.h"
  20. #include "components/drive/drive_api_util.h"
  21. #include "components/drive/file_system_core_util.h"
  22. #include "third_party/leveldatabase/env_chromium.h"
  23. #include "third_party/leveldatabase/leveldb_chrome.h"
  24. #include "third_party/leveldatabase/src/include/leveldb/db.h"
  25. #include "third_party/leveldatabase/src/include/leveldb/write_batch.h"
  26. namespace drive {
  27. namespace internal {
  28. namespace {
  29. // Enum to describe DB initialization status.
  30. enum DBInitStatus {
  31. DB_INIT_SUCCESS,
  32. DB_INIT_NOT_FOUND,
  33. DB_INIT_CORRUPTION,
  34. DB_INIT_IO_ERROR,
  35. DB_INIT_FAILED,
  36. DB_INIT_INCOMPATIBLE,
  37. DB_INIT_BROKEN,
  38. DB_INIT_OPENED_EXISTING_DB,
  39. DB_INIT_CREATED_NEW_DB,
  40. DB_INIT_REPLACED_EXISTING_DB_WITH_NEW_DB,
  41. DB_INIT_MAX_VALUE,
  42. };
  43. // Enum to describe DB validity check failure reason.
  44. enum CheckValidityFailureReason {
  45. CHECK_VALIDITY_FAILURE_INVALID_HEADER,
  46. CHECK_VALIDITY_FAILURE_BROKEN_ID_ENTRY,
  47. CHECK_VALIDITY_FAILURE_BROKEN_ENTRY,
  48. CHECK_VALIDITY_FAILURE_INVALID_LOCAL_ID,
  49. CHECK_VALIDITY_FAILURE_INVALID_PARENT_ID,
  50. CHECK_VALIDITY_FAILURE_BROKEN_CHILD_MAP,
  51. CHECK_VALIDITY_FAILURE_CHILD_ENTRY_COUNT_MISMATCH,
  52. CHECK_VALIDITY_FAILURE_ITERATOR_ERROR,
  53. CHECK_VALIDITY_FAILURE_MAX_VALUE,
  54. };
  55. // The name of the DB which stores the metadata.
  56. const base::FilePath::CharType kResourceMapDBName[] =
  57. FILE_PATH_LITERAL("resource_metadata_resource_map.db");
  58. // The name of the DB which couldn't be opened, but is preserved just in case.
  59. const base::FilePath::CharType kPreservedResourceMapDBName[] =
  60. FILE_PATH_LITERAL("resource_metadata_preserved_resource_map.db");
  61. // The name of the DB which couldn't be opened, and was replaced with a new one.
  62. const base::FilePath::CharType kTrashedResourceMapDBName[] =
  63. FILE_PATH_LITERAL("resource_metadata_trashed_resource_map.db");
  64. // Meant to be a character which never happen to be in real IDs.
  65. const char kDBKeyDelimeter = '\0';
  66. // String used as a suffix of a key for a cache entry.
  67. const char kCacheEntryKeySuffix[] = "CACHE";
  68. // String used as a prefix of a key for a resource-ID-to-local-ID entry.
  69. const char kIdEntryKeyPrefix[] = "ID";
  70. // Returns a string to be used as the key for the header.
  71. std::string GetHeaderDBKey() {
  72. std::string key;
  73. key.push_back(kDBKeyDelimeter);
  74. key.append("HEADER");
  75. return key;
  76. }
  77. // Returns true if |key| is a key for a child entry.
  78. bool IsChildEntryKey(const leveldb::Slice& key) {
  79. return !key.empty() && key[key.size() - 1] == kDBKeyDelimeter;
  80. }
  81. // Returns true if |key| is a key for a cache entry.
  82. bool IsCacheEntryKey(const leveldb::Slice& key) {
  83. // A cache entry key should end with |kDBKeyDelimeter + kCacheEntryKeySuffix|.
  84. const leveldb::Slice expected_suffix(kCacheEntryKeySuffix,
  85. std::size(kCacheEntryKeySuffix) - 1);
  86. if (key.size() < 1 + expected_suffix.size() ||
  87. key[key.size() - expected_suffix.size() - 1] != kDBKeyDelimeter)
  88. return false;
  89. const leveldb::Slice key_substring(
  90. key.data() + key.size() - expected_suffix.size(), expected_suffix.size());
  91. return key_substring.compare(expected_suffix) == 0;
  92. }
  93. // Returns ID extracted from a cache entry key.
  94. std::string GetIdFromCacheEntryKey(const leveldb::Slice& key) {
  95. DCHECK(IsCacheEntryKey(key));
  96. // Drop the suffix |kDBKeyDelimeter + kCacheEntryKeySuffix| from the key.
  97. const size_t kSuffixLength = std::size(kCacheEntryKeySuffix) - 1;
  98. const int id_length = key.size() - 1 - kSuffixLength;
  99. return std::string(key.data(), id_length);
  100. }
  101. // Returns a string to be used as a key for a resource-ID-to-local-ID entry.
  102. std::string GetIdEntryKey(const std::string& resource_id) {
  103. std::string key;
  104. key.push_back(kDBKeyDelimeter);
  105. key.append(kIdEntryKeyPrefix);
  106. key.push_back(kDBKeyDelimeter);
  107. key.append(resource_id);
  108. return key;
  109. }
  110. // Returns true if |key| is a key for a resource-ID-to-local-ID entry.
  111. bool IsIdEntryKey(const leveldb::Slice& key) {
  112. // A resource-ID-to-local-ID entry key should start with
  113. // |kDBKeyDelimeter + kIdEntryKeyPrefix + kDBKeyDelimeter|.
  114. const leveldb::Slice expected_prefix(kIdEntryKeyPrefix,
  115. std::size(kIdEntryKeyPrefix) - 1);
  116. if (key.size() < 2 + expected_prefix.size())
  117. return false;
  118. const leveldb::Slice key_substring(key.data() + 1, expected_prefix.size());
  119. return key[0] == kDBKeyDelimeter &&
  120. key_substring.compare(expected_prefix) == 0 &&
  121. key[expected_prefix.size() + 1] == kDBKeyDelimeter;
  122. }
  123. // Returns the resource ID extracted from a resource-ID-to-local-ID entry key.
  124. std::string GetResourceIdFromIdEntryKey(const leveldb::Slice& key) {
  125. DCHECK(IsIdEntryKey(key));
  126. // Drop the prefix |kDBKeyDelimeter + kIdEntryKeyPrefix + kDBKeyDelimeter|
  127. // from the key.
  128. const size_t kPrefixLength = std::size(kIdEntryKeyPrefix) - 1;
  129. const int offset = kPrefixLength + 2;
  130. return std::string(key.data() + offset, key.size() - offset);
  131. }
  132. // Converts leveldb::Status to DBInitStatus.
  133. DBInitStatus LevelDBStatusToDBInitStatus(const leveldb::Status& status) {
  134. if (status.ok())
  135. return DB_INIT_SUCCESS;
  136. if (status.IsNotFound())
  137. return DB_INIT_NOT_FOUND;
  138. if (status.IsCorruption())
  139. return DB_INIT_CORRUPTION;
  140. if (status.IsIOError())
  141. return DB_INIT_IO_ERROR;
  142. return DB_INIT_FAILED;
  143. }
  144. // Converts leveldb::Status to FileError.
  145. FileError LevelDBStatusToFileError(const leveldb::Status& status) {
  146. if (status.ok())
  147. return FILE_ERROR_OK;
  148. if (status.IsNotFound())
  149. return FILE_ERROR_NOT_FOUND;
  150. if (leveldb_env::IndicatesDiskFull(status))
  151. return FILE_ERROR_NO_LOCAL_SPACE;
  152. return FILE_ERROR_FAILED;
  153. }
  154. ResourceMetadataHeader GetDefaultHeaderEntry() {
  155. ResourceMetadataHeader header;
  156. header.set_version(ResourceMetadataStorage::kDBVersion);
  157. return header;
  158. }
  159. bool MoveIfPossible(const base::FilePath& from, const base::FilePath& to) {
  160. return !base::PathExists(from) || base::Move(from, to);
  161. }
  162. void RecordCheckValidityFailure(CheckValidityFailureReason reason) {
  163. UMA_HISTOGRAM_ENUMERATION("Drive.MetadataDBValidityCheckFailureReason",
  164. reason,
  165. CHECK_VALIDITY_FAILURE_MAX_VALUE);
  166. }
  167. bool UpgradeOldDBVersions6To10(leveldb::DB* resource_map) {
  168. // Cache entries can be reused.
  169. leveldb::ReadOptions options;
  170. options.verify_checksums = true;
  171. std::unique_ptr<leveldb::Iterator> it(resource_map->NewIterator(options));
  172. leveldb::WriteBatch batch;
  173. // First, remove all entries.
  174. for (it->SeekToFirst(); it->Valid(); it->Next())
  175. batch.Delete(it->key());
  176. // Put ID entries and cache entries.
  177. for (it->SeekToFirst(); it->Valid(); it->Next()) {
  178. if (!IsCacheEntryKey(it->key()))
  179. continue;
  180. FileCacheEntry cache_entry;
  181. if (!cache_entry.ParseFromArray(it->value().data(), it->value().size()))
  182. return false;
  183. // The resource ID might be in old WAPI format. We need to canonicalize
  184. // to the format of API service currently in use.
  185. const std::string& id = GetIdFromCacheEntryKey(it->key());
  186. const std::string& id_new = util::CanonicalizeResourceId(id);
  187. // Before v11, resource ID was directly used as local ID. Such entries
  188. // can be migrated by adding an identity ID mapping.
  189. batch.Put(GetIdEntryKey(id_new), id_new);
  190. // Put cache state into a ResourceEntry.
  191. ResourceEntry entry;
  192. entry.set_local_id(id_new);
  193. entry.set_resource_id(id_new);
  194. *entry.mutable_file_specific_info()->mutable_cache_state() = cache_entry;
  195. std::string serialized_entry;
  196. if (!entry.SerializeToString(&serialized_entry)) {
  197. DLOG(ERROR) << "Failed to serialize the entry: " << id;
  198. return false;
  199. }
  200. batch.Put(id_new, serialized_entry);
  201. }
  202. if (!it->status().ok())
  203. return false;
  204. // Put header with the latest version number. This also clears
  205. // largest_changestamp and triggers refresh of metadata.
  206. std::string serialized_header;
  207. if (!GetDefaultHeaderEntry().SerializeToString(&serialized_header))
  208. return false;
  209. batch.Put(GetHeaderDBKey(), serialized_header);
  210. return resource_map->Write(leveldb::WriteOptions(), &batch).ok();
  211. }
  212. bool UpgradeOldDBVersion11(leveldb::DB* resource_map) {
  213. // Cache and ID map entries are reusable.
  214. leveldb::ReadOptions options;
  215. options.verify_checksums = true;
  216. std::unique_ptr<leveldb::Iterator> it(resource_map->NewIterator(options));
  217. // First, get the set of local IDs associated with cache entries.
  218. std::set<std::string> cached_entry_ids;
  219. for (it->SeekToFirst(); it->Valid(); it->Next()) {
  220. if (IsCacheEntryKey(it->key()))
  221. cached_entry_ids.insert(GetIdFromCacheEntryKey(it->key()));
  222. }
  223. if (!it->status().ok())
  224. return false;
  225. // Remove all entries except used ID entries.
  226. leveldb::WriteBatch batch;
  227. std::map<std::string, std::string> local_id_to_resource_id;
  228. for (it->SeekToFirst(); it->Valid(); it->Next()) {
  229. const bool is_used_id = IsIdEntryKey(it->key()) &&
  230. cached_entry_ids.count(it->value().ToString());
  231. if (is_used_id) {
  232. local_id_to_resource_id[it->value().ToString()] =
  233. GetResourceIdFromIdEntryKey(it->key());
  234. } else {
  235. batch.Delete(it->key());
  236. }
  237. }
  238. if (!it->status().ok())
  239. return false;
  240. // Put cache entries.
  241. for (it->SeekToFirst(); it->Valid(); it->Next()) {
  242. if (!IsCacheEntryKey(it->key()))
  243. continue;
  244. const std::string& id = GetIdFromCacheEntryKey(it->key());
  245. const auto iter_resource_id = local_id_to_resource_id.find(id);
  246. if (iter_resource_id == local_id_to_resource_id.end())
  247. continue;
  248. FileCacheEntry cache_entry;
  249. if (!cache_entry.ParseFromArray(it->value().data(), it->value().size()))
  250. return false;
  251. // Put cache state into a ResourceEntry.
  252. ResourceEntry entry;
  253. entry.set_local_id(id);
  254. entry.set_resource_id(iter_resource_id->second);
  255. *entry.mutable_file_specific_info()->mutable_cache_state() = cache_entry;
  256. std::string serialized_entry;
  257. if (!entry.SerializeToString(&serialized_entry)) {
  258. DLOG(ERROR) << "Failed to serialize the entry: " << id;
  259. return false;
  260. }
  261. batch.Put(id, serialized_entry);
  262. }
  263. if (!it->status().ok())
  264. return false;
  265. // Put header with the latest version number. This also clears
  266. // largest_changestamp and triggers refresh of metadata.
  267. std::string serialized_header;
  268. if (!GetDefaultHeaderEntry().SerializeToString(&serialized_header))
  269. return false;
  270. batch.Put(GetHeaderDBKey(), serialized_header);
  271. return resource_map->Write(leveldb::WriteOptions(), &batch).ok();
  272. }
  273. bool UpgradeOldDBVersion12(leveldb::DB* resource_map) {
  274. // Reuse all entries.
  275. leveldb::ReadOptions options;
  276. options.verify_checksums = true;
  277. std::unique_ptr<leveldb::Iterator> it(resource_map->NewIterator(options));
  278. // First, get local ID to resource ID map.
  279. std::map<std::string, std::string> local_id_to_resource_id;
  280. for (it->SeekToFirst(); it->Valid(); it->Next()) {
  281. if (IsIdEntryKey(it->key())) {
  282. local_id_to_resource_id[it->value().ToString()] =
  283. GetResourceIdFromIdEntryKey(it->key());
  284. }
  285. }
  286. if (!it->status().ok())
  287. return false;
  288. leveldb::WriteBatch batch;
  289. // Merge cache entries to ResourceEntry.
  290. for (it->SeekToFirst(); it->Valid(); it->Next()) {
  291. if (!IsCacheEntryKey(it->key()))
  292. continue;
  293. const std::string& id = GetIdFromCacheEntryKey(it->key());
  294. FileCacheEntry cache_entry;
  295. if (!cache_entry.ParseFromArray(it->value().data(), it->value().size()))
  296. return false;
  297. std::string serialized_entry;
  298. leveldb::Status status =
  299. resource_map->Get(options, leveldb::Slice(id), &serialized_entry);
  300. const auto iter_resource_id = local_id_to_resource_id.find(id);
  301. // No need to keep cache-only entries without resource ID.
  302. if (status.IsNotFound() &&
  303. iter_resource_id == local_id_to_resource_id.end())
  304. continue;
  305. ResourceEntry entry;
  306. if (status.ok()) {
  307. if (!entry.ParseFromString(serialized_entry))
  308. return false;
  309. } else if (status.IsNotFound()) {
  310. entry.set_local_id(id);
  311. entry.set_resource_id(iter_resource_id->second);
  312. } else {
  313. DLOG(ERROR) << "Failed to get the entry: " << id;
  314. return false;
  315. }
  316. *entry.mutable_file_specific_info()->mutable_cache_state() = cache_entry;
  317. if (!entry.SerializeToString(&serialized_entry)) {
  318. DLOG(ERROR) << "Failed to serialize the entry: " << id;
  319. return false;
  320. }
  321. batch.Delete(it->key());
  322. batch.Put(id, serialized_entry);
  323. }
  324. if (!it->status().ok())
  325. return false;
  326. // Put header with the latest version number. This also clears
  327. // largest_changestamp and triggers refresh of metadata.
  328. std::string serialized_header;
  329. if (!GetDefaultHeaderEntry().SerializeToString(&serialized_header))
  330. return false;
  331. batch.Put(GetHeaderDBKey(), serialized_header);
  332. return resource_map->Write(leveldb::WriteOptions(), &batch).ok();
  333. }
  334. bool UpgradeOldDBVersion13(leveldb::DB* resource_map) {
  335. // Before r272134, UpgradeOldDB() was not deleting unused ID entries.
  336. // Delete unused ID entries to fix crbug.com/374648.
  337. std::set<std::string> used_ids;
  338. std::unique_ptr<leveldb::Iterator> it(
  339. resource_map->NewIterator(leveldb::ReadOptions()));
  340. it->Seek(leveldb::Slice(GetHeaderDBKey()));
  341. it->Next();
  342. for (; it->Valid(); it->Next()) {
  343. if (IsCacheEntryKey(it->key()))
  344. used_ids.insert(GetIdFromCacheEntryKey(it->key()));
  345. else if (!IsChildEntryKey(it->key()) && !IsIdEntryKey(it->key()))
  346. used_ids.insert(it->key().ToString());
  347. }
  348. if (!it->status().ok())
  349. return false;
  350. leveldb::WriteBatch batch;
  351. for (it->SeekToFirst(); it->Valid(); it->Next()) {
  352. if (IsIdEntryKey(it->key()) && !used_ids.count(it->value().ToString()))
  353. batch.Delete(it->key());
  354. }
  355. if (!it->status().ok())
  356. return false;
  357. // Put header with the latest version number. This also clears
  358. // largest_changestamp and triggers refresh of metadata.
  359. std::string serialized_header;
  360. if (!GetDefaultHeaderEntry().SerializeToString(&serialized_header))
  361. return false;
  362. batch.Put(GetHeaderDBKey(), serialized_header);
  363. return resource_map->Write(leveldb::WriteOptions(), &batch).ok();
  364. }
  365. bool UpgradeOldDBVersion14(leveldb::DB* resource_map) {
  366. // Just need to clear largest_changestamp.
  367. // Put header with the latest version number.
  368. std::string serialized_header;
  369. if (!GetDefaultHeaderEntry().SerializeToString(&serialized_header))
  370. return false;
  371. leveldb::WriteBatch batch;
  372. batch.Put(GetHeaderDBKey(), serialized_header);
  373. return resource_map->Write(leveldb::WriteOptions(), &batch).ok();
  374. }
  375. bool UpgradeOldDBVersion15(leveldb::DB* resource_map) {
  376. leveldb::ReadOptions read_options;
  377. read_options.verify_checksums = true;
  378. leveldb::WriteBatch batch;
  379. std::unique_ptr<leveldb::Iterator> it(
  380. resource_map->NewIterator(read_options));
  381. it->SeekToFirst();
  382. ResourceMetadataHeader header;
  383. if (!it->Valid() || it->key() != GetHeaderDBKey()) {
  384. DLOG(ERROR) << "Header not detected.";
  385. return false;
  386. }
  387. if (!header.ParseFromArray(it->value().data(), it->value().size())) {
  388. DLOG(ERROR) << "Could not parse header.";
  389. return false;
  390. }
  391. header.set_version(ResourceMetadataStorage::kDBVersion);
  392. header.set_start_page_token(drive::util::ConvertChangestampToStartPageToken(
  393. header.largest_changestamp()));
  394. std::string serialized_header;
  395. header.SerializeToString(&serialized_header);
  396. batch.Put(GetHeaderDBKey(), serialized_header);
  397. for (it->Next(); it->Valid(); it->Next()) {
  398. if (IsIdEntryKey(it->key()))
  399. continue;
  400. ResourceEntry entry;
  401. if (!entry.ParseFromArray(it->value().data(), it->value().size()))
  402. return false;
  403. if (entry.has_directory_specific_info()) {
  404. int64_t changestamp = entry.directory_specific_info().changestamp();
  405. entry.mutable_directory_specific_info()->set_start_page_token(
  406. drive::util::ConvertChangestampToStartPageToken(changestamp));
  407. std::string serialized_entry;
  408. if (!entry.SerializeToString(&serialized_entry)) {
  409. DLOG(ERROR) << "Failed to serialize the entry";
  410. return false;
  411. }
  412. batch.Put(entry.local_id(), serialized_entry);
  413. }
  414. }
  415. return resource_map->Write(leveldb::WriteOptions(), &batch).ok();
  416. }
  417. bool UpgradeOldDBVersions16To18(leveldb::DB* resource_map) {
  418. // From 15->16, the field |alternate_url| was moved from FileSpecificData
  419. // to ResourceEntry. Since it isn't saved for directories, we need to do a
  420. // full fetch to get the |alternate_url| fetched for each directory.
  421. // Put a new header with the latest version number, and clear the start page
  422. // token.
  423. std::string serialized_header;
  424. if (!GetDefaultHeaderEntry().SerializeToString(&serialized_header))
  425. return false;
  426. leveldb::WriteBatch batch;
  427. batch.Put(GetHeaderDBKey(), serialized_header);
  428. return resource_map->Write(leveldb::WriteOptions(), &batch).ok();
  429. }
  430. } // namespace
  431. ResourceMetadataStorage::Iterator::Iterator(
  432. std::unique_ptr<leveldb::Iterator> it)
  433. : it_(std::move(it)) {
  434. base::ScopedBlockingCall scoped_blocking_call(FROM_HERE,
  435. base::BlockingType::MAY_BLOCK);
  436. DCHECK(it_);
  437. // Skip the header entry.
  438. // Note: The header entry comes before all other entries because its key
  439. // starts with kDBKeyDelimeter. (i.e. '\0')
  440. it_->Seek(leveldb::Slice(GetHeaderDBKey()));
  441. Advance();
  442. }
  443. ResourceMetadataStorage::Iterator::~Iterator() {
  444. base::ScopedBlockingCall scoped_blocking_call(FROM_HERE,
  445. base::BlockingType::MAY_BLOCK);
  446. }
  447. bool ResourceMetadataStorage::Iterator::IsAtEnd() const {
  448. base::ScopedBlockingCall scoped_blocking_call(FROM_HERE,
  449. base::BlockingType::MAY_BLOCK);
  450. return !it_->Valid();
  451. }
  452. std::string ResourceMetadataStorage::Iterator::GetID() const {
  453. return it_->key().ToString();
  454. }
  455. const ResourceEntry& ResourceMetadataStorage::Iterator::GetValue() const {
  456. base::ScopedBlockingCall scoped_blocking_call(FROM_HERE,
  457. base::BlockingType::MAY_BLOCK);
  458. DCHECK(!IsAtEnd());
  459. return entry_;
  460. }
  461. void ResourceMetadataStorage::Iterator::Advance() {
  462. base::ScopedBlockingCall scoped_blocking_call(FROM_HERE,
  463. base::BlockingType::MAY_BLOCK);
  464. DCHECK(!IsAtEnd());
  465. for (it_->Next() ; it_->Valid(); it_->Next()) {
  466. if (!IsChildEntryKey(it_->key()) &&
  467. !IsIdEntryKey(it_->key()) &&
  468. entry_.ParseFromArray(it_->value().data(), it_->value().size())) {
  469. break;
  470. }
  471. }
  472. }
  473. bool ResourceMetadataStorage::Iterator::HasError() const {
  474. base::ScopedBlockingCall scoped_blocking_call(FROM_HERE,
  475. base::BlockingType::MAY_BLOCK);
  476. return !it_->status().ok();
  477. }
  478. // static
  479. bool ResourceMetadataStorage::UpgradeOldDB(
  480. const base::FilePath& directory_path) {
  481. base::ScopedBlockingCall scoped_blocking_call(FROM_HERE,
  482. base::BlockingType::MAY_BLOCK);
  483. const base::FilePath resource_map_path =
  484. directory_path.Append(kResourceMapDBName);
  485. const base::FilePath preserved_resource_map_path =
  486. directory_path.Append(kPreservedResourceMapDBName);
  487. leveldb_env::Options options;
  488. options.max_open_files = 0; // Use minimum.
  489. options.create_if_missing = false;
  490. if (base::PathExists(preserved_resource_map_path)) {
  491. // Preserved DB is found. The previous attempt to create a new DB should not
  492. // be successful. Discard the imperfect new DB and restore the old DB.
  493. leveldb::Status status =
  494. leveldb_chrome::DeleteDB(resource_map_path, options);
  495. if (!status.ok()) {
  496. LOG(ERROR) << "ERROR deleting " << resource_map_path
  497. << ", err:" << status.ToString();
  498. return false;
  499. }
  500. if (!base::Move(preserved_resource_map_path, resource_map_path))
  501. return false;
  502. }
  503. if (!base::PathExists(resource_map_path))
  504. return false;
  505. // Open DB.
  506. std::unique_ptr<leveldb::DB> resource_map;
  507. leveldb::Status status = leveldb_env::OpenDB(
  508. options, resource_map_path.AsUTF8Unsafe(), &resource_map);
  509. if (!status.ok())
  510. return false;
  511. // Check DB version.
  512. std::string serialized_header;
  513. ResourceMetadataHeader header;
  514. if (!resource_map->Get(leveldb::ReadOptions(),
  515. leveldb::Slice(GetHeaderDBKey()),
  516. &serialized_header).ok() ||
  517. !header.ParseFromString(serialized_header))
  518. return false;
  519. base::UmaHistogramSparse("Drive.MetadataDBVersionBeforeUpgradeCheck",
  520. header.version());
  521. switch (header.version()) {
  522. case 1:
  523. case 2:
  524. case 3:
  525. case 4:
  526. case 5:
  527. return false; // Too old, nothing can be done.
  528. case 6:
  529. case 7:
  530. case 8:
  531. case 9:
  532. case 10:
  533. return UpgradeOldDBVersions6To10(resource_map.get());
  534. case 11:
  535. return UpgradeOldDBVersion11(resource_map.get());
  536. case 12:
  537. return UpgradeOldDBVersion12(resource_map.get());
  538. case 13:
  539. return UpgradeOldDBVersion13(resource_map.get());
  540. case 14:
  541. return UpgradeOldDBVersion14(resource_map.get());
  542. case 15:
  543. return UpgradeOldDBVersion15(resource_map.get());
  544. case 16:
  545. case 17:
  546. case 18:
  547. return UpgradeOldDBVersions16To18(resource_map.get());
  548. case kDBVersion:
  549. static_assert(
  550. kDBVersion == 19,
  551. "database version and this function must be updated together");
  552. return true;
  553. default:
  554. LOG(WARNING) << "Unexpected DB version: " << header.version();
  555. return false;
  556. }
  557. }
  558. ResourceMetadataStorage::ResourceMetadataStorage(
  559. const base::FilePath& directory_path,
  560. base::SequencedTaskRunner* blocking_task_runner)
  561. : directory_path_(directory_path),
  562. cache_file_scan_is_needed_(true),
  563. blocking_task_runner_(blocking_task_runner) {
  564. }
  565. bool ResourceMetadataStorage::Initialize() {
  566. base::ScopedBlockingCall scoped_blocking_call(FROM_HERE,
  567. base::BlockingType::MAY_BLOCK);
  568. resource_map_.reset();
  569. const base::FilePath resource_map_path =
  570. directory_path_.Append(kResourceMapDBName);
  571. const base::FilePath preserved_resource_map_path =
  572. directory_path_.Append(kPreservedResourceMapDBName);
  573. const base::FilePath trashed_resource_map_path =
  574. directory_path_.Append(kTrashedResourceMapDBName);
  575. leveldb_env::Options options;
  576. options.max_open_files = 0; // Use minimum.
  577. options.create_if_missing = false;
  578. // Discard unneeded DBs.
  579. if (!leveldb_chrome::DeleteDB(preserved_resource_map_path, options).ok() ||
  580. !leveldb_chrome::DeleteDB(trashed_resource_map_path, options).ok()) {
  581. LOG(ERROR) << "Failed to remove unneeded DBs.";
  582. return false;
  583. }
  584. // Try to open the existing DB.
  585. DBInitStatus open_existing_result = DB_INIT_NOT_FOUND;
  586. leveldb::Status status;
  587. if (base::PathExists(resource_map_path)) {
  588. status = leveldb_env::OpenDB(options, resource_map_path.AsUTF8Unsafe(),
  589. &resource_map_);
  590. open_existing_result = LevelDBStatusToDBInitStatus(status);
  591. }
  592. if (open_existing_result == DB_INIT_SUCCESS) {
  593. // Check the validity of existing DB.
  594. int db_version = -1;
  595. ResourceMetadataHeader header;
  596. if (GetHeader(&header) == FILE_ERROR_OK)
  597. db_version = header.version();
  598. bool should_discard_db = true;
  599. if (db_version != kDBVersion) {
  600. open_existing_result = DB_INIT_INCOMPATIBLE;
  601. DVLOG(1) << "Reject incompatible DB.";
  602. } else if (!CheckValidity()) {
  603. open_existing_result = DB_INIT_BROKEN;
  604. LOG(ERROR) << "Reject invalid DB.";
  605. } else {
  606. should_discard_db = false;
  607. }
  608. if (should_discard_db)
  609. resource_map_.reset();
  610. else
  611. cache_file_scan_is_needed_ = false;
  612. }
  613. UMA_HISTOGRAM_ENUMERATION("Drive.MetadataDBOpenExistingResult",
  614. open_existing_result,
  615. DB_INIT_MAX_VALUE);
  616. DBInitStatus init_result = DB_INIT_OPENED_EXISTING_DB;
  617. // Failed to open the existing DB, create new DB.
  618. if (!resource_map_) {
  619. // Move the existing DB to the preservation path. The moved old DB is
  620. // deleted once the new DB creation succeeds, or is restored later in
  621. // UpgradeOldDB() when the creation fails.
  622. MoveIfPossible(resource_map_path, preserved_resource_map_path);
  623. // Create DB.
  624. options = leveldb_env::Options();
  625. options.max_open_files = 0; // Use minimum.
  626. options.create_if_missing = true;
  627. options.error_if_exists = true;
  628. status = leveldb_env::OpenDB(options, resource_map_path.AsUTF8Unsafe(),
  629. &resource_map_);
  630. if (status.ok()) {
  631. // Set up header and trash the old DB.
  632. if (PutHeader(GetDefaultHeaderEntry()) == FILE_ERROR_OK &&
  633. MoveIfPossible(preserved_resource_map_path,
  634. trashed_resource_map_path)) {
  635. init_result = open_existing_result == DB_INIT_NOT_FOUND ?
  636. DB_INIT_CREATED_NEW_DB : DB_INIT_REPLACED_EXISTING_DB_WITH_NEW_DB;
  637. } else {
  638. init_result = DB_INIT_FAILED;
  639. resource_map_.reset();
  640. }
  641. } else {
  642. LOG(ERROR) << "Failed to create resource map DB: " << status.ToString();
  643. init_result = LevelDBStatusToDBInitStatus(status);
  644. }
  645. }
  646. UMA_HISTOGRAM_ENUMERATION("Drive.MetadataDBInitResult",
  647. init_result,
  648. DB_INIT_MAX_VALUE);
  649. return !!resource_map_;
  650. }
  651. void ResourceMetadataStorage::Destroy() {
  652. blocking_task_runner_->PostTask(
  653. FROM_HERE, base::BindOnce(&ResourceMetadataStorage::DestroyOnBlockingPool,
  654. base::Unretained(this)));
  655. }
  656. void ResourceMetadataStorage::RecoverCacheInfoFromTrashedResourceMap(
  657. RecoveredCacheInfoMap* out_info) {
  658. const base::FilePath trashed_resource_map_path =
  659. directory_path_.Append(kTrashedResourceMapDBName);
  660. if (!base::PathExists(trashed_resource_map_path))
  661. return;
  662. leveldb_env::Options options;
  663. options.max_open_files = 0; // Use minimum.
  664. options.create_if_missing = false;
  665. options.reuse_logs = false;
  666. // Trashed DB may be broken, repair it first.
  667. leveldb::Status status;
  668. status = leveldb::RepairDB(trashed_resource_map_path.AsUTF8Unsafe(), options);
  669. if (!status.ok()) {
  670. LOG(ERROR) << "Failed to repair trashed DB: " << status.ToString();
  671. return;
  672. }
  673. // Open it.
  674. std::unique_ptr<leveldb::DB> resource_map;
  675. status = leveldb_env::OpenDB(
  676. options, trashed_resource_map_path.AsUTF8Unsafe(), &resource_map);
  677. if (!status.ok()) {
  678. LOG(ERROR) << "Failed to open trashed DB: " << status.ToString();
  679. return;
  680. }
  681. // Check DB version.
  682. std::string serialized_header;
  683. ResourceMetadataHeader header;
  684. if (!resource_map->Get(leveldb::ReadOptions(),
  685. leveldb::Slice(GetHeaderDBKey()),
  686. &serialized_header).ok() ||
  687. !header.ParseFromString(serialized_header) ||
  688. header.version() != kDBVersion) {
  689. LOG(ERROR) << "Incompatible DB version: " << header.version();
  690. return;
  691. }
  692. // Collect cache entries.
  693. std::unique_ptr<leveldb::Iterator> it(
  694. resource_map->NewIterator(leveldb::ReadOptions()));
  695. for (it->SeekToFirst(); it->Valid(); it->Next()) {
  696. if (!IsChildEntryKey(it->key()) &&
  697. !IsIdEntryKey(it->key())) {
  698. const std::string id = it->key().ToString();
  699. ResourceEntry entry;
  700. if (entry.ParseFromArray(it->value().data(), it->value().size()) &&
  701. entry.file_specific_info().has_cache_state()) {
  702. RecoveredCacheInfo* info = &(*out_info)[id];
  703. info->is_dirty = entry.file_specific_info().cache_state().is_dirty();
  704. info->md5 = entry.file_specific_info().cache_state().md5();
  705. info->title = entry.title();
  706. }
  707. }
  708. }
  709. }
  710. FileError ResourceMetadataStorage::SetLargestChangestamp(
  711. int64_t largest_changestamp) {
  712. base::ScopedBlockingCall scoped_blocking_call(FROM_HERE,
  713. base::BlockingType::MAY_BLOCK);
  714. ResourceMetadataHeader header;
  715. FileError error = GetHeader(&header);
  716. if (error != FILE_ERROR_OK) {
  717. DLOG(ERROR) << "Failed to get the header.";
  718. return error;
  719. }
  720. header.set_largest_changestamp(largest_changestamp);
  721. return PutHeader(header);
  722. }
  723. FileError ResourceMetadataStorage::GetLargestChangestamp(
  724. int64_t* largest_changestamp) {
  725. base::ScopedBlockingCall scoped_blocking_call(FROM_HERE,
  726. base::BlockingType::MAY_BLOCK);
  727. ResourceMetadataHeader header;
  728. FileError error = GetHeader(&header);
  729. if (error != FILE_ERROR_OK) {
  730. DLOG(ERROR) << "Failed to get the header.";
  731. return error;
  732. }
  733. *largest_changestamp = header.largest_changestamp();
  734. return FILE_ERROR_OK;
  735. }
  736. FileError ResourceMetadataStorage::GetStartPageToken(
  737. std::string* start_page_token) {
  738. base::ScopedBlockingCall scoped_blocking_call(FROM_HERE,
  739. base::BlockingType::MAY_BLOCK);
  740. ResourceMetadataHeader header;
  741. FileError error = GetHeader(&header);
  742. if (error != FILE_ERROR_OK) {
  743. DLOG(ERROR) << "Failed to get the header.";
  744. return error;
  745. }
  746. *start_page_token = header.start_page_token();
  747. return FILE_ERROR_OK;
  748. }
  749. FileError ResourceMetadataStorage::SetStartPageToken(
  750. const std::string& start_page_token) {
  751. base::ScopedBlockingCall scoped_blocking_call(FROM_HERE,
  752. base::BlockingType::MAY_BLOCK);
  753. ResourceMetadataHeader header;
  754. FileError error = GetHeader(&header);
  755. if (error != FILE_ERROR_OK) {
  756. DLOG(ERROR) << "Failed to get the header.";
  757. return error;
  758. }
  759. header.set_start_page_token(start_page_token);
  760. return PutHeader(header);
  761. }
  762. FileError ResourceMetadataStorage::PutEntry(const ResourceEntry& entry) {
  763. base::ScopedBlockingCall scoped_blocking_call(FROM_HERE,
  764. base::BlockingType::MAY_BLOCK);
  765. const std::string& id = entry.local_id();
  766. DCHECK(!id.empty());
  767. // Try to get existing entry.
  768. std::string serialized_entry;
  769. leveldb::Status status = resource_map_->Get(leveldb::ReadOptions(),
  770. leveldb::Slice(id),
  771. &serialized_entry);
  772. if (!status.ok() && !status.IsNotFound()) // Unexpected errors.
  773. return LevelDBStatusToFileError(status);
  774. ResourceEntry old_entry;
  775. if (status.ok() && !old_entry.ParseFromString(serialized_entry))
  776. return FILE_ERROR_FAILED;
  777. // Construct write batch.
  778. leveldb::WriteBatch batch;
  779. // Remove from the old parent.
  780. if (!old_entry.parent_local_id().empty()) {
  781. batch.Delete(GetChildEntryKey(old_entry.parent_local_id(),
  782. old_entry.base_name()));
  783. }
  784. // Add to the new parent.
  785. if (!entry.parent_local_id().empty())
  786. batch.Put(GetChildEntryKey(entry.parent_local_id(), entry.base_name()), id);
  787. // Refresh resource-ID-to-local-ID mapping entry.
  788. if (old_entry.resource_id() != entry.resource_id()) {
  789. // Resource ID should not change.
  790. DCHECK(old_entry.resource_id().empty() || entry.resource_id().empty());
  791. if (!old_entry.resource_id().empty())
  792. batch.Delete(GetIdEntryKey(old_entry.resource_id()));
  793. if (!entry.resource_id().empty())
  794. batch.Put(GetIdEntryKey(entry.resource_id()), id);
  795. }
  796. // Put the entry itself.
  797. if (!entry.SerializeToString(&serialized_entry)) {
  798. DLOG(ERROR) << "Failed to serialize the entry: " << id;
  799. return FILE_ERROR_FAILED;
  800. }
  801. batch.Put(id, serialized_entry);
  802. status = resource_map_->Write(leveldb::WriteOptions(), &batch);
  803. return LevelDBStatusToFileError(status);
  804. }
  805. FileError ResourceMetadataStorage::GetEntry(const std::string& id,
  806. ResourceEntry* out_entry) {
  807. base::ScopedBlockingCall scoped_blocking_call(FROM_HERE,
  808. base::BlockingType::MAY_BLOCK);
  809. DCHECK(!id.empty());
  810. std::string serialized_entry;
  811. const leveldb::Status status = resource_map_->Get(leveldb::ReadOptions(),
  812. leveldb::Slice(id),
  813. &serialized_entry);
  814. if (!status.ok())
  815. return LevelDBStatusToFileError(status);
  816. if (!out_entry->ParseFromString(serialized_entry))
  817. return FILE_ERROR_FAILED;
  818. return FILE_ERROR_OK;
  819. }
  820. FileError ResourceMetadataStorage::RemoveEntry(const std::string& id) {
  821. base::ScopedBlockingCall scoped_blocking_call(FROM_HERE,
  822. base::BlockingType::MAY_BLOCK);
  823. DCHECK(!id.empty());
  824. ResourceEntry entry;
  825. FileError error = GetEntry(id, &entry);
  826. if (error != FILE_ERROR_OK)
  827. return error;
  828. leveldb::WriteBatch batch;
  829. // Remove from the parent.
  830. if (!entry.parent_local_id().empty())
  831. batch.Delete(GetChildEntryKey(entry.parent_local_id(), entry.base_name()));
  832. // Remove resource ID-local ID mapping entry.
  833. if (!entry.resource_id().empty())
  834. batch.Delete(GetIdEntryKey(entry.resource_id()));
  835. // Remove the entry itself.
  836. batch.Delete(id);
  837. const leveldb::Status status = resource_map_->Write(leveldb::WriteOptions(),
  838. &batch);
  839. return LevelDBStatusToFileError(status);
  840. }
  841. std::unique_ptr<ResourceMetadataStorage::Iterator>
  842. ResourceMetadataStorage::GetIterator() {
  843. base::ScopedBlockingCall scoped_blocking_call(FROM_HERE,
  844. base::BlockingType::MAY_BLOCK);
  845. std::unique_ptr<leveldb::Iterator> it(
  846. resource_map_->NewIterator(leveldb::ReadOptions()));
  847. return std::make_unique<Iterator>(std::move(it));
  848. }
  849. FileError ResourceMetadataStorage::GetChild(const std::string& parent_id,
  850. const std::string& child_name,
  851. std::string* child_id) const {
  852. base::ScopedBlockingCall scoped_blocking_call(FROM_HERE,
  853. base::BlockingType::MAY_BLOCK);
  854. DCHECK(!parent_id.empty());
  855. DCHECK(!child_name.empty());
  856. const leveldb::Status status =
  857. resource_map_->Get(
  858. leveldb::ReadOptions(),
  859. leveldb::Slice(GetChildEntryKey(parent_id, child_name)),
  860. child_id);
  861. return LevelDBStatusToFileError(status);
  862. }
  863. FileError ResourceMetadataStorage::GetChildren(
  864. const std::string& parent_id,
  865. std::vector<std::string>* children) const {
  866. base::ScopedBlockingCall scoped_blocking_call(FROM_HERE,
  867. base::BlockingType::MAY_BLOCK);
  868. DCHECK(!parent_id.empty());
  869. // Iterate over all entries with keys starting with |parent_id|.
  870. std::unique_ptr<leveldb::Iterator> it(
  871. resource_map_->NewIterator(leveldb::ReadOptions()));
  872. for (it->Seek(parent_id);
  873. it->Valid() && it->key().starts_with(leveldb::Slice(parent_id));
  874. it->Next()) {
  875. if (IsChildEntryKey(it->key()))
  876. children->push_back(it->value().ToString());
  877. }
  878. return LevelDBStatusToFileError(it->status());
  879. }
  880. FileError ResourceMetadataStorage::GetIdByResourceId(
  881. const std::string& resource_id,
  882. std::string* out_id) const {
  883. base::ScopedBlockingCall scoped_blocking_call(FROM_HERE,
  884. base::BlockingType::MAY_BLOCK);
  885. DCHECK(!resource_id.empty());
  886. const leveldb::Status status = resource_map_->Get(
  887. leveldb::ReadOptions(),
  888. leveldb::Slice(GetIdEntryKey(resource_id)),
  889. out_id);
  890. return LevelDBStatusToFileError(status);
  891. }
  892. ResourceMetadataStorage::RecoveredCacheInfo::RecoveredCacheInfo()
  893. : is_dirty(false) {}
  894. ResourceMetadataStorage::RecoveredCacheInfo::~RecoveredCacheInfo() = default;
  895. ResourceMetadataStorage::~ResourceMetadataStorage() {
  896. base::ScopedBlockingCall scoped_blocking_call(FROM_HERE,
  897. base::BlockingType::MAY_BLOCK);
  898. }
  899. void ResourceMetadataStorage::DestroyOnBlockingPool() {
  900. delete this;
  901. }
  902. // static
  903. std::string ResourceMetadataStorage::GetChildEntryKey(
  904. const std::string& parent_id,
  905. const std::string& child_name) {
  906. DCHECK(!parent_id.empty());
  907. DCHECK(!child_name.empty());
  908. std::string key = parent_id;
  909. key.push_back(kDBKeyDelimeter);
  910. key.append(child_name);
  911. key.push_back(kDBKeyDelimeter);
  912. return key;
  913. }
  914. FileError ResourceMetadataStorage::PutHeader(
  915. const ResourceMetadataHeader& header) {
  916. base::ScopedBlockingCall scoped_blocking_call(FROM_HERE,
  917. base::BlockingType::MAY_BLOCK);
  918. std::string serialized_header;
  919. if (!header.SerializeToString(&serialized_header)) {
  920. DLOG(ERROR) << "Failed to serialize the header";
  921. return FILE_ERROR_FAILED;
  922. }
  923. const leveldb::Status status = resource_map_->Put(
  924. leveldb::WriteOptions(),
  925. leveldb::Slice(GetHeaderDBKey()),
  926. leveldb::Slice(serialized_header));
  927. return LevelDBStatusToFileError(status);
  928. }
  929. FileError ResourceMetadataStorage::GetHeader(
  930. ResourceMetadataHeader* header) const {
  931. base::ScopedBlockingCall scoped_blocking_call(FROM_HERE,
  932. base::BlockingType::MAY_BLOCK);
  933. std::string serialized_header;
  934. const leveldb::Status status = resource_map_->Get(
  935. leveldb::ReadOptions(),
  936. leveldb::Slice(GetHeaderDBKey()),
  937. &serialized_header);
  938. if (!status.ok())
  939. return LevelDBStatusToFileError(status);
  940. return header->ParseFromString(serialized_header) ?
  941. FILE_ERROR_OK : FILE_ERROR_FAILED;
  942. }
  943. bool ResourceMetadataStorage::CheckValidity() {
  944. base::ScopedBlockingCall scoped_blocking_call(FROM_HERE,
  945. base::BlockingType::MAY_BLOCK);
  946. // Perform read with checksums verification enabled.
  947. leveldb::ReadOptions options;
  948. options.verify_checksums = true;
  949. std::unique_ptr<leveldb::Iterator> it(resource_map_->NewIterator(options));
  950. it->SeekToFirst();
  951. // DB is organized like this:
  952. //
  953. // <key> : <value>
  954. // "\0HEADER" : ResourceMetadataHeader
  955. // "\0ID\0|resource ID 1|" : Local ID associated to resource ID 1.
  956. // "\0ID\0|resource ID 2|" : Local ID associated to resource ID 2.
  957. // ...
  958. // "|ID of A|" : ResourceEntry for entry A.
  959. // "|ID of A|\0|child name 1|\0" : ID of the 1st child entry of entry A.
  960. // "|ID of A|\0|child name 2|\0" : ID of the 2nd child entry of entry A.
  961. // ...
  962. // "|ID of A|\0|child name n|\0" : ID of the nth child entry of entry A.
  963. // "|ID of B|" : ResourceEntry for entry B.
  964. // ...
  965. // Check the header.
  966. ResourceMetadataHeader header;
  967. if (!it->Valid() ||
  968. it->key() != GetHeaderDBKey() || // Header entry must come first.
  969. !header.ParseFromArray(it->value().data(), it->value().size()) ||
  970. header.version() != kDBVersion) {
  971. DLOG(ERROR) << "Invalid header detected. version = " << header.version();
  972. RecordCheckValidityFailure(CHECK_VALIDITY_FAILURE_INVALID_HEADER);
  973. return false;
  974. }
  975. // First scan. Remember relationships between IDs.
  976. typedef std::unordered_map<std::string, std::string> KeyToIdMapping;
  977. KeyToIdMapping local_id_to_resource_id_map;
  978. KeyToIdMapping child_key_to_local_id_map;
  979. std::set<std::string> resource_entries;
  980. std::string first_resource_entry_key;
  981. for (it->Next(); it->Valid(); it->Next()) {
  982. if (IsChildEntryKey(it->key())) {
  983. child_key_to_local_id_map[it->key().ToString()] = it->value().ToString();
  984. continue;
  985. }
  986. if (IsIdEntryKey(it->key())) {
  987. const auto result = local_id_to_resource_id_map.insert(std::make_pair(
  988. it->value().ToString(),
  989. GetResourceIdFromIdEntryKey(it->key().ToString())));
  990. // Check that no local ID is associated with more than one resource ID.
  991. if (!result.second) {
  992. DLOG(ERROR) << "Broken ID entry.";
  993. RecordCheckValidityFailure(CHECK_VALIDITY_FAILURE_BROKEN_ID_ENTRY);
  994. return false;
  995. }
  996. continue;
  997. }
  998. // Remember the key of the first resource entry record, so the second scan
  999. // can start from this point.
  1000. if (first_resource_entry_key.empty())
  1001. first_resource_entry_key = it->key().ToString();
  1002. resource_entries.insert(it->key().ToString());
  1003. }
  1004. // Second scan. Verify relationships and resource entry correctness.
  1005. size_t num_entries_with_parent = 0;
  1006. ResourceEntry entry;
  1007. for (it->Seek(first_resource_entry_key); it->Valid(); it->Next()) {
  1008. if (IsChildEntryKey(it->key()))
  1009. continue;
  1010. if (!entry.ParseFromArray(it->value().data(), it->value().size())) {
  1011. DLOG(ERROR) << "Broken entry detected.";
  1012. RecordCheckValidityFailure(CHECK_VALIDITY_FAILURE_BROKEN_ENTRY);
  1013. return false;
  1014. }
  1015. // Resource-ID-to-local-ID mapping without entry for the local ID is OK,
  1016. // but if it exists, then the resource ID must be consistent.
  1017. const auto mapping_it =
  1018. local_id_to_resource_id_map.find(it->key().ToString());
  1019. if (mapping_it != local_id_to_resource_id_map.end() &&
  1020. entry.resource_id() != mapping_it->second) {
  1021. DLOG(ERROR) << "Broken ID entry.";
  1022. RecordCheckValidityFailure(CHECK_VALIDITY_FAILURE_BROKEN_ID_ENTRY);
  1023. return false;
  1024. }
  1025. // If the parent is referenced, then confirm that it exists and check the
  1026. // parent-child relationships.
  1027. if (!entry.parent_local_id().empty()) {
  1028. const auto parent_mapping_it =
  1029. resource_entries.find(entry.parent_local_id());
  1030. if (parent_mapping_it == resource_entries.end()) {
  1031. DLOG(ERROR) << "Parent entry not found.";
  1032. RecordCheckValidityFailure(CHECK_VALIDITY_FAILURE_INVALID_PARENT_ID);
  1033. return false;
  1034. }
  1035. // Check if parent-child relationship is stored correctly.
  1036. const auto child_mapping_it = child_key_to_local_id_map.find(
  1037. GetChildEntryKey(entry.parent_local_id(), entry.base_name()));
  1038. if (child_mapping_it == child_key_to_local_id_map.end() ||
  1039. leveldb::Slice(child_mapping_it->second) != it->key()) {
  1040. DLOG(ERROR) << "Child map is broken.";
  1041. RecordCheckValidityFailure(CHECK_VALIDITY_FAILURE_BROKEN_CHILD_MAP);
  1042. return false;
  1043. }
  1044. ++num_entries_with_parent;
  1045. }
  1046. }
  1047. if (!it->status().ok()) {
  1048. DLOG(ERROR) << "Error during checking resource map. status = "
  1049. << it->status().ToString();
  1050. RecordCheckValidityFailure(CHECK_VALIDITY_FAILURE_ITERATOR_ERROR);
  1051. return false;
  1052. }
  1053. if (child_key_to_local_id_map.size() != num_entries_with_parent) {
  1054. DLOG(ERROR) << "Child entry count mismatch.";
  1055. RecordCheckValidityFailure(
  1056. CHECK_VALIDITY_FAILURE_CHILD_ENTRY_COUNT_MISMATCH);
  1057. return false;
  1058. }
  1059. return true;
  1060. }
  1061. } // namespace internal
  1062. } // namespace drive