12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493 |
- // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
- // Use of this source code is governed by a BSD-style license that can be
- // found in the LICENSE file. See the AUTHORS file for names of contributors.
- #include "third_party/leveldatabase/env_chromium.h"
- #include <atomic>
- #include <iterator>
- #include <limits>
- #include <utility>
- #include "base/bind.h"
- #include "base/check_op.h"
- #include "base/files/file_error_or.h"
- #include "base/files/file_util.h"
- #include "base/format_macros.h"
- #include "base/metrics/histogram_functions.h"
- #include "base/no_destructor.h"
- #include "base/notreached.h"
- #include "base/process/process_metrics.h"
- #include "base/strings/string_number_conversions.h"
- #include "base/strings/string_util.h"
- #include "base/strings/stringprintf.h"
- #include "base/strings/utf_string_conversions.h"
- #include "base/system/sys_info.h"
- #include "base/task/task_traits.h"
- #include "base/task/thread_pool.h"
- #include "base/time/time.h"
- #include "base/time/time_override.h"
- #include "base/trace_event/memory_dump_manager.h"
- #include "base/trace_event/memory_dump_provider.h"
- #include "base/trace_event/process_memory_dump.h"
- #include "base/trace_event/trace_event.h"
- #include "build/build_config.h"
- #include "build/chromeos_buildflags.h"
- #include "components/services/storage/public/cpp/filesystem/filesystem_proxy.h"
- #include "third_party/leveldatabase/chromium_logger.h"
- #include "third_party/leveldatabase/leveldb_chrome.h"
- #include "third_party/leveldatabase/port/port_chromium.h"
- #include "third_party/leveldatabase/src/include/leveldb/options.h"
- #include "third_party/re2/src/re2/re2.h"
- using base::FilePath;
- using base::trace_event::MemoryAllocatorDump;
- using base::trace_event::MemoryDumpArgs;
- using base::trace_event::ProcessMemoryDump;
- using leveldb::FileLock;
- using leveldb::Slice;
- using leveldb::Status;
- namespace leveldb_env {
- namespace {
- template <typename ValueType>
- using FileErrorOr = base::FileErrorOr<ValueType>;
- // After this limit we don't bother doing file eviction for leveldb for speed,
- // memory usage, and simplicity.
- const constexpr size_t kFileLimitToDisableEviction = 10'000;
- // The maximum time for the |Retrier| to indicate that an operation should
- // be retried.
- constexpr auto kMaxRetryDuration = base::Milliseconds(1000);
- const FilePath::CharType table_extension[] = FILE_PATH_LITERAL(".ldb");
- static const FilePath::CharType kLevelDBTestDirectoryPrefix[] =
- FILE_PATH_LITERAL("leveldb-test-");
- // This name should not be changed or users involved in a crash might not be
- // able to recover data.
- static const char kDatabaseNameSuffixForRebuildDB[] = "__tmp_for_rebuild";
- class ChromiumFileLock : public FileLock {
- public:
- ChromiumFileLock(std::unique_ptr<storage::FilesystemProxy::FileLock> lock,
- const std::string& name)
- : lock(std::move(lock)), name(name) {}
- ChromiumFileLock(const ChromiumFileLock&) = delete;
- ChromiumFileLock& operator=(const ChromiumFileLock&) = delete;
- const std::unique_ptr<storage::FilesystemProxy::FileLock> lock;
- const std::string name;
- };
- class Retrier {
- public:
- Retrier()
- // TODO(crbug.com/1059965): figure out a better way to handle time for
- // tests.
- : start_(base::subtle::TimeTicksNowIgnoringOverride()),
- limit_(start_ + kMaxRetryDuration),
- last_(start_),
- time_to_sleep_(base::Milliseconds(10)) {}
- Retrier(const Retrier&) = delete;
- Retrier& operator=(const Retrier&) = delete;
- ~Retrier() = default;
- bool ShouldKeepTrying() {
- if (last_ < limit_) {
- base::PlatformThread::Sleep(time_to_sleep_);
- // TODO(crbug.com/1059965): figure out a better way to handle time for
- // tests.
- last_ = base::subtle::TimeTicksNowIgnoringOverride();
- return true;
- }
- return false;
- }
- private:
- const base::TimeTicks start_;
- base::TimeTicks limit_;
- base::TimeTicks last_;
- base::TimeDelta time_to_sleep_;
- };
- class ChromiumSequentialFile : public leveldb::SequentialFile {
- public:
- ChromiumSequentialFile(const std::string& fname, base::File f)
- : filename_(fname), file_(std::move(f)) {}
- ChromiumSequentialFile(const ChromiumSequentialFile&) = delete;
- ChromiumSequentialFile& operator=(const ChromiumSequentialFile&) = delete;
- ~ChromiumSequentialFile() override = default;
- // Note: This method is relatively hot during leveldb database
- // compaction. Please avoid making them slower.
- Status Read(size_t n, Slice* result, char* scratch) override {
- int bytes_read = file_.ReadAtCurrentPosNoBestEffort(scratch, n);
- if (bytes_read == -1) {
- base::File::Error error = base::File::GetLastFileError();
- return MakeIOError(filename_, base::File::ErrorToString(error),
- kSequentialFileRead, error);
- }
- *result = Slice(scratch, bytes_read);
- return Status::OK();
- }
- Status Skip(uint64_t n) override {
- if (file_.Seek(base::File::FROM_CURRENT, n) == -1) {
- base::File::Error error = base::File::GetLastFileError();
- return MakeIOError(filename_, base::File::ErrorToString(error),
- kSequentialFileSkip, error);
- } else {
- return Status::OK();
- }
- }
- private:
- std::string filename_;
- base::File file_;
- };
- void RemoveFile(const Slice& key, void* value) {
- delete static_cast<base::File*>(value);
- }
- Status ReadFromFileToScratch(uint64_t offset,
- size_t n,
- Slice* result,
- char* scratch,
- base::File* file,
- const base::FilePath& file_path) {
- int bytes_read = file->Read(offset, scratch, n);
- if (bytes_read < 0) {
- return MakeIOError(file_path.AsUTF8Unsafe(), "Could not perform read",
- kRandomAccessFileRead);
- }
- *result = Slice(scratch, (bytes_read < 0) ? 0 : bytes_read);
- return Status::OK();
- }
- // The cache mechanism uses leveldb's LRU cache, which is a threadsafe sharded
- // LRU cache. The keys use the pointer value of |this|, and the values are file
- // objects.
- // Each object will only use its own cache entry, so the |Erase| call in the
- // object destructor should synchronously delete the file from the cache. This
- // ensures that pointer location re-use won't re-use an entry in the cache as
- // the entry at |this| will always have been deleted.
- // Files are always cleaned up with |RemoveFile|, which will be called when the
- // ChromiumEvictableRandomAccessFile is deleted, the cache is deleted, or the
- // file is evicted.
- class ChromiumEvictableRandomAccessFile : public leveldb::RandomAccessFile {
- public:
- ChromiumEvictableRandomAccessFile(base::FilePath file_path,
- base::File file,
- storage::FilesystemProxy* filesystem,
- leveldb::Cache* file_cache)
- : filepath_(std::move(file_path)),
- filesystem_(filesystem),
- file_cache_(file_cache),
- cache_key_data_(this),
- cache_key_(
- leveldb::Slice(reinterpret_cast<const char*>(&cache_key_data_),
- sizeof(cache_key_data_))) {
- DCHECK(file_cache_);
- base::File* heap_file = new base::File(std::move(file));
- // A |charge| of '1' is used because the capacity is the file handle limit,
- // and each entry is one file.
- file_cache_->Release(file_cache_->Insert(cache_key_, heap_file,
- 1 /* charge */, &RemoveFile));
- }
- ChromiumEvictableRandomAccessFile(const ChromiumEvictableRandomAccessFile&) =
- delete;
- ChromiumEvictableRandomAccessFile& operator=(
- const ChromiumEvictableRandomAccessFile&) = delete;
- virtual ~ChromiumEvictableRandomAccessFile() {
- file_cache_->Erase(cache_key_);
- }
- // Note: This method is relatively hot during leveldb database
- // compaction. Please avoid making them slower.
- Status Read(uint64_t offset,
- size_t n,
- Slice* result,
- char* scratch) const override {
- leveldb::Cache::Handle* handle = file_cache_->Lookup(cache_key_);
- if (!handle) {
- int flags = base::File::FLAG_READ | base::File::FLAG_OPEN;
- FileErrorOr<base::File> result = filesystem_->OpenFile(filepath_, flags);
- if (result.is_error()) {
- return MakeIOError(filepath_.AsUTF8Unsafe(), "Could not perform read",
- kRandomAccessFileRead);
- }
- handle = file_cache_->Insert(cache_key_,
- new base::File(std::move(result.value())),
- sizeof(base::File), &RemoveFile);
- }
- base::File* file = static_cast<base::File*>(file_cache_->Value(handle));
- Status status =
- ReadFromFileToScratch(offset, n, result, scratch, file, filepath_);
- file_cache_->Release(handle);
- return status;
- }
- private:
- const base::FilePath filepath_;
- storage::FilesystemProxy* const filesystem_;
- mutable leveldb::Cache* file_cache_;
- const ChromiumEvictableRandomAccessFile* cache_key_data_;
- leveldb::Slice cache_key_;
- };
- class ChromiumRandomAccessFile : public leveldb::RandomAccessFile {
- public:
- ChromiumRandomAccessFile(base::FilePath file_path, base::File file)
- : filepath_(std::move(file_path)), file_(std::move(file)) {}
- ChromiumRandomAccessFile(const ChromiumRandomAccessFile&) = delete;
- ChromiumRandomAccessFile& operator=(const ChromiumRandomAccessFile&) = delete;
- virtual ~ChromiumRandomAccessFile() {}
- // Note: This method is relatively hot during leveldb database
- // compaction. Please avoid making them slower.
- Status Read(uint64_t offset,
- size_t n,
- Slice* result,
- char* scratch) const override {
- return ReadFromFileToScratch(offset, n, result, scratch, &file_, filepath_);
- }
- private:
- const base::FilePath filepath_;
- mutable base::File file_;
- };
- class ChromiumWritableFile : public leveldb::WritableFile {
- public:
- ChromiumWritableFile(const std::string& fname,
- base::File f,
- storage::FilesystemProxy* filesystem);
- ChromiumWritableFile(const ChromiumWritableFile&) = delete;
- ChromiumWritableFile& operator=(const ChromiumWritableFile&) = delete;
- ~ChromiumWritableFile() override = default;
- leveldb::Status Append(const leveldb::Slice& data) override;
- leveldb::Status Close() override;
- leveldb::Status Flush() override;
- leveldb::Status Sync() override;
- private:
- enum Type { kManifest, kTable, kOther };
- leveldb::Status SyncParent();
- std::string filename_;
- base::File file_;
- #if defined(OS_POSIX) || defined(OS_FUCHSIA)
- storage::FilesystemProxy* const filesystem_;
- #endif
- Type file_type_;
- std::string parent_dir_;
- };
- ChromiumWritableFile::ChromiumWritableFile(const std::string& fname,
- base::File f,
- storage::FilesystemProxy* filesystem)
- : filename_(fname),
- file_(std::move(f)),
- #if defined(OS_POSIX) || defined(OS_FUCHSIA)
- filesystem_(filesystem),
- #endif
- file_type_(kOther) {
- FilePath path = FilePath::FromUTF8Unsafe(fname);
- if (path.BaseName().AsUTF8Unsafe().find("MANIFEST") == 0)
- file_type_ = kManifest;
- else if (path.MatchesExtension(table_extension))
- file_type_ = kTable;
- parent_dir_ = FilePath::FromUTF8Unsafe(fname).DirName().AsUTF8Unsafe();
- }
- Status ChromiumWritableFile::SyncParent() {
- TRACE_EVENT0("leveldb", "SyncParent");
- #if defined(OS_POSIX) || defined(OS_FUCHSIA)
- FilePath path = FilePath::FromUTF8Unsafe(parent_dir_);
- FileErrorOr<base::File> result = filesystem_->OpenFile(
- path, base::File::FLAG_OPEN | base::File::FLAG_READ);
- if (result.is_error()) {
- return MakeIOError(parent_dir_, "Unable to open directory", kSyncParent,
- result.error());
- }
- if (!result->Flush()) {
- base::File::Error error = base::File::GetLastFileError();
- return MakeIOError(parent_dir_, base::File::ErrorToString(error),
- kSyncParent, error);
- }
- #endif
- return Status::OK();
- }
- Status ChromiumWritableFile::Append(const Slice& data) {
- DCHECK(file_.IsValid());
- int bytes_written = file_.WriteAtCurrentPos(data.data(), data.size());
- if (static_cast<size_t>(bytes_written) != data.size()) {
- base::File::Error error = base::File::GetLastFileError();
- return MakeIOError(filename_, base::File::ErrorToString(error),
- kWritableFileAppend, error);
- }
- return Status::OK();
- }
- Status ChromiumWritableFile::Close() {
- file_.Close();
- return Status::OK();
- }
- Status ChromiumWritableFile::Flush() {
- // base::File doesn't do buffered I/O (i.e. POSIX FILE streams) so nothing to
- // flush.
- return Status::OK();
- }
- Status ChromiumWritableFile::Sync() {
- TRACE_EVENT0("leveldb", "WritableFile::Sync");
- // leveldb's implicit contract for Sync() is that if this instance is for a
- // manifest file then the directory is also sync'ed, to ensure new files
- // referred to by the manifest are in the filesystem.
- //
- // This needs to happen before the manifest file is flushed to disk, to
- // avoid crashing in a state where the manifest refers to files that are not
- // yet on disk.
- //
- // See leveldb's env_posix.cc.
- if (file_type_ == kManifest) {
- Status status = SyncParent();
- if (!status.ok())
- return status;
- }
- if (!file_.Flush()) {
- base::File::Error error = base::File::GetLastFileError();
- return MakeIOError(filename_, base::File::ErrorToString(error),
- kWritableFileSync, error);
- }
- return Status::OK();
- }
- // Return the maximum number of read-only files to keep open.
- size_t GetLevelDBFileLimit(size_t max_file_descriptors) {
- // Allow use of 20% of available file descriptors for read-only files.
- return max_file_descriptors / 5;
- }
- std::string GetDumpNameForDB(const leveldb::DB* db) {
- return base::StringPrintf("leveldatabase/db_0x%" PRIXPTR,
- reinterpret_cast<uintptr_t>(db));
- }
- std::string GetDumpNameForCache(DBTracker::SharedReadCacheUse cache) {
- switch (cache) {
- case DBTracker::SharedReadCacheUse_Browser:
- return "leveldatabase/block_cache/browser";
- case DBTracker::SharedReadCacheUse_Web:
- return "leveldatabase/block_cache/web";
- case DBTracker::SharedReadCacheUse_Unified:
- return "leveldatabase/block_cache/unified";
- case DBTracker::SharedReadCacheUse_InMemory:
- return "leveldatabase/block_cache/in_memory";
- case DBTracker::SharedReadCacheUse_NumCacheUses:
- NOTREACHED();
- }
- NOTREACHED();
- return "";
- }
- MemoryAllocatorDump* CreateDumpMalloced(ProcessMemoryDump* pmd,
- const std::string& name,
- size_t size) {
- auto* dump = pmd->CreateAllocatorDump(name);
- dump->AddScalar(MemoryAllocatorDump::kNameSize,
- MemoryAllocatorDump::kUnitsBytes, size);
- static const char* system_allocator_name =
- base::trace_event::MemoryDumpManager::GetInstance()
- ->system_allocator_pool_name();
- if (system_allocator_name)
- pmd->AddSuballocation(dump->guid(), system_allocator_name);
- return dump;
- }
- void RecordCacheUsageInTracing(ProcessMemoryDump* pmd,
- DBTracker::SharedReadCacheUse cache) {
- std::string name = GetDumpNameForCache(cache);
- leveldb::Cache* cache_ptr = nullptr;
- switch (cache) {
- case DBTracker::SharedReadCacheUse_Browser:
- cache_ptr = leveldb_chrome::GetSharedBrowserBlockCache();
- break;
- case DBTracker::SharedReadCacheUse_Web:
- cache_ptr = leveldb_chrome::GetSharedWebBlockCache();
- break;
- case DBTracker::SharedReadCacheUse_Unified:
- cache_ptr = leveldb_chrome::GetSharedBrowserBlockCache();
- break;
- case DBTracker::SharedReadCacheUse_InMemory:
- cache_ptr = leveldb_chrome::GetSharedInMemoryBlockCache();
- break;
- case DBTracker::SharedReadCacheUse_NumCacheUses:
- NOTREACHED();
- }
- if (!cache_ptr)
- return;
- CreateDumpMalloced(pmd, name, cache_ptr->TotalCharge());
- }
- } // namespace
- Options::Options() {
- // Note: Ensure that these default values correspond to those in
- // components/services/leveldb/public/mojom/leveldb.mojom.
- // TODO(cmumford) Create struct-trait for leveldb.mojom.OpenOptions to force
- // users to pass in a leveldb_env::Options instance (and it's defaults).
- //
- // Currently log reuse is an experimental feature in leveldb. More info at:
- // https://github.com/google/leveldb/commit/251ebf5dc70129ad3
- #if BUILDFLAG(IS_CHROMEOS_ASH)
- // Reusing logs on Chrome OS resulted in an unacceptably high leveldb
- // corruption rate (at least for Indexed DB). More info at
- // https://crbug.com/460568
- reuse_logs = false;
- #else
- // Low end devices have limited RAM. Reusing logs will prevent the database
- // from being compacted on open and instead load the log file back into the
- // memory buffer which won't be written until it hits the maximum size
- // (leveldb::Options::write_buffer_size - 4MB by default). The downside here
- // is that databases opens take longer as the open is blocked on compaction.
- reuse_logs = !base::SysInfo::IsLowEndDevice();
- #endif
- // By default use a single shared block cache to conserve memory. The owner of
- // this object can create their own, or set to NULL to have leveldb create a
- // new db-specific block cache.
- block_cache = leveldb_chrome::GetSharedBrowserBlockCache();
- }
- const char* MethodIDToString(MethodID method) {
- switch (method) {
- case kSequentialFileRead:
- return "SequentialFileRead";
- case kSequentialFileSkip:
- return "SequentialFileSkip";
- case kRandomAccessFileRead:
- return "RandomAccessFileRead";
- case kWritableFileAppend:
- return "WritableFileAppend";
- case kWritableFileClose:
- return "WritableFileClose";
- case kWritableFileFlush:
- return "WritableFileFlush";
- case kWritableFileSync:
- return "WritableFileSync";
- case kNewSequentialFile:
- return "NewSequentialFile";
- case kNewRandomAccessFile:
- return "NewRandomAccessFile";
- case kNewWritableFile:
- return "NewWritableFile";
- case kNewAppendableFile:
- return "NewAppendableFile";
- case kCreateDir:
- return "CreateDir";
- case kGetFileSize:
- return "GetFileSize";
- case kRenameFile:
- return "RenameFile";
- case kLockFile:
- return "LockFile";
- case kUnlockFile:
- return "UnlockFile";
- case kGetTestDirectory:
- return "GetTestDirectory";
- case kNewLogger:
- return "NewLogger";
- case kSyncParent:
- return "SyncParent";
- case kGetChildren:
- return "GetChildren";
- case kRemoveFile:
- return "RemoveFile";
- case kRemoveDir:
- return "RemoveDir";
- case kObsoleteDeleteFile:
- case kObsoleteDeleteDir:
- case kNumEntries:
- NOTREACHED();
- return "Unknown";
- }
- NOTREACHED();
- return "Unknown";
- }
- Status MakeIOError(Slice filename,
- const std::string& message,
- MethodID method,
- base::File::Error error) {
- DCHECK_LT(error, 0);
- char buf[512];
- base::snprintf(buf, sizeof(buf), "%s (ChromeMethodBFE: %d::%s::%d)",
- message.c_str(), method, MethodIDToString(method), -error);
- // TOOD(crbug.com/760362): Map base::File::FILE_ERROR_NOT_FOUND to
- // Status::NotFound, after fixing LevelDB to handle
- // the NotFound correctly.
- return Status::IOError(filename, buf);
- }
- Status MakeIOError(Slice filename,
- const std::string& message,
- MethodID method) {
- char buf[512];
- base::snprintf(buf, sizeof(buf), "%s (ChromeMethodOnly: %d::%s)",
- message.c_str(), method, MethodIDToString(method));
- return Status::IOError(filename, buf);
- }
- ErrorParsingResult ParseMethodAndError(const leveldb::Status& status,
- MethodID* method_param,
- base::File::Error* error) {
- const std::string status_string = status.ToString();
- int method;
- if (RE2::PartialMatch(status_string.c_str(), "ChromeMethodOnly: (\\d+)",
- &method)) {
- *method_param = static_cast<MethodID>(method);
- return METHOD_ONLY;
- }
- int parsed_error;
- if (RE2::PartialMatch(status_string.c_str(),
- "ChromeMethodBFE: (\\d+)::.*::(\\d+)", &method,
- &parsed_error)) {
- *method_param = static_cast<MethodID>(method);
- *error = static_cast<base::File::Error>(-parsed_error);
- DCHECK_LT(*error, base::File::FILE_OK);
- DCHECK_GT(*error, base::File::FILE_ERROR_MAX);
- return METHOD_AND_BFE;
- }
- return NONE;
- }
- // Keep in sync with LevelDBCorruptionTypes in histograms.xml. Also, don't
- // change the order because indices into this array have been recorded in uma
- // histograms.
- const char* patterns[] = {
- "missing files",
- "log record too small",
- "corrupted internal key",
- "partial record",
- "missing start of fragmented record",
- "error in middle of record",
- "unknown record type",
- "truncated record at end",
- "bad record length",
- "VersionEdit",
- "FileReader invoked with unexpected value",
- "corrupted key",
- "CURRENT file does not end with newline",
- "no meta-nextfile entry",
- "no meta-lognumber entry",
- "no last-sequence-number entry",
- "malformed WriteBatch",
- "bad WriteBatch Put",
- "bad WriteBatch Delete",
- "unknown WriteBatch tag",
- "WriteBatch has wrong count",
- "bad entry in block",
- "bad block contents",
- "bad block handle",
- "truncated block read",
- "block checksum mismatch",
- "checksum mismatch",
- "corrupted compressed block contents",
- "bad block type",
- "bad magic number",
- "file is too short",
- };
- // Returns 1-based index into the above array or 0 if nothing matches.
- int GetCorruptionCode(const leveldb::Status& status) {
- DCHECK(!status.IsIOError());
- DCHECK(!status.ok());
- const int kOtherError = 0;
- int error = kOtherError;
- const std::string& str_error = status.ToString();
- const size_t kNumPatterns = std::size(patterns);
- for (size_t i = 0; i < kNumPatterns; ++i) {
- if (str_error.find(patterns[i]) != std::string::npos) {
- error = i + 1;
- break;
- }
- }
- return error;
- }
- int GetNumCorruptionCodes() {
- // + 1 for the "other" error that is returned when a corruption message
- // doesn't match any of the patterns.
- return std::size(patterns) + 1;
- }
- std::string GetCorruptionMessage(const leveldb::Status& status) {
- int code = GetCorruptionCode(status);
- if (code == 0)
- return "Unknown corruption";
- return patterns[code - 1];
- }
- bool IndicatesDiskFull(const leveldb::Status& status) {
- if (status.ok())
- return false;
- leveldb_env::MethodID method;
- base::File::Error error = base::File::FILE_OK;
- leveldb_env::ErrorParsingResult result =
- leveldb_env::ParseMethodAndError(status, &method, &error);
- return (result == leveldb_env::METHOD_AND_BFE &&
- static_cast<base::File::Error>(error) ==
- base::File::FILE_ERROR_NO_SPACE);
- }
- std::string DatabaseNameForRewriteDB(const std::string& original_name) {
- return original_name + kDatabaseNameSuffixForRebuildDB;
- }
- // Given the size of the disk, identified by |disk_size| in bytes, determine the
- // appropriate write_buffer_size. Ignoring snapshots, if the current set of
- // tables in a database contains a set of key/value pairs identified by {A}, and
- // a set of key/value pairs identified by {B} has been written and is in the log
- // file, then during compaction you will have {A} + {B} + {A, B} = 2A + 2B.
- // There is no way to know the size of A, so minimizing the size of B will
- // maximize the likelihood of a successful compaction.
- size_t WriteBufferSize(int64_t disk_size) {
- const leveldb_env::Options default_options;
- const int64_t kMinBufferSize = 1024 * 1024;
- const int64_t kMaxBufferSize = default_options.write_buffer_size;
- const int64_t kDiskMinBuffSize = 10 * 1024 * 1024;
- const int64_t kDiskMaxBuffSize = 40 * 1024 * 1024;
- if (disk_size == -1)
- return default_options.write_buffer_size;
- if (disk_size <= kDiskMinBuffSize)
- return kMinBufferSize;
- if (disk_size >= kDiskMaxBuffSize)
- return kMaxBufferSize;
- // A linear equation to intersect (kDiskMinBuffSize, kMinBufferSize) and
- // (kDiskMaxBuffSize, kMaxBufferSize).
- return static_cast<size_t>(
- kMinBufferSize +
- ((kMaxBufferSize - kMinBufferSize) * (disk_size - kDiskMinBuffSize)) /
- (kDiskMaxBuffSize - kDiskMinBuffSize));
- }
- ChromiumEnv::ChromiumEnv() : ChromiumEnv("LevelDBEnv") {}
- ChromiumEnv::ChromiumEnv(std::unique_ptr<storage::FilesystemProxy> filesystem)
- : ChromiumEnv("LevelDBEnv", std::move(filesystem)) {}
- ChromiumEnv::ChromiumEnv(const std::string& name)
- : ChromiumEnv(name,
- std::make_unique<storage::FilesystemProxy>(
- storage::FilesystemProxy::UNRESTRICTED,
- base::FilePath())) {}
- ChromiumEnv::ChromiumEnv(const std::string& name,
- std::unique_ptr<storage::FilesystemProxy> filesystem)
- : filesystem_(std::move(filesystem)), name_(name) {
- DCHECK(filesystem_);
- size_t max_open_files = base::GetMaxFds();
- if (max_open_files < kFileLimitToDisableEviction) {
- file_cache_.reset(
- leveldb::NewLRUCache(GetLevelDBFileLimit(max_open_files)));
- }
- }
- ChromiumEnv::~ChromiumEnv() {
- // In chromium, ChromiumEnv is leaked. It'd be nice to add NOTREACHED here to
- // ensure that behavior isn't accidentally changed, but there's an instance in
- // a unit test that is deleted.
- }
- bool ChromiumEnv::FileExists(const std::string& fname) {
- return filesystem_->PathExists(FilePath::FromUTF8Unsafe(fname));
- }
- const char* ChromiumEnv::FileErrorString(base::File::Error error) {
- switch (error) {
- case base::File::FILE_ERROR_FAILED:
- return "No further details.";
- case base::File::FILE_ERROR_IN_USE:
- return "File currently in use.";
- case base::File::FILE_ERROR_EXISTS:
- return "File already exists.";
- case base::File::FILE_ERROR_NOT_FOUND:
- return "File not found.";
- case base::File::FILE_ERROR_ACCESS_DENIED:
- return "Access denied.";
- case base::File::FILE_ERROR_TOO_MANY_OPENED:
- return "Too many files open.";
- case base::File::FILE_ERROR_NO_MEMORY:
- return "Out of memory.";
- case base::File::FILE_ERROR_NO_SPACE:
- return "No space left on drive.";
- case base::File::FILE_ERROR_NOT_A_DIRECTORY:
- return "Not a directory.";
- case base::File::FILE_ERROR_INVALID_OPERATION:
- return "Invalid operation.";
- case base::File::FILE_ERROR_SECURITY:
- return "Security error.";
- case base::File::FILE_ERROR_ABORT:
- return "File operation aborted.";
- case base::File::FILE_ERROR_NOT_A_FILE:
- return "The supplied path was not a file.";
- case base::File::FILE_ERROR_NOT_EMPTY:
- return "The file was not empty.";
- case base::File::FILE_ERROR_INVALID_URL:
- return "Invalid URL.";
- case base::File::FILE_ERROR_IO:
- return "OS or hardware error.";
- case base::File::FILE_OK:
- return "OK.";
- case base::File::FILE_ERROR_MAX:
- NOTREACHED();
- }
- NOTIMPLEMENTED();
- return "Unknown error.";
- }
- // Delete unused table backup files - a feature no longer supported.
- // TODO(cmumford): Delete this function once found backup files drop below some
- // very small (TBD) number.
- void ChromiumEnv::RemoveBackupFiles(const FilePath& dir) {
- base::HistogramBase* histogram = base::BooleanHistogram::FactoryGet(
- "LevelDBEnv.DeleteTableBackupFile",
- base::Histogram::kUmaTargetedHistogramFlag);
- FileErrorOr<std::vector<base::FilePath>> result =
- filesystem_->GetDirectoryEntries(
- dir, storage::FilesystemProxy::DirectoryEntryType::kFilesOnly);
- if (result.is_error())
- return;
- for (const auto& path : result.value()) {
- if (path.Extension() == FILE_PATH_LITERAL(".bak"))
- histogram->AddBoolean(filesystem_->DeleteFile(path));
- }
- }
- // Test must call this *before* opening any random-access files.
- void ChromiumEnv::SetReadOnlyFileLimitForTesting(int max_open_files) {
- DCHECK(!file_cache_ || file_cache_->TotalCharge() == 0);
- file_cache_.reset(leveldb::NewLRUCache(max_open_files));
- }
- Status ChromiumEnv::GetChildren(const std::string& dir,
- std::vector<std::string>* result) {
- FilePath dir_path = FilePath::FromUTF8Unsafe(dir);
- RemoveBackupFiles(dir_path);
- FileErrorOr<std::vector<FilePath>> entries_result =
- filesystem_->GetDirectoryEntries(
- dir_path,
- storage::FilesystemProxy::DirectoryEntryType::kFilesAndDirectories);
- if (entries_result.is_error()) {
- return MakeIOError(dir, "Could not open/read directory", kGetChildren,
- entries_result.error());
- }
- result->clear();
- for (const auto& entry : entries_result.value())
- result->push_back(entry.BaseName().AsUTF8Unsafe());
- return Status::OK();
- }
- Status ChromiumEnv::RemoveFile(const std::string& fname) {
- Status result;
- FilePath fname_filepath = FilePath::FromUTF8Unsafe(fname);
- if (!filesystem_->DeleteFile(fname_filepath)) {
- result = MakeIOError(fname, "Could not delete file.", kRemoveFile);
- }
- return result;
- }
- Status ChromiumEnv::CreateDir(const std::string& name) {
- Status result;
- base::File::Error error = base::File::FILE_OK;
- Retrier retrier;
- do {
- error = filesystem_->CreateDirectory(base::FilePath::FromUTF8Unsafe(name));
- if (error == base::File::FILE_OK)
- return result;
- } while (retrier.ShouldKeepTrying());
- return MakeIOError(name, "Could not create directory.", kCreateDir, error);
- }
- Status ChromiumEnv::RemoveDir(const std::string& name) {
- Status result;
- if (!filesystem_->DeleteFile(FilePath::FromUTF8Unsafe(name))) {
- result = MakeIOError(name, "Could not delete directory.", kRemoveDir);
- }
- return result;
- }
- Status ChromiumEnv::GetFileSize(const std::string& fname, uint64_t* size) {
- Status s;
- absl::optional<base::File::Info> info =
- filesystem_->GetFileInfo(base::FilePath::FromUTF8Unsafe(fname));
- if (!info) {
- *size = 0;
- s = MakeIOError(fname, "Could not determine file size.", kGetFileSize);
- } else {
- *size = static_cast<uint64_t>(info->size);
- }
- return s;
- }
- Status ChromiumEnv::RenameFile(const std::string& src, const std::string& dst) {
- Status result;
- FilePath src_file_path = FilePath::FromUTF8Unsafe(src);
- if (!filesystem_->PathExists(src_file_path))
- return result;
- FilePath destination = FilePath::FromUTF8Unsafe(dst);
- Retrier retrier;
- base::File::Error error = base::File::FILE_OK;
- do {
- error = filesystem_->RenameFile(src_file_path, destination);
- if (error == base::File::FILE_OK)
- return result;
- } while (retrier.ShouldKeepTrying());
- DCHECK(error != base::File::FILE_OK);
- char buf[100];
- base::snprintf(buf,
- sizeof(buf),
- "Could not rename file: %s",
- FileErrorString(error));
- return MakeIOError(src, buf, kRenameFile, error);
- }
- Status ChromiumEnv::LockFile(const std::string& fname, FileLock** lock) {
- *lock = nullptr;
- Status result;
- const base::FilePath path = base::FilePath::FromUTF8Unsafe(fname);
- Retrier retrier;
- FileErrorOr<std::unique_ptr<storage::FilesystemProxy::FileLock>> lock_result =
- base::File::Error::FILE_ERROR_FAILED;
- do {
- lock_result = filesystem_->LockFile(path);
- } while (lock_result.is_error() && retrier.ShouldKeepTrying());
- if (lock_result.is_error()) {
- return MakeIOError(fname, FileErrorString(lock_result.error()), kLockFile,
- lock_result.error());
- }
- *lock = new ChromiumFileLock(std::move(lock_result.value()), fname);
- return result;
- }
- Status ChromiumEnv::UnlockFile(FileLock* lock) {
- std::unique_ptr<ChromiumFileLock> my_lock(
- reinterpret_cast<ChromiumFileLock*>(lock));
- Status result = Status::OK();
- base::File::Error error_code = my_lock->lock->Release();
- if (error_code != base::File::FILE_OK) {
- result =
- MakeIOError(my_lock->name, "Could not unlock lock file.", kUnlockFile);
- }
- return result;
- }
- Status ChromiumEnv::GetTestDirectory(std::string* path) {
- mu_.Acquire();
- if (test_directory_.empty()) {
- if (!base::CreateNewTempDirectory(kLevelDBTestDirectoryPrefix,
- &test_directory_)) {
- mu_.Release();
- return MakeIOError(
- "Could not create temp directory.", "", kGetTestDirectory);
- }
- }
- *path = test_directory_.AsUTF8Unsafe();
- mu_.Release();
- return Status::OK();
- }
- Status ChromiumEnv::NewLogger(const std::string& fname,
- leveldb::Logger** result) {
- FilePath path = FilePath::FromUTF8Unsafe(fname);
- FileErrorOr<base::File> open_result = filesystem_->OpenFile(
- path, base::File::FLAG_CREATE_ALWAYS | base::File::FLAG_WRITE);
- if (open_result.is_error()) {
- *result = nullptr;
- return MakeIOError(fname, "Unable to create log file", kNewLogger,
- open_result.error());
- } else {
- *result = new leveldb::ChromiumLogger(std::move(open_result.value()));
- return Status::OK();
- }
- }
- Status ChromiumEnv::NewSequentialFile(const std::string& fname,
- leveldb::SequentialFile** result) {
- FilePath path = FilePath::FromUTF8Unsafe(fname);
- FileErrorOr<base::File> open_result = filesystem_->OpenFile(
- path, base::File::FLAG_OPEN | base::File::FLAG_READ);
- if (open_result.is_error()) {
- *result = nullptr;
- return MakeIOError(fname, "Unable to create sequential file",
- kNewSequentialFile, open_result.error());
- } else {
- *result = new ChromiumSequentialFile(fname, std::move(open_result.value()));
- return Status::OK();
- }
- }
- Status ChromiumEnv::NewRandomAccessFile(const std::string& fname,
- leveldb::RandomAccessFile** result) {
- base::FilePath file_path = FilePath::FromUTF8Unsafe(fname);
- FileErrorOr<base::File> open_result = filesystem_->OpenFile(
- file_path, base::File::FLAG_READ | base::File::FLAG_OPEN);
- if (!open_result.is_error()) {
- base::File file = std::move(open_result.value());
- if (file_cache_) {
- *result = new ChromiumEvictableRandomAccessFile(
- std::move(file_path), std::move(file), filesystem_.get(),
- file_cache_.get());
- } else {
- *result =
- new ChromiumRandomAccessFile(std::move(file_path), std::move(file));
- }
- return Status::OK();
- }
- *result = nullptr;
- return MakeIOError(fname, FileErrorString(open_result.error()),
- kNewRandomAccessFile, open_result.error());
- }
- Status ChromiumEnv::NewWritableFile(const std::string& fname,
- leveldb::WritableFile** result) {
- FilePath path = FilePath::FromUTF8Unsafe(fname);
- FileErrorOr<base::File> open_result = filesystem_->OpenFile(
- path, base::File::FLAG_CREATE_ALWAYS | base::File::FLAG_WRITE);
- if (open_result.is_error()) {
- *result = nullptr;
- return MakeIOError(fname, "Unable to create writable file",
- kNewWritableFile, open_result.error());
- }
- *result = new ChromiumWritableFile(fname, std::move(open_result.value()),
- filesystem_.get());
- return Status::OK();
- }
- Status ChromiumEnv::NewAppendableFile(const std::string& fname,
- leveldb::WritableFile** result) {
- FilePath path = FilePath::FromUTF8Unsafe(fname);
- FileErrorOr<base::File> open_result = filesystem_->OpenFile(
- path, base::File::FLAG_OPEN_ALWAYS | base::File::FLAG_APPEND);
- if (open_result.is_error()) {
- *result = nullptr;
- return MakeIOError(fname, "Unable to create appendable file",
- kNewAppendableFile, open_result.error());
- }
- *result = new ChromiumWritableFile(fname, std::move(open_result.value()),
- filesystem_.get());
- return Status::OK();
- }
- uint64_t ChromiumEnv::NowMicros() {
- return base::TimeTicks::Now().ToInternalValue();
- }
- void ChromiumEnv::SleepForMicroseconds(int micros) {
- // Round up to the next millisecond.
- base::PlatformThread::Sleep(base::Microseconds(micros));
- }
- class Thread : public base::PlatformThread::Delegate {
- public:
- Thread(void (*function)(void* arg), void* arg)
- : function_(function), arg_(arg) {
- base::PlatformThreadHandle handle;
- bool success = base::PlatformThread::Create(0, this, &handle);
- DCHECK(success);
- }
- Thread(const Thread&) = delete;
- Thread& operator=(const Thread&) = delete;
- virtual ~Thread() {}
- void ThreadMain() override {
- (*function_)(arg_);
- delete this;
- }
- private:
- void (*function_)(void* arg);
- void* arg_;
- };
- void ChromiumEnv::Schedule(ScheduleFunc* function, void* arg) {
- // The BLOCK_SHUTDOWN is required to avoid shutdown hangs. The scheduled
- // tasks may be blocking foreground threads waiting for their completions.
- // see: https://crbug.com/1086185.
- base::ThreadPool::PostTask(FROM_HERE,
- {base::MayBlock(), base::WithBaseSyncPrimitives(),
- base::TaskShutdownBehavior::BLOCK_SHUTDOWN},
- base::BindOnce(function, arg));
- }
- void ChromiumEnv::StartThread(void (*function)(void* arg), void* arg) {
- new Thread(function, arg); // Will self-delete.
- }
- LevelDBStatusValue GetLevelDBStatusUMAValue(const leveldb::Status& s) {
- if (s.ok())
- return LEVELDB_STATUS_OK;
- if (s.IsNotFound())
- return LEVELDB_STATUS_NOT_FOUND;
- if (s.IsCorruption())
- return LEVELDB_STATUS_CORRUPTION;
- if (s.IsNotSupportedError())
- return LEVELDB_STATUS_NOT_SUPPORTED;
- if (s.IsIOError())
- return LEVELDB_STATUS_IO_ERROR;
- // TODO(cmumford): IsInvalidArgument() was just added to leveldb. Use this
- // function once that change goes to the public repository.
- return LEVELDB_STATUS_INVALID_ARGUMENT;
- }
- // Forwards all calls to the underlying leveldb::DB instance.
- // Adds / removes itself in the DBTracker it's created with.
- class DBTracker::TrackedDBImpl : public base::LinkNode<TrackedDBImpl>,
- public TrackedDB {
- public:
- TrackedDBImpl(DBTracker* tracker,
- const std::string name,
- leveldb::DB* db,
- const leveldb::Cache* block_cache,
- DatabaseErrorReportingCallback on_get_error,
- DatabaseErrorReportingCallback on_write_error)
- : tracker_(tracker),
- name_(name),
- db_(db),
- on_get_error_(std::move(on_get_error)),
- on_write_error_(std::move(on_write_error)) {
- if (leveldb_chrome::GetSharedWebBlockCache() ==
- leveldb_chrome::GetSharedBrowserBlockCache()) {
- shared_read_cache_use_ = SharedReadCacheUse_Unified;
- } else if (block_cache == leveldb_chrome::GetSharedBrowserBlockCache()) {
- shared_read_cache_use_ = SharedReadCacheUse_Browser;
- } else if (block_cache == leveldb_chrome::GetSharedWebBlockCache()) {
- shared_read_cache_use_ = SharedReadCacheUse_Web;
- } else if (block_cache == leveldb_chrome::GetSharedInMemoryBlockCache()) {
- shared_read_cache_use_ = SharedReadCacheUse_InMemory;
- } else {
- NOTREACHED();
- }
- tracker_->DatabaseOpened(this, shared_read_cache_use_);
- }
- TrackedDBImpl(const TrackedDBImpl&) = delete;
- TrackedDBImpl& operator=(const TrackedDBImpl&) = delete;
- ~TrackedDBImpl() override {
- tracker_->DatabaseDestroyed(this, shared_read_cache_use_);
- leveldb::port::ScopedAllowWait scoped_allow_wait;
- db_.reset();
- }
- const std::string& name() const override { return name_; }
- SharedReadCacheUse block_cache_type() const override {
- return shared_read_cache_use_;
- }
- leveldb::Status Put(const leveldb::WriteOptions& options,
- const leveldb::Slice& key,
- const leveldb::Slice& value) override {
- return db_->Put(options, key, value);
- }
- leveldb::Status Delete(const leveldb::WriteOptions& options,
- const leveldb::Slice& key) override {
- return db_->Delete(options, key);
- }
- leveldb::Status Write(const leveldb::WriteOptions& options,
- leveldb::WriteBatch* updates) override {
- leveldb::Status status = db_->Write(options, updates);
- if (LIKELY(status.ok()))
- return status;
- if (on_write_error_)
- on_write_error_.Run(status);
- return status;
- }
- leveldb::Status Get(const leveldb::ReadOptions& options,
- const leveldb::Slice& key,
- std::string* value) override {
- leveldb::Status status = db_->Get(options, key, value);
- if (LIKELY(status.ok() || status.IsNotFound()))
- return status;
- if (on_get_error_)
- on_get_error_.Run(status);
- return status;
- }
- const leveldb::Snapshot* GetSnapshot() override { return db_->GetSnapshot(); }
- void ReleaseSnapshot(const leveldb::Snapshot* snapshot) override {
- return db_->ReleaseSnapshot(snapshot);
- }
- bool GetProperty(const leveldb::Slice& property,
- std::string* value) override {
- return db_->GetProperty(property, value);
- }
- void GetApproximateSizes(const leveldb::Range* range,
- int n,
- uint64_t* sizes) override {
- return db_->GetApproximateSizes(range, n, sizes);
- }
- void CompactRange(const leveldb::Slice* begin,
- const leveldb::Slice* end) override {
- return db_->CompactRange(begin, end);
- }
- leveldb::Iterator* NewIterator(const leveldb::ReadOptions& options) override {
- return db_->NewIterator(options);
- }
- private:
- DBTracker* tracker_;
- std::string name_;
- std::unique_ptr<leveldb::DB> db_;
- SharedReadCacheUse shared_read_cache_use_;
- const DatabaseErrorReportingCallback on_get_error_;
- const DatabaseErrorReportingCallback on_write_error_;
- };
- // Reports live databases and in-memory env's to memory-infra. For each live
- // database the following information is reported:
- // 1. Instance pointer (to disambiguate databases).
- // 2. Memory taken by the database, with the shared cache being attributed
- // equally to each database sharing 3. The name of the database (when not in
- // BACKGROUND mode to avoid exposing
- // PIIs in slow reports).
- //
- // Example report (as seen after clicking "leveldatabase" in "Overview" pane
- // in Chrome tracing UI):
- //
- // Component effective_size size name
- // ---------------------------------------------------------------------------
- // leveldatabase 390 KiB 490 KiB
- // db_0x7FE70F2040A0 100 KiB 100 KiB Users/.../Sync
- // block_cache (browser) 40 KiB 40 KiB
- // db_0x7FE70F530D80 150 KiB 150 KiB Users/.../Data Proxy
- // block_cache (web) 30 KiB 30 KiB
- // db_0x7FE70F530D80 140 KiB 140 KiB Users/.../Extensions
- // block_cache (web) 30 KiB 30 KiB
- // block_cache 0 KiB 100 KiB
- // browser 0 KiB 40 KiB
- // web 0 KiB 60 KiB
- // memenv_0x7FE80F2040A0 4 KiB 4 KiB
- // memenv_0x7FE80F3040A0 4 KiB 4 KiB
- //
- class DBTracker::MemoryDumpProvider
- : public base::trace_event::MemoryDumpProvider {
- public:
- void DumpAllDatabases(ProcessMemoryDump* pmd);
- bool OnMemoryDump(const MemoryDumpArgs& args,
- ProcessMemoryDump* pmd) override {
- DumpAllDatabases(pmd);
- return true;
- }
- void DatabaseOpened(const TrackedDBImpl* database) {
- database_use_count_[database->block_cache_type()]++;
- }
- void DatabaseDestroyed(const TrackedDBImpl* database) {
- database_use_count_[database->block_cache_type()]--;
- DCHECK_GE(database_use_count_[database->block_cache_type()], 0);
- }
- private:
- void DumpVisitor(ProcessMemoryDump* pmd, TrackedDB* db);
- int database_use_count_[SharedReadCacheUse_NumCacheUses] = {};
- };
- void DBTracker::MemoryDumpProvider::DumpAllDatabases(ProcessMemoryDump* pmd) {
- if (pmd->GetAllocatorDump("leveldatabase"))
- return;
- pmd->CreateAllocatorDump("leveldatabase");
- const auto* browser_cache = leveldb_chrome::GetSharedBrowserBlockCache();
- const auto* web_cache = leveldb_chrome::GetSharedWebBlockCache();
- if (browser_cache == web_cache) {
- RecordCacheUsageInTracing(pmd, SharedReadCacheUse_Unified);
- } else {
- RecordCacheUsageInTracing(pmd, SharedReadCacheUse_Browser);
- RecordCacheUsageInTracing(pmd, SharedReadCacheUse_Web);
- }
- RecordCacheUsageInTracing(pmd, SharedReadCacheUse_InMemory);
- DBTracker::GetInstance()->VisitDatabases(
- base::BindRepeating(&DBTracker::MemoryDumpProvider::DumpVisitor,
- base::Unretained(this), base::Unretained(pmd)));
- leveldb_chrome::DumpAllTrackedEnvs(pmd);
- }
- void DBTracker::MemoryDumpProvider::DumpVisitor(ProcessMemoryDump* pmd,
- TrackedDB* db) {
- std::string db_dump_name = GetDumpNameForDB(db);
- auto* db_cache_dump = pmd->CreateAllocatorDump(db_dump_name + "/block_cache");
- const std::string cache_dump_name =
- GetDumpNameForCache(db->block_cache_type());
- pmd->AddSuballocation(db_cache_dump->guid(), cache_dump_name);
- size_t cache_usage =
- pmd->GetAllocatorDump(cache_dump_name)->GetSizeInternal();
- // The |database_use_count_| can be accessed by the visitor because the
- // visitor is called holding the lock at DBTracker.
- size_t cache_usage_pss =
- cache_usage / database_use_count_[db->block_cache_type()];
- db_cache_dump->AddScalar(MemoryAllocatorDump::kNameSize,
- MemoryAllocatorDump::kUnitsBytes, cache_usage_pss);
- auto* db_dump = pmd->CreateAllocatorDump(db_dump_name);
- uint64_t total_usage = 0;
- std::string usage_string;
- bool success =
- db->GetProperty("leveldb.approximate-memory-usage", &usage_string) &&
- base::StringToUint64(usage_string, &total_usage);
- DCHECK(success);
- db_dump->AddScalar(MemoryAllocatorDump::kNameSize,
- MemoryAllocatorDump::kUnitsBytes,
- total_usage - cache_usage + cache_usage_pss);
- if (pmd->dump_args().level_of_detail !=
- base::trace_event::MemoryDumpLevelOfDetail::BACKGROUND) {
- db_dump->AddString("name", "", db->name());
- }
- }
- DBTracker::DBTracker() : mdp_(new MemoryDumpProvider()) {
- base::trace_event::MemoryDumpManager::GetInstance()->RegisterDumpProvider(
- mdp_.get(), "LevelDB", nullptr);
- }
- DBTracker::~DBTracker() {
- NOTREACHED(); // DBTracker is a singleton
- }
- // static
- DBTracker* DBTracker::GetInstance() {
- static DBTracker* instance = new DBTracker();
- return instance;
- }
- // static
- MemoryAllocatorDump* DBTracker::GetOrCreateAllocatorDump(
- ProcessMemoryDump* pmd,
- leveldb::DB* tracked_db) {
- DCHECK(GetInstance()->IsTrackedDB(tracked_db))
- << std::hex << tracked_db << " is not tracked";
- // Create dumps for all databases to make sure the shared cache is equally
- // attributed to each database sharing it.
- GetInstance()->mdp_->DumpAllDatabases(pmd);
- return pmd->GetAllocatorDump(GetDumpNameForDB(tracked_db));
- }
- // static
- MemoryAllocatorDump* DBTracker::GetOrCreateAllocatorDump(
- ProcessMemoryDump* pmd,
- leveldb::Env* tracked_memenv) {
- GetInstance()->mdp_->DumpAllDatabases(pmd);
- return leveldb_chrome::GetEnvAllocatorDump(pmd, tracked_memenv);
- }
- bool DBTracker::IsTrackedDB(const leveldb::DB* db) const {
- base::AutoLock lock(databases_lock_);
- for (auto* i = databases_.head(); i != databases_.end(); i = i->next()) {
- if (i->value() == db)
- return true;
- }
- return false;
- }
- leveldb::Status DBTracker::OpenDatabase(const leveldb_env::Options& options,
- const std::string& name,
- TrackedDB** dbptr) {
- leveldb::DB* db = nullptr;
- auto status = leveldb::DB::Open(options, name, &db);
- // Enforce expectations: either we succeed, and get a valid object in |db|,
- // or we fail, and |db| is still NULL.
- CHECK((status.ok() && db) || (!status.ok() && !db));
- if (status.ok()) {
- // TrackedDBImpl ctor adds the instance to the tracker.
- *dbptr = new TrackedDBImpl(GetInstance(), name, db, options.block_cache,
- std::move(options.on_get_error),
- std::move(options.on_write_error));
- }
- return status;
- }
- void DBTracker::VisitDatabases(const DatabaseVisitor& visitor) {
- base::AutoLock lock(databases_lock_);
- for (auto* i = databases_.head(); i != databases_.end(); i = i->next())
- visitor.Run(i->value());
- }
- void DBTracker::DatabaseOpened(TrackedDBImpl* database,
- SharedReadCacheUse cache_use) {
- base::AutoLock lock(databases_lock_);
- databases_.Append(database);
- mdp_->DatabaseOpened(database);
- }
- void DBTracker::DatabaseDestroyed(TrackedDBImpl* database,
- SharedReadCacheUse cache_use) {
- base::AutoLock lock(databases_lock_);
- mdp_->DatabaseDestroyed(database);
- database->RemoveFromList();
- }
- leveldb::Status OpenDB(const leveldb_env::Options& options,
- const std::string& name,
- std::unique_ptr<leveldb::DB>* dbptr) {
- // For UMA logging purposes we need the block cache to be created outside of
- // leveldb so that the size can be logged and it can be pruned.
- DCHECK(options.block_cache != nullptr);
- DBTracker::TrackedDB* tracked_db = nullptr;
- leveldb::Status s;
- if (options.env && leveldb_chrome::IsMemEnv(options.env)) {
- Options mem_options = options;
- mem_options.block_cache = leveldb_chrome::GetSharedInMemoryBlockCache();
- mem_options.write_buffer_size = 0; // minimum size.
- s = DBTracker::GetInstance()->OpenDatabase(mem_options, name, &tracked_db);
- } else {
- std::string tmp_name = DatabaseNameForRewriteDB(name);
- // If Chrome crashes during rewrite, there might be a temporary db but
- // no actual db.
- if (options.env->FileExists(tmp_name) &&
- !options.env->FileExists(name + "/CURRENT")) {
- s = leveldb::DestroyDB(name, options);
- if (!s.ok())
- return s;
- s = options.env->RenameFile(tmp_name, name);
- if (!s.ok())
- return s;
- }
- s = DBTracker::GetInstance()->OpenDatabase(options, name, &tracked_db);
- // It is possible that the database was partially deleted during a
- // rewrite and can't be opened anymore.
- if (!s.ok() && options.env->FileExists(tmp_name)) {
- s = leveldb::DestroyDB(name, options);
- if (!s.ok())
- return s;
- s = options.env->RenameFile(tmp_name, name);
- if (!s.ok())
- return s;
- s = DBTracker::GetInstance()->OpenDatabase(options, name, &tracked_db);
- }
- // There might be a temporary database that needs to be cleaned up.
- if (options.env->FileExists(tmp_name)) {
- leveldb::DestroyDB(tmp_name, options);
- }
- }
- if (s.ok())
- dbptr->reset(tracked_db);
- return s;
- }
- leveldb::Status RewriteDB(const leveldb_env::Options& options,
- const std::string& name,
- std::unique_ptr<leveldb::DB>* dbptr) {
- DCHECK(options.create_if_missing);
- if (leveldb_chrome::IsMemEnv(options.env))
- return Status::OK();
- TRACE_EVENT1("leveldb", "ChromiumEnv::RewriteDB", "name", name);
- leveldb::Status s;
- std::string tmp_name = DatabaseNameForRewriteDB(name);
- if (options.env->FileExists(tmp_name)) {
- s = leveldb::DestroyDB(tmp_name, options);
- if (!s.ok())
- return s;
- }
- // Copy all data from *dbptr to a temporary db.
- std::unique_ptr<leveldb::DB> tmp_db;
- s = leveldb_env::OpenDB(options, tmp_name, &tmp_db);
- if (!s.ok())
- return s;
- std::unique_ptr<leveldb::Iterator> it(
- (*dbptr)->NewIterator(leveldb::ReadOptions()));
- for (it->SeekToFirst(); it->Valid(); it->Next()) {
- s = tmp_db->Put(leveldb::WriteOptions(), it->key(), it->value());
- if (!s.ok())
- break;
- }
- it.reset();
- tmp_db.reset();
- if (!s.ok()) {
- leveldb::DestroyDB(tmp_name, options);
- return s;
- }
- // Replace the old database with tmp_db.
- (*dbptr).reset();
- s = leveldb::DestroyDB(name, options);
- if (!s.ok())
- return s;
- s = options.env->RenameFile(tmp_name, name);
- if (!s.ok())
- return s;
- return leveldb_env::OpenDB(options, name, dbptr);
- }
- base::StringPiece MakeStringPiece(const leveldb::Slice& s) {
- return base::StringPiece(s.data(), s.size());
- }
- leveldb::Slice MakeSlice(const base::StringPiece& s) {
- return leveldb::Slice(s.begin(), s.size());
- }
- leveldb::Slice MakeSlice(base::span<const uint8_t> s) {
- return MakeSlice(
- base::StringPiece(reinterpret_cast<const char*>(s.data()), s.size()));
- }
- } // namespace leveldb_env
- namespace leveldb {
- Env* Env::Default() {
- static base::NoDestructor<leveldb_env::ChromiumEnv> default_env;
- return default_env.get();
- }
- } // namespace leveldb
|