123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340 |
- // Copyright 2019 The Chromium Authors. All rights reserved.
- // Use of this source code is governed by a BSD-style license that can be
- // found in the LICENSE file.
- #include "base/memory/madv_free_discardable_memory_posix.h"
- #include <errno.h>
- #include <inttypes.h>
- #include <sys/mman.h>
- #include <sys/types.h>
- #include <sys/utsname.h>
- #include <atomic>
- #include "base/atomicops.h"
- #include "base/bits.h"
- #include "base/callback.h"
- #include "base/logging.h"
- #include "base/memory/madv_free_discardable_memory_allocator_posix.h"
- #include "base/memory/page_size.h"
- #include "base/notreached.h"
- #include "base/strings/string_number_conversions.h"
- #include "base/strings/stringprintf.h"
- #include "base/tracing_buildflags.h"
- #include "build/build_config.h"
- #if BUILDFLAG(ENABLE_BASE_TRACING)
- #include "base/trace_event/memory_allocator_dump.h" // no-presubmit-check
- #include "base/trace_event/memory_dump_manager.h" // no-presubmit-check
- #endif // BUILDFLAG(ENABLE_BASE_TRACING)
- #if defined(ADDRESS_SANITIZER)
- #include <sanitizer/asan_interface.h>
- #endif // defined(ADDRESS_SANITIZER)
- namespace {
- constexpr intptr_t kPageMagicCookie = 1;
- void* AllocatePages(size_t size_in_pages) {
- void* data = mmap(nullptr, size_in_pages * base::GetPageSize(),
- PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
- PCHECK(data != MAP_FAILED);
- return data;
- }
- // Checks if the system supports usage of MADV_FREE as a backing for discardable
- // memory.
- base::MadvFreeSupport ProbePlatformMadvFreeSupport() {
- // Note: If the compiling system does not have headers for Linux 4.5+, then
- // the MADV_FREE define will not exist and the probe will default to
- // unsupported, regardless of whether the target system actually supports
- // MADV_FREE.
- #if !BUILDFLAG(IS_APPLE) && defined(MADV_FREE)
- uint8_t* dummy_page = static_cast<uint8_t*>(AllocatePages(1));
- dummy_page[0] = 1;
- base::MadvFreeSupport support = base::MadvFreeSupport::kUnsupported;
- // Check if the MADV_FREE advice value exists.
- int retval = madvise(dummy_page, base::GetPageSize(), MADV_FREE);
- if (!retval) {
- // For Linux 4.5 to 4.12, MADV_FREE on a swapless system will lead to memory
- // being immediately discarded. Verify that the memory was not discarded.
- if (dummy_page[0]) {
- support = base::MadvFreeSupport::kSupported;
- }
- }
- PCHECK(!munmap(dummy_page, base::GetPageSize()));
- return support;
- #else
- return base::MadvFreeSupport::kUnsupported;
- #endif
- }
- } // namespace
- namespace base {
- MadvFreeDiscardableMemoryPosix::MadvFreeDiscardableMemoryPosix(
- size_t size_in_bytes,
- std::atomic<size_t>* allocator_byte_count)
- : size_in_bytes_(size_in_bytes),
- allocated_pages_((size_in_bytes_ + base::GetPageSize() - 1) /
- base::GetPageSize()),
- allocator_byte_count_(allocator_byte_count),
- page_first_word_((size_in_bytes_ + base::GetPageSize() - 1) /
- base::GetPageSize()) {
- data_ = AllocatePages(allocated_pages_);
- (*allocator_byte_count_) += size_in_bytes_;
- }
- MadvFreeDiscardableMemoryPosix::~MadvFreeDiscardableMemoryPosix() {
- if (Deallocate()) {
- DVLOG(1) << "Region evicted during destructor with " << allocated_pages_
- << " pages";
- }
- }
- bool MadvFreeDiscardableMemoryPosix::Lock() {
- DFAKE_SCOPED_LOCK(thread_collision_warner_);
- DCHECK(!is_locked_);
- // Locking fails if the memory has been deallocated.
- if (!data_)
- return false;
- #if defined(ADDRESS_SANITIZER)
- // We need to unpoison here since locking pages writes to them.
- // Note that even if locking fails, we want to unpoison anyways after
- // deallocation.
- ASAN_UNPOISON_MEMORY_REGION(data_, allocated_pages_ * base::GetPageSize());
- #endif // defined(ADDRESS_SANITIZER)
- size_t page_index;
- for (page_index = 0; page_index < allocated_pages_; ++page_index) {
- if (!LockPage(page_index))
- break;
- }
- if (page_index < allocated_pages_) {
- DVLOG(1) << "Region eviction discovered during lock with "
- << allocated_pages_ << " pages";
- Deallocate();
- return false;
- }
- DCHECK(IsResident());
- is_locked_ = true;
- return true;
- }
- void MadvFreeDiscardableMemoryPosix::Unlock() {
- DFAKE_SCOPED_LOCK(thread_collision_warner_);
- DCHECK(is_locked_);
- DCHECK(data_ != nullptr);
- for (size_t page_index = 0; page_index < allocated_pages_; ++page_index) {
- UnlockPage(page_index);
- }
- #ifdef MADV_FREE
- if (!keep_memory_for_testing_) {
- int retval =
- madvise(data_, allocated_pages_ * base::GetPageSize(), MADV_FREE);
- DPCHECK(!retval);
- }
- #endif
- #if defined(ADDRESS_SANITIZER)
- ASAN_POISON_MEMORY_REGION(data_, allocated_pages_ * base::GetPageSize());
- #endif // defined(ADDRESS_SANITIZER)
- is_locked_ = false;
- }
- void* MadvFreeDiscardableMemoryPosix::data() const {
- DFAKE_SCOPED_LOCK(thread_collision_warner_);
- DCHECK(is_locked_);
- DCHECK(data_ != nullptr);
- return data_;
- }
- bool MadvFreeDiscardableMemoryPosix::LockPage(size_t page_index) {
- // We require the byte-level representation of std::atomic<intptr_t> to be
- // equivalent to that of an intptr_t. Since std::atomic<intptr_t> has standard
- // layout, having equal size is sufficient but not necessary for them to have
- // the same byte-level representation.
- static_assert(sizeof(intptr_t) == sizeof(std::atomic<intptr_t>),
- "Incompatible layout of std::atomic.");
- DCHECK(std::atomic<intptr_t>{}.is_lock_free());
- std::atomic<intptr_t>* page_as_atomic =
- reinterpret_cast<std::atomic<intptr_t>*>(
- static_cast<uint8_t*>(data_) + page_index * base::GetPageSize());
- intptr_t expected = kPageMagicCookie;
- // Recall that we set the first word of the page to |kPageMagicCookie|
- // (non-zero) during unlocking. Thus, if the value has changed, the page has
- // been discarded. Restore the page's original first word from before
- // unlocking only if the page has not been discarded.
- if (!std::atomic_compare_exchange_strong_explicit(
- page_as_atomic, &expected,
- static_cast<intptr_t>(page_first_word_[page_index]),
- std::memory_order_relaxed, std::memory_order_relaxed)) {
- return false;
- }
- return true;
- }
- void MadvFreeDiscardableMemoryPosix::UnlockPage(size_t page_index) {
- DCHECK(std::atomic<intptr_t>{}.is_lock_free());
- std::atomic<intptr_t>* page_as_atomic =
- reinterpret_cast<std::atomic<intptr_t>*>(
- static_cast<uint8_t*>(data_) + page_index * base::GetPageSize());
- // Store the first word of the page for use during unlocking.
- page_first_word_[page_index].store(*page_as_atomic,
- std::memory_order_relaxed);
- // Store a non-zero value into the first word of the page, so we can tell when
- // the page is discarded during locking.
- page_as_atomic->store(kPageMagicCookie, std::memory_order_relaxed);
- }
- void MadvFreeDiscardableMemoryPosix::DiscardPage(size_t page_index) {
- DFAKE_SCOPED_LOCK(thread_collision_warner_);
- DCHECK(!is_locked_);
- DCHECK(page_index < allocated_pages_);
- int retval =
- madvise(static_cast<uint8_t*>(data_) + base::GetPageSize() * page_index,
- base::GetPageSize(), MADV_DONTNEED);
- DPCHECK(!retval);
- }
- bool MadvFreeDiscardableMemoryPosix::IsLockedForTesting() const {
- DFAKE_SCOPED_LOCK(thread_collision_warner_);
- return is_locked_;
- }
- void MadvFreeDiscardableMemoryPosix::DiscardForTesting() {
- DFAKE_SCOPED_LOCK(thread_collision_warner_);
- DCHECK(!is_locked_);
- int retval =
- madvise(data_, base::GetPageSize() * allocated_pages_, MADV_DONTNEED);
- DPCHECK(!retval);
- }
- trace_event::MemoryAllocatorDump*
- MadvFreeDiscardableMemoryPosix::CreateMemoryAllocatorDump(
- const char* name,
- trace_event::ProcessMemoryDump* pmd) const {
- #if BUILDFLAG(ENABLE_BASE_TRACING)
- DFAKE_SCOPED_LOCK(thread_collision_warner_);
- using base::trace_event::MemoryAllocatorDump;
- std::string allocator_dump_name = base::StringPrintf(
- "discardable/segment_0x%" PRIXPTR, reinterpret_cast<uintptr_t>(this));
- MemoryAllocatorDump* allocator_dump =
- pmd->CreateAllocatorDump(allocator_dump_name);
- bool is_discarded = IsDiscarded();
- MemoryAllocatorDump* dump = pmd->CreateAllocatorDump(name);
- // The effective_size is the amount of unused space as a result of being
- // page-aligned.
- dump->AddScalar(MemoryAllocatorDump::kNameSize,
- MemoryAllocatorDump::kUnitsBytes,
- is_discarded ? 0U : static_cast<uint64_t>(size_in_bytes_));
- allocator_dump->AddScalar(
- MemoryAllocatorDump::kNameSize, MemoryAllocatorDump::kUnitsBytes,
- is_discarded
- ? 0U
- : static_cast<uint64_t>(allocated_pages_ * base::GetPageSize()));
- allocator_dump->AddScalar(MemoryAllocatorDump::kNameObjectCount,
- MemoryAllocatorDump::kUnitsObjects, 1U);
- allocator_dump->AddScalar(
- "wasted_size", MemoryAllocatorDump::kUnitsBytes,
- static_cast<uint64_t>(allocated_pages_ * base::GetPageSize() -
- size_in_bytes_));
- allocator_dump->AddScalar("locked_size", MemoryAllocatorDump::kUnitsBytes,
- is_locked_ ? size_in_bytes_ : 0U);
- allocator_dump->AddScalar("page_count", MemoryAllocatorDump::kUnitsObjects,
- static_cast<uint64_t>(allocated_pages_));
- // The amount of space that is discarded, but not unmapped (i.e. the memory
- // was discarded while unlocked, but the pages are still mapped in memory
- // since Deallocate() has not been called yet). This instance is discarded if
- // it is unlocked and not all pages are resident in memory.
- allocator_dump->AddScalar(
- "discarded_size", MemoryAllocatorDump::kUnitsBytes,
- is_discarded ? allocated_pages_ * base::GetPageSize() : 0U);
- pmd->AddSuballocation(dump->guid(), allocator_dump_name);
- return dump;
- #else // BUILDFLAG(ENABLE_BASE_TRACING)
- NOTREACHED();
- return nullptr;
- #endif // BUILDFLAG(ENABLE_BASE_TRACING)
- }
- bool MadvFreeDiscardableMemoryPosix::IsValid() const {
- DFAKE_SCOPED_RECURSIVE_LOCK(thread_collision_warner_);
- return data_ != nullptr;
- }
- void MadvFreeDiscardableMemoryPosix::SetKeepMemoryForTesting(bool keep_memory) {
- DFAKE_SCOPED_LOCK(thread_collision_warner_);
- DCHECK(is_locked_);
- keep_memory_for_testing_ = keep_memory;
- }
- bool MadvFreeDiscardableMemoryPosix::IsResident() const {
- DFAKE_SCOPED_RECURSIVE_LOCK(thread_collision_warner_);
- #if BUILDFLAG(IS_APPLE)
- std::vector<char> vec(allocated_pages_);
- #else
- std::vector<unsigned char> vec(allocated_pages_);
- #endif
- int retval =
- mincore(data_, allocated_pages_ * base::GetPageSize(), vec.data());
- DPCHECK(retval == 0 || errno == EAGAIN);
- for (size_t i = 0; i < allocated_pages_; ++i) {
- if (!(vec[i] & 1))
- return false;
- }
- return true;
- }
- bool MadvFreeDiscardableMemoryPosix::IsDiscarded() const {
- return !is_locked_ && !IsResident();
- }
- bool MadvFreeDiscardableMemoryPosix::Deallocate() {
- DFAKE_SCOPED_RECURSIVE_LOCK(thread_collision_warner_);
- if (data_) {
- #if defined(ADDRESS_SANITIZER)
- ASAN_UNPOISON_MEMORY_REGION(data_, allocated_pages_ * base::GetPageSize());
- #endif // defined(ADDRESS_SANITIZER)
- int retval = munmap(data_, allocated_pages_ * base::GetPageSize());
- PCHECK(!retval);
- data_ = nullptr;
- (*allocator_byte_count_) -= size_in_bytes_;
- return true;
- }
- return false;
- }
- MadvFreeSupport GetMadvFreeSupport() {
- static MadvFreeSupport kMadvFreeSupport = ProbePlatformMadvFreeSupport();
- return kMadvFreeSupport;
- }
- } // namespace base
|