madv_free_discardable_memory_posix.cc 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340
  1. // Copyright 2019 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "base/memory/madv_free_discardable_memory_posix.h"
  5. #include <errno.h>
  6. #include <inttypes.h>
  7. #include <sys/mman.h>
  8. #include <sys/types.h>
  9. #include <sys/utsname.h>
  10. #include <atomic>
  11. #include "base/atomicops.h"
  12. #include "base/bits.h"
  13. #include "base/callback.h"
  14. #include "base/logging.h"
  15. #include "base/memory/madv_free_discardable_memory_allocator_posix.h"
  16. #include "base/memory/page_size.h"
  17. #include "base/notreached.h"
  18. #include "base/strings/string_number_conversions.h"
  19. #include "base/strings/stringprintf.h"
  20. #include "base/tracing_buildflags.h"
  21. #include "build/build_config.h"
  22. #if BUILDFLAG(ENABLE_BASE_TRACING)
  23. #include "base/trace_event/memory_allocator_dump.h" // no-presubmit-check
  24. #include "base/trace_event/memory_dump_manager.h" // no-presubmit-check
  25. #endif // BUILDFLAG(ENABLE_BASE_TRACING)
  26. #if defined(ADDRESS_SANITIZER)
  27. #include <sanitizer/asan_interface.h>
  28. #endif // defined(ADDRESS_SANITIZER)
  29. namespace {
  30. constexpr intptr_t kPageMagicCookie = 1;
  31. void* AllocatePages(size_t size_in_pages) {
  32. void* data = mmap(nullptr, size_in_pages * base::GetPageSize(),
  33. PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
  34. PCHECK(data != MAP_FAILED);
  35. return data;
  36. }
  37. // Checks if the system supports usage of MADV_FREE as a backing for discardable
  38. // memory.
  39. base::MadvFreeSupport ProbePlatformMadvFreeSupport() {
  40. // Note: If the compiling system does not have headers for Linux 4.5+, then
  41. // the MADV_FREE define will not exist and the probe will default to
  42. // unsupported, regardless of whether the target system actually supports
  43. // MADV_FREE.
  44. #if !BUILDFLAG(IS_APPLE) && defined(MADV_FREE)
  45. uint8_t* dummy_page = static_cast<uint8_t*>(AllocatePages(1));
  46. dummy_page[0] = 1;
  47. base::MadvFreeSupport support = base::MadvFreeSupport::kUnsupported;
  48. // Check if the MADV_FREE advice value exists.
  49. int retval = madvise(dummy_page, base::GetPageSize(), MADV_FREE);
  50. if (!retval) {
  51. // For Linux 4.5 to 4.12, MADV_FREE on a swapless system will lead to memory
  52. // being immediately discarded. Verify that the memory was not discarded.
  53. if (dummy_page[0]) {
  54. support = base::MadvFreeSupport::kSupported;
  55. }
  56. }
  57. PCHECK(!munmap(dummy_page, base::GetPageSize()));
  58. return support;
  59. #else
  60. return base::MadvFreeSupport::kUnsupported;
  61. #endif
  62. }
  63. } // namespace
  64. namespace base {
  65. MadvFreeDiscardableMemoryPosix::MadvFreeDiscardableMemoryPosix(
  66. size_t size_in_bytes,
  67. std::atomic<size_t>* allocator_byte_count)
  68. : size_in_bytes_(size_in_bytes),
  69. allocated_pages_((size_in_bytes_ + base::GetPageSize() - 1) /
  70. base::GetPageSize()),
  71. allocator_byte_count_(allocator_byte_count),
  72. page_first_word_((size_in_bytes_ + base::GetPageSize() - 1) /
  73. base::GetPageSize()) {
  74. data_ = AllocatePages(allocated_pages_);
  75. (*allocator_byte_count_) += size_in_bytes_;
  76. }
  77. MadvFreeDiscardableMemoryPosix::~MadvFreeDiscardableMemoryPosix() {
  78. if (Deallocate()) {
  79. DVLOG(1) << "Region evicted during destructor with " << allocated_pages_
  80. << " pages";
  81. }
  82. }
  83. bool MadvFreeDiscardableMemoryPosix::Lock() {
  84. DFAKE_SCOPED_LOCK(thread_collision_warner_);
  85. DCHECK(!is_locked_);
  86. // Locking fails if the memory has been deallocated.
  87. if (!data_)
  88. return false;
  89. #if defined(ADDRESS_SANITIZER)
  90. // We need to unpoison here since locking pages writes to them.
  91. // Note that even if locking fails, we want to unpoison anyways after
  92. // deallocation.
  93. ASAN_UNPOISON_MEMORY_REGION(data_, allocated_pages_ * base::GetPageSize());
  94. #endif // defined(ADDRESS_SANITIZER)
  95. size_t page_index;
  96. for (page_index = 0; page_index < allocated_pages_; ++page_index) {
  97. if (!LockPage(page_index))
  98. break;
  99. }
  100. if (page_index < allocated_pages_) {
  101. DVLOG(1) << "Region eviction discovered during lock with "
  102. << allocated_pages_ << " pages";
  103. Deallocate();
  104. return false;
  105. }
  106. DCHECK(IsResident());
  107. is_locked_ = true;
  108. return true;
  109. }
  110. void MadvFreeDiscardableMemoryPosix::Unlock() {
  111. DFAKE_SCOPED_LOCK(thread_collision_warner_);
  112. DCHECK(is_locked_);
  113. DCHECK(data_ != nullptr);
  114. for (size_t page_index = 0; page_index < allocated_pages_; ++page_index) {
  115. UnlockPage(page_index);
  116. }
  117. #ifdef MADV_FREE
  118. if (!keep_memory_for_testing_) {
  119. int retval =
  120. madvise(data_, allocated_pages_ * base::GetPageSize(), MADV_FREE);
  121. DPCHECK(!retval);
  122. }
  123. #endif
  124. #if defined(ADDRESS_SANITIZER)
  125. ASAN_POISON_MEMORY_REGION(data_, allocated_pages_ * base::GetPageSize());
  126. #endif // defined(ADDRESS_SANITIZER)
  127. is_locked_ = false;
  128. }
  129. void* MadvFreeDiscardableMemoryPosix::data() const {
  130. DFAKE_SCOPED_LOCK(thread_collision_warner_);
  131. DCHECK(is_locked_);
  132. DCHECK(data_ != nullptr);
  133. return data_;
  134. }
  135. bool MadvFreeDiscardableMemoryPosix::LockPage(size_t page_index) {
  136. // We require the byte-level representation of std::atomic<intptr_t> to be
  137. // equivalent to that of an intptr_t. Since std::atomic<intptr_t> has standard
  138. // layout, having equal size is sufficient but not necessary for them to have
  139. // the same byte-level representation.
  140. static_assert(sizeof(intptr_t) == sizeof(std::atomic<intptr_t>),
  141. "Incompatible layout of std::atomic.");
  142. DCHECK(std::atomic<intptr_t>{}.is_lock_free());
  143. std::atomic<intptr_t>* page_as_atomic =
  144. reinterpret_cast<std::atomic<intptr_t>*>(
  145. static_cast<uint8_t*>(data_) + page_index * base::GetPageSize());
  146. intptr_t expected = kPageMagicCookie;
  147. // Recall that we set the first word of the page to |kPageMagicCookie|
  148. // (non-zero) during unlocking. Thus, if the value has changed, the page has
  149. // been discarded. Restore the page's original first word from before
  150. // unlocking only if the page has not been discarded.
  151. if (!std::atomic_compare_exchange_strong_explicit(
  152. page_as_atomic, &expected,
  153. static_cast<intptr_t>(page_first_word_[page_index]),
  154. std::memory_order_relaxed, std::memory_order_relaxed)) {
  155. return false;
  156. }
  157. return true;
  158. }
  159. void MadvFreeDiscardableMemoryPosix::UnlockPage(size_t page_index) {
  160. DCHECK(std::atomic<intptr_t>{}.is_lock_free());
  161. std::atomic<intptr_t>* page_as_atomic =
  162. reinterpret_cast<std::atomic<intptr_t>*>(
  163. static_cast<uint8_t*>(data_) + page_index * base::GetPageSize());
  164. // Store the first word of the page for use during unlocking.
  165. page_first_word_[page_index].store(*page_as_atomic,
  166. std::memory_order_relaxed);
  167. // Store a non-zero value into the first word of the page, so we can tell when
  168. // the page is discarded during locking.
  169. page_as_atomic->store(kPageMagicCookie, std::memory_order_relaxed);
  170. }
  171. void MadvFreeDiscardableMemoryPosix::DiscardPage(size_t page_index) {
  172. DFAKE_SCOPED_LOCK(thread_collision_warner_);
  173. DCHECK(!is_locked_);
  174. DCHECK(page_index < allocated_pages_);
  175. int retval =
  176. madvise(static_cast<uint8_t*>(data_) + base::GetPageSize() * page_index,
  177. base::GetPageSize(), MADV_DONTNEED);
  178. DPCHECK(!retval);
  179. }
  180. bool MadvFreeDiscardableMemoryPosix::IsLockedForTesting() const {
  181. DFAKE_SCOPED_LOCK(thread_collision_warner_);
  182. return is_locked_;
  183. }
  184. void MadvFreeDiscardableMemoryPosix::DiscardForTesting() {
  185. DFAKE_SCOPED_LOCK(thread_collision_warner_);
  186. DCHECK(!is_locked_);
  187. int retval =
  188. madvise(data_, base::GetPageSize() * allocated_pages_, MADV_DONTNEED);
  189. DPCHECK(!retval);
  190. }
  191. trace_event::MemoryAllocatorDump*
  192. MadvFreeDiscardableMemoryPosix::CreateMemoryAllocatorDump(
  193. const char* name,
  194. trace_event::ProcessMemoryDump* pmd) const {
  195. #if BUILDFLAG(ENABLE_BASE_TRACING)
  196. DFAKE_SCOPED_LOCK(thread_collision_warner_);
  197. using base::trace_event::MemoryAllocatorDump;
  198. std::string allocator_dump_name = base::StringPrintf(
  199. "discardable/segment_0x%" PRIXPTR, reinterpret_cast<uintptr_t>(this));
  200. MemoryAllocatorDump* allocator_dump =
  201. pmd->CreateAllocatorDump(allocator_dump_name);
  202. bool is_discarded = IsDiscarded();
  203. MemoryAllocatorDump* dump = pmd->CreateAllocatorDump(name);
  204. // The effective_size is the amount of unused space as a result of being
  205. // page-aligned.
  206. dump->AddScalar(MemoryAllocatorDump::kNameSize,
  207. MemoryAllocatorDump::kUnitsBytes,
  208. is_discarded ? 0U : static_cast<uint64_t>(size_in_bytes_));
  209. allocator_dump->AddScalar(
  210. MemoryAllocatorDump::kNameSize, MemoryAllocatorDump::kUnitsBytes,
  211. is_discarded
  212. ? 0U
  213. : static_cast<uint64_t>(allocated_pages_ * base::GetPageSize()));
  214. allocator_dump->AddScalar(MemoryAllocatorDump::kNameObjectCount,
  215. MemoryAllocatorDump::kUnitsObjects, 1U);
  216. allocator_dump->AddScalar(
  217. "wasted_size", MemoryAllocatorDump::kUnitsBytes,
  218. static_cast<uint64_t>(allocated_pages_ * base::GetPageSize() -
  219. size_in_bytes_));
  220. allocator_dump->AddScalar("locked_size", MemoryAllocatorDump::kUnitsBytes,
  221. is_locked_ ? size_in_bytes_ : 0U);
  222. allocator_dump->AddScalar("page_count", MemoryAllocatorDump::kUnitsObjects,
  223. static_cast<uint64_t>(allocated_pages_));
  224. // The amount of space that is discarded, but not unmapped (i.e. the memory
  225. // was discarded while unlocked, but the pages are still mapped in memory
  226. // since Deallocate() has not been called yet). This instance is discarded if
  227. // it is unlocked and not all pages are resident in memory.
  228. allocator_dump->AddScalar(
  229. "discarded_size", MemoryAllocatorDump::kUnitsBytes,
  230. is_discarded ? allocated_pages_ * base::GetPageSize() : 0U);
  231. pmd->AddSuballocation(dump->guid(), allocator_dump_name);
  232. return dump;
  233. #else // BUILDFLAG(ENABLE_BASE_TRACING)
  234. NOTREACHED();
  235. return nullptr;
  236. #endif // BUILDFLAG(ENABLE_BASE_TRACING)
  237. }
  238. bool MadvFreeDiscardableMemoryPosix::IsValid() const {
  239. DFAKE_SCOPED_RECURSIVE_LOCK(thread_collision_warner_);
  240. return data_ != nullptr;
  241. }
  242. void MadvFreeDiscardableMemoryPosix::SetKeepMemoryForTesting(bool keep_memory) {
  243. DFAKE_SCOPED_LOCK(thread_collision_warner_);
  244. DCHECK(is_locked_);
  245. keep_memory_for_testing_ = keep_memory;
  246. }
  247. bool MadvFreeDiscardableMemoryPosix::IsResident() const {
  248. DFAKE_SCOPED_RECURSIVE_LOCK(thread_collision_warner_);
  249. #if BUILDFLAG(IS_APPLE)
  250. std::vector<char> vec(allocated_pages_);
  251. #else
  252. std::vector<unsigned char> vec(allocated_pages_);
  253. #endif
  254. int retval =
  255. mincore(data_, allocated_pages_ * base::GetPageSize(), vec.data());
  256. DPCHECK(retval == 0 || errno == EAGAIN);
  257. for (size_t i = 0; i < allocated_pages_; ++i) {
  258. if (!(vec[i] & 1))
  259. return false;
  260. }
  261. return true;
  262. }
  263. bool MadvFreeDiscardableMemoryPosix::IsDiscarded() const {
  264. return !is_locked_ && !IsResident();
  265. }
  266. bool MadvFreeDiscardableMemoryPosix::Deallocate() {
  267. DFAKE_SCOPED_RECURSIVE_LOCK(thread_collision_warner_);
  268. if (data_) {
  269. #if defined(ADDRESS_SANITIZER)
  270. ASAN_UNPOISON_MEMORY_REGION(data_, allocated_pages_ * base::GetPageSize());
  271. #endif // defined(ADDRESS_SANITIZER)
  272. int retval = munmap(data_, allocated_pages_ * base::GetPageSize());
  273. PCHECK(!retval);
  274. data_ = nullptr;
  275. (*allocator_byte_count_) -= size_in_bytes_;
  276. return true;
  277. }
  278. return false;
  279. }
  280. MadvFreeSupport GetMadvFreeSupport() {
  281. static MadvFreeSupport kMadvFreeSupport = ProbePlatformMadvFreeSupport();
  282. return kMadvFreeSupport;
  283. }
  284. } // namespace base