icu_mergeable_data_file.cc 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365
  1. // Copyright 2022 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "base/i18n/icu_mergeable_data_file.h"
  5. #include <sys/mman.h>
  6. #include "base/hash/hash.h"
  7. #include "base/strings/string_number_conversions.h"
  8. #include "base/threading/scoped_blocking_call.h"
  9. #include "build/chromeos_buildflags.h"
  10. namespace base::i18n {
  11. // Enable merging of `icudtl.dat` in Lacros.
  12. const base::Feature kLacrosMergeIcuDataFile{"LacrosMergeIcuDataFile",
  13. base::FEATURE_ENABLED_BY_DEFAULT};
  14. namespace {
  15. #if BUILDFLAG(IS_CHROMEOS_DEVICE)
  16. // Path of Ash's ICU data file.
  17. constexpr char kIcuDataFileAshPath[] = "/opt/google/chrome/icudtl.dat";
  18. #endif // BUILDFLAG(IS_CHROMEOS_DEVICE)
  19. // Expected size of a system page.
  20. constexpr int64_t kPageSize = 0x1000;
  21. // Size of a page hash. Changing this will break compatibility
  22. // with existing `icudtl.dat.hash` files, so be careful.
  23. constexpr size_t kHashBytes = 8;
  24. static_assert(sizeof(IcuMergeableDataFile::HashType) == kHashBytes);
  25. inline IcuMergeableDataFile::HashType HashPage(const uint8_t* page) {
  26. return FastHash(base::make_span(page, kPageSize));
  27. }
  28. IcuMergeableDataFile::HashType ReadHash(const uint8_t* data, size_t offset) {
  29. DCHECK_EQ(0ul, offset % kHashBytes);
  30. IcuMergeableDataFile::HashType hash = 0;
  31. for (size_t i = 0; i < kHashBytes; i++) {
  32. IcuMergeableDataFile::HashType byte = data[offset + i];
  33. hash |= byte << (i * 8);
  34. }
  35. return hash;
  36. }
  37. constexpr size_t NPages(size_t length) {
  38. return (length + kPageSize - 1) / kPageSize;
  39. }
  40. } // namespace
  41. class AshMemoryMappedFile {
  42. public:
  43. bool Initialize(File ash_file) {
  44. fd_ = ash_file.GetPlatformFile();
  45. return memory_mapped_file_.Initialize(std::move(ash_file));
  46. }
  47. PlatformFile fd() const { return fd_; }
  48. const uint8_t* data() const { return memory_mapped_file_.data(); }
  49. size_t length() const { return memory_mapped_file_.length(); }
  50. private:
  51. PlatformFile fd_;
  52. MemoryMappedFile memory_mapped_file_;
  53. };
  54. std::unique_ptr<AshMemoryMappedFile> MmapAshFile(
  55. const FilePath& ash_file_path) {
  56. ScopedBlockingCall scoped_blocking_call(FROM_HERE, BlockingType::MAY_BLOCK);
  57. // Open Ash's data file.
  58. File ash_file(FilePath(ash_file_path), File::FLAG_OPEN | File::FLAG_READ);
  59. // Mmap Ash's data file.
  60. auto ash_mapped_file = std::make_unique<AshMemoryMappedFile>();
  61. bool map_successful = ash_mapped_file->Initialize(std::move(ash_file));
  62. if (!map_successful) {
  63. PLOG(DFATAL) << "Failed to mmap Ash's icudtl.dat";
  64. return nullptr;
  65. }
  66. return ash_mapped_file;
  67. }
  68. // Class wrapping the memory-merging logic for `icudtl.dat`.
  69. IcuMergeableDataFile::IcuMergeableDataFile() = default;
  70. IcuMergeableDataFile::~IcuMergeableDataFile() {
  71. if (lacros_data_) {
  72. ScopedBlockingCall scoped_blocking_call(FROM_HERE, BlockingType::MAY_BLOCK);
  73. munmap(lacros_data_, lacros_length_);
  74. }
  75. }
  76. IcuMergeableDataFile::Hashes::Hashes() = default;
  77. IcuMergeableDataFile::Hashes::Hashes(HashToOffsetMap ash,
  78. std::vector<HashType> lacros)
  79. : ash(std::move(ash)), lacros(std::move(lacros)) {}
  80. IcuMergeableDataFile::Hashes::Hashes(Hashes&& other) = default;
  81. IcuMergeableDataFile::Hashes& IcuMergeableDataFile::Hashes::operator=(
  82. Hashes&& other) = default;
  83. IcuMergeableDataFile::Hashes::~Hashes() = default;
  84. bool IcuMergeableDataFile::Initialize(File lacros_file,
  85. MemoryMappedFile::Region region) {
  86. DCHECK(region == MemoryMappedFile::Region::kWholeFile);
  87. DCHECK(!lacros_file_.IsValid()) << "ICUDataFile::Initialize called twice";
  88. lacros_file_ = std::move(lacros_file);
  89. lacros_length_ = lacros_file_.GetLength();
  90. if (lacros_length_ < 0)
  91. return false;
  92. // Map Lacros's version of `icudtl.dat`, then attempt merging with Ash.
  93. bool map_successful = MmapLacrosFile(/*remap=*/false);
  94. #if BUILDFLAG(IS_CHROMEOS_DEVICE)
  95. // If we're inside an actual ChromeOS system (i.e. not just in
  96. // linux-lacros-rel) then we can expect Ash Chrome (and its version of
  97. // `icudtl.dat`) to be present in the default directory.
  98. // In that case, we can attempt merging.
  99. if (map_successful && base::FeatureList::IsEnabled(kLacrosMergeIcuDataFile)) {
  100. bool merge_successful = MergeWithAshVersion(FilePath(kIcuDataFileAshPath));
  101. // If we hit a critical failure while merging, remap Lacros's version.
  102. if (!merge_successful) {
  103. PLOG(DFATAL) << "Attempt to merge Lacros's icudtl.dat with Ash's failed";
  104. map_successful = MmapLacrosFile(/*remap=*/true);
  105. }
  106. }
  107. #endif // BUILDFLAG(IS_CHROMEOS_DEVICE)
  108. return map_successful;
  109. }
  110. const uint8_t* IcuMergeableDataFile::data() const {
  111. return static_cast<const uint8_t*>(lacros_data_);
  112. }
  113. bool IcuMergeableDataFile::MergeWithAshVersion(const FilePath& ash_file_path) {
  114. // Verify the assumption that page size is 4K.
  115. DCHECK_EQ(sysconf(_SC_PAGESIZE), kPageSize);
  116. // Mmap Ash's data file.
  117. auto ash_file = MmapAshFile(ash_file_path);
  118. if (!ash_file)
  119. return true; // Non-critical failure.
  120. // Calculate hashes for each page in Ash and Lacros's data files.
  121. Hashes hashes = CalculateHashes(*ash_file, ash_file_path);
  122. // Find Lacros's ICU pages that are duplicated in Ash.
  123. int64_t lacros_offset = 0;
  124. while (lacros_offset < lacros_length_) {
  125. Slice ash_overlap = FindOverlap(*ash_file, hashes, lacros_offset);
  126. // If there's no overlap, move to the next page and keep scanning.
  127. if (ash_overlap.length == 0) {
  128. lacros_offset += kPageSize;
  129. continue;
  130. }
  131. // Found a sequence of equal pages, merge them with Ash.
  132. bool merge_successful = MergeArea(*ash_file, ash_overlap, lacros_offset);
  133. if (!merge_successful)
  134. return false; // Critical failure.
  135. lacros_offset += ash_overlap.length;
  136. }
  137. return true; // Success.
  138. }
  139. bool IcuMergeableDataFile::MmapLacrosFile(bool remap) {
  140. ScopedBlockingCall scoped_blocking_call(FROM_HERE, BlockingType::MAY_BLOCK);
  141. if (remap) {
  142. // If `remap` == true, we add the MAP_FIXED option to unmap the
  143. // existing map and replace it with the new one in a single operation.
  144. DCHECK_NE(lacros_data_, nullptr);
  145. lacros_data_ = static_cast<uint8_t*>(
  146. mmap(lacros_data_, lacros_length_, PROT_READ, MAP_FIXED | MAP_PRIVATE,
  147. lacros_file_.GetPlatformFile(), 0));
  148. } else {
  149. // Otherwise, simply map the file.
  150. lacros_data_ = static_cast<uint8_t*>(
  151. mmap(nullptr, lacros_length_, PROT_READ, MAP_PRIVATE,
  152. lacros_file_.GetPlatformFile(), 0));
  153. }
  154. if (lacros_data_ == MAP_FAILED) {
  155. lacros_data_ = nullptr;
  156. PLOG(DFATAL) << "Failed to mmap Lacros's icudtl.dat";
  157. return false;
  158. }
  159. return true;
  160. }
  161. IcuMergeableDataFile::Slice IcuMergeableDataFile::FindOverlap(
  162. const AshMemoryMappedFile& ash_file,
  163. const Hashes& hashes,
  164. size_t lacros_offset) const {
  165. // Search for equal pages by hash.
  166. HashType hash = hashes.lacros[lacros_offset / kPageSize];
  167. auto search = hashes.ash.find(hash);
  168. if (search == hashes.ash.end())
  169. return {0, 0};
  170. // Count how many pages (if any) have the same content.
  171. size_t ash_offset = search->second;
  172. size_t overlap_length =
  173. kPageSize * CountEqualPages(ash_file, ash_file.data() + ash_offset,
  174. lacros_data_ + lacros_offset);
  175. return {ash_offset, overlap_length};
  176. }
  177. bool IcuMergeableDataFile::MergeArea(const AshMemoryMappedFile& ash_file,
  178. const Slice& ash_overlap,
  179. size_t lacros_offset) {
  180. ScopedBlockingCall scoped_blocking_call(FROM_HERE, BlockingType::MAY_BLOCK);
  181. // Unmap from Lacros's file and map from Ash's file instead.
  182. // NOTE: "[...] If the memory region specified by addr and length overlaps
  183. // pages of any existing mapping(s), then the overlapped part of the
  184. // existing mapping(s) will be discarded. If the specified address
  185. // cannot be used, mmap() will fail."
  186. // Reference: https://man7.org/linux/man-pages/man2/mmap.2.html
  187. uint8_t* map_result = static_cast<uint8_t*>(
  188. mmap(lacros_data_ + lacros_offset, ash_overlap.length, PROT_READ,
  189. MAP_FIXED | MAP_PRIVATE, ash_file.fd(), ash_overlap.offset));
  190. if (map_result == MAP_FAILED) {
  191. PLOG(DFATAL) << "Couldn't mmap Ash's icudtl.dat while merging";
  192. return false;
  193. }
  194. return true;
  195. }
  196. size_t IcuMergeableDataFile::CountEqualPages(
  197. const AshMemoryMappedFile& ash_file,
  198. const uint8_t* ash_page,
  199. const uint8_t* lacros_page) const {
  200. size_t pages = 0;
  201. const uint8_t* ash_end = ash_file.data() + ash_file.length();
  202. const uint8_t* lacros_end = lacros_data_ + lacros_length_;
  203. while (ash_page < ash_end && lacros_page < lacros_end &&
  204. memcmp(ash_page, lacros_page, kPageSize) == 0) {
  205. ash_page += kPageSize;
  206. lacros_page += kPageSize;
  207. pages++;
  208. }
  209. return pages;
  210. }
  211. IcuMergeableDataFile::Hashes IcuMergeableDataFile::CalculateHashes(
  212. const AshMemoryMappedFile& ash_file,
  213. const FilePath& ash_file_path) {
  214. // Try loading hashes from the pre-computed files first.
  215. Hashes hashes;
  216. used_cached_hashes_ = MaybeLoadCachedHashes(ash_file, ash_file_path, hashes);
  217. if (!used_cached_hashes_) {
  218. // Calculate hashes for each page in Ash's data file.
  219. std::vector<HashOffset> ash_hashes;
  220. ash_hashes.reserve(NPages(ash_file.length()));
  221. for (size_t offset = 0; offset < ash_file.length(); offset += kPageSize) {
  222. // NOTE: "POSIX specifies that the system shall always zero fill any
  223. // partial page at the end of the object [...]".
  224. // Reference: https://man7.org/linux/man-pages/man2/mmap.2.html
  225. //
  226. // Therefore this code works even if the size of Ash's `icudtl.dat` is not
  227. // a multiple of the page size.
  228. HashType hash = HashPage(ash_file.data() + offset);
  229. ash_hashes.emplace_back(hash, offset);
  230. }
  231. // Calculate hashes for each page in Lacros's data file.
  232. hashes.lacros.reserve(NPages(lacros_length_));
  233. for (int64_t offset = 0; offset < lacros_length_; offset += kPageSize) {
  234. HashType hash = HashPage(lacros_data_ + offset);
  235. hashes.lacros.emplace_back(hash);
  236. }
  237. hashes.ash = HashToOffsetMap(std::move(ash_hashes));
  238. }
  239. return hashes;
  240. }
  241. bool IcuMergeableDataFile::MaybeLoadCachedHashes(
  242. const AshMemoryMappedFile& ash_file,
  243. const FilePath& ash_file_path,
  244. Hashes& hashes) {
  245. FilePath ash_hash_path =
  246. ash_file_path.AddExtensionASCII(kIcuDataFileHashExtension);
  247. FilePath lacros_hash_path =
  248. GetLacrosFilePath().AddExtensionASCII(kIcuDataFileHashExtension);
  249. // Memory map Ash's `icudtl.dat.hash`. Ensure its size is valid and consistent
  250. // with the current version of `icudtl.dat`.
  251. MemoryMappedFile ash_hash_file;
  252. size_t ash_pages = NPages(ash_file.length());
  253. bool result = ash_hash_file.Initialize(ash_hash_path);
  254. if (!result || (ash_hash_file.length() % kHashBytes) ||
  255. ((ash_hash_file.length() / kHashBytes) != ash_pages)) {
  256. return false;
  257. }
  258. // Same for Lacros's `icudtl.dat.hash`.
  259. MemoryMappedFile lacros_hash_file;
  260. size_t lacros_pages = NPages(lacros_length_);
  261. result = lacros_hash_file.Initialize(lacros_hash_path);
  262. if (!result || (lacros_hash_file.length() % kHashBytes) ||
  263. ((lacros_hash_file.length() / kHashBytes) != lacros_pages)) {
  264. return false;
  265. }
  266. // Load Ash's hashes.
  267. std::vector<HashOffset> ash_hashes;
  268. ash_hashes.reserve(ash_pages);
  269. for (size_t i = 0; i < ash_hash_file.length(); i += kHashBytes) {
  270. HashType hash = ReadHash(ash_hash_file.data(), i);
  271. size_t offset = (i / kHashBytes) * kPageSize;
  272. ash_hashes.emplace_back(hash, offset);
  273. }
  274. // Load Lacros's hashes.
  275. hashes.lacros.reserve(lacros_pages);
  276. for (size_t i = 0; i < lacros_hash_file.length(); i += kHashBytes) {
  277. HashType hash = ReadHash(lacros_hash_file.data(), i);
  278. hashes.lacros.emplace_back(hash);
  279. }
  280. hashes.ash = HashToOffsetMap(std::move(ash_hashes));
  281. return true;
  282. }
  283. FilePath IcuMergeableDataFile::GetLacrosFilePath() {
  284. // /proc/self/fd/<fd>
  285. // This is a subdirectory containing one entry for each file
  286. // which the process has open, named by its file descriptor,
  287. // and which is a symbolic link to the actual file.
  288. // Reference: proc(5) - Linux manual page.
  289. char path[PATH_MAX];
  290. FilePath proc_path =
  291. FilePath("/proc/self/fd/")
  292. .AppendASCII(base::NumberToString(lacros_file_.GetPlatformFile()));
  293. // We read the content of the symbolic link to find the path of the
  294. // file associated with the file descriptor.
  295. int64_t path_len = readlink(proc_path.value().c_str(), path, sizeof(path));
  296. DCHECK_NE(path_len, -1);
  297. DCHECK_LT(path_len, PATH_MAX);
  298. return FilePath(std::string(path, 0, path_len));
  299. }
  300. } // namespace base::i18n