zdata.h 4.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190
  1. /* SPDX-License-Identifier: GPL-2.0-only */
  2. /*
  3. * Copyright (C) 2018 HUAWEI, Inc.
  4. * https://www.huawei.com/
  5. * Created by Gao Xiang <gaoxiang25@huawei.com>
  6. */
  7. #ifndef __EROFS_FS_ZDATA_H
  8. #define __EROFS_FS_ZDATA_H
  9. #include "internal.h"
  10. #include "zpvec.h"
  11. #define Z_EROFS_PCLUSTER_MAX_PAGES (Z_EROFS_PCLUSTER_MAX_SIZE / PAGE_SIZE)
  12. #define Z_EROFS_NR_INLINE_PAGEVECS 3
  13. /*
  14. * Structure fields follow one of the following exclusion rules.
  15. *
  16. * I: Modifiable by initialization/destruction paths and read-only
  17. * for everyone else;
  18. *
  19. * L: Field should be protected by pageset lock;
  20. *
  21. * A: Field should be accessed / updated in atomic for parallelized code.
  22. */
  23. struct z_erofs_collection {
  24. struct mutex lock;
  25. /* I: page offset of start position of decompression */
  26. unsigned short pageofs;
  27. /* L: maximum relative page index in pagevec[] */
  28. unsigned short nr_pages;
  29. /* L: total number of pages in pagevec[] */
  30. unsigned int vcnt;
  31. union {
  32. /* L: inline a certain number of pagevecs for bootstrap */
  33. erofs_vtptr_t pagevec[Z_EROFS_NR_INLINE_PAGEVECS];
  34. /* I: can be used to free the pcluster by RCU. */
  35. struct rcu_head rcu;
  36. };
  37. };
  38. #define Z_EROFS_PCLUSTER_FULL_LENGTH 0x00000001
  39. #define Z_EROFS_PCLUSTER_LENGTH_BIT 1
  40. /*
  41. * let's leave a type here in case of introducing
  42. * another tagged pointer later.
  43. */
  44. typedef void *z_erofs_next_pcluster_t;
  45. struct z_erofs_pcluster {
  46. struct erofs_workgroup obj;
  47. struct z_erofs_collection primary_collection;
  48. /* A: point to next chained pcluster or TAILs */
  49. z_erofs_next_pcluster_t next;
  50. /* A: lower limit of decompressed length and if full length or not */
  51. unsigned int length;
  52. /* I: physical cluster size in pages */
  53. unsigned short pclusterpages;
  54. /* I: compression algorithm format */
  55. unsigned char algorithmformat;
  56. /* A: compressed pages (can be cached or inplaced pages) */
  57. struct page *compressed_pages[];
  58. };
  59. #define z_erofs_primarycollection(pcluster) (&(pcluster)->primary_collection)
  60. /* let's avoid the valid 32-bit kernel addresses */
  61. /* the chained workgroup has't submitted io (still open) */
  62. #define Z_EROFS_PCLUSTER_TAIL ((void *)0x5F0ECAFE)
  63. /* the chained workgroup has already submitted io */
  64. #define Z_EROFS_PCLUSTER_TAIL_CLOSED ((void *)0x5F0EDEAD)
  65. #define Z_EROFS_PCLUSTER_NIL (NULL)
  66. struct z_erofs_decompressqueue {
  67. struct super_block *sb;
  68. atomic_t pending_bios;
  69. z_erofs_next_pcluster_t head;
  70. union {
  71. wait_queue_head_t wait;
  72. struct work_struct work;
  73. } u;
  74. };
  75. #define MNGD_MAPPING(sbi) ((sbi)->managed_cache->i_mapping)
  76. static inline bool erofs_page_is_managed(const struct erofs_sb_info *sbi,
  77. struct page *page)
  78. {
  79. return page->mapping == MNGD_MAPPING(sbi);
  80. }
  81. #define Z_EROFS_ONLINEPAGE_COUNT_BITS 2
  82. #define Z_EROFS_ONLINEPAGE_COUNT_MASK ((1 << Z_EROFS_ONLINEPAGE_COUNT_BITS) - 1)
  83. #define Z_EROFS_ONLINEPAGE_INDEX_SHIFT (Z_EROFS_ONLINEPAGE_COUNT_BITS)
  84. /*
  85. * waiters (aka. ongoing_packs): # to unlock the page
  86. * sub-index: 0 - for partial page, >= 1 full page sub-index
  87. */
  88. typedef atomic_t z_erofs_onlinepage_t;
  89. /* type punning */
  90. union z_erofs_onlinepage_converter {
  91. z_erofs_onlinepage_t *o;
  92. unsigned long *v;
  93. };
  94. static inline unsigned int z_erofs_onlinepage_index(struct page *page)
  95. {
  96. union z_erofs_onlinepage_converter u;
  97. DBG_BUGON(!PagePrivate(page));
  98. u.v = &page_private(page);
  99. return atomic_read(u.o) >> Z_EROFS_ONLINEPAGE_INDEX_SHIFT;
  100. }
  101. static inline void z_erofs_onlinepage_init(struct page *page)
  102. {
  103. union {
  104. z_erofs_onlinepage_t o;
  105. unsigned long v;
  106. /* keep from being unlocked in advance */
  107. } u = { .o = ATOMIC_INIT(1) };
  108. set_page_private(page, u.v);
  109. smp_wmb();
  110. SetPagePrivate(page);
  111. }
  112. static inline void z_erofs_onlinepage_fixup(struct page *page,
  113. uintptr_t index, bool down)
  114. {
  115. union z_erofs_onlinepage_converter u = { .v = &page_private(page) };
  116. int orig, orig_index, val;
  117. repeat:
  118. orig = atomic_read(u.o);
  119. orig_index = orig >> Z_EROFS_ONLINEPAGE_INDEX_SHIFT;
  120. if (orig_index) {
  121. if (!index)
  122. return;
  123. DBG_BUGON(orig_index != index);
  124. }
  125. val = (index << Z_EROFS_ONLINEPAGE_INDEX_SHIFT) |
  126. ((orig & Z_EROFS_ONLINEPAGE_COUNT_MASK) + (unsigned int)down);
  127. if (atomic_cmpxchg(u.o, orig, val) != orig)
  128. goto repeat;
  129. }
  130. static inline void z_erofs_onlinepage_endio(struct page *page)
  131. {
  132. union z_erofs_onlinepage_converter u;
  133. unsigned int v;
  134. DBG_BUGON(!PagePrivate(page));
  135. u.v = &page_private(page);
  136. v = atomic_dec_return(u.o);
  137. if (!(v & Z_EROFS_ONLINEPAGE_COUNT_MASK)) {
  138. set_page_private(page, 0);
  139. ClearPagePrivate(page);
  140. if (!PageError(page))
  141. SetPageUptodate(page);
  142. unlock_page(page);
  143. }
  144. erofs_dbg("%s, page %p value %x", __func__, page, atomic_read(u.o));
  145. }
  146. #define Z_EROFS_VMAP_ONSTACK_PAGES \
  147. min_t(unsigned int, THREAD_SIZE / 8 / sizeof(struct page *), 96U)
  148. #define Z_EROFS_VMAP_GLOBAL_PAGES 2048
  149. #endif