format.h 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. /*
  3. * Copyright 2018 Google LLC
  4. */
  5. /*
  6. * Overview
  7. * --------
  8. * The backbone of the incremental-fs ondisk format is an append only linked
  9. * list of metadata blocks. Each metadata block contains an offset of the next
  10. * one. These blocks describe files and directories on the
  11. * file system. They also represent actions of adding and removing file names
  12. * (hard links).
  13. *
  14. * Every time incremental-fs instance is mounted, it reads through this list
  15. * to recreate filesystem's state in memory. An offset of the first record in
  16. * the metadata list is stored in the superblock at the beginning of the backing
  17. * file.
  18. *
  19. * Most of the backing file is taken by data areas and blockmaps.
  20. * Since data blocks can be compressed and have different sizes,
  21. * single per-file data area can't be pre-allocated. That's why blockmaps are
  22. * needed in order to find a location and size of each data block in
  23. * the backing file. Each time a file is created, a corresponding block map is
  24. * allocated to store future offsets of data blocks.
  25. *
  26. * Whenever a data block is given by data loader to incremental-fs:
  27. * - A data area with the given block is appended to the end of
  28. * the backing file.
  29. * - A record in the blockmap for the given block index is updated to reflect
  30. * its location, size, and compression algorithm.
  31. * Metadata records
  32. * ----------------
  33. * incfs_blockmap - metadata record that specifies size and location
  34. * of a blockmap area for a given file. This area
  35. * contains an array of incfs_blockmap_entry-s.
  36. * incfs_file_signature - metadata record that specifies where file signature
  37. * and its hash tree can be found in the backing file.
  38. *
  39. * incfs_file_attr - metadata record that specifies where additional file
  40. * attributes blob can be found.
  41. *
  42. * Metadata header
  43. * ---------------
  44. * incfs_md_header - header of a metadata record. It's always a part
  45. * of other structures and served purpose of metadata
  46. * bookkeeping.
  47. *
  48. * +-----------------------------------------------+ ^
  49. * | incfs_md_header | |
  50. * | 1. type of body(BLOCKMAP, FILE_ATTR..) | |
  51. * | 2. size of the whole record header + body | |
  52. * | 3. CRC the whole record header + body | |
  53. * | 4. offset of the previous md record |]------+
  54. * | 5. offset of the next md record (md link) |]---+
  55. * +-----------------------------------------------+ |
  56. * | Metadata record body with useful data | |
  57. * +-----------------------------------------------+ |
  58. * +--->
  59. *
  60. * Other ondisk structures
  61. * -----------------------
  62. * incfs_super_block - backing file header
  63. * incfs_blockmap_entry - a record in a blockmap area that describes size
  64. * and location of a data block.
  65. * Data blocks dont have any particular structure, they are written to the
  66. * backing file in a raw form as they come from a data loader.
  67. *
  68. * Backing file layout
  69. * -------------------
  70. *
  71. *
  72. * +-------------------------------------------+
  73. * | incfs_file_header |]---+
  74. * +-------------------------------------------+ |
  75. * | metadata |<---+
  76. * | incfs_file_signature |]---+
  77. * +-------------------------------------------+ |
  78. * ......................... |
  79. * +-------------------------------------------+ | metadata
  80. * +------->| blockmap area | | list links
  81. * | | [incfs_blockmap_entry] | |
  82. * | | [incfs_blockmap_entry] | |
  83. * | | [incfs_blockmap_entry] | |
  84. * | +--[| [incfs_blockmap_entry] | |
  85. * | | | [incfs_blockmap_entry] | |
  86. * | | | [incfs_blockmap_entry] | |
  87. * | | +-------------------------------------------+ |
  88. * | | ......................... |
  89. * | | +-------------------------------------------+ |
  90. * | | | metadata |<---+
  91. * +----|--[| incfs_blockmap |]---+
  92. * | +-------------------------------------------+ |
  93. * | ......................... |
  94. * | +-------------------------------------------+ |
  95. * +-->| data block | |
  96. * +-------------------------------------------+ |
  97. * ......................... |
  98. * +-------------------------------------------+ |
  99. * | metadata |<---+
  100. * | incfs_file_attr |
  101. * +-------------------------------------------+
  102. */
  103. #ifndef _INCFS_FORMAT_H
  104. #define _INCFS_FORMAT_H
  105. #include <linux/types.h>
  106. #include <linux/kernel.h>
  107. #include <uapi/linux/incrementalfs.h>
  108. #include "internal.h"
  109. #define INCFS_MAX_NAME_LEN 255
  110. #define INCFS_FORMAT_V1 1
  111. #define INCFS_FORMAT_CURRENT_VER INCFS_FORMAT_V1
  112. enum incfs_metadata_type {
  113. INCFS_MD_NONE = 0,
  114. INCFS_MD_BLOCK_MAP = 1,
  115. INCFS_MD_FILE_ATTR = 2,
  116. INCFS_MD_SIGNATURE = 3,
  117. INCFS_MD_STATUS = 4,
  118. INCFS_MD_VERITY_SIGNATURE = 5,
  119. };
  120. enum incfs_file_header_flags {
  121. INCFS_FILE_MAPPED = 1 << 1,
  122. };
  123. /* Header included at the beginning of all metadata records on the disk. */
  124. struct incfs_md_header {
  125. __u8 h_md_entry_type;
  126. /*
  127. * Size of the metadata record.
  128. * (e.g. inode, dir entry etc) not just this struct.
  129. */
  130. __le16 h_record_size;
  131. /*
  132. * Was: CRC32 of the metadata record.
  133. * (e.g. inode, dir entry etc) not just this struct.
  134. */
  135. __le32 h_unused1;
  136. /* Offset of the next metadata entry if any */
  137. __le64 h_next_md_offset;
  138. /* Was: Offset of the previous metadata entry if any */
  139. __le64 h_unused2;
  140. } __packed;
  141. /* Backing file header */
  142. struct incfs_file_header {
  143. /* Magic number: INCFS_MAGIC_NUMBER */
  144. __le64 fh_magic;
  145. /* Format version: INCFS_FORMAT_CURRENT_VER */
  146. __le64 fh_version;
  147. /* sizeof(incfs_file_header) */
  148. __le16 fh_header_size;
  149. /* INCFS_DATA_FILE_BLOCK_SIZE */
  150. __le16 fh_data_block_size;
  151. /* File flags, from incfs_file_header_flags */
  152. __le32 fh_flags;
  153. union {
  154. /* Standard incfs file */
  155. struct {
  156. /* Offset of the first metadata record */
  157. __le64 fh_first_md_offset;
  158. /* Full size of the file's content */
  159. __le64 fh_file_size;
  160. /* File uuid */
  161. incfs_uuid_t fh_uuid;
  162. };
  163. /* Mapped file - INCFS_FILE_MAPPED set in fh_flags */
  164. struct {
  165. /* Offset in original file */
  166. __le64 fh_original_offset;
  167. /* Full size of the file's content */
  168. __le64 fh_mapped_file_size;
  169. /* Original file's uuid */
  170. incfs_uuid_t fh_original_uuid;
  171. };
  172. };
  173. } __packed;
  174. enum incfs_block_map_entry_flags {
  175. INCFS_BLOCK_COMPRESSED_LZ4 = 1,
  176. INCFS_BLOCK_COMPRESSED_ZSTD = 2,
  177. /* Reserve 3 bits for compression alg */
  178. INCFS_BLOCK_COMPRESSED_MASK = 7,
  179. };
  180. /* Block map entry pointing to an actual location of the data block. */
  181. struct incfs_blockmap_entry {
  182. /* Offset of the actual data block. Lower 32 bits */
  183. __le32 me_data_offset_lo;
  184. /* Offset of the actual data block. Higher 16 bits */
  185. __le16 me_data_offset_hi;
  186. /* How many bytes the data actually occupies in the backing file */
  187. __le16 me_data_size;
  188. /* Block flags from incfs_block_map_entry_flags */
  189. __le16 me_flags;
  190. } __packed;
  191. /* Metadata record for locations of file blocks. Type = INCFS_MD_BLOCK_MAP */
  192. struct incfs_blockmap {
  193. struct incfs_md_header m_header;
  194. /* Base offset of the array of incfs_blockmap_entry */
  195. __le64 m_base_offset;
  196. /* Size of the map entry array in blocks */
  197. __le32 m_block_count;
  198. } __packed;
  199. /*
  200. * Metadata record for file signature. Type = INCFS_MD_SIGNATURE
  201. *
  202. * The signature stored here is the APK V4 signature data blob. See the
  203. * definition of incfs_new_file_args::signature_info for an explanation of this
  204. * blob. Specifically, it contains the root hash, but it does *not* contain
  205. * anything that the kernel treats as a signature.
  206. *
  207. * When FS_IOC_ENABLE_VERITY is called on a file without this record, an APK V4
  208. * signature blob and a hash tree are added to the file, and then this metadata
  209. * record is created to record their locations.
  210. */
  211. struct incfs_file_signature {
  212. struct incfs_md_header sg_header;
  213. __le32 sg_sig_size; /* The size of the signature. */
  214. __le64 sg_sig_offset; /* Signature's offset in the backing file */
  215. __le32 sg_hash_tree_size; /* The size of the hash tree. */
  216. __le64 sg_hash_tree_offset; /* Hash tree offset in the backing file */
  217. } __packed;
  218. /* In memory version of above */
  219. struct incfs_df_signature {
  220. u32 sig_size;
  221. u64 sig_offset;
  222. u32 hash_size;
  223. u64 hash_offset;
  224. };
  225. struct incfs_status {
  226. struct incfs_md_header is_header;
  227. __le32 is_data_blocks_written; /* Number of data blocks written */
  228. __le32 is_hash_blocks_written; /* Number of hash blocks written */
  229. __le32 is_dummy[6]; /* Spare fields */
  230. } __packed;
  231. /*
  232. * Metadata record for verity signature. Type = INCFS_MD_VERITY_SIGNATURE
  233. *
  234. * This record will only exist for verity-enabled files with signatures. Verity
  235. * enabled files without signatures do not have this record. This signature is
  236. * checked by fs-verity identically to any other fs-verity signature.
  237. */
  238. struct incfs_file_verity_signature {
  239. struct incfs_md_header vs_header;
  240. /* The size of the signature */
  241. __le32 vs_size;
  242. /* Signature's offset in the backing file */
  243. __le64 vs_offset;
  244. } __packed;
  245. /* In memory version of above */
  246. struct incfs_df_verity_signature {
  247. u32 size;
  248. u64 offset;
  249. };
  250. /* State of the backing file. */
  251. struct backing_file_context {
  252. /* Protects writes to bc_file */
  253. struct mutex bc_mutex;
  254. /* File object to read data from */
  255. struct file *bc_file;
  256. /*
  257. * Offset of the last known metadata record in the backing file.
  258. * 0 means there are no metadata records.
  259. */
  260. loff_t bc_last_md_record_offset;
  261. /*
  262. * Credentials to set before reads/writes
  263. * Note that this is a pointer to the mount_info mi_owner field so
  264. * there is no need to get/put the creds
  265. */
  266. const struct cred *bc_cred;
  267. };
  268. struct metadata_handler {
  269. loff_t md_record_offset;
  270. loff_t md_prev_record_offset;
  271. void *context;
  272. union {
  273. struct incfs_md_header md_header;
  274. struct incfs_blockmap blockmap;
  275. struct incfs_file_signature signature;
  276. struct incfs_status status;
  277. struct incfs_file_verity_signature verity_signature;
  278. } md_buffer;
  279. int (*handle_blockmap)(struct incfs_blockmap *bm,
  280. struct metadata_handler *handler);
  281. int (*handle_signature)(struct incfs_file_signature *sig,
  282. struct metadata_handler *handler);
  283. int (*handle_status)(struct incfs_status *sig,
  284. struct metadata_handler *handler);
  285. int (*handle_verity_signature)(struct incfs_file_verity_signature *s,
  286. struct metadata_handler *handler);
  287. };
  288. #define INCFS_MAX_METADATA_RECORD_SIZE \
  289. sizeof_field(struct metadata_handler, md_buffer)
  290. /* Backing file context management */
  291. struct mount_info;
  292. struct backing_file_context *incfs_alloc_bfc(struct mount_info *mi,
  293. struct file *backing_file);
  294. void incfs_free_bfc(struct backing_file_context *bfc);
  295. /* Writing stuff */
  296. int incfs_write_blockmap_to_backing_file(struct backing_file_context *bfc,
  297. u32 block_count);
  298. int incfs_write_fh_to_backing_file(struct backing_file_context *bfc,
  299. incfs_uuid_t *uuid, u64 file_size);
  300. int incfs_write_mapping_fh_to_backing_file(struct backing_file_context *bfc,
  301. incfs_uuid_t *uuid, u64 file_size, u64 offset);
  302. int incfs_write_data_block_to_backing_file(struct backing_file_context *bfc,
  303. struct mem_range block,
  304. int block_index, loff_t bm_base_off,
  305. u16 flags);
  306. int incfs_write_hash_block_to_backing_file(struct backing_file_context *bfc,
  307. struct mem_range block,
  308. int block_index,
  309. loff_t hash_area_off,
  310. loff_t bm_base_off,
  311. loff_t file_size);
  312. int incfs_write_signature_to_backing_file(struct backing_file_context *bfc,
  313. struct mem_range sig, u32 tree_size,
  314. loff_t *tree_offset, loff_t *sig_offset);
  315. int incfs_write_status_to_backing_file(struct backing_file_context *bfc,
  316. loff_t status_offset,
  317. u32 data_blocks_written,
  318. u32 hash_blocks_written);
  319. int incfs_write_verity_signature_to_backing_file(
  320. struct backing_file_context *bfc, struct mem_range signature,
  321. loff_t *offset);
  322. /* Reading stuff */
  323. int incfs_read_file_header(struct backing_file_context *bfc,
  324. loff_t *first_md_off, incfs_uuid_t *uuid,
  325. u64 *file_size, u32 *flags);
  326. int incfs_read_blockmap_entry(struct backing_file_context *bfc, int block_index,
  327. loff_t bm_base_off,
  328. struct incfs_blockmap_entry *bm_entry);
  329. int incfs_read_blockmap_entries(struct backing_file_context *bfc,
  330. struct incfs_blockmap_entry *entries,
  331. int start_index, int blocks_number,
  332. loff_t bm_base_off);
  333. int incfs_read_next_metadata_record(struct backing_file_context *bfc,
  334. struct metadata_handler *handler);
  335. ssize_t incfs_kread(struct backing_file_context *bfc, void *buf, size_t size,
  336. loff_t pos);
  337. ssize_t incfs_kwrite(struct backing_file_context *bfc, const void *buf,
  338. size_t size, loff_t pos);
  339. #endif /* _INCFS_FORMAT_H */