data_mgmt.h 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. /*
  3. * Copyright 2019 Google LLC
  4. */
  5. #ifndef _INCFS_DATA_MGMT_H
  6. #define _INCFS_DATA_MGMT_H
  7. #include <linux/cred.h>
  8. #include <linux/fs.h>
  9. #include <linux/types.h>
  10. #include <linux/mutex.h>
  11. #include <linux/spinlock.h>
  12. #include <linux/rcupdate.h>
  13. #include <linux/completion.h>
  14. #include <linux/wait.h>
  15. #include <linux/zstd.h>
  16. #include <crypto/hash.h>
  17. #include <linux/rwsem.h>
  18. #include <uapi/linux/incrementalfs.h>
  19. #include "internal.h"
  20. #include "pseudo_files.h"
  21. #define SEGMENTS_PER_FILE 3
  22. enum LOG_RECORD_TYPE {
  23. FULL,
  24. SAME_FILE,
  25. SAME_FILE_CLOSE_BLOCK,
  26. SAME_FILE_CLOSE_BLOCK_SHORT,
  27. SAME_FILE_NEXT_BLOCK,
  28. SAME_FILE_NEXT_BLOCK_SHORT,
  29. };
  30. struct full_record {
  31. enum LOG_RECORD_TYPE type : 3; /* FULL */
  32. u32 block_index : 29;
  33. incfs_uuid_t file_id;
  34. u64 absolute_ts_us;
  35. uid_t uid;
  36. } __packed; /* 32 bytes */
  37. struct same_file {
  38. enum LOG_RECORD_TYPE type : 3; /* SAME_FILE */
  39. u32 block_index : 29;
  40. uid_t uid;
  41. u16 relative_ts_us; /* max 2^16 us ~= 64 ms */
  42. } __packed; /* 10 bytes */
  43. struct same_file_close_block {
  44. enum LOG_RECORD_TYPE type : 3; /* SAME_FILE_CLOSE_BLOCK */
  45. u16 relative_ts_us : 13; /* max 2^13 us ~= 8 ms */
  46. s16 block_index_delta;
  47. } __packed; /* 4 bytes */
  48. struct same_file_close_block_short {
  49. enum LOG_RECORD_TYPE type : 3; /* SAME_FILE_CLOSE_BLOCK_SHORT */
  50. u8 relative_ts_tens_us : 5; /* max 2^5*10 us ~= 320 us */
  51. s8 block_index_delta;
  52. } __packed; /* 2 bytes */
  53. struct same_file_next_block {
  54. enum LOG_RECORD_TYPE type : 3; /* SAME_FILE_NEXT_BLOCK */
  55. u16 relative_ts_us : 13; /* max 2^13 us ~= 8 ms */
  56. } __packed; /* 2 bytes */
  57. struct same_file_next_block_short {
  58. enum LOG_RECORD_TYPE type : 3; /* SAME_FILE_NEXT_BLOCK_SHORT */
  59. u8 relative_ts_tens_us : 5; /* max 2^5*10 us ~= 320 us */
  60. } __packed; /* 1 byte */
  61. union log_record {
  62. struct full_record full_record;
  63. struct same_file same_file;
  64. struct same_file_close_block same_file_close_block;
  65. struct same_file_close_block_short same_file_close_block_short;
  66. struct same_file_next_block same_file_next_block;
  67. struct same_file_next_block_short same_file_next_block_short;
  68. };
  69. struct read_log_state {
  70. /* Log buffer generation id, incremented on configuration changes */
  71. u32 generation_id;
  72. /* Offset in rl_ring_buf to write into. */
  73. u32 next_offset;
  74. /* Current number of writer passes over rl_ring_buf */
  75. u32 current_pass_no;
  76. /* Current full_record to diff against */
  77. struct full_record base_record;
  78. /* Current record number counting from configuration change */
  79. u64 current_record_no;
  80. };
  81. /* A ring buffer to save records about data blocks which were recently read. */
  82. struct read_log {
  83. void *rl_ring_buf;
  84. int rl_size;
  85. struct read_log_state rl_head;
  86. struct read_log_state rl_tail;
  87. /* A lock to protect the above fields */
  88. spinlock_t rl_lock;
  89. /* A queue of waiters who want to be notified about reads */
  90. wait_queue_head_t ml_notif_wq;
  91. /* A work item to wake up those waiters without slowing down readers */
  92. struct delayed_work ml_wakeup_work;
  93. };
  94. struct mount_options {
  95. unsigned int read_timeout_ms;
  96. unsigned int readahead_pages;
  97. unsigned int read_log_pages;
  98. unsigned int read_log_wakeup_count;
  99. bool report_uid;
  100. char *sysfs_name;
  101. };
  102. struct mount_info {
  103. struct super_block *mi_sb;
  104. struct path mi_backing_dir_path;
  105. struct dentry *mi_index_dir;
  106. /* For stacking mounts, if true, this indicates if the index dir needs
  107. * to be freed for this SB otherwise it was created by lower level SB */
  108. bool mi_index_free;
  109. struct dentry *mi_incomplete_dir;
  110. /* For stacking mounts, if true, this indicates if the incomplete dir
  111. * needs to be freed for this SB. Similar to mi_index_free */
  112. bool mi_incomplete_free;
  113. const struct cred *mi_owner;
  114. struct mount_options mi_options;
  115. /* This mutex is to be taken before create, rename, delete */
  116. struct mutex mi_dir_struct_mutex;
  117. /*
  118. * A queue of waiters who want to be notified about new pending reads.
  119. */
  120. wait_queue_head_t mi_pending_reads_notif_wq;
  121. /*
  122. * Protects - RCU safe:
  123. * - reads_list_head
  124. * - mi_pending_reads_count
  125. * - mi_last_pending_read_number
  126. * - data_file_segment.reads_list_head
  127. */
  128. spinlock_t pending_read_lock;
  129. /* List of active pending_read objects */
  130. struct list_head mi_reads_list_head;
  131. /* Total number of items in reads_list_head */
  132. int mi_pending_reads_count;
  133. /*
  134. * Last serial number that was assigned to a pending read.
  135. * 0 means no pending reads have been seen yet.
  136. */
  137. int mi_last_pending_read_number;
  138. /* Temporary buffer for read logger. */
  139. struct read_log mi_log;
  140. /* SELinux needs special xattrs on our pseudo files */
  141. struct mem_range pseudo_file_xattr[PSEUDO_FILE_COUNT];
  142. /* A queue of waiters who want to be notified about blocks_written */
  143. wait_queue_head_t mi_blocks_written_notif_wq;
  144. /* Number of blocks written since mount */
  145. atomic_t mi_blocks_written;
  146. /* Per UID read timeouts */
  147. spinlock_t mi_per_uid_read_timeouts_lock;
  148. struct incfs_per_uid_read_timeouts *mi_per_uid_read_timeouts;
  149. int mi_per_uid_read_timeouts_size;
  150. /* zstd workspace */
  151. struct mutex mi_zstd_workspace_mutex;
  152. void *mi_zstd_workspace;
  153. ZSTD_DStream *mi_zstd_stream;
  154. struct delayed_work mi_zstd_cleanup_work;
  155. /* sysfs node */
  156. struct incfs_sysfs_node *mi_sysfs_node;
  157. /* Last error information */
  158. struct mutex mi_le_mutex;
  159. incfs_uuid_t mi_le_file_id;
  160. u64 mi_le_time_us;
  161. u32 mi_le_page;
  162. u32 mi_le_errno;
  163. uid_t mi_le_uid;
  164. /* Number of reads timed out */
  165. u32 mi_reads_failed_timed_out;
  166. /* Number of reads failed because hash verification failed */
  167. u32 mi_reads_failed_hash_verification;
  168. /* Number of reads failed for another reason */
  169. u32 mi_reads_failed_other;
  170. /* Number of reads delayed because page had to be fetched */
  171. u32 mi_reads_delayed_pending;
  172. /* Total time waiting for pages to be fetched */
  173. u64 mi_reads_delayed_pending_us;
  174. /*
  175. * Number of reads delayed because of per-uid min_time_us or
  176. * min_pending_time_us settings
  177. */
  178. u32 mi_reads_delayed_min;
  179. /* Total time waiting because of per-uid min_time_us or
  180. * min_pending_time_us settings.
  181. *
  182. * Note that if a read is initially delayed because we have to wait for
  183. * the page, then further delayed because of min_pending_time_us
  184. * setting, this counter gets incremented by only the further delay
  185. * time.
  186. */
  187. u64 mi_reads_delayed_min_us;
  188. };
  189. struct data_file_block {
  190. loff_t db_backing_file_data_offset;
  191. size_t db_stored_size;
  192. enum incfs_compression_alg db_comp_alg;
  193. };
  194. struct pending_read {
  195. incfs_uuid_t file_id;
  196. s64 timestamp_us;
  197. atomic_t done;
  198. int block_index;
  199. int serial_number;
  200. uid_t uid;
  201. struct list_head mi_reads_list;
  202. struct list_head segment_reads_list;
  203. struct rcu_head rcu;
  204. };
  205. struct data_file_segment {
  206. wait_queue_head_t new_data_arrival_wq;
  207. /* Protects reads and writes from the blockmap */
  208. struct rw_semaphore rwsem;
  209. /* List of active pending_read objects belonging to this segment */
  210. /* Protected by mount_info.pending_reads_mutex */
  211. struct list_head reads_list_head;
  212. };
  213. /*
  214. * Extra info associated with a file. Just a few bytes set by a user.
  215. */
  216. struct file_attr {
  217. loff_t fa_value_offset;
  218. size_t fa_value_size;
  219. u32 fa_crc;
  220. };
  221. struct data_file {
  222. struct backing_file_context *df_backing_file_context;
  223. struct mount_info *df_mount_info;
  224. incfs_uuid_t df_id;
  225. /*
  226. * Array of segments used to reduce lock contention for the file.
  227. * Segment is chosen for a block depends on the block's index.
  228. */
  229. struct data_file_segment df_segments[SEGMENTS_PER_FILE];
  230. /* Base offset of the first metadata record. */
  231. loff_t df_metadata_off;
  232. /* Base offset of the block map. */
  233. loff_t df_blockmap_off;
  234. /* File size in bytes */
  235. loff_t df_size;
  236. /* File header flags */
  237. u32 df_header_flags;
  238. /* File size in DATA_FILE_BLOCK_SIZE blocks */
  239. int df_data_block_count;
  240. /* Total number of blocks, data + hash */
  241. int df_total_block_count;
  242. /* For mapped files, the offset into the actual file */
  243. loff_t df_mapped_offset;
  244. /* Number of data blocks written to file */
  245. atomic_t df_data_blocks_written;
  246. /* Number of data blocks in the status block */
  247. u32 df_initial_data_blocks_written;
  248. /* Number of hash blocks written to file */
  249. atomic_t df_hash_blocks_written;
  250. /* Number of hash blocks in the status block */
  251. u32 df_initial_hash_blocks_written;
  252. /* Offset to status metadata header */
  253. loff_t df_status_offset;
  254. /*
  255. * Mutex acquired while enabling verity. Note that df_hash_tree is set
  256. * by enable verity.
  257. *
  258. * The backing file mutex bc_mutex may be taken while this mutex is
  259. * held.
  260. */
  261. struct mutex df_enable_verity;
  262. /*
  263. * Set either at construction time or during enabling verity. In the
  264. * latter case, set via smp_store_release, so use smp_load_acquire to
  265. * read it.
  266. */
  267. struct mtree *df_hash_tree;
  268. /* Guaranteed set if df_hash_tree is set. */
  269. struct incfs_df_signature *df_signature;
  270. /*
  271. * The verity file digest, set when verity is enabled and the file has
  272. * been opened
  273. */
  274. struct mem_range df_verity_file_digest;
  275. struct incfs_df_verity_signature *df_verity_signature;
  276. };
  277. struct dir_file {
  278. struct mount_info *mount_info;
  279. struct file *backing_dir;
  280. };
  281. struct inode_info {
  282. struct mount_info *n_mount_info; /* A mount, this file belongs to */
  283. struct inode *n_backing_inode;
  284. struct data_file *n_file;
  285. struct inode n_vfs_inode;
  286. };
  287. struct dentry_info {
  288. struct path backing_path;
  289. };
  290. enum FILL_PERMISSION {
  291. CANT_FILL = 0,
  292. CAN_FILL = 1,
  293. };
  294. struct incfs_file_data {
  295. /* Does this file handle have INCFS_IOC_FILL_BLOCKS permission */
  296. enum FILL_PERMISSION fd_fill_permission;
  297. /* If INCFS_IOC_GET_FILLED_BLOCKS has been called, where are we */
  298. int fd_get_block_pos;
  299. /* And how many filled blocks are there up to that point */
  300. int fd_filled_data_blocks;
  301. int fd_filled_hash_blocks;
  302. };
  303. struct mount_info *incfs_alloc_mount_info(struct super_block *sb,
  304. struct mount_options *options,
  305. struct path *backing_dir_path);
  306. int incfs_realloc_mount_info(struct mount_info *mi,
  307. struct mount_options *options);
  308. void incfs_free_mount_info(struct mount_info *mi);
  309. char *file_id_to_str(incfs_uuid_t id);
  310. struct dentry *incfs_lookup_dentry(struct dentry *parent, const char *name);
  311. struct data_file *incfs_open_data_file(struct mount_info *mi, struct file *bf);
  312. void incfs_free_data_file(struct data_file *df);
  313. struct dir_file *incfs_open_dir_file(struct mount_info *mi, struct file *bf);
  314. void incfs_free_dir_file(struct dir_file *dir);
  315. struct incfs_read_data_file_timeouts {
  316. u32 min_time_us;
  317. u32 min_pending_time_us;
  318. u32 max_pending_time_us;
  319. };
  320. ssize_t incfs_read_data_file_block(struct mem_range dst, struct file *f,
  321. int index, struct mem_range tmp,
  322. struct incfs_read_data_file_timeouts *timeouts);
  323. ssize_t incfs_read_merkle_tree_blocks(struct mem_range dst,
  324. struct data_file *df, size_t offset);
  325. int incfs_get_filled_blocks(struct data_file *df,
  326. struct incfs_file_data *fd,
  327. struct incfs_get_filled_blocks_args *arg);
  328. int incfs_read_file_signature(struct data_file *df, struct mem_range dst);
  329. int incfs_process_new_data_block(struct data_file *df,
  330. struct incfs_fill_block *block, u8 *data);
  331. int incfs_process_new_hash_block(struct data_file *df,
  332. struct incfs_fill_block *block, u8 *data);
  333. bool incfs_fresh_pending_reads_exist(struct mount_info *mi, int last_number);
  334. /*
  335. * Collects pending reads and saves them into the array (reads/reads_size).
  336. * Only reads with serial_number > sn_lowerbound are reported.
  337. * Returns how many reads were saved into the array.
  338. */
  339. int incfs_collect_pending_reads(struct mount_info *mi, int sn_lowerbound,
  340. struct incfs_pending_read_info *reads,
  341. struct incfs_pending_read_info2 *reads2,
  342. int reads_size, int *new_max_sn);
  343. int incfs_collect_logged_reads(struct mount_info *mi,
  344. struct read_log_state *start_state,
  345. struct incfs_pending_read_info *reads,
  346. struct incfs_pending_read_info2 *reads2,
  347. int reads_size);
  348. struct read_log_state incfs_get_log_state(struct mount_info *mi);
  349. int incfs_get_uncollected_logs_count(struct mount_info *mi,
  350. const struct read_log_state *state);
  351. static inline struct inode_info *get_incfs_node(struct inode *inode)
  352. {
  353. if (!inode)
  354. return NULL;
  355. if (inode->i_sb->s_magic != INCFS_MAGIC_NUMBER) {
  356. /* This inode doesn't belong to us. */
  357. pr_warn_once("incfs: %s on an alien inode.", __func__);
  358. return NULL;
  359. }
  360. return container_of(inode, struct inode_info, n_vfs_inode);
  361. }
  362. static inline struct data_file *get_incfs_data_file(struct file *f)
  363. {
  364. struct inode_info *node = NULL;
  365. if (!f)
  366. return NULL;
  367. if (!S_ISREG(f->f_inode->i_mode))
  368. return NULL;
  369. node = get_incfs_node(f->f_inode);
  370. if (!node)
  371. return NULL;
  372. return node->n_file;
  373. }
  374. static inline struct dir_file *get_incfs_dir_file(struct file *f)
  375. {
  376. if (!f)
  377. return NULL;
  378. if (!S_ISDIR(f->f_inode->i_mode))
  379. return NULL;
  380. return (struct dir_file *)f->private_data;
  381. }
  382. /*
  383. * Make sure that inode_info.n_file is initialized and inode can be used
  384. * for reading and writing data from/to the backing file.
  385. */
  386. int make_inode_ready_for_data_ops(struct mount_info *mi,
  387. struct inode *inode,
  388. struct file *backing_file);
  389. static inline struct dentry_info *get_incfs_dentry(const struct dentry *d)
  390. {
  391. if (!d)
  392. return NULL;
  393. return (struct dentry_info *)d->d_fsdata;
  394. }
  395. static inline void get_incfs_backing_path(const struct dentry *d,
  396. struct path *path)
  397. {
  398. struct dentry_info *di = get_incfs_dentry(d);
  399. if (!di) {
  400. *path = (struct path) {};
  401. return;
  402. }
  403. *path = di->backing_path;
  404. path_get(path);
  405. }
  406. static inline int get_blocks_count_for_size(u64 size)
  407. {
  408. if (size == 0)
  409. return 0;
  410. return 1 + (size - 1) / INCFS_DATA_FILE_BLOCK_SIZE;
  411. }
  412. #endif /* _INCFS_DATA_MGMT_H */