crypto.c 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * This contains encryption functions for per-file encryption.
  4. *
  5. * Copyright (C) 2015, Google, Inc.
  6. * Copyright (C) 2015, Motorola Mobility
  7. *
  8. * Written by Michael Halcrow, 2014.
  9. *
  10. * Filename encryption additions
  11. * Uday Savagaonkar, 2014
  12. * Encryption policy handling additions
  13. * Ildar Muslukhov, 2014
  14. * Add fscrypt_pullback_bio_page()
  15. * Jaegeuk Kim, 2015.
  16. *
  17. * This has not yet undergone a rigorous security audit.
  18. *
  19. * The usage of AES-XTS should conform to recommendations in NIST
  20. * Special Publication 800-38E and IEEE P1619/D16.
  21. */
  22. #include <linux/pagemap.h>
  23. #include <linux/mempool.h>
  24. #include <linux/module.h>
  25. #include <linux/scatterlist.h>
  26. #include <linux/ratelimit.h>
  27. #include <crypto/skcipher.h>
  28. #include "fscrypt_private.h"
  29. static unsigned int num_prealloc_crypto_pages = 32;
  30. module_param(num_prealloc_crypto_pages, uint, 0444);
  31. MODULE_PARM_DESC(num_prealloc_crypto_pages,
  32. "Number of crypto pages to preallocate");
  33. static mempool_t *fscrypt_bounce_page_pool = NULL;
  34. static struct workqueue_struct *fscrypt_read_workqueue;
  35. static DEFINE_MUTEX(fscrypt_init_mutex);
  36. struct kmem_cache *fscrypt_info_cachep;
  37. void fscrypt_enqueue_decrypt_work(struct work_struct *work)
  38. {
  39. queue_work(fscrypt_read_workqueue, work);
  40. }
  41. EXPORT_SYMBOL(fscrypt_enqueue_decrypt_work);
  42. struct page *fscrypt_alloc_bounce_page(gfp_t gfp_flags)
  43. {
  44. return mempool_alloc(fscrypt_bounce_page_pool, gfp_flags);
  45. }
  46. /**
  47. * fscrypt_free_bounce_page() - free a ciphertext bounce page
  48. * @bounce_page: the bounce page to free, or NULL
  49. *
  50. * Free a bounce page that was allocated by fscrypt_encrypt_pagecache_blocks(),
  51. * or by fscrypt_alloc_bounce_page() directly.
  52. */
  53. void fscrypt_free_bounce_page(struct page *bounce_page)
  54. {
  55. if (!bounce_page)
  56. return;
  57. set_page_private(bounce_page, (unsigned long)NULL);
  58. ClearPagePrivate(bounce_page);
  59. mempool_free(bounce_page, fscrypt_bounce_page_pool);
  60. }
  61. EXPORT_SYMBOL(fscrypt_free_bounce_page);
  62. /*
  63. * Generate the IV for the given logical block number within the given file.
  64. * For filenames encryption, lblk_num == 0.
  65. *
  66. * Keep this in sync with fscrypt_limit_io_blocks(). fscrypt_limit_io_blocks()
  67. * needs to know about any IV generation methods where the low bits of IV don't
  68. * simply contain the lblk_num (e.g., IV_INO_LBLK_32).
  69. */
  70. void fscrypt_generate_iv(union fscrypt_iv *iv, u64 lblk_num,
  71. const struct fscrypt_info *ci)
  72. {
  73. u8 flags = fscrypt_policy_flags(&ci->ci_policy);
  74. memset(iv, 0, ci->ci_mode->ivsize);
  75. if (flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64) {
  76. WARN_ON_ONCE(lblk_num > U32_MAX);
  77. WARN_ON_ONCE(ci->ci_inode->i_ino > U32_MAX);
  78. lblk_num |= (u64)ci->ci_inode->i_ino << 32;
  79. } else if (flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32) {
  80. WARN_ON_ONCE(lblk_num > U32_MAX);
  81. lblk_num = (u32)(ci->ci_hashed_ino + lblk_num);
  82. } else if (flags & FSCRYPT_POLICY_FLAG_DIRECT_KEY) {
  83. memcpy(iv->nonce, ci->ci_nonce, FSCRYPT_FILE_NONCE_SIZE);
  84. }
  85. iv->lblk_num = cpu_to_le64(lblk_num);
  86. }
  87. /* Encrypt or decrypt a single filesystem block of file contents */
  88. int fscrypt_crypt_block(const struct inode *inode, fscrypt_direction_t rw,
  89. u64 lblk_num, struct page *src_page,
  90. struct page *dest_page, unsigned int len,
  91. unsigned int offs, gfp_t gfp_flags)
  92. {
  93. union fscrypt_iv iv;
  94. struct skcipher_request *req = NULL;
  95. DECLARE_CRYPTO_WAIT(wait);
  96. struct scatterlist dst, src;
  97. struct fscrypt_info *ci = inode->i_crypt_info;
  98. struct crypto_skcipher *tfm = ci->ci_enc_key.tfm;
  99. int res = 0;
  100. if (WARN_ON_ONCE(len <= 0))
  101. return -EINVAL;
  102. if (WARN_ON_ONCE(len % FS_CRYPTO_BLOCK_SIZE != 0))
  103. return -EINVAL;
  104. fscrypt_generate_iv(&iv, lblk_num, ci);
  105. req = skcipher_request_alloc(tfm, gfp_flags);
  106. if (!req)
  107. return -ENOMEM;
  108. skcipher_request_set_callback(
  109. req, CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP,
  110. crypto_req_done, &wait);
  111. sg_init_table(&dst, 1);
  112. sg_set_page(&dst, dest_page, len, offs);
  113. sg_init_table(&src, 1);
  114. sg_set_page(&src, src_page, len, offs);
  115. skcipher_request_set_crypt(req, &src, &dst, len, &iv);
  116. if (rw == FS_DECRYPT)
  117. res = crypto_wait_req(crypto_skcipher_decrypt(req), &wait);
  118. else
  119. res = crypto_wait_req(crypto_skcipher_encrypt(req), &wait);
  120. skcipher_request_free(req);
  121. if (res) {
  122. fscrypt_err(inode, "%scryption failed for block %llu: %d",
  123. (rw == FS_DECRYPT ? "De" : "En"), lblk_num, res);
  124. return res;
  125. }
  126. return 0;
  127. }
  128. /**
  129. * fscrypt_encrypt_pagecache_blocks() - Encrypt filesystem blocks from a
  130. * pagecache page
  131. * @page: The locked pagecache page containing the block(s) to encrypt
  132. * @len: Total size of the block(s) to encrypt. Must be a nonzero
  133. * multiple of the filesystem's block size.
  134. * @offs: Byte offset within @page of the first block to encrypt. Must be
  135. * a multiple of the filesystem's block size.
  136. * @gfp_flags: Memory allocation flags. See details below.
  137. *
  138. * A new bounce page is allocated, and the specified block(s) are encrypted into
  139. * it. In the bounce page, the ciphertext block(s) will be located at the same
  140. * offsets at which the plaintext block(s) were located in the source page; any
  141. * other parts of the bounce page will be left uninitialized. However, normally
  142. * blocksize == PAGE_SIZE and the whole page is encrypted at once.
  143. *
  144. * This is for use by the filesystem's ->writepages() method.
  145. *
  146. * The bounce page allocation is mempool-backed, so it will always succeed when
  147. * @gfp_flags includes __GFP_DIRECT_RECLAIM, e.g. when it's GFP_NOFS. However,
  148. * only the first page of each bio can be allocated this way. To prevent
  149. * deadlocks, for any additional pages a mask like GFP_NOWAIT must be used.
  150. *
  151. * Return: the new encrypted bounce page on success; an ERR_PTR() on failure
  152. */
  153. struct page *fscrypt_encrypt_pagecache_blocks(struct page *page,
  154. unsigned int len,
  155. unsigned int offs,
  156. gfp_t gfp_flags)
  157. {
  158. const struct inode *inode = page->mapping->host;
  159. const unsigned int blockbits = inode->i_blkbits;
  160. const unsigned int blocksize = 1 << blockbits;
  161. struct page *ciphertext_page;
  162. u64 lblk_num = ((u64)page->index << (PAGE_SHIFT - blockbits)) +
  163. (offs >> blockbits);
  164. unsigned int i;
  165. int err;
  166. if (WARN_ON_ONCE(!PageLocked(page)))
  167. return ERR_PTR(-EINVAL);
  168. if (WARN_ON_ONCE(len <= 0 || !IS_ALIGNED(len | offs, blocksize)))
  169. return ERR_PTR(-EINVAL);
  170. ciphertext_page = fscrypt_alloc_bounce_page(gfp_flags);
  171. if (!ciphertext_page)
  172. return ERR_PTR(-ENOMEM);
  173. for (i = offs; i < offs + len; i += blocksize, lblk_num++) {
  174. err = fscrypt_crypt_block(inode, FS_ENCRYPT, lblk_num,
  175. page, ciphertext_page,
  176. blocksize, i, gfp_flags);
  177. if (err) {
  178. fscrypt_free_bounce_page(ciphertext_page);
  179. return ERR_PTR(err);
  180. }
  181. }
  182. SetPagePrivate(ciphertext_page);
  183. set_page_private(ciphertext_page, (unsigned long)page);
  184. return ciphertext_page;
  185. }
  186. EXPORT_SYMBOL(fscrypt_encrypt_pagecache_blocks);
  187. /**
  188. * fscrypt_encrypt_block_inplace() - Encrypt a filesystem block in-place
  189. * @inode: The inode to which this block belongs
  190. * @page: The page containing the block to encrypt
  191. * @len: Size of block to encrypt. Doesn't need to be a multiple of the
  192. * fs block size, but must be a multiple of FS_CRYPTO_BLOCK_SIZE.
  193. * @offs: Byte offset within @page at which the block to encrypt begins
  194. * @lblk_num: Filesystem logical block number of the block, i.e. the 0-based
  195. * number of the block within the file
  196. * @gfp_flags: Memory allocation flags
  197. *
  198. * Encrypt a possibly-compressed filesystem block that is located in an
  199. * arbitrary page, not necessarily in the original pagecache page. The @inode
  200. * and @lblk_num must be specified, as they can't be determined from @page.
  201. *
  202. * Return: 0 on success; -errno on failure
  203. */
  204. int fscrypt_encrypt_block_inplace(const struct inode *inode, struct page *page,
  205. unsigned int len, unsigned int offs,
  206. u64 lblk_num, gfp_t gfp_flags)
  207. {
  208. return fscrypt_crypt_block(inode, FS_ENCRYPT, lblk_num, page, page,
  209. len, offs, gfp_flags);
  210. }
  211. EXPORT_SYMBOL(fscrypt_encrypt_block_inplace);
  212. /**
  213. * fscrypt_decrypt_pagecache_blocks() - Decrypt filesystem blocks in a
  214. * pagecache page
  215. * @page: The locked pagecache page containing the block(s) to decrypt
  216. * @len: Total size of the block(s) to decrypt. Must be a nonzero
  217. * multiple of the filesystem's block size.
  218. * @offs: Byte offset within @page of the first block to decrypt. Must be
  219. * a multiple of the filesystem's block size.
  220. *
  221. * The specified block(s) are decrypted in-place within the pagecache page,
  222. * which must still be locked and not uptodate. Normally, blocksize ==
  223. * PAGE_SIZE and the whole page is decrypted at once.
  224. *
  225. * This is for use by the filesystem's ->readpages() method.
  226. *
  227. * Return: 0 on success; -errno on failure
  228. */
  229. int fscrypt_decrypt_pagecache_blocks(struct page *page, unsigned int len,
  230. unsigned int offs)
  231. {
  232. const struct inode *inode = page->mapping->host;
  233. const unsigned int blockbits = inode->i_blkbits;
  234. const unsigned int blocksize = 1 << blockbits;
  235. u64 lblk_num = ((u64)page->index << (PAGE_SHIFT - blockbits)) +
  236. (offs >> blockbits);
  237. unsigned int i;
  238. int err;
  239. if (WARN_ON_ONCE(!PageLocked(page)))
  240. return -EINVAL;
  241. if (WARN_ON_ONCE(len <= 0 || !IS_ALIGNED(len | offs, blocksize)))
  242. return -EINVAL;
  243. for (i = offs; i < offs + len; i += blocksize, lblk_num++) {
  244. err = fscrypt_crypt_block(inode, FS_DECRYPT, lblk_num, page,
  245. page, blocksize, i, GFP_NOFS);
  246. if (err)
  247. return err;
  248. }
  249. return 0;
  250. }
  251. EXPORT_SYMBOL(fscrypt_decrypt_pagecache_blocks);
  252. /**
  253. * fscrypt_decrypt_block_inplace() - Decrypt a filesystem block in-place
  254. * @inode: The inode to which this block belongs
  255. * @page: The page containing the block to decrypt
  256. * @len: Size of block to decrypt. Doesn't need to be a multiple of the
  257. * fs block size, but must be a multiple of FS_CRYPTO_BLOCK_SIZE.
  258. * @offs: Byte offset within @page at which the block to decrypt begins
  259. * @lblk_num: Filesystem logical block number of the block, i.e. the 0-based
  260. * number of the block within the file
  261. *
  262. * Decrypt a possibly-compressed filesystem block that is located in an
  263. * arbitrary page, not necessarily in the original pagecache page. The @inode
  264. * and @lblk_num must be specified, as they can't be determined from @page.
  265. *
  266. * Return: 0 on success; -errno on failure
  267. */
  268. int fscrypt_decrypt_block_inplace(const struct inode *inode, struct page *page,
  269. unsigned int len, unsigned int offs,
  270. u64 lblk_num)
  271. {
  272. return fscrypt_crypt_block(inode, FS_DECRYPT, lblk_num, page, page,
  273. len, offs, GFP_NOFS);
  274. }
  275. EXPORT_SYMBOL(fscrypt_decrypt_block_inplace);
  276. /**
  277. * fscrypt_initialize() - allocate major buffers for fs encryption.
  278. * @cop_flags: fscrypt operations flags
  279. *
  280. * We only call this when we start accessing encrypted files, since it
  281. * results in memory getting allocated that wouldn't otherwise be used.
  282. *
  283. * Return: 0 on success; -errno on failure
  284. */
  285. int fscrypt_initialize(unsigned int cop_flags)
  286. {
  287. int err = 0;
  288. /* No need to allocate a bounce page pool if this FS won't use it. */
  289. if (cop_flags & FS_CFLG_OWN_PAGES)
  290. return 0;
  291. mutex_lock(&fscrypt_init_mutex);
  292. if (fscrypt_bounce_page_pool)
  293. goto out_unlock;
  294. err = -ENOMEM;
  295. fscrypt_bounce_page_pool =
  296. mempool_create_page_pool(num_prealloc_crypto_pages, 0);
  297. if (!fscrypt_bounce_page_pool)
  298. goto out_unlock;
  299. err = 0;
  300. out_unlock:
  301. mutex_unlock(&fscrypt_init_mutex);
  302. return err;
  303. }
  304. void fscrypt_msg(const struct inode *inode, const char *level,
  305. const char *fmt, ...)
  306. {
  307. static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
  308. DEFAULT_RATELIMIT_BURST);
  309. struct va_format vaf;
  310. va_list args;
  311. if (!__ratelimit(&rs))
  312. return;
  313. va_start(args, fmt);
  314. vaf.fmt = fmt;
  315. vaf.va = &args;
  316. if (inode && inode->i_ino)
  317. printk("%sfscrypt (%s, inode %lu): %pV\n",
  318. level, inode->i_sb->s_id, inode->i_ino, &vaf);
  319. else if (inode)
  320. printk("%sfscrypt (%s): %pV\n", level, inode->i_sb->s_id, &vaf);
  321. else
  322. printk("%sfscrypt: %pV\n", level, &vaf);
  323. va_end(args);
  324. }
  325. /**
  326. * fscrypt_init() - Set up for fs encryption.
  327. *
  328. * Return: 0 on success; -errno on failure
  329. */
  330. static int __init fscrypt_init(void)
  331. {
  332. int err = -ENOMEM;
  333. /*
  334. * Use an unbound workqueue to allow bios to be decrypted in parallel
  335. * even when they happen to complete on the same CPU. This sacrifices
  336. * locality, but it's worthwhile since decryption is CPU-intensive.
  337. *
  338. * Also use a high-priority workqueue to prioritize decryption work,
  339. * which blocks reads from completing, over regular application tasks.
  340. */
  341. fscrypt_read_workqueue = alloc_workqueue("fscrypt_read_queue",
  342. WQ_UNBOUND | WQ_HIGHPRI,
  343. num_online_cpus());
  344. if (!fscrypt_read_workqueue)
  345. goto fail;
  346. fscrypt_info_cachep = KMEM_CACHE(fscrypt_info, SLAB_RECLAIM_ACCOUNT);
  347. if (!fscrypt_info_cachep)
  348. goto fail_free_queue;
  349. err = fscrypt_init_keyring();
  350. if (err)
  351. goto fail_free_info;
  352. return 0;
  353. fail_free_info:
  354. kmem_cache_destroy(fscrypt_info_cachep);
  355. fail_free_queue:
  356. destroy_workqueue(fscrypt_read_workqueue);
  357. fail:
  358. return err;
  359. }
  360. late_initcall(fscrypt_init)