zlib.c 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Copyright (C) 2008 Oracle. All rights reserved.
  4. *
  5. * Based on jffs2 zlib code:
  6. * Copyright © 2001-2007 Red Hat, Inc.
  7. * Created by David Woodhouse <dwmw2@infradead.org>
  8. */
  9. #include <linux/kernel.h>
  10. #include <linux/slab.h>
  11. #include <linux/zlib.h>
  12. #include <linux/zutil.h>
  13. #include <linux/mm.h>
  14. #include <linux/init.h>
  15. #include <linux/err.h>
  16. #include <linux/sched.h>
  17. #include <linux/pagemap.h>
  18. #include <linux/bio.h>
  19. #include <linux/refcount.h>
  20. #include "compression.h"
  21. /* workspace buffer size for s390 zlib hardware support */
  22. #define ZLIB_DFLTCC_BUF_SIZE (4 * PAGE_SIZE)
  23. struct workspace {
  24. z_stream strm;
  25. char *buf;
  26. unsigned int buf_size;
  27. struct list_head list;
  28. int level;
  29. };
  30. static struct workspace_manager wsm;
  31. struct list_head *zlib_get_workspace(unsigned int level)
  32. {
  33. struct list_head *ws = btrfs_get_workspace(BTRFS_COMPRESS_ZLIB, level);
  34. struct workspace *workspace = list_entry(ws, struct workspace, list);
  35. workspace->level = level;
  36. return ws;
  37. }
  38. void zlib_free_workspace(struct list_head *ws)
  39. {
  40. struct workspace *workspace = list_entry(ws, struct workspace, list);
  41. kvfree(workspace->strm.workspace);
  42. kfree(workspace->buf);
  43. kfree(workspace);
  44. }
  45. struct list_head *zlib_alloc_workspace(unsigned int level)
  46. {
  47. struct workspace *workspace;
  48. int workspacesize;
  49. workspace = kzalloc(sizeof(*workspace), GFP_KERNEL);
  50. if (!workspace)
  51. return ERR_PTR(-ENOMEM);
  52. workspacesize = max(zlib_deflate_workspacesize(MAX_WBITS, MAX_MEM_LEVEL),
  53. zlib_inflate_workspacesize());
  54. workspace->strm.workspace = kvmalloc(workspacesize, GFP_KERNEL);
  55. workspace->level = level;
  56. workspace->buf = NULL;
  57. /*
  58. * In case of s390 zlib hardware support, allocate lager workspace
  59. * buffer. If allocator fails, fall back to a single page buffer.
  60. */
  61. if (zlib_deflate_dfltcc_enabled()) {
  62. workspace->buf = kmalloc(ZLIB_DFLTCC_BUF_SIZE,
  63. __GFP_NOMEMALLOC | __GFP_NORETRY |
  64. __GFP_NOWARN | GFP_NOIO);
  65. workspace->buf_size = ZLIB_DFLTCC_BUF_SIZE;
  66. }
  67. if (!workspace->buf) {
  68. workspace->buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
  69. workspace->buf_size = PAGE_SIZE;
  70. }
  71. if (!workspace->strm.workspace || !workspace->buf)
  72. goto fail;
  73. INIT_LIST_HEAD(&workspace->list);
  74. return &workspace->list;
  75. fail:
  76. zlib_free_workspace(&workspace->list);
  77. return ERR_PTR(-ENOMEM);
  78. }
  79. int zlib_compress_pages(struct list_head *ws, struct address_space *mapping,
  80. u64 start, struct page **pages, unsigned long *out_pages,
  81. unsigned long *total_in, unsigned long *total_out)
  82. {
  83. struct workspace *workspace = list_entry(ws, struct workspace, list);
  84. int ret;
  85. char *data_in;
  86. char *cpage_out;
  87. int nr_pages = 0;
  88. struct page *in_page = NULL;
  89. struct page *out_page = NULL;
  90. unsigned long bytes_left;
  91. unsigned int in_buf_pages;
  92. unsigned long len = *total_out;
  93. unsigned long nr_dest_pages = *out_pages;
  94. const unsigned long max_out = nr_dest_pages * PAGE_SIZE;
  95. *out_pages = 0;
  96. *total_out = 0;
  97. *total_in = 0;
  98. if (Z_OK != zlib_deflateInit(&workspace->strm, workspace->level)) {
  99. pr_warn("BTRFS: deflateInit failed\n");
  100. ret = -EIO;
  101. goto out;
  102. }
  103. workspace->strm.total_in = 0;
  104. workspace->strm.total_out = 0;
  105. out_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
  106. if (out_page == NULL) {
  107. ret = -ENOMEM;
  108. goto out;
  109. }
  110. cpage_out = kmap(out_page);
  111. pages[0] = out_page;
  112. nr_pages = 1;
  113. workspace->strm.next_in = workspace->buf;
  114. workspace->strm.avail_in = 0;
  115. workspace->strm.next_out = cpage_out;
  116. workspace->strm.avail_out = PAGE_SIZE;
  117. while (workspace->strm.total_in < len) {
  118. /*
  119. * Get next input pages and copy the contents to
  120. * the workspace buffer if required.
  121. */
  122. if (workspace->strm.avail_in == 0) {
  123. bytes_left = len - workspace->strm.total_in;
  124. in_buf_pages = min(DIV_ROUND_UP(bytes_left, PAGE_SIZE),
  125. workspace->buf_size / PAGE_SIZE);
  126. if (in_buf_pages > 1) {
  127. int i;
  128. for (i = 0; i < in_buf_pages; i++) {
  129. if (in_page) {
  130. kunmap(in_page);
  131. put_page(in_page);
  132. }
  133. in_page = find_get_page(mapping,
  134. start >> PAGE_SHIFT);
  135. data_in = kmap(in_page);
  136. memcpy(workspace->buf + i * PAGE_SIZE,
  137. data_in, PAGE_SIZE);
  138. start += PAGE_SIZE;
  139. }
  140. workspace->strm.next_in = workspace->buf;
  141. } else {
  142. if (in_page) {
  143. kunmap(in_page);
  144. put_page(in_page);
  145. }
  146. in_page = find_get_page(mapping,
  147. start >> PAGE_SHIFT);
  148. data_in = kmap(in_page);
  149. start += PAGE_SIZE;
  150. workspace->strm.next_in = data_in;
  151. }
  152. workspace->strm.avail_in = min(bytes_left,
  153. (unsigned long) workspace->buf_size);
  154. }
  155. ret = zlib_deflate(&workspace->strm, Z_SYNC_FLUSH);
  156. if (ret != Z_OK) {
  157. pr_debug("BTRFS: deflate in loop returned %d\n",
  158. ret);
  159. zlib_deflateEnd(&workspace->strm);
  160. ret = -EIO;
  161. goto out;
  162. }
  163. /* we're making it bigger, give up */
  164. if (workspace->strm.total_in > 8192 &&
  165. workspace->strm.total_in <
  166. workspace->strm.total_out) {
  167. ret = -E2BIG;
  168. goto out;
  169. }
  170. /* we need another page for writing out. Test this
  171. * before the total_in so we will pull in a new page for
  172. * the stream end if required
  173. */
  174. if (workspace->strm.avail_out == 0) {
  175. kunmap(out_page);
  176. if (nr_pages == nr_dest_pages) {
  177. out_page = NULL;
  178. ret = -E2BIG;
  179. goto out;
  180. }
  181. out_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
  182. if (out_page == NULL) {
  183. ret = -ENOMEM;
  184. goto out;
  185. }
  186. cpage_out = kmap(out_page);
  187. pages[nr_pages] = out_page;
  188. nr_pages++;
  189. workspace->strm.avail_out = PAGE_SIZE;
  190. workspace->strm.next_out = cpage_out;
  191. }
  192. /* we're all done */
  193. if (workspace->strm.total_in >= len)
  194. break;
  195. if (workspace->strm.total_out > max_out)
  196. break;
  197. }
  198. workspace->strm.avail_in = 0;
  199. /*
  200. * Call deflate with Z_FINISH flush parameter providing more output
  201. * space but no more input data, until it returns with Z_STREAM_END.
  202. */
  203. while (ret != Z_STREAM_END) {
  204. ret = zlib_deflate(&workspace->strm, Z_FINISH);
  205. if (ret == Z_STREAM_END)
  206. break;
  207. if (ret != Z_OK && ret != Z_BUF_ERROR) {
  208. zlib_deflateEnd(&workspace->strm);
  209. ret = -EIO;
  210. goto out;
  211. } else if (workspace->strm.avail_out == 0) {
  212. /* get another page for the stream end */
  213. kunmap(out_page);
  214. if (nr_pages == nr_dest_pages) {
  215. out_page = NULL;
  216. ret = -E2BIG;
  217. goto out;
  218. }
  219. out_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
  220. if (out_page == NULL) {
  221. ret = -ENOMEM;
  222. goto out;
  223. }
  224. cpage_out = kmap(out_page);
  225. pages[nr_pages] = out_page;
  226. nr_pages++;
  227. workspace->strm.avail_out = PAGE_SIZE;
  228. workspace->strm.next_out = cpage_out;
  229. }
  230. }
  231. zlib_deflateEnd(&workspace->strm);
  232. if (workspace->strm.total_out >= workspace->strm.total_in) {
  233. ret = -E2BIG;
  234. goto out;
  235. }
  236. ret = 0;
  237. *total_out = workspace->strm.total_out;
  238. *total_in = workspace->strm.total_in;
  239. out:
  240. *out_pages = nr_pages;
  241. if (out_page)
  242. kunmap(out_page);
  243. if (in_page) {
  244. kunmap(in_page);
  245. put_page(in_page);
  246. }
  247. return ret;
  248. }
  249. int zlib_decompress_bio(struct list_head *ws, struct compressed_bio *cb)
  250. {
  251. struct workspace *workspace = list_entry(ws, struct workspace, list);
  252. int ret = 0, ret2;
  253. int wbits = MAX_WBITS;
  254. char *data_in;
  255. size_t total_out = 0;
  256. unsigned long page_in_index = 0;
  257. size_t srclen = cb->compressed_len;
  258. unsigned long total_pages_in = DIV_ROUND_UP(srclen, PAGE_SIZE);
  259. unsigned long buf_start;
  260. struct page **pages_in = cb->compressed_pages;
  261. u64 disk_start = cb->start;
  262. struct bio *orig_bio = cb->orig_bio;
  263. data_in = kmap(pages_in[page_in_index]);
  264. workspace->strm.next_in = data_in;
  265. workspace->strm.avail_in = min_t(size_t, srclen, PAGE_SIZE);
  266. workspace->strm.total_in = 0;
  267. workspace->strm.total_out = 0;
  268. workspace->strm.next_out = workspace->buf;
  269. workspace->strm.avail_out = workspace->buf_size;
  270. /* If it's deflate, and it's got no preset dictionary, then
  271. we can tell zlib to skip the adler32 check. */
  272. if (srclen > 2 && !(data_in[1] & PRESET_DICT) &&
  273. ((data_in[0] & 0x0f) == Z_DEFLATED) &&
  274. !(((data_in[0]<<8) + data_in[1]) % 31)) {
  275. wbits = -((data_in[0] >> 4) + 8);
  276. workspace->strm.next_in += 2;
  277. workspace->strm.avail_in -= 2;
  278. }
  279. if (Z_OK != zlib_inflateInit2(&workspace->strm, wbits)) {
  280. pr_warn("BTRFS: inflateInit failed\n");
  281. kunmap(pages_in[page_in_index]);
  282. return -EIO;
  283. }
  284. while (workspace->strm.total_in < srclen) {
  285. ret = zlib_inflate(&workspace->strm, Z_NO_FLUSH);
  286. if (ret != Z_OK && ret != Z_STREAM_END)
  287. break;
  288. buf_start = total_out;
  289. total_out = workspace->strm.total_out;
  290. /* we didn't make progress in this inflate call, we're done */
  291. if (buf_start == total_out)
  292. break;
  293. ret2 = btrfs_decompress_buf2page(workspace->buf, buf_start,
  294. total_out, disk_start,
  295. orig_bio);
  296. if (ret2 == 0) {
  297. ret = 0;
  298. goto done;
  299. }
  300. workspace->strm.next_out = workspace->buf;
  301. workspace->strm.avail_out = workspace->buf_size;
  302. if (workspace->strm.avail_in == 0) {
  303. unsigned long tmp;
  304. kunmap(pages_in[page_in_index]);
  305. page_in_index++;
  306. if (page_in_index >= total_pages_in) {
  307. data_in = NULL;
  308. break;
  309. }
  310. data_in = kmap(pages_in[page_in_index]);
  311. workspace->strm.next_in = data_in;
  312. tmp = srclen - workspace->strm.total_in;
  313. workspace->strm.avail_in = min(tmp,
  314. PAGE_SIZE);
  315. }
  316. }
  317. if (ret != Z_STREAM_END)
  318. ret = -EIO;
  319. else
  320. ret = 0;
  321. done:
  322. zlib_inflateEnd(&workspace->strm);
  323. if (data_in)
  324. kunmap(pages_in[page_in_index]);
  325. if (!ret)
  326. zero_fill_bio(orig_bio);
  327. return ret;
  328. }
  329. int zlib_decompress(struct list_head *ws, unsigned char *data_in,
  330. struct page *dest_page, unsigned long start_byte, size_t srclen,
  331. size_t destlen)
  332. {
  333. struct workspace *workspace = list_entry(ws, struct workspace, list);
  334. int ret = 0;
  335. int wbits = MAX_WBITS;
  336. unsigned long bytes_left;
  337. unsigned long total_out = 0;
  338. unsigned long pg_offset = 0;
  339. char *kaddr;
  340. destlen = min_t(unsigned long, destlen, PAGE_SIZE);
  341. bytes_left = destlen;
  342. workspace->strm.next_in = data_in;
  343. workspace->strm.avail_in = srclen;
  344. workspace->strm.total_in = 0;
  345. workspace->strm.next_out = workspace->buf;
  346. workspace->strm.avail_out = workspace->buf_size;
  347. workspace->strm.total_out = 0;
  348. /* If it's deflate, and it's got no preset dictionary, then
  349. we can tell zlib to skip the adler32 check. */
  350. if (srclen > 2 && !(data_in[1] & PRESET_DICT) &&
  351. ((data_in[0] & 0x0f) == Z_DEFLATED) &&
  352. !(((data_in[0]<<8) + data_in[1]) % 31)) {
  353. wbits = -((data_in[0] >> 4) + 8);
  354. workspace->strm.next_in += 2;
  355. workspace->strm.avail_in -= 2;
  356. }
  357. if (Z_OK != zlib_inflateInit2(&workspace->strm, wbits)) {
  358. pr_warn("BTRFS: inflateInit failed\n");
  359. return -EIO;
  360. }
  361. while (bytes_left > 0) {
  362. unsigned long buf_start;
  363. unsigned long buf_offset;
  364. unsigned long bytes;
  365. ret = zlib_inflate(&workspace->strm, Z_NO_FLUSH);
  366. if (ret != Z_OK && ret != Z_STREAM_END)
  367. break;
  368. buf_start = total_out;
  369. total_out = workspace->strm.total_out;
  370. if (total_out == buf_start) {
  371. ret = -EIO;
  372. break;
  373. }
  374. if (total_out <= start_byte)
  375. goto next;
  376. if (total_out > start_byte && buf_start < start_byte)
  377. buf_offset = start_byte - buf_start;
  378. else
  379. buf_offset = 0;
  380. bytes = min(PAGE_SIZE - pg_offset,
  381. PAGE_SIZE - (buf_offset % PAGE_SIZE));
  382. bytes = min(bytes, bytes_left);
  383. kaddr = kmap_atomic(dest_page);
  384. memcpy(kaddr + pg_offset, workspace->buf + buf_offset, bytes);
  385. kunmap_atomic(kaddr);
  386. pg_offset += bytes;
  387. bytes_left -= bytes;
  388. next:
  389. workspace->strm.next_out = workspace->buf;
  390. workspace->strm.avail_out = workspace->buf_size;
  391. }
  392. if (ret != Z_STREAM_END && bytes_left != 0)
  393. ret = -EIO;
  394. else
  395. ret = 0;
  396. zlib_inflateEnd(&workspace->strm);
  397. /*
  398. * this should only happen if zlib returned fewer bytes than we
  399. * expected. btrfs_get_block is responsible for zeroing from the
  400. * end of the inline extent (destlen) to the end of the page
  401. */
  402. if (pg_offset < destlen) {
  403. kaddr = kmap_atomic(dest_page);
  404. memset(kaddr + pg_offset, 0, destlen - pg_offset);
  405. kunmap_atomic(kaddr);
  406. }
  407. return ret;
  408. }
  409. const struct btrfs_compress_op btrfs_zlib_compress = {
  410. .workspace_manager = &wsm,
  411. .max_level = 9,
  412. .default_level = BTRFS_ZLIB_DEFAULT_LEVEL,
  413. };