maccess.c 8.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Access kernel or user memory without faulting.
  4. */
  5. #include <linux/export.h>
  6. #include <linux/mm.h>
  7. #include <linux/uaccess.h>
  8. bool __weak copy_from_kernel_nofault_allowed(const void *unsafe_src,
  9. size_t size)
  10. {
  11. return true;
  12. }
  13. #ifdef HAVE_GET_KERNEL_NOFAULT
  14. #define copy_from_kernel_nofault_loop(dst, src, len, type, err_label) \
  15. while (len >= sizeof(type)) { \
  16. __get_kernel_nofault(dst, src, type, err_label); \
  17. dst += sizeof(type); \
  18. src += sizeof(type); \
  19. len -= sizeof(type); \
  20. }
  21. long copy_from_kernel_nofault(void *dst, const void *src, size_t size)
  22. {
  23. if (!copy_from_kernel_nofault_allowed(src, size))
  24. return -ERANGE;
  25. pagefault_disable();
  26. copy_from_kernel_nofault_loop(dst, src, size, u64, Efault);
  27. copy_from_kernel_nofault_loop(dst, src, size, u32, Efault);
  28. copy_from_kernel_nofault_loop(dst, src, size, u16, Efault);
  29. copy_from_kernel_nofault_loop(dst, src, size, u8, Efault);
  30. pagefault_enable();
  31. return 0;
  32. Efault:
  33. pagefault_enable();
  34. return -EFAULT;
  35. }
  36. EXPORT_SYMBOL_GPL(copy_from_kernel_nofault);
  37. #define copy_to_kernel_nofault_loop(dst, src, len, type, err_label) \
  38. while (len >= sizeof(type)) { \
  39. __put_kernel_nofault(dst, src, type, err_label); \
  40. dst += sizeof(type); \
  41. src += sizeof(type); \
  42. len -= sizeof(type); \
  43. }
  44. long copy_to_kernel_nofault(void *dst, const void *src, size_t size)
  45. {
  46. pagefault_disable();
  47. copy_to_kernel_nofault_loop(dst, src, size, u64, Efault);
  48. copy_to_kernel_nofault_loop(dst, src, size, u32, Efault);
  49. copy_to_kernel_nofault_loop(dst, src, size, u16, Efault);
  50. copy_to_kernel_nofault_loop(dst, src, size, u8, Efault);
  51. pagefault_enable();
  52. return 0;
  53. Efault:
  54. pagefault_enable();
  55. return -EFAULT;
  56. }
  57. long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count)
  58. {
  59. const void *src = unsafe_addr;
  60. if (unlikely(count <= 0))
  61. return 0;
  62. if (!copy_from_kernel_nofault_allowed(unsafe_addr, count))
  63. return -ERANGE;
  64. pagefault_disable();
  65. do {
  66. __get_kernel_nofault(dst, src, u8, Efault);
  67. dst++;
  68. src++;
  69. } while (dst[-1] && src - unsafe_addr < count);
  70. pagefault_enable();
  71. dst[-1] = '\0';
  72. return src - unsafe_addr;
  73. Efault:
  74. pagefault_enable();
  75. dst[-1] = '\0';
  76. return -EFAULT;
  77. }
  78. #else /* HAVE_GET_KERNEL_NOFAULT */
  79. /**
  80. * copy_from_kernel_nofault(): safely attempt to read from kernel-space
  81. * @dst: pointer to the buffer that shall take the data
  82. * @src: address to read from
  83. * @size: size of the data chunk
  84. *
  85. * Safely read from kernel address @src to the buffer at @dst. If a kernel
  86. * fault happens, handle that and return -EFAULT. If @src is not a valid kernel
  87. * address, return -ERANGE.
  88. *
  89. * We ensure that the copy_from_user is executed in atomic context so that
  90. * do_page_fault() doesn't attempt to take mmap_lock. This makes
  91. * copy_from_kernel_nofault() suitable for use within regions where the caller
  92. * already holds mmap_lock, or other locks which nest inside mmap_lock.
  93. */
  94. long copy_from_kernel_nofault(void *dst, const void *src, size_t size)
  95. {
  96. long ret;
  97. mm_segment_t old_fs = get_fs();
  98. if (!copy_from_kernel_nofault_allowed(src, size))
  99. return -ERANGE;
  100. set_fs(KERNEL_DS);
  101. pagefault_disable();
  102. ret = __copy_from_user_inatomic(dst, (__force const void __user *)src,
  103. size);
  104. pagefault_enable();
  105. set_fs(old_fs);
  106. if (ret)
  107. return -EFAULT;
  108. return 0;
  109. }
  110. EXPORT_SYMBOL_GPL(copy_from_kernel_nofault);
  111. /**
  112. * copy_to_kernel_nofault(): safely attempt to write to a location
  113. * @dst: address to write to
  114. * @src: pointer to the data that shall be written
  115. * @size: size of the data chunk
  116. *
  117. * Safely write to address @dst from the buffer at @src. If a kernel fault
  118. * happens, handle that and return -EFAULT.
  119. */
  120. long copy_to_kernel_nofault(void *dst, const void *src, size_t size)
  121. {
  122. long ret;
  123. mm_segment_t old_fs = get_fs();
  124. set_fs(KERNEL_DS);
  125. pagefault_disable();
  126. ret = __copy_to_user_inatomic((__force void __user *)dst, src, size);
  127. pagefault_enable();
  128. set_fs(old_fs);
  129. if (ret)
  130. return -EFAULT;
  131. return 0;
  132. }
  133. /**
  134. * strncpy_from_kernel_nofault: - Copy a NUL terminated string from unsafe
  135. * address.
  136. * @dst: Destination address, in kernel space. This buffer must be at
  137. * least @count bytes long.
  138. * @unsafe_addr: Unsafe address.
  139. * @count: Maximum number of bytes to copy, including the trailing NUL.
  140. *
  141. * Copies a NUL-terminated string from unsafe address to kernel buffer.
  142. *
  143. * On success, returns the length of the string INCLUDING the trailing NUL.
  144. *
  145. * If access fails, returns -EFAULT (some data may have been copied and the
  146. * trailing NUL added). If @unsafe_addr is not a valid kernel address, return
  147. * -ERANGE.
  148. *
  149. * If @count is smaller than the length of the string, copies @count-1 bytes,
  150. * sets the last byte of @dst buffer to NUL and returns @count.
  151. */
  152. long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count)
  153. {
  154. mm_segment_t old_fs = get_fs();
  155. const void *src = unsafe_addr;
  156. long ret;
  157. if (unlikely(count <= 0))
  158. return 0;
  159. if (!copy_from_kernel_nofault_allowed(unsafe_addr, count))
  160. return -ERANGE;
  161. set_fs(KERNEL_DS);
  162. pagefault_disable();
  163. do {
  164. ret = __get_user(*dst++, (const char __user __force *)src++);
  165. } while (dst[-1] && ret == 0 && src - unsafe_addr < count);
  166. dst[-1] = '\0';
  167. pagefault_enable();
  168. set_fs(old_fs);
  169. return ret ? -EFAULT : src - unsafe_addr;
  170. }
  171. #endif /* HAVE_GET_KERNEL_NOFAULT */
  172. /**
  173. * copy_from_user_nofault(): safely attempt to read from a user-space location
  174. * @dst: pointer to the buffer that shall take the data
  175. * @src: address to read from. This must be a user address.
  176. * @size: size of the data chunk
  177. *
  178. * Safely read from user address @src to the buffer at @dst. If a kernel fault
  179. * happens, handle that and return -EFAULT.
  180. */
  181. long copy_from_user_nofault(void *dst, const void __user *src, size_t size)
  182. {
  183. long ret = -EFAULT;
  184. mm_segment_t old_fs = force_uaccess_begin();
  185. if (access_ok(src, size)) {
  186. pagefault_disable();
  187. ret = __copy_from_user_inatomic(dst, src, size);
  188. pagefault_enable();
  189. }
  190. force_uaccess_end(old_fs);
  191. if (ret)
  192. return -EFAULT;
  193. return 0;
  194. }
  195. EXPORT_SYMBOL_GPL(copy_from_user_nofault);
  196. /**
  197. * copy_to_user_nofault(): safely attempt to write to a user-space location
  198. * @dst: address to write to
  199. * @src: pointer to the data that shall be written
  200. * @size: size of the data chunk
  201. *
  202. * Safely write to address @dst from the buffer at @src. If a kernel fault
  203. * happens, handle that and return -EFAULT.
  204. */
  205. long copy_to_user_nofault(void __user *dst, const void *src, size_t size)
  206. {
  207. long ret = -EFAULT;
  208. mm_segment_t old_fs = force_uaccess_begin();
  209. if (access_ok(dst, size)) {
  210. pagefault_disable();
  211. ret = __copy_to_user_inatomic(dst, src, size);
  212. pagefault_enable();
  213. }
  214. force_uaccess_end(old_fs);
  215. if (ret)
  216. return -EFAULT;
  217. return 0;
  218. }
  219. EXPORT_SYMBOL_GPL(copy_to_user_nofault);
  220. /**
  221. * strncpy_from_user_nofault: - Copy a NUL terminated string from unsafe user
  222. * address.
  223. * @dst: Destination address, in kernel space. This buffer must be at
  224. * least @count bytes long.
  225. * @unsafe_addr: Unsafe user address.
  226. * @count: Maximum number of bytes to copy, including the trailing NUL.
  227. *
  228. * Copies a NUL-terminated string from unsafe user address to kernel buffer.
  229. *
  230. * On success, returns the length of the string INCLUDING the trailing NUL.
  231. *
  232. * If access fails, returns -EFAULT (some data may have been copied
  233. * and the trailing NUL added).
  234. *
  235. * If @count is smaller than the length of the string, copies @count-1 bytes,
  236. * sets the last byte of @dst buffer to NUL and returns @count.
  237. */
  238. long strncpy_from_user_nofault(char *dst, const void __user *unsafe_addr,
  239. long count)
  240. {
  241. mm_segment_t old_fs;
  242. long ret;
  243. if (unlikely(count <= 0))
  244. return 0;
  245. old_fs = force_uaccess_begin();
  246. pagefault_disable();
  247. ret = strncpy_from_user(dst, unsafe_addr, count);
  248. pagefault_enable();
  249. force_uaccess_end(old_fs);
  250. if (ret >= count) {
  251. ret = count;
  252. dst[ret - 1] = '\0';
  253. } else if (ret > 0) {
  254. ret++;
  255. }
  256. return ret;
  257. }
  258. /**
  259. * strnlen_user_nofault: - Get the size of a user string INCLUDING final NUL.
  260. * @unsafe_addr: The string to measure.
  261. * @count: Maximum count (including NUL)
  262. *
  263. * Get the size of a NUL-terminated string in user space without pagefault.
  264. *
  265. * Returns the size of the string INCLUDING the terminating NUL.
  266. *
  267. * If the string is too long, returns a number larger than @count. User
  268. * has to check the return value against "> count".
  269. * On exception (or invalid count), returns 0.
  270. *
  271. * Unlike strnlen_user, this can be used from IRQ handler etc. because
  272. * it disables pagefaults.
  273. */
  274. long strnlen_user_nofault(const void __user *unsafe_addr, long count)
  275. {
  276. mm_segment_t old_fs;
  277. int ret;
  278. old_fs = force_uaccess_begin();
  279. pagefault_disable();
  280. ret = strnlen_user(unsafe_addr, count);
  281. pagefault_enable();
  282. force_uaccess_end(old_fs);
  283. return ret;
  284. }