ion.h 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419
  1. /* SPDX-License-Identifier: GPL-2.0-only */
  2. /*
  3. * Copyright (c) 2019, The Linux Foundation. All rights reserved.
  4. */
  5. #ifndef _ION_KERNEL_H
  6. #define _ION_KERNEL_H
  7. #include <linux/dma-buf.h>
  8. #include <linux/err.h>
  9. #include <linux/device.h>
  10. #include <linux/dma-direction.h>
  11. #include <linux/kref.h>
  12. #include <linux/mm_types.h>
  13. #include <linux/module.h>
  14. #include <linux/mutex.h>
  15. #include <linux/rbtree.h>
  16. #include <linux/sched.h>
  17. #include <linux/shrinker.h>
  18. #include <linux/types.h>
  19. #include <uapi/linux/ion.h>
  20. /**
  21. * struct ion_buffer - metadata for a particular buffer
  22. * @list: element in list of deferred freeable buffers
  23. * @heap: back pointer to the heap the buffer came from
  24. * @flags: buffer specific flags
  25. * @private_flags: internal buffer specific flags
  26. * @size: size of the buffer
  27. * @priv_virt: private data to the buffer representable as
  28. * a void *
  29. * @lock: protects the buffers cnt fields
  30. * @kmap_cnt: number of times the buffer is mapped to the kernel
  31. * @vaddr: the kernel mapping if kmap_cnt is not zero
  32. * @sg_table: the sg table for the buffer
  33. * @attachments: list of devices attached to this buffer
  34. */
  35. struct ion_buffer {
  36. struct list_head list;
  37. struct ion_heap *heap;
  38. unsigned long flags;
  39. unsigned long private_flags;
  40. size_t size;
  41. void *priv_virt;
  42. struct mutex lock;
  43. int kmap_cnt;
  44. void *vaddr;
  45. struct sg_table *sg_table;
  46. struct list_head attachments;
  47. };
  48. /**
  49. * struct ion_heap_ops - ops to operate on a given heap
  50. * @allocate: allocate memory
  51. * @free: free memory
  52. * @get_pool_size: get pool size in pages
  53. *
  54. * allocate returns 0 on success, -errno on error.
  55. * map_dma and map_kernel return pointer on success, ERR_PTR on
  56. * error. @free will be called with ION_PRIV_FLAG_SHRINKER_FREE set in
  57. * the buffer's private_flags when called from a shrinker. In that
  58. * case, the pages being free'd must be truly free'd back to the
  59. * system, not put in a page pool or otherwise cached.
  60. */
  61. struct ion_heap_ops {
  62. int (*allocate)(struct ion_heap *heap,
  63. struct ion_buffer *buffer, unsigned long len,
  64. unsigned long flags);
  65. void (*free)(struct ion_buffer *buffer);
  66. int (*shrink)(struct ion_heap *heap, gfp_t gfp_mask, int nr_to_scan);
  67. long (*get_pool_size)(struct ion_heap *heap);
  68. };
  69. /**
  70. * heap flags - flags between the heaps and core ion code
  71. */
  72. #define ION_HEAP_FLAG_DEFER_FREE BIT(0)
  73. /**
  74. * private flags - flags internal to ion
  75. */
  76. /*
  77. * Buffer is being freed from a shrinker function. Skip any possible
  78. * heap-specific caching mechanism (e.g. page pools). Guarantees that
  79. * any buffer storage that came from the system allocator will be
  80. * returned to the system allocator.
  81. */
  82. #define ION_PRIV_FLAG_SHRINKER_FREE BIT(0)
  83. /**
  84. * struct ion_heap - represents a heap in the system
  85. * @node: rb node to put the heap on the device's tree of heaps
  86. * @type: type of heap
  87. * @ops: ops struct as above
  88. * @buf_ops: dma_buf ops specific to the heap implementation.
  89. * @flags: flags
  90. * @id: id of heap, also indicates priority of this heap when
  91. * allocating. These are specified by platform data and
  92. * MUST be unique
  93. * @name: used for debugging
  94. * @owner: kernel module that implements this heap
  95. * @shrinker: a shrinker for the heap
  96. * @free_list: free list head if deferred free is used
  97. * @free_list_size size of the deferred free list in bytes
  98. * @lock: protects the free list
  99. * @waitqueue: queue to wait on from deferred free thread
  100. * @task: task struct of deferred free thread
  101. * @num_of_buffers the number of currently allocated buffers
  102. * @num_of_alloc_bytes the number of allocated bytes
  103. * @alloc_bytes_wm the number of allocated bytes watermark
  104. *
  105. * Represents a pool of memory from which buffers can be made. In some
  106. * systems the only heap is regular system memory allocated via vmalloc.
  107. * On others, some blocks might require large physically contiguous buffers
  108. * that are allocated from a specially reserved heap.
  109. */
  110. struct ion_heap {
  111. struct plist_node node;
  112. enum ion_heap_type type;
  113. struct ion_heap_ops *ops;
  114. struct dma_buf_ops buf_ops;
  115. unsigned long flags;
  116. unsigned int id;
  117. const char *name;
  118. struct module *owner;
  119. /* deferred free support */
  120. struct shrinker shrinker;
  121. struct list_head free_list;
  122. size_t free_list_size;
  123. spinlock_t free_lock;
  124. wait_queue_head_t waitqueue;
  125. struct task_struct *task;
  126. /* heap statistics */
  127. u64 num_of_buffers;
  128. u64 num_of_alloc_bytes;
  129. u64 alloc_bytes_wm;
  130. /* protect heap statistics */
  131. spinlock_t stat_lock;
  132. /* heap's debugfs root */
  133. struct dentry *debugfs_dir;
  134. };
  135. #define ion_device_add_heap(heap) __ion_device_add_heap(heap, THIS_MODULE)
  136. /**
  137. * struct ion_dma_buf_attachment - hold device-table attachment data for buffer
  138. * @dev: device attached to the buffer.
  139. * @table: cached mapping.
  140. * @list: list of ion_dma_buf_attachment.
  141. */
  142. struct ion_dma_buf_attachment {
  143. struct device *dev;
  144. struct sg_table *table;
  145. struct list_head list;
  146. bool mapped:1;
  147. };
  148. #ifdef CONFIG_ION
  149. /**
  150. * __ion_device_add_heap - adds a heap to the ion device
  151. *
  152. * @heap: the heap to add
  153. *
  154. * Returns 0 on success, negative error otherwise.
  155. */
  156. int __ion_device_add_heap(struct ion_heap *heap, struct module *owner);
  157. /**
  158. * ion_device_remove_heap - removes a heap from ion device
  159. *
  160. * @heap: pointer to the heap to be removed
  161. */
  162. void ion_device_remove_heap(struct ion_heap *heap);
  163. /**
  164. * ion_heap_init_shrinker
  165. * @heap: the heap
  166. *
  167. * If a heap sets the ION_HEAP_FLAG_DEFER_FREE flag or defines the shrink op
  168. * this function will be called to setup a shrinker to shrink the freelists
  169. * and call the heap's shrink op.
  170. */
  171. int ion_heap_init_shrinker(struct ion_heap *heap);
  172. /**
  173. * ion_heap_init_deferred_free -- initialize deferred free functionality
  174. * @heap: the heap
  175. *
  176. * If a heap sets the ION_HEAP_FLAG_DEFER_FREE flag this function will
  177. * be called to setup deferred frees. Calls to free the buffer will
  178. * return immediately and the actual free will occur some time later
  179. */
  180. int ion_heap_init_deferred_free(struct ion_heap *heap);
  181. /**
  182. * ion_heap_freelist_add - add a buffer to the deferred free list
  183. * @heap: the heap
  184. * @buffer: the buffer
  185. *
  186. * Adds an item to the deferred freelist.
  187. */
  188. void ion_heap_freelist_add(struct ion_heap *heap, struct ion_buffer *buffer);
  189. /**
  190. * ion_heap_freelist_drain - drain the deferred free list
  191. * @heap: the heap
  192. * @size: amount of memory to drain in bytes
  193. *
  194. * Drains the indicated amount of memory from the deferred freelist immediately.
  195. * Returns the total amount freed. The total freed may be higher depending
  196. * on the size of the items in the list, or lower if there is insufficient
  197. * total memory on the freelist.
  198. */
  199. size_t ion_heap_freelist_drain(struct ion_heap *heap, size_t size);
  200. /**
  201. * ion_heap_freelist_shrink - drain the deferred free
  202. * list, skipping any heap-specific
  203. * pooling or caching mechanisms
  204. *
  205. * @heap: the heap
  206. * @size: amount of memory to drain in bytes
  207. *
  208. * Drains the indicated amount of memory from the deferred freelist immediately.
  209. * Returns the total amount freed. The total freed may be higher depending
  210. * on the size of the items in the list, or lower if there is insufficient
  211. * total memory on the freelist.
  212. *
  213. * Unlike with @ion_heap_freelist_drain, don't put any pages back into
  214. * page pools or otherwise cache the pages. Everything must be
  215. * genuinely free'd back to the system. If you're free'ing from a
  216. * shrinker you probably want to use this. Note that this relies on
  217. * the heap.ops.free callback honoring the ION_PRIV_FLAG_SHRINKER_FREE
  218. * flag.
  219. */
  220. size_t ion_heap_freelist_shrink(struct ion_heap *heap,
  221. size_t size);
  222. /**
  223. * ion_heap_freelist_size - returns the size of the freelist in bytes
  224. * @heap: the heap
  225. */
  226. size_t ion_heap_freelist_size(struct ion_heap *heap);
  227. /**
  228. * ion_heap_map_kernel - map the ion_buffer in kernel virtual address space.
  229. *
  230. * @heap: the heap
  231. * @buffer: buffer to be mapped
  232. *
  233. * Maps the buffer using vmap(). The function respects cache flags for the
  234. * buffer and creates the page table entries accordingly. Returns virtual
  235. * address at the beginning of the buffer or ERR_PTR.
  236. */
  237. void *ion_heap_map_kernel(struct ion_heap *heap, struct ion_buffer *buffer);
  238. /**
  239. * ion_heap_unmap_kernel - unmap ion_buffer
  240. *
  241. * @buffer: buffer to be unmapped
  242. *
  243. * ION wrapper for vunmap() of the ion buffer.
  244. */
  245. void ion_heap_unmap_kernel(struct ion_heap *heap, struct ion_buffer *buffer);
  246. /**
  247. * ion_heap_map_user - map given ion buffer in provided vma
  248. *
  249. * @heap: the heap this buffer belongs to
  250. * @buffer: Ion buffer to be mapped
  251. * @vma: vma of the process where buffer should be mapped.
  252. *
  253. * Maps the buffer using remap_pfn_range() into specific process's vma starting
  254. * with vma->vm_start. The vma size is expected to be >= ion buffer size.
  255. * If not, a partial buffer mapping may be created. Returns 0 on success.
  256. */
  257. int ion_heap_map_user(struct ion_heap *heap, struct ion_buffer *buffer,
  258. struct vm_area_struct *vma);
  259. /* ion_buffer_zero - zeroes out an ion buffer respecting the ION_FLAGs.
  260. *
  261. * @buffer: ion_buffer to zero
  262. *
  263. * Returns 0 on success, negative error otherwise.
  264. */
  265. int ion_buffer_zero(struct ion_buffer *buffer);
  266. /**
  267. * ion_buffer_prep_noncached - flush cache before non-cached mapping
  268. *
  269. * @buffer: ion_buffer to flush
  270. *
  271. * The memory allocated by the heap could be in the CPU cache. To map
  272. * this memory as non-cached, we need to flush the associated cache
  273. * first. Without the flush, it is possible for stale dirty cache lines
  274. * to be evicted after the ION client started writing into this buffer,
  275. * leading to data corruption.
  276. */
  277. void ion_buffer_prep_noncached(struct ion_buffer *buffer);
  278. /**
  279. * ion_alloc - Allocates an ion buffer of given size from given heap
  280. *
  281. * @len: size of the buffer to be allocated.
  282. * @heap_id_mask: a bitwise maks of heap ids to allocate from
  283. * @flags: ION_BUFFER_XXXX flags for the new buffer.
  284. *
  285. * The function exports a dma_buf object for the new ion buffer internally
  286. * and returns that to the caller. So, the buffer is ready to be used by other
  287. * drivers immediately. Returns ERR_PTR in case of failure.
  288. */
  289. struct dma_buf *ion_alloc(size_t len, unsigned int heap_id_mask,
  290. unsigned int flags);
  291. /**
  292. * ion_free - Releases the ion buffer.
  293. *
  294. * @buffer: ion buffer to be released
  295. */
  296. int ion_free(struct ion_buffer *buffer);
  297. /**
  298. * ion_query_heaps_kernel - Returns information about available heaps to
  299. * in-kernel clients.
  300. *
  301. * @hdata: pointer to array of struct ion_heap_data.
  302. * @size: size of @hdata array.
  303. *
  304. * Returns the number of available heaps and populates @hdata with information
  305. * regarding the same. When invoked with @size as 0, the function with return
  306. * the number of available heaps without modifying @hdata. When the number of
  307. * available heaps is higher than @size, @size is returned instead of the
  308. * actual number of available heaps.
  309. */
  310. size_t ion_query_heaps_kernel(struct ion_heap_data *hdata, size_t size);
  311. #else
  312. static inline int __ion_device_add_heap(struct ion_heap *heap,
  313. struct module *owner)
  314. {
  315. return -ENODEV;
  316. }
  317. static inline int ion_heap_init_shrinker(struct ion_heap *heap)
  318. {
  319. return -ENODEV;
  320. }
  321. static inline int ion_heap_init_deferred_free(struct ion_heap *heap)
  322. {
  323. return -ENODEV;
  324. }
  325. static inline void ion_heap_freelist_add(struct ion_heap *heap,
  326. struct ion_buffer *buffer) {}
  327. static inline size_t ion_heap_freelist_drain(struct ion_heap *heap, size_t size)
  328. {
  329. return -ENODEV;
  330. }
  331. static inline size_t ion_heap_freelist_shrink(struct ion_heap *heap,
  332. size_t size)
  333. {
  334. return -ENODEV;
  335. }
  336. static inline size_t ion_heap_freelist_size(struct ion_heap *heap)
  337. {
  338. return -ENODEV;
  339. }
  340. static inline void *ion_heap_map_kernel(struct ion_heap *heap,
  341. struct ion_buffer *buffer)
  342. {
  343. return ERR_PTR(-ENODEV);
  344. }
  345. static inline void ion_heap_unmap_kernel(struct ion_heap *heap,
  346. struct ion_buffer *buffer) {}
  347. static inline int ion_heap_map_user(struct ion_heap *heap,
  348. struct ion_buffer *buffer,
  349. struct vm_area_struct *vma)
  350. {
  351. return -ENODEV;
  352. }
  353. static inline int ion_buffer_zero(struct ion_buffer *buffer)
  354. {
  355. return -EINVAL;
  356. }
  357. static inline void ion_buffer_prep_noncached(struct ion_buffer *buffer) {}
  358. static inline struct dma_buf *ion_alloc(size_t len, unsigned int heap_id_mask,
  359. unsigned int flags)
  360. {
  361. return ERR_PTR(-ENOMEM);
  362. }
  363. static inline int ion_free(struct ion_buffer *buffer)
  364. {
  365. return 0;
  366. }
  367. static inline size_t ion_query_heaps_kernel(struct ion_heap_data *hdata,
  368. size_t size)
  369. {
  370. return 0;
  371. }
  372. #endif /* CONFIG_ION */
  373. #endif /* _ION_KERNEL_H */