mspec.c 7.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (C) 2001-2006 Silicon Graphics, Inc. All rights
  4. * reserved.
  5. */
  6. /*
  7. * SN Platform Special Memory (mspec) Support
  8. *
  9. * This driver exports the SN special memory (mspec) facility to user
  10. * processes.
  11. * There are two types of memory made available thru this driver:
  12. * uncached and cached.
  13. *
  14. * Uncached are used for memory write combining feature of the ia64
  15. * cpu.
  16. *
  17. * Cached are used for areas of memory that are used as cached addresses
  18. * on our partition and used as uncached addresses from other partitions.
  19. * Due to a design constraint of the SN2 Shub, you can not have processors
  20. * on the same FSB perform both a cached and uncached reference to the
  21. * same cache line. These special memory cached regions prevent the
  22. * kernel from ever dropping in a TLB entry and therefore prevent the
  23. * processor from ever speculating a cache line from this page.
  24. */
  25. #include <linux/types.h>
  26. #include <linux/kernel.h>
  27. #include <linux/module.h>
  28. #include <linux/init.h>
  29. #include <linux/errno.h>
  30. #include <linux/miscdevice.h>
  31. #include <linux/spinlock.h>
  32. #include <linux/mm.h>
  33. #include <linux/fs.h>
  34. #include <linux/vmalloc.h>
  35. #include <linux/string.h>
  36. #include <linux/slab.h>
  37. #include <linux/numa.h>
  38. #include <linux/refcount.h>
  39. #include <asm/page.h>
  40. #include <linux/atomic.h>
  41. #include <asm/tlbflush.h>
  42. #include <asm/uncached.h>
  43. #define CACHED_ID "Cached,"
  44. #define UNCACHED_ID "Uncached"
  45. #define REVISION "4.0"
  46. #define MSPEC_BASENAME "mspec"
  47. /*
  48. * Page types allocated by the device.
  49. */
  50. enum mspec_page_type {
  51. MSPEC_CACHED = 2,
  52. MSPEC_UNCACHED
  53. };
  54. /*
  55. * One of these structures is allocated when an mspec region is mmaped. The
  56. * structure is pointed to by the vma->vm_private_data field in the vma struct.
  57. * This structure is used to record the addresses of the mspec pages.
  58. * This structure is shared by all vma's that are split off from the
  59. * original vma when split_vma()'s are done.
  60. *
  61. * The refcnt is incremented atomically because mm->mmap_lock does not
  62. * protect in fork case where multiple tasks share the vma_data.
  63. */
  64. struct vma_data {
  65. refcount_t refcnt; /* Number of vmas sharing the data. */
  66. spinlock_t lock; /* Serialize access to this structure. */
  67. int count; /* Number of pages allocated. */
  68. enum mspec_page_type type; /* Type of pages allocated. */
  69. unsigned long vm_start; /* Original (unsplit) base. */
  70. unsigned long vm_end; /* Original (unsplit) end. */
  71. unsigned long maddr[]; /* Array of MSPEC addresses. */
  72. };
  73. /*
  74. * mspec_open
  75. *
  76. * Called when a device mapping is created by a means other than mmap
  77. * (via fork, munmap, etc.). Increments the reference count on the
  78. * underlying mspec data so it is not freed prematurely.
  79. */
  80. static void
  81. mspec_open(struct vm_area_struct *vma)
  82. {
  83. struct vma_data *vdata;
  84. vdata = vma->vm_private_data;
  85. refcount_inc(&vdata->refcnt);
  86. }
  87. /*
  88. * mspec_close
  89. *
  90. * Called when unmapping a device mapping. Frees all mspec pages
  91. * belonging to all the vma's sharing this vma_data structure.
  92. */
  93. static void
  94. mspec_close(struct vm_area_struct *vma)
  95. {
  96. struct vma_data *vdata;
  97. int index, last_index;
  98. unsigned long my_page;
  99. vdata = vma->vm_private_data;
  100. if (!refcount_dec_and_test(&vdata->refcnt))
  101. return;
  102. last_index = (vdata->vm_end - vdata->vm_start) >> PAGE_SHIFT;
  103. for (index = 0; index < last_index; index++) {
  104. if (vdata->maddr[index] == 0)
  105. continue;
  106. /*
  107. * Clear the page before sticking it back
  108. * into the pool.
  109. */
  110. my_page = vdata->maddr[index];
  111. vdata->maddr[index] = 0;
  112. memset((char *)my_page, 0, PAGE_SIZE);
  113. uncached_free_page(my_page, 1);
  114. }
  115. kvfree(vdata);
  116. }
  117. /*
  118. * mspec_fault
  119. *
  120. * Creates a mspec page and maps it to user space.
  121. */
  122. static vm_fault_t
  123. mspec_fault(struct vm_fault *vmf)
  124. {
  125. unsigned long paddr, maddr;
  126. unsigned long pfn;
  127. pgoff_t index = vmf->pgoff;
  128. struct vma_data *vdata = vmf->vma->vm_private_data;
  129. maddr = (volatile unsigned long) vdata->maddr[index];
  130. if (maddr == 0) {
  131. maddr = uncached_alloc_page(numa_node_id(), 1);
  132. if (maddr == 0)
  133. return VM_FAULT_OOM;
  134. spin_lock(&vdata->lock);
  135. if (vdata->maddr[index] == 0) {
  136. vdata->count++;
  137. vdata->maddr[index] = maddr;
  138. } else {
  139. uncached_free_page(maddr, 1);
  140. maddr = vdata->maddr[index];
  141. }
  142. spin_unlock(&vdata->lock);
  143. }
  144. paddr = maddr & ~__IA64_UNCACHED_OFFSET;
  145. pfn = paddr >> PAGE_SHIFT;
  146. return vmf_insert_pfn(vmf->vma, vmf->address, pfn);
  147. }
  148. static const struct vm_operations_struct mspec_vm_ops = {
  149. .open = mspec_open,
  150. .close = mspec_close,
  151. .fault = mspec_fault,
  152. };
  153. /*
  154. * mspec_mmap
  155. *
  156. * Called when mmapping the device. Initializes the vma with a fault handler
  157. * and private data structure necessary to allocate, track, and free the
  158. * underlying pages.
  159. */
  160. static int
  161. mspec_mmap(struct file *file, struct vm_area_struct *vma,
  162. enum mspec_page_type type)
  163. {
  164. struct vma_data *vdata;
  165. int pages, vdata_size;
  166. if (vma->vm_pgoff != 0)
  167. return -EINVAL;
  168. if ((vma->vm_flags & VM_SHARED) == 0)
  169. return -EINVAL;
  170. if ((vma->vm_flags & VM_WRITE) == 0)
  171. return -EPERM;
  172. pages = vma_pages(vma);
  173. vdata_size = sizeof(struct vma_data) + pages * sizeof(long);
  174. vdata = kvzalloc(vdata_size, GFP_KERNEL);
  175. if (!vdata)
  176. return -ENOMEM;
  177. vdata->vm_start = vma->vm_start;
  178. vdata->vm_end = vma->vm_end;
  179. vdata->type = type;
  180. spin_lock_init(&vdata->lock);
  181. refcount_set(&vdata->refcnt, 1);
  182. vma->vm_private_data = vdata;
  183. vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP;
  184. if (vdata->type == MSPEC_UNCACHED)
  185. vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
  186. vma->vm_ops = &mspec_vm_ops;
  187. return 0;
  188. }
  189. static int
  190. cached_mmap(struct file *file, struct vm_area_struct *vma)
  191. {
  192. return mspec_mmap(file, vma, MSPEC_CACHED);
  193. }
  194. static int
  195. uncached_mmap(struct file *file, struct vm_area_struct *vma)
  196. {
  197. return mspec_mmap(file, vma, MSPEC_UNCACHED);
  198. }
  199. static const struct file_operations cached_fops = {
  200. .owner = THIS_MODULE,
  201. .mmap = cached_mmap,
  202. .llseek = noop_llseek,
  203. };
  204. static struct miscdevice cached_miscdev = {
  205. .minor = MISC_DYNAMIC_MINOR,
  206. .name = "mspec_cached",
  207. .fops = &cached_fops
  208. };
  209. static const struct file_operations uncached_fops = {
  210. .owner = THIS_MODULE,
  211. .mmap = uncached_mmap,
  212. .llseek = noop_llseek,
  213. };
  214. static struct miscdevice uncached_miscdev = {
  215. .minor = MISC_DYNAMIC_MINOR,
  216. .name = "mspec_uncached",
  217. .fops = &uncached_fops
  218. };
  219. /*
  220. * mspec_init
  221. *
  222. * Called at boot time to initialize the mspec facility.
  223. */
  224. static int __init
  225. mspec_init(void)
  226. {
  227. int ret;
  228. ret = misc_register(&cached_miscdev);
  229. if (ret) {
  230. printk(KERN_ERR "%s: failed to register device %i\n",
  231. CACHED_ID, ret);
  232. return ret;
  233. }
  234. ret = misc_register(&uncached_miscdev);
  235. if (ret) {
  236. printk(KERN_ERR "%s: failed to register device %i\n",
  237. UNCACHED_ID, ret);
  238. misc_deregister(&cached_miscdev);
  239. return ret;
  240. }
  241. printk(KERN_INFO "%s %s initialized devices: %s %s\n",
  242. MSPEC_BASENAME, REVISION, CACHED_ID, UNCACHED_ID);
  243. return 0;
  244. }
  245. static void __exit
  246. mspec_exit(void)
  247. {
  248. misc_deregister(&uncached_miscdev);
  249. misc_deregister(&cached_miscdev);
  250. }
  251. module_init(mspec_init);
  252. module_exit(mspec_exit);
  253. MODULE_AUTHOR("Silicon Graphics, Inc. <linux-altix@sgi.com>");
  254. MODULE_DESCRIPTION("Driver for SGI SN special memory operations");
  255. MODULE_LICENSE("GPL");