mspec.c 9.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426
  1. /*
  2. * Copyright (C) 2001-2006 Silicon Graphics, Inc. All rights
  3. * reserved.
  4. *
  5. * This program is free software; you can redistribute it and/or modify it
  6. * under the terms of version 2 of the GNU General Public License
  7. * as published by the Free Software Foundation.
  8. */
  9. /*
  10. * SN Platform Special Memory (mspec) Support
  11. *
  12. * This driver exports the SN special memory (mspec) facility to user
  13. * processes.
  14. * There are three types of memory made available thru this driver:
  15. * fetchops, uncached and cached.
  16. *
  17. * Fetchops are atomic memory operations that are implemented in the
  18. * memory controller on SGI SN hardware.
  19. *
  20. * Uncached are used for memory write combining feature of the ia64
  21. * cpu.
  22. *
  23. * Cached are used for areas of memory that are used as cached addresses
  24. * on our partition and used as uncached addresses from other partitions.
  25. * Due to a design constraint of the SN2 Shub, you can not have processors
  26. * on the same FSB perform both a cached and uncached reference to the
  27. * same cache line. These special memory cached regions prevent the
  28. * kernel from ever dropping in a TLB entry and therefore prevent the
  29. * processor from ever speculating a cache line from this page.
  30. */
  31. #include <linux/types.h>
  32. #include <linux/kernel.h>
  33. #include <linux/module.h>
  34. #include <linux/init.h>
  35. #include <linux/errno.h>
  36. #include <linux/miscdevice.h>
  37. #include <linux/spinlock.h>
  38. #include <linux/mm.h>
  39. #include <linux/vmalloc.h>
  40. #include <linux/string.h>
  41. #include <linux/slab.h>
  42. #include <linux/numa.h>
  43. #include <asm/page.h>
  44. #include <asm/system.h>
  45. #include <asm/pgtable.h>
  46. #include <asm/atomic.h>
  47. #include <asm/tlbflush.h>
  48. #include <asm/uncached.h>
  49. #include <asm/sn/addrs.h>
  50. #include <asm/sn/arch.h>
  51. #include <asm/sn/mspec.h>
  52. #include <asm/sn/sn_cpuid.h>
  53. #include <asm/sn/io.h>
  54. #include <asm/sn/bte.h>
  55. #include <asm/sn/shubio.h>
  56. #define FETCHOP_ID "SGI Fetchop,"
  57. #define CACHED_ID "Cached,"
  58. #define UNCACHED_ID "Uncached"
  59. #define REVISION "4.0"
  60. #define MSPEC_BASENAME "mspec"
  61. /*
  62. * Page types allocated by the device.
  63. */
  64. enum {
  65. MSPEC_FETCHOP = 1,
  66. MSPEC_CACHED,
  67. MSPEC_UNCACHED
  68. };
  69. #ifdef CONFIG_SGI_SN
  70. static int is_sn2;
  71. #else
  72. #define is_sn2 0
  73. #endif
  74. /*
  75. * One of these structures is allocated when an mspec region is mmaped. The
  76. * structure is pointed to by the vma->vm_private_data field in the vma struct.
  77. * This structure is used to record the addresses of the mspec pages.
  78. */
  79. struct vma_data {
  80. atomic_t refcnt; /* Number of vmas sharing the data. */
  81. spinlock_t lock; /* Serialize access to the vma. */
  82. int count; /* Number of pages allocated. */
  83. int type; /* Type of pages allocated. */
  84. unsigned long maddr[0]; /* Array of MSPEC addresses. */
  85. };
  86. /* used on shub2 to clear FOP cache in the HUB */
  87. static unsigned long scratch_page[MAX_NUMNODES];
  88. #define SH2_AMO_CACHE_ENTRIES 4
  89. static inline int
  90. mspec_zero_block(unsigned long addr, int len)
  91. {
  92. int status;
  93. if (is_sn2) {
  94. if (is_shub2()) {
  95. int nid;
  96. void *p;
  97. int i;
  98. nid = nasid_to_cnodeid(get_node_number(__pa(addr)));
  99. p = (void *)TO_AMO(scratch_page[nid]);
  100. for (i=0; i < SH2_AMO_CACHE_ENTRIES; i++) {
  101. FETCHOP_LOAD_OP(p, FETCHOP_LOAD);
  102. p += FETCHOP_VAR_SIZE;
  103. }
  104. }
  105. status = bte_copy(0, addr & ~__IA64_UNCACHED_OFFSET, len,
  106. BTE_WACQUIRE | BTE_ZERO_FILL, NULL);
  107. } else {
  108. memset((char *) addr, 0, len);
  109. status = 0;
  110. }
  111. return status;
  112. }
  113. /*
  114. * mspec_open
  115. *
  116. * Called when a device mapping is created by a means other than mmap
  117. * (via fork, etc.). Increments the reference count on the underlying
  118. * mspec data so it is not freed prematurely.
  119. */
  120. static void
  121. mspec_open(struct vm_area_struct *vma)
  122. {
  123. struct vma_data *vdata;
  124. vdata = vma->vm_private_data;
  125. atomic_inc(&vdata->refcnt);
  126. }
  127. /*
  128. * mspec_close
  129. *
  130. * Called when unmapping a device mapping. Frees all mspec pages
  131. * belonging to the vma.
  132. */
  133. static void
  134. mspec_close(struct vm_area_struct *vma)
  135. {
  136. struct vma_data *vdata;
  137. int i, pages, result, vdata_size;
  138. vdata = vma->vm_private_data;
  139. if (!atomic_dec_and_test(&vdata->refcnt))
  140. return;
  141. pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
  142. vdata_size = sizeof(struct vma_data) + pages * sizeof(long);
  143. for (i = 0; i < pages; i++) {
  144. if (vdata->maddr[i] == 0)
  145. continue;
  146. /*
  147. * Clear the page before sticking it back
  148. * into the pool.
  149. */
  150. result = mspec_zero_block(vdata->maddr[i], PAGE_SIZE);
  151. if (!result)
  152. uncached_free_page(vdata->maddr[i]);
  153. else
  154. printk(KERN_WARNING "mspec_close(): "
  155. "failed to zero page %i\n",
  156. result);
  157. }
  158. if (vdata_size <= PAGE_SIZE)
  159. kfree(vdata);
  160. else
  161. vfree(vdata);
  162. }
  163. /*
  164. * mspec_nopfn
  165. *
  166. * Creates a mspec page and maps it to user space.
  167. */
  168. static unsigned long
  169. mspec_nopfn(struct vm_area_struct *vma, unsigned long address)
  170. {
  171. unsigned long paddr, maddr;
  172. unsigned long pfn;
  173. int index;
  174. struct vma_data *vdata = vma->vm_private_data;
  175. index = (address - vma->vm_start) >> PAGE_SHIFT;
  176. maddr = (volatile unsigned long) vdata->maddr[index];
  177. if (maddr == 0) {
  178. maddr = uncached_alloc_page(numa_node_id());
  179. if (maddr == 0)
  180. return NOPFN_OOM;
  181. spin_lock(&vdata->lock);
  182. if (vdata->maddr[index] == 0) {
  183. vdata->count++;
  184. vdata->maddr[index] = maddr;
  185. } else {
  186. uncached_free_page(maddr);
  187. maddr = vdata->maddr[index];
  188. }
  189. spin_unlock(&vdata->lock);
  190. }
  191. if (vdata->type == MSPEC_FETCHOP)
  192. paddr = TO_AMO(maddr);
  193. else
  194. paddr = maddr & ~__IA64_UNCACHED_OFFSET;
  195. pfn = paddr >> PAGE_SHIFT;
  196. return pfn;
  197. }
  198. static struct vm_operations_struct mspec_vm_ops = {
  199. .open = mspec_open,
  200. .close = mspec_close,
  201. .nopfn = mspec_nopfn
  202. };
  203. /*
  204. * mspec_mmap
  205. *
  206. * Called when mmaping the device. Initializes the vma with a fault handler
  207. * and private data structure necessary to allocate, track, and free the
  208. * underlying pages.
  209. */
  210. static int
  211. mspec_mmap(struct file *file, struct vm_area_struct *vma, int type)
  212. {
  213. struct vma_data *vdata;
  214. int pages, vdata_size;
  215. if (vma->vm_pgoff != 0)
  216. return -EINVAL;
  217. if ((vma->vm_flags & VM_SHARED) == 0)
  218. return -EINVAL;
  219. if ((vma->vm_flags & VM_WRITE) == 0)
  220. return -EPERM;
  221. pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
  222. vdata_size = sizeof(struct vma_data) + pages * sizeof(long);
  223. if (vdata_size <= PAGE_SIZE)
  224. vdata = kmalloc(vdata_size, GFP_KERNEL);
  225. else
  226. vdata = vmalloc(vdata_size);
  227. if (!vdata)
  228. return -ENOMEM;
  229. memset(vdata, 0, vdata_size);
  230. vdata->type = type;
  231. spin_lock_init(&vdata->lock);
  232. vdata->refcnt = ATOMIC_INIT(1);
  233. vma->vm_private_data = vdata;
  234. vma->vm_flags |= (VM_IO | VM_LOCKED | VM_RESERVED | VM_PFNMAP);
  235. if (vdata->type == MSPEC_FETCHOP || vdata->type == MSPEC_UNCACHED)
  236. vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
  237. vma->vm_ops = &mspec_vm_ops;
  238. return 0;
  239. }
  240. static int
  241. fetchop_mmap(struct file *file, struct vm_area_struct *vma)
  242. {
  243. return mspec_mmap(file, vma, MSPEC_FETCHOP);
  244. }
  245. static int
  246. cached_mmap(struct file *file, struct vm_area_struct *vma)
  247. {
  248. return mspec_mmap(file, vma, MSPEC_CACHED);
  249. }
  250. static int
  251. uncached_mmap(struct file *file, struct vm_area_struct *vma)
  252. {
  253. return mspec_mmap(file, vma, MSPEC_UNCACHED);
  254. }
  255. static const struct file_operations fetchop_fops = {
  256. .owner = THIS_MODULE,
  257. .mmap = fetchop_mmap
  258. };
  259. static struct miscdevice fetchop_miscdev = {
  260. .minor = MISC_DYNAMIC_MINOR,
  261. .name = "sgi_fetchop",
  262. .fops = &fetchop_fops
  263. };
  264. static const struct file_operations cached_fops = {
  265. .owner = THIS_MODULE,
  266. .mmap = cached_mmap
  267. };
  268. static struct miscdevice cached_miscdev = {
  269. .minor = MISC_DYNAMIC_MINOR,
  270. .name = "mspec_cached",
  271. .fops = &cached_fops
  272. };
  273. static const struct file_operations uncached_fops = {
  274. .owner = THIS_MODULE,
  275. .mmap = uncached_mmap
  276. };
  277. static struct miscdevice uncached_miscdev = {
  278. .minor = MISC_DYNAMIC_MINOR,
  279. .name = "mspec_uncached",
  280. .fops = &uncached_fops
  281. };
  282. /*
  283. * mspec_init
  284. *
  285. * Called at boot time to initialize the mspec facility.
  286. */
  287. static int __init
  288. mspec_init(void)
  289. {
  290. int ret;
  291. int nid;
  292. /*
  293. * The fetchop device only works on SN2 hardware, uncached and cached
  294. * memory drivers should both be valid on all ia64 hardware
  295. */
  296. #ifdef CONFIG_SGI_SN
  297. if (ia64_platform_is("sn2")) {
  298. is_sn2 = 1;
  299. if (is_shub2()) {
  300. ret = -ENOMEM;
  301. for_each_online_node(nid) {
  302. int actual_nid;
  303. int nasid;
  304. unsigned long phys;
  305. scratch_page[nid] = uncached_alloc_page(nid);
  306. if (scratch_page[nid] == 0)
  307. goto free_scratch_pages;
  308. phys = __pa(scratch_page[nid]);
  309. nasid = get_node_number(phys);
  310. actual_nid = nasid_to_cnodeid(nasid);
  311. if (actual_nid != nid)
  312. goto free_scratch_pages;
  313. }
  314. }
  315. ret = misc_register(&fetchop_miscdev);
  316. if (ret) {
  317. printk(KERN_ERR
  318. "%s: failed to register device %i\n",
  319. FETCHOP_ID, ret);
  320. goto free_scratch_pages;
  321. }
  322. }
  323. #endif
  324. ret = misc_register(&cached_miscdev);
  325. if (ret) {
  326. printk(KERN_ERR "%s: failed to register device %i\n",
  327. CACHED_ID, ret);
  328. if (is_sn2)
  329. misc_deregister(&fetchop_miscdev);
  330. goto free_scratch_pages;
  331. }
  332. ret = misc_register(&uncached_miscdev);
  333. if (ret) {
  334. printk(KERN_ERR "%s: failed to register device %i\n",
  335. UNCACHED_ID, ret);
  336. misc_deregister(&cached_miscdev);
  337. if (is_sn2)
  338. misc_deregister(&fetchop_miscdev);
  339. goto free_scratch_pages;
  340. }
  341. printk(KERN_INFO "%s %s initialized devices: %s %s %s\n",
  342. MSPEC_BASENAME, REVISION, is_sn2 ? FETCHOP_ID : "",
  343. CACHED_ID, UNCACHED_ID);
  344. return 0;
  345. free_scratch_pages:
  346. for_each_node(nid) {
  347. if (scratch_page[nid] != 0)
  348. uncached_free_page(scratch_page[nid]);
  349. }
  350. return ret;
  351. }
  352. static void __exit
  353. mspec_exit(void)
  354. {
  355. int nid;
  356. misc_deregister(&uncached_miscdev);
  357. misc_deregister(&cached_miscdev);
  358. if (is_sn2) {
  359. misc_deregister(&fetchop_miscdev);
  360. for_each_node(nid) {
  361. if (scratch_page[nid] != 0)
  362. uncached_free_page(scratch_page[nid]);
  363. }
  364. }
  365. }
  366. module_init(mspec_init);
  367. module_exit(mspec_exit);
  368. MODULE_AUTHOR("Silicon Graphics, Inc. <linux-altix@sgi.com>");
  369. MODULE_DESCRIPTION("Driver for SGI SN special memory operations");
  370. MODULE_LICENSE("GPL");