tlb.c 9.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390
  1. /*
  2. * Copyright (C) 2000, 2001, 2002 Jeff Dike (jdike@karaya.com)
  3. * Licensed under the GPL
  4. */
  5. #include "linux/mm.h"
  6. #include "asm/page.h"
  7. #include "asm/pgalloc.h"
  8. #include "asm/tlbflush.h"
  9. #include "choose-mode.h"
  10. #include "mode_kern.h"
  11. #include "user_util.h"
  12. #include "tlb.h"
  13. #include "mem.h"
  14. #include "mem_user.h"
  15. #include "os.h"
  16. static int add_mmap(unsigned long virt, unsigned long phys, unsigned long len,
  17. int r, int w, int x, struct host_vm_op *ops, int *index,
  18. int last_filled, union mm_context *mmu, void **flush,
  19. int (*do_ops)(union mm_context *, struct host_vm_op *,
  20. int, int, void **))
  21. {
  22. __u64 offset;
  23. struct host_vm_op *last;
  24. int fd, ret = 0;
  25. fd = phys_mapping(phys, &offset);
  26. if(*index != -1){
  27. last = &ops[*index];
  28. if((last->type == MMAP) &&
  29. (last->u.mmap.addr + last->u.mmap.len == virt) &&
  30. (last->u.mmap.r == r) && (last->u.mmap.w == w) &&
  31. (last->u.mmap.x == x) && (last->u.mmap.fd == fd) &&
  32. (last->u.mmap.offset + last->u.mmap.len == offset)){
  33. last->u.mmap.len += len;
  34. return 0;
  35. }
  36. }
  37. if(*index == last_filled){
  38. ret = (*do_ops)(mmu, ops, last_filled, 0, flush);
  39. *index = -1;
  40. }
  41. ops[++*index] = ((struct host_vm_op) { .type = MMAP,
  42. .u = { .mmap = {
  43. .addr = virt,
  44. .len = len,
  45. .r = r,
  46. .w = w,
  47. .x = x,
  48. .fd = fd,
  49. .offset = offset }
  50. } });
  51. return ret;
  52. }
  53. static int add_munmap(unsigned long addr, unsigned long len,
  54. struct host_vm_op *ops, int *index, int last_filled,
  55. union mm_context *mmu, void **flush,
  56. int (*do_ops)(union mm_context *, struct host_vm_op *,
  57. int, int, void **))
  58. {
  59. struct host_vm_op *last;
  60. int ret = 0;
  61. if(*index != -1){
  62. last = &ops[*index];
  63. if((last->type == MUNMAP) &&
  64. (last->u.munmap.addr + last->u.mmap.len == addr)){
  65. last->u.munmap.len += len;
  66. return 0;
  67. }
  68. }
  69. if(*index == last_filled){
  70. ret = (*do_ops)(mmu, ops, last_filled, 0, flush);
  71. *index = -1;
  72. }
  73. ops[++*index] = ((struct host_vm_op) { .type = MUNMAP,
  74. .u = { .munmap = {
  75. .addr = addr,
  76. .len = len } } });
  77. return ret;
  78. }
  79. static int add_mprotect(unsigned long addr, unsigned long len, int r, int w,
  80. int x, struct host_vm_op *ops, int *index,
  81. int last_filled, union mm_context *mmu, void **flush,
  82. int (*do_ops)(union mm_context *, struct host_vm_op *,
  83. int, int, void **))
  84. {
  85. struct host_vm_op *last;
  86. int ret = 0;
  87. if(*index != -1){
  88. last = &ops[*index];
  89. if((last->type == MPROTECT) &&
  90. (last->u.mprotect.addr + last->u.mprotect.len == addr) &&
  91. (last->u.mprotect.r == r) && (last->u.mprotect.w == w) &&
  92. (last->u.mprotect.x == x)){
  93. last->u.mprotect.len += len;
  94. return 0;
  95. }
  96. }
  97. if(*index == last_filled){
  98. ret = (*do_ops)(mmu, ops, last_filled, 0, flush);
  99. *index = -1;
  100. }
  101. ops[++*index] = ((struct host_vm_op) { .type = MPROTECT,
  102. .u = { .mprotect = {
  103. .addr = addr,
  104. .len = len,
  105. .r = r,
  106. .w = w,
  107. .x = x } } });
  108. return ret;
  109. }
  110. #define ADD_ROUND(n, inc) (((n) + (inc)) & ~((inc) - 1))
  111. void fix_range_common(struct mm_struct *mm, unsigned long start_addr,
  112. unsigned long end_addr, int force,
  113. int (*do_ops)(union mm_context *, struct host_vm_op *,
  114. int, int, void **))
  115. {
  116. pgd_t *npgd;
  117. pud_t *npud;
  118. pmd_t *npmd;
  119. pte_t *npte;
  120. union mm_context *mmu = &mm->context;
  121. unsigned long addr, end;
  122. int r, w, x;
  123. struct host_vm_op ops[1];
  124. void *flush = NULL;
  125. int op_index = -1, last_op = ARRAY_SIZE(ops) - 1;
  126. int ret = 0;
  127. if(mm == NULL)
  128. return;
  129. ops[0].type = NONE;
  130. for(addr = start_addr; addr < end_addr && !ret;){
  131. npgd = pgd_offset(mm, addr);
  132. if(!pgd_present(*npgd)){
  133. end = ADD_ROUND(addr, PGDIR_SIZE);
  134. if(end > end_addr)
  135. end = end_addr;
  136. if(force || pgd_newpage(*npgd)){
  137. ret = add_munmap(addr, end - addr, ops,
  138. &op_index, last_op, mmu,
  139. &flush, do_ops);
  140. pgd_mkuptodate(*npgd);
  141. }
  142. addr = end;
  143. continue;
  144. }
  145. npud = pud_offset(npgd, addr);
  146. if(!pud_present(*npud)){
  147. end = ADD_ROUND(addr, PUD_SIZE);
  148. if(end > end_addr)
  149. end = end_addr;
  150. if(force || pud_newpage(*npud)){
  151. ret = add_munmap(addr, end - addr, ops,
  152. &op_index, last_op, mmu,
  153. &flush, do_ops);
  154. pud_mkuptodate(*npud);
  155. }
  156. addr = end;
  157. continue;
  158. }
  159. npmd = pmd_offset(npud, addr);
  160. if(!pmd_present(*npmd)){
  161. end = ADD_ROUND(addr, PMD_SIZE);
  162. if(end > end_addr)
  163. end = end_addr;
  164. if(force || pmd_newpage(*npmd)){
  165. ret = add_munmap(addr, end - addr, ops,
  166. &op_index, last_op, mmu,
  167. &flush, do_ops);
  168. pmd_mkuptodate(*npmd);
  169. }
  170. addr = end;
  171. continue;
  172. }
  173. npte = pte_offset_kernel(npmd, addr);
  174. r = pte_read(*npte);
  175. w = pte_write(*npte);
  176. x = pte_exec(*npte);
  177. if (!pte_young(*npte)) {
  178. r = 0;
  179. w = 0;
  180. } else if (!pte_dirty(*npte)) {
  181. w = 0;
  182. }
  183. if(force || pte_newpage(*npte)){
  184. if(pte_present(*npte))
  185. ret = add_mmap(addr,
  186. pte_val(*npte) & PAGE_MASK,
  187. PAGE_SIZE, r, w, x, ops,
  188. &op_index, last_op, mmu,
  189. &flush, do_ops);
  190. else ret = add_munmap(addr, PAGE_SIZE, ops,
  191. &op_index, last_op, mmu,
  192. &flush, do_ops);
  193. }
  194. else if(pte_newprot(*npte))
  195. ret = add_mprotect(addr, PAGE_SIZE, r, w, x, ops,
  196. &op_index, last_op, mmu,
  197. &flush, do_ops);
  198. *npte = pte_mkuptodate(*npte);
  199. addr += PAGE_SIZE;
  200. }
  201. if(!ret)
  202. ret = (*do_ops)(mmu, ops, op_index, 1, &flush);
  203. /* This is not an else because ret is modified above */
  204. if(ret) {
  205. printk("fix_range_common: failed, killing current process\n");
  206. force_sig(SIGKILL, current);
  207. }
  208. }
  209. int flush_tlb_kernel_range_common(unsigned long start, unsigned long end)
  210. {
  211. struct mm_struct *mm;
  212. pgd_t *pgd;
  213. pud_t *pud;
  214. pmd_t *pmd;
  215. pte_t *pte;
  216. unsigned long addr, last;
  217. int updated = 0, err;
  218. mm = &init_mm;
  219. for(addr = start; addr < end;){
  220. pgd = pgd_offset(mm, addr);
  221. if(!pgd_present(*pgd)){
  222. last = ADD_ROUND(addr, PGDIR_SIZE);
  223. if(last > end)
  224. last = end;
  225. if(pgd_newpage(*pgd)){
  226. updated = 1;
  227. err = os_unmap_memory((void *) addr,
  228. last - addr);
  229. if(err < 0)
  230. panic("munmap failed, errno = %d\n",
  231. -err);
  232. }
  233. addr = last;
  234. continue;
  235. }
  236. pud = pud_offset(pgd, addr);
  237. if(!pud_present(*pud)){
  238. last = ADD_ROUND(addr, PUD_SIZE);
  239. if(last > end)
  240. last = end;
  241. if(pud_newpage(*pud)){
  242. updated = 1;
  243. err = os_unmap_memory((void *) addr,
  244. last - addr);
  245. if(err < 0)
  246. panic("munmap failed, errno = %d\n",
  247. -err);
  248. }
  249. addr = last;
  250. continue;
  251. }
  252. pmd = pmd_offset(pud, addr);
  253. if(!pmd_present(*pmd)){
  254. last = ADD_ROUND(addr, PMD_SIZE);
  255. if(last > end)
  256. last = end;
  257. if(pmd_newpage(*pmd)){
  258. updated = 1;
  259. err = os_unmap_memory((void *) addr,
  260. last - addr);
  261. if(err < 0)
  262. panic("munmap failed, errno = %d\n",
  263. -err);
  264. }
  265. addr = last;
  266. continue;
  267. }
  268. pte = pte_offset_kernel(pmd, addr);
  269. if(!pte_present(*pte) || pte_newpage(*pte)){
  270. updated = 1;
  271. err = os_unmap_memory((void *) addr,
  272. PAGE_SIZE);
  273. if(err < 0)
  274. panic("munmap failed, errno = %d\n",
  275. -err);
  276. if(pte_present(*pte))
  277. map_memory(addr,
  278. pte_val(*pte) & PAGE_MASK,
  279. PAGE_SIZE, 1, 1, 1);
  280. }
  281. else if(pte_newprot(*pte)){
  282. updated = 1;
  283. os_protect_memory((void *) addr, PAGE_SIZE, 1, 1, 1);
  284. }
  285. addr += PAGE_SIZE;
  286. }
  287. return(updated);
  288. }
  289. pgd_t *pgd_offset_proc(struct mm_struct *mm, unsigned long address)
  290. {
  291. return(pgd_offset(mm, address));
  292. }
  293. pud_t *pud_offset_proc(pgd_t *pgd, unsigned long address)
  294. {
  295. return(pud_offset(pgd, address));
  296. }
  297. pmd_t *pmd_offset_proc(pud_t *pud, unsigned long address)
  298. {
  299. return(pmd_offset(pud, address));
  300. }
  301. pte_t *pte_offset_proc(pmd_t *pmd, unsigned long address)
  302. {
  303. return(pte_offset_kernel(pmd, address));
  304. }
  305. pte_t *addr_pte(struct task_struct *task, unsigned long addr)
  306. {
  307. pgd_t *pgd = pgd_offset(task->mm, addr);
  308. pud_t *pud = pud_offset(pgd, addr);
  309. pmd_t *pmd = pmd_offset(pud, addr);
  310. return(pte_offset_map(pmd, addr));
  311. }
  312. void flush_tlb_page(struct vm_area_struct *vma, unsigned long address)
  313. {
  314. address &= PAGE_MASK;
  315. flush_tlb_range(vma, address, address + PAGE_SIZE);
  316. }
  317. void flush_tlb_all(void)
  318. {
  319. flush_tlb_mm(current->mm);
  320. }
  321. void flush_tlb_kernel_range(unsigned long start, unsigned long end)
  322. {
  323. CHOOSE_MODE_PROC(flush_tlb_kernel_range_tt,
  324. flush_tlb_kernel_range_common, start, end);
  325. }
  326. void flush_tlb_kernel_vm(void)
  327. {
  328. CHOOSE_MODE(flush_tlb_kernel_vm_tt(),
  329. flush_tlb_kernel_range_common(start_vm, end_vm));
  330. }
  331. void __flush_tlb_one(unsigned long addr)
  332. {
  333. CHOOSE_MODE_PROC(__flush_tlb_one_tt, __flush_tlb_one_skas, addr);
  334. }
  335. void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
  336. unsigned long end)
  337. {
  338. CHOOSE_MODE_PROC(flush_tlb_range_tt, flush_tlb_range_skas, vma, start,
  339. end);
  340. }
  341. void flush_tlb_mm(struct mm_struct *mm)
  342. {
  343. CHOOSE_MODE_PROC(flush_tlb_mm_tt, flush_tlb_mm_skas, mm);
  344. }
  345. void force_flush_all(void)
  346. {
  347. CHOOSE_MODE(force_flush_all_tt(), force_flush_all_skas());
  348. }