vmacache.c 2.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Copyright (C) 2014 Davidlohr Bueso.
  4. */
  5. #include <linux/sched/signal.h>
  6. #include <linux/sched/task.h>
  7. #include <linux/mm.h>
  8. #include <linux/vmacache.h>
  9. /*
  10. * Hash based on the pmd of addr if configured with MMU, which provides a good
  11. * hit rate for workloads with spatial locality. Otherwise, use pages.
  12. */
  13. #ifdef CONFIG_MMU
  14. #define VMACACHE_SHIFT PMD_SHIFT
  15. #else
  16. #define VMACACHE_SHIFT PAGE_SHIFT
  17. #endif
  18. #define VMACACHE_HASH(addr) ((addr >> VMACACHE_SHIFT) & VMACACHE_MASK)
  19. /*
  20. * This task may be accessing a foreign mm via (for example)
  21. * get_user_pages()->find_vma(). The vmacache is task-local and this
  22. * task's vmacache pertains to a different mm (ie, its own). There is
  23. * nothing we can do here.
  24. *
  25. * Also handle the case where a kernel thread has adopted this mm via
  26. * kthread_use_mm(). That kernel thread's vmacache is not applicable to this mm.
  27. */
  28. static inline bool vmacache_valid_mm(struct mm_struct *mm)
  29. {
  30. return current->mm == mm && !(current->flags & PF_KTHREAD);
  31. }
  32. void vmacache_update(unsigned long addr, struct vm_area_struct *newvma)
  33. {
  34. if (vmacache_valid_mm(newvma->vm_mm))
  35. current->vmacache.vmas[VMACACHE_HASH(addr)] = newvma;
  36. }
  37. static bool vmacache_valid(struct mm_struct *mm)
  38. {
  39. struct task_struct *curr;
  40. if (!vmacache_valid_mm(mm))
  41. return false;
  42. curr = current;
  43. if (mm->vmacache_seqnum != curr->vmacache.seqnum) {
  44. /*
  45. * First attempt will always be invalid, initialize
  46. * the new cache for this task here.
  47. */
  48. curr->vmacache.seqnum = mm->vmacache_seqnum;
  49. vmacache_flush(curr);
  50. return false;
  51. }
  52. return true;
  53. }
  54. struct vm_area_struct *vmacache_find(struct mm_struct *mm, unsigned long addr)
  55. {
  56. int idx = VMACACHE_HASH(addr);
  57. int i;
  58. count_vm_vmacache_event(VMACACHE_FIND_CALLS);
  59. if (!vmacache_valid(mm))
  60. return NULL;
  61. for (i = 0; i < VMACACHE_SIZE; i++) {
  62. struct vm_area_struct *vma = current->vmacache.vmas[idx];
  63. if (vma) {
  64. #ifdef CONFIG_DEBUG_VM_VMACACHE
  65. if (WARN_ON_ONCE(vma->vm_mm != mm))
  66. break;
  67. #endif
  68. if (vma->vm_start <= addr && vma->vm_end > addr) {
  69. count_vm_vmacache_event(VMACACHE_FIND_HITS);
  70. return vma;
  71. }
  72. }
  73. if (++idx == VMACACHE_SIZE)
  74. idx = 0;
  75. }
  76. return NULL;
  77. }
  78. #ifndef CONFIG_MMU
  79. struct vm_area_struct *vmacache_find_exact(struct mm_struct *mm,
  80. unsigned long start,
  81. unsigned long end)
  82. {
  83. int idx = VMACACHE_HASH(start);
  84. int i;
  85. count_vm_vmacache_event(VMACACHE_FIND_CALLS);
  86. if (!vmacache_valid(mm))
  87. return NULL;
  88. for (i = 0; i < VMACACHE_SIZE; i++) {
  89. struct vm_area_struct *vma = current->vmacache.vmas[idx];
  90. if (vma && vma->vm_start == start && vma->vm_end == end) {
  91. count_vm_vmacache_event(VMACACHE_FIND_HITS);
  92. return vma;
  93. }
  94. if (++idx == VMACACHE_SIZE)
  95. idx = 0;
  96. }
  97. return NULL;
  98. }
  99. #endif