drm_cache.c 5.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178
  1. /**************************************************************************
  2. *
  3. * Copyright (c) 2006-2007 Tungsten Graphics, Inc., Cedar Park, TX., USA
  4. * All Rights Reserved.
  5. *
  6. * Permission is hereby granted, free of charge, to any person obtaining a
  7. * copy of this software and associated documentation files (the
  8. * "Software"), to deal in the Software without restriction, including
  9. * without limitation the rights to use, copy, modify, merge, publish,
  10. * distribute, sub license, and/or sell copies of the Software, and to
  11. * permit persons to whom the Software is furnished to do so, subject to
  12. * the following conditions:
  13. *
  14. * The above copyright notice and this permission notice (including the
  15. * next paragraph) shall be included in all copies or substantial portions
  16. * of the Software.
  17. *
  18. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20. * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  21. * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  22. * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  23. * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  24. * USE OR OTHER DEALINGS IN THE SOFTWARE.
  25. *
  26. **************************************************************************/
  27. /*
  28. * Authors: Thomas Hellström <thomas-at-tungstengraphics-dot-com>
  29. */
  30. #include <linux/export.h>
  31. #include <linux/highmem.h>
  32. #include <drm/drm_cache.h>
  33. #if defined(CONFIG_X86)
  34. #include <asm/smp.h>
  35. /*
  36. * clflushopt is an unordered instruction which needs fencing with mfence or
  37. * sfence to avoid ordering issues. For drm_clflush_page this fencing happens
  38. * in the caller.
  39. */
  40. static void
  41. drm_clflush_page(struct page *page)
  42. {
  43. uint8_t *page_virtual;
  44. unsigned int i;
  45. const int size = boot_cpu_data.x86_clflush_size;
  46. if (unlikely(page == NULL))
  47. return;
  48. page_virtual = kmap_atomic(page);
  49. for (i = 0; i < PAGE_SIZE; i += size)
  50. clflushopt(page_virtual + i);
  51. kunmap_atomic(page_virtual);
  52. }
  53. static void drm_cache_flush_clflush(struct page *pages[],
  54. unsigned long num_pages)
  55. {
  56. unsigned long i;
  57. mb(); /*Full memory barrier used before so that CLFLUSH is ordered*/
  58. for (i = 0; i < num_pages; i++)
  59. drm_clflush_page(*pages++);
  60. mb(); /*Also used after CLFLUSH so that all cache is flushed*/
  61. }
  62. #endif
  63. /**
  64. * drm_clflush_pages - Flush dcache lines of a set of pages.
  65. * @pages: List of pages to be flushed.
  66. * @num_pages: Number of pages in the array.
  67. *
  68. * Flush every data cache line entry that points to an address belonging
  69. * to a page in the array.
  70. */
  71. void
  72. drm_clflush_pages(struct page *pages[], unsigned long num_pages)
  73. {
  74. #if defined(CONFIG_X86)
  75. if (static_cpu_has(X86_FEATURE_CLFLUSH)) {
  76. drm_cache_flush_clflush(pages, num_pages);
  77. return;
  78. }
  79. if (wbinvd_on_all_cpus())
  80. pr_err("Timed out waiting for cache flush\n");
  81. #elif defined(__powerpc__)
  82. unsigned long i;
  83. for (i = 0; i < num_pages; i++) {
  84. struct page *page = pages[i];
  85. void *page_virtual;
  86. if (unlikely(page == NULL))
  87. continue;
  88. page_virtual = kmap_atomic(page);
  89. flush_dcache_range((unsigned long)page_virtual,
  90. (unsigned long)page_virtual + PAGE_SIZE);
  91. kunmap_atomic(page_virtual);
  92. }
  93. #else
  94. pr_err("Architecture has no drm_cache.c support\n");
  95. WARN_ON_ONCE(1);
  96. #endif
  97. }
  98. EXPORT_SYMBOL(drm_clflush_pages);
  99. /**
  100. * drm_clflush_sg - Flush dcache lines pointing to a scather-gather.
  101. * @st: struct sg_table.
  102. *
  103. * Flush every data cache line entry that points to an address in the
  104. * sg.
  105. */
  106. void
  107. drm_clflush_sg(struct sg_table *st)
  108. {
  109. #if defined(CONFIG_X86)
  110. if (static_cpu_has(X86_FEATURE_CLFLUSH)) {
  111. struct sg_page_iter sg_iter;
  112. mb(); /*CLFLUSH is ordered only by using memory barriers*/
  113. for_each_sgtable_page(st, &sg_iter, 0)
  114. drm_clflush_page(sg_page_iter_page(&sg_iter));
  115. mb(); /*Make sure that all cache line entry is flushed*/
  116. return;
  117. }
  118. if (wbinvd_on_all_cpus())
  119. pr_err("Timed out waiting for cache flush\n");
  120. #else
  121. pr_err("Architecture has no drm_cache.c support\n");
  122. WARN_ON_ONCE(1);
  123. #endif
  124. }
  125. EXPORT_SYMBOL(drm_clflush_sg);
  126. /**
  127. * drm_clflush_virt_range - Flush dcache lines of a region
  128. * @addr: Initial kernel memory address.
  129. * @length: Region size.
  130. *
  131. * Flush every data cache line entry that points to an address in the
  132. * region requested.
  133. */
  134. void
  135. drm_clflush_virt_range(void *addr, unsigned long length)
  136. {
  137. #if defined(CONFIG_X86)
  138. if (static_cpu_has(X86_FEATURE_CLFLUSH)) {
  139. const int size = boot_cpu_data.x86_clflush_size;
  140. void *end = addr + length;
  141. addr = (void *)(((unsigned long)addr) & -size);
  142. mb(); /*CLFLUSH is only ordered with a full memory barrier*/
  143. for (; addr < end; addr += size)
  144. clflushopt(addr);
  145. clflushopt(end - 1); /* force serialisation */
  146. mb(); /*Ensure that evry data cache line entry is flushed*/
  147. return;
  148. }
  149. if (wbinvd_on_all_cpus())
  150. pr_err("Timed out waiting for cache flush\n");
  151. #else
  152. pr_err("Architecture has no drm_cache.c support\n");
  153. WARN_ON_ONCE(1);
  154. #endif
  155. }
  156. EXPORT_SYMBOL(drm_clflush_virt_range);