gruhandles.c 4.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * GRU KERNEL MCS INSTRUCTIONS
  4. *
  5. * Copyright (c) 2008 Silicon Graphics, Inc. All Rights Reserved.
  6. */
  7. #include <linux/kernel.h>
  8. #include "gru.h"
  9. #include "grulib.h"
  10. #include "grutables.h"
  11. /* 10 sec */
  12. #ifdef CONFIG_IA64
  13. #include <asm/processor.h>
  14. #define GRU_OPERATION_TIMEOUT (((cycles_t) local_cpu_data->itc_freq)*10)
  15. #define CLKS2NSEC(c) ((c) *1000000000 / local_cpu_data->itc_freq)
  16. #else
  17. #include <linux/sync_core.h>
  18. #include <asm/tsc.h>
  19. #define GRU_OPERATION_TIMEOUT ((cycles_t) tsc_khz*10*1000)
  20. #define CLKS2NSEC(c) ((c) * 1000000 / tsc_khz)
  21. #endif
  22. /* Extract the status field from a kernel handle */
  23. #define GET_MSEG_HANDLE_STATUS(h) (((*(unsigned long *)(h)) >> 16) & 3)
  24. struct mcs_op_statistic mcs_op_statistics[mcsop_last];
  25. static void update_mcs_stats(enum mcs_op op, unsigned long clks)
  26. {
  27. unsigned long nsec;
  28. nsec = CLKS2NSEC(clks);
  29. atomic_long_inc(&mcs_op_statistics[op].count);
  30. atomic_long_add(nsec, &mcs_op_statistics[op].total);
  31. if (mcs_op_statistics[op].max < nsec)
  32. mcs_op_statistics[op].max = nsec;
  33. }
  34. static void start_instruction(void *h)
  35. {
  36. unsigned long *w0 = h;
  37. wmb(); /* setting CMD/STATUS bits must be last */
  38. *w0 = *w0 | 0x20001;
  39. gru_flush_cache(h);
  40. }
  41. static void report_instruction_timeout(void *h)
  42. {
  43. unsigned long goff = GSEGPOFF((unsigned long)h);
  44. char *id = "???";
  45. if (TYPE_IS(CCH, goff))
  46. id = "CCH";
  47. else if (TYPE_IS(TGH, goff))
  48. id = "TGH";
  49. else if (TYPE_IS(TFH, goff))
  50. id = "TFH";
  51. panic(KERN_ALERT "GRU %p (%s) is malfunctioning\n", h, id);
  52. }
  53. static int wait_instruction_complete(void *h, enum mcs_op opc)
  54. {
  55. int status;
  56. unsigned long start_time = get_cycles();
  57. while (1) {
  58. cpu_relax();
  59. status = GET_MSEG_HANDLE_STATUS(h);
  60. if (status != CCHSTATUS_ACTIVE)
  61. break;
  62. if (GRU_OPERATION_TIMEOUT < (get_cycles() - start_time)) {
  63. report_instruction_timeout(h);
  64. start_time = get_cycles();
  65. }
  66. }
  67. if (gru_options & OPT_STATS)
  68. update_mcs_stats(opc, get_cycles() - start_time);
  69. return status;
  70. }
  71. int cch_allocate(struct gru_context_configuration_handle *cch)
  72. {
  73. int ret;
  74. cch->opc = CCHOP_ALLOCATE;
  75. start_instruction(cch);
  76. ret = wait_instruction_complete(cch, cchop_allocate);
  77. /*
  78. * Stop speculation into the GSEG being mapped by the previous ALLOCATE.
  79. * The GSEG memory does not exist until the ALLOCATE completes.
  80. */
  81. sync_core();
  82. return ret;
  83. }
  84. int cch_start(struct gru_context_configuration_handle *cch)
  85. {
  86. cch->opc = CCHOP_START;
  87. start_instruction(cch);
  88. return wait_instruction_complete(cch, cchop_start);
  89. }
  90. int cch_interrupt(struct gru_context_configuration_handle *cch)
  91. {
  92. cch->opc = CCHOP_INTERRUPT;
  93. start_instruction(cch);
  94. return wait_instruction_complete(cch, cchop_interrupt);
  95. }
  96. int cch_deallocate(struct gru_context_configuration_handle *cch)
  97. {
  98. int ret;
  99. cch->opc = CCHOP_DEALLOCATE;
  100. start_instruction(cch);
  101. ret = wait_instruction_complete(cch, cchop_deallocate);
  102. /*
  103. * Stop speculation into the GSEG being unmapped by the previous
  104. * DEALLOCATE.
  105. */
  106. sync_core();
  107. return ret;
  108. }
  109. int cch_interrupt_sync(struct gru_context_configuration_handle
  110. *cch)
  111. {
  112. cch->opc = CCHOP_INTERRUPT_SYNC;
  113. start_instruction(cch);
  114. return wait_instruction_complete(cch, cchop_interrupt_sync);
  115. }
  116. int tgh_invalidate(struct gru_tlb_global_handle *tgh,
  117. unsigned long vaddr, unsigned long vaddrmask,
  118. int asid, int pagesize, int global, int n,
  119. unsigned short ctxbitmap)
  120. {
  121. tgh->vaddr = vaddr;
  122. tgh->asid = asid;
  123. tgh->pagesize = pagesize;
  124. tgh->n = n;
  125. tgh->global = global;
  126. tgh->vaddrmask = vaddrmask;
  127. tgh->ctxbitmap = ctxbitmap;
  128. tgh->opc = TGHOP_TLBINV;
  129. start_instruction(tgh);
  130. return wait_instruction_complete(tgh, tghop_invalidate);
  131. }
  132. int tfh_write_only(struct gru_tlb_fault_handle *tfh,
  133. unsigned long paddr, int gaa,
  134. unsigned long vaddr, int asid, int dirty,
  135. int pagesize)
  136. {
  137. tfh->fillasid = asid;
  138. tfh->fillvaddr = vaddr;
  139. tfh->pfn = paddr >> GRU_PADDR_SHIFT;
  140. tfh->gaa = gaa;
  141. tfh->dirty = dirty;
  142. tfh->pagesize = pagesize;
  143. tfh->opc = TFHOP_WRITE_ONLY;
  144. start_instruction(tfh);
  145. return wait_instruction_complete(tfh, tfhop_write_only);
  146. }
  147. void tfh_write_restart(struct gru_tlb_fault_handle *tfh,
  148. unsigned long paddr, int gaa,
  149. unsigned long vaddr, int asid, int dirty,
  150. int pagesize)
  151. {
  152. tfh->fillasid = asid;
  153. tfh->fillvaddr = vaddr;
  154. tfh->pfn = paddr >> GRU_PADDR_SHIFT;
  155. tfh->gaa = gaa;
  156. tfh->dirty = dirty;
  157. tfh->pagesize = pagesize;
  158. tfh->opc = TFHOP_WRITE_RESTART;
  159. start_instruction(tfh);
  160. }
  161. void tfh_user_polling_mode(struct gru_tlb_fault_handle *tfh)
  162. {
  163. tfh->opc = TFHOP_USER_POLLING_MODE;
  164. start_instruction(tfh);
  165. }
  166. void tfh_exception(struct gru_tlb_fault_handle *tfh)
  167. {
  168. tfh->opc = TFHOP_EXCEPTION;
  169. start_instruction(tfh);
  170. }