srmmu.c 49 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * srmmu.c: SRMMU specific routines for memory management.
  4. *
  5. * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu)
  6. * Copyright (C) 1995,2002 Pete Zaitcev (zaitcev@yahoo.com)
  7. * Copyright (C) 1996 Eddie C. Dost (ecd@skynet.be)
  8. * Copyright (C) 1997,1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
  9. * Copyright (C) 1999,2000 Anton Blanchard (anton@samba.org)
  10. */
  11. #include <linux/seq_file.h>
  12. #include <linux/spinlock.h>
  13. #include <linux/memblock.h>
  14. #include <linux/pagemap.h>
  15. #include <linux/vmalloc.h>
  16. #include <linux/kdebug.h>
  17. #include <linux/export.h>
  18. #include <linux/kernel.h>
  19. #include <linux/init.h>
  20. #include <linux/log2.h>
  21. #include <linux/gfp.h>
  22. #include <linux/fs.h>
  23. #include <linux/mm.h>
  24. #include <asm/mmu_context.h>
  25. #include <asm/cacheflush.h>
  26. #include <asm/tlbflush.h>
  27. #include <asm/io-unit.h>
  28. #include <asm/pgalloc.h>
  29. #include <asm/pgtable.h>
  30. #include <asm/bitext.h>
  31. #include <asm/vaddrs.h>
  32. #include <asm/cache.h>
  33. #include <asm/traps.h>
  34. #include <asm/oplib.h>
  35. #include <asm/mbus.h>
  36. #include <asm/page.h>
  37. #include <asm/asi.h>
  38. #include <asm/smp.h>
  39. #include <asm/io.h>
  40. /* Now the cpu specific definitions. */
  41. #include <asm/turbosparc.h>
  42. #include <asm/tsunami.h>
  43. #include <asm/viking.h>
  44. #include <asm/swift.h>
  45. #include <asm/leon.h>
  46. #include <asm/mxcc.h>
  47. #include <asm/ross.h>
  48. #include "mm_32.h"
  49. enum mbus_module srmmu_modtype;
  50. static unsigned int hwbug_bitmask;
  51. int vac_cache_size;
  52. EXPORT_SYMBOL(vac_cache_size);
  53. int vac_line_size;
  54. extern struct resource sparc_iomap;
  55. extern unsigned long last_valid_pfn;
  56. static pgd_t *srmmu_swapper_pg_dir;
  57. const struct sparc32_cachetlb_ops *sparc32_cachetlb_ops;
  58. EXPORT_SYMBOL(sparc32_cachetlb_ops);
  59. #ifdef CONFIG_SMP
  60. const struct sparc32_cachetlb_ops *local_ops;
  61. #define FLUSH_BEGIN(mm)
  62. #define FLUSH_END
  63. #else
  64. #define FLUSH_BEGIN(mm) if ((mm)->context != NO_CONTEXT) {
  65. #define FLUSH_END }
  66. #endif
  67. int flush_page_for_dma_global = 1;
  68. char *srmmu_name;
  69. ctxd_t *srmmu_ctx_table_phys;
  70. static ctxd_t *srmmu_context_table;
  71. int viking_mxcc_present;
  72. static DEFINE_SPINLOCK(srmmu_context_spinlock);
  73. static int is_hypersparc;
  74. static int srmmu_cache_pagetables;
  75. /* these will be initialized in srmmu_nocache_calcsize() */
  76. static unsigned long srmmu_nocache_size;
  77. static unsigned long srmmu_nocache_end;
  78. /* 1 bit <=> 256 bytes of nocache <=> 64 PTEs */
  79. #define SRMMU_NOCACHE_BITMAP_SHIFT (PAGE_SHIFT - 4)
  80. /* The context table is a nocache user with the biggest alignment needs. */
  81. #define SRMMU_NOCACHE_ALIGN_MAX (sizeof(ctxd_t)*SRMMU_MAX_CONTEXTS)
  82. void *srmmu_nocache_pool;
  83. static struct bit_map srmmu_nocache_map;
  84. static inline int srmmu_pmd_none(pmd_t pmd)
  85. { return !(pmd_val(pmd) & 0xFFFFFFF); }
  86. /* XXX should we hyper_flush_whole_icache here - Anton */
  87. static inline void srmmu_ctxd_set(ctxd_t *ctxp, pgd_t *pgdp)
  88. {
  89. pte_t pte;
  90. pte = __pte((SRMMU_ET_PTD | (__nocache_pa(pgdp) >> 4)));
  91. set_pte((pte_t *)ctxp, pte);
  92. }
  93. /*
  94. * Locations of MSI Registers.
  95. */
  96. #define MSI_MBUS_ARBEN 0xe0001008 /* MBus Arbiter Enable register */
  97. /*
  98. * Useful bits in the MSI Registers.
  99. */
  100. #define MSI_ASYNC_MODE 0x80000000 /* Operate the MSI asynchronously */
  101. static void msi_set_sync(void)
  102. {
  103. __asm__ __volatile__ ("lda [%0] %1, %%g3\n\t"
  104. "andn %%g3, %2, %%g3\n\t"
  105. "sta %%g3, [%0] %1\n\t" : :
  106. "r" (MSI_MBUS_ARBEN),
  107. "i" (ASI_M_CTL), "r" (MSI_ASYNC_MODE) : "g3");
  108. }
  109. void pmd_set(pmd_t *pmdp, pte_t *ptep)
  110. {
  111. unsigned long ptp = __nocache_pa(ptep) >> 4;
  112. set_pte((pte_t *)&pmd_val(*pmdp), __pte(SRMMU_ET_PTD | ptp));
  113. }
  114. /*
  115. * size: bytes to allocate in the nocache area.
  116. * align: bytes, number to align at.
  117. * Returns the virtual address of the allocated area.
  118. */
  119. static void *__srmmu_get_nocache(int size, int align)
  120. {
  121. int offset, minsz = 1 << SRMMU_NOCACHE_BITMAP_SHIFT;
  122. unsigned long addr;
  123. if (size < minsz) {
  124. printk(KERN_ERR "Size 0x%x too small for nocache request\n",
  125. size);
  126. size = minsz;
  127. }
  128. if (size & (minsz - 1)) {
  129. printk(KERN_ERR "Size 0x%x unaligned in nocache request\n",
  130. size);
  131. size += minsz - 1;
  132. }
  133. BUG_ON(align > SRMMU_NOCACHE_ALIGN_MAX);
  134. offset = bit_map_string_get(&srmmu_nocache_map,
  135. size >> SRMMU_NOCACHE_BITMAP_SHIFT,
  136. align >> SRMMU_NOCACHE_BITMAP_SHIFT);
  137. if (offset == -1) {
  138. printk(KERN_ERR "srmmu: out of nocache %d: %d/%d\n",
  139. size, (int) srmmu_nocache_size,
  140. srmmu_nocache_map.used << SRMMU_NOCACHE_BITMAP_SHIFT);
  141. return NULL;
  142. }
  143. addr = SRMMU_NOCACHE_VADDR + (offset << SRMMU_NOCACHE_BITMAP_SHIFT);
  144. return (void *)addr;
  145. }
  146. void *srmmu_get_nocache(int size, int align)
  147. {
  148. void *tmp;
  149. tmp = __srmmu_get_nocache(size, align);
  150. if (tmp)
  151. memset(tmp, 0, size);
  152. return tmp;
  153. }
  154. void srmmu_free_nocache(void *addr, int size)
  155. {
  156. unsigned long vaddr;
  157. int offset;
  158. vaddr = (unsigned long)addr;
  159. if (vaddr < SRMMU_NOCACHE_VADDR) {
  160. printk("Vaddr %lx is smaller than nocache base 0x%lx\n",
  161. vaddr, (unsigned long)SRMMU_NOCACHE_VADDR);
  162. BUG();
  163. }
  164. if (vaddr + size > srmmu_nocache_end) {
  165. printk("Vaddr %lx is bigger than nocache end 0x%lx\n",
  166. vaddr, srmmu_nocache_end);
  167. BUG();
  168. }
  169. if (!is_power_of_2(size)) {
  170. printk("Size 0x%x is not a power of 2\n", size);
  171. BUG();
  172. }
  173. if (size < SRMMU_NOCACHE_BITMAP_SHIFT) {
  174. printk("Size 0x%x is too small\n", size);
  175. BUG();
  176. }
  177. if (vaddr & (size - 1)) {
  178. printk("Vaddr %lx is not aligned to size 0x%x\n", vaddr, size);
  179. BUG();
  180. }
  181. offset = (vaddr - SRMMU_NOCACHE_VADDR) >> SRMMU_NOCACHE_BITMAP_SHIFT;
  182. size = size >> SRMMU_NOCACHE_BITMAP_SHIFT;
  183. bit_map_clear(&srmmu_nocache_map, offset, size);
  184. }
  185. static void srmmu_early_allocate_ptable_skeleton(unsigned long start,
  186. unsigned long end);
  187. /* Return how much physical memory we have. */
  188. static unsigned long __init probe_memory(void)
  189. {
  190. unsigned long total = 0;
  191. int i;
  192. for (i = 0; sp_banks[i].num_bytes; i++)
  193. total += sp_banks[i].num_bytes;
  194. return total;
  195. }
  196. /*
  197. * Reserve nocache dynamically proportionally to the amount of
  198. * system RAM. -- Tomas Szepe <szepe@pinerecords.com>, June 2002
  199. */
  200. static void __init srmmu_nocache_calcsize(void)
  201. {
  202. unsigned long sysmemavail = probe_memory() / 1024;
  203. int srmmu_nocache_npages;
  204. srmmu_nocache_npages =
  205. sysmemavail / SRMMU_NOCACHE_ALCRATIO / 1024 * 256;
  206. /* P3 XXX The 4x overuse: corroborated by /proc/meminfo. */
  207. // if (srmmu_nocache_npages < 256) srmmu_nocache_npages = 256;
  208. if (srmmu_nocache_npages < SRMMU_MIN_NOCACHE_PAGES)
  209. srmmu_nocache_npages = SRMMU_MIN_NOCACHE_PAGES;
  210. /* anything above 1280 blows up */
  211. if (srmmu_nocache_npages > SRMMU_MAX_NOCACHE_PAGES)
  212. srmmu_nocache_npages = SRMMU_MAX_NOCACHE_PAGES;
  213. srmmu_nocache_size = srmmu_nocache_npages * PAGE_SIZE;
  214. srmmu_nocache_end = SRMMU_NOCACHE_VADDR + srmmu_nocache_size;
  215. }
  216. static void __init srmmu_nocache_init(void)
  217. {
  218. void *srmmu_nocache_bitmap;
  219. unsigned int bitmap_bits;
  220. pgd_t *pgd;
  221. p4d_t *p4d;
  222. pud_t *pud;
  223. pmd_t *pmd;
  224. pte_t *pte;
  225. unsigned long paddr, vaddr;
  226. unsigned long pteval;
  227. bitmap_bits = srmmu_nocache_size >> SRMMU_NOCACHE_BITMAP_SHIFT;
  228. srmmu_nocache_pool = memblock_alloc(srmmu_nocache_size,
  229. SRMMU_NOCACHE_ALIGN_MAX);
  230. if (!srmmu_nocache_pool)
  231. panic("%s: Failed to allocate %lu bytes align=0x%x\n",
  232. __func__, srmmu_nocache_size, SRMMU_NOCACHE_ALIGN_MAX);
  233. memset(srmmu_nocache_pool, 0, srmmu_nocache_size);
  234. srmmu_nocache_bitmap =
  235. memblock_alloc(BITS_TO_LONGS(bitmap_bits) * sizeof(long),
  236. SMP_CACHE_BYTES);
  237. if (!srmmu_nocache_bitmap)
  238. panic("%s: Failed to allocate %zu bytes\n", __func__,
  239. BITS_TO_LONGS(bitmap_bits) * sizeof(long));
  240. bit_map_init(&srmmu_nocache_map, srmmu_nocache_bitmap, bitmap_bits);
  241. srmmu_swapper_pg_dir = __srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE);
  242. memset(__nocache_fix(srmmu_swapper_pg_dir), 0, SRMMU_PGD_TABLE_SIZE);
  243. init_mm.pgd = srmmu_swapper_pg_dir;
  244. srmmu_early_allocate_ptable_skeleton(SRMMU_NOCACHE_VADDR, srmmu_nocache_end);
  245. paddr = __pa((unsigned long)srmmu_nocache_pool);
  246. vaddr = SRMMU_NOCACHE_VADDR;
  247. while (vaddr < srmmu_nocache_end) {
  248. pgd = pgd_offset_k(vaddr);
  249. p4d = p4d_offset(pgd, vaddr);
  250. pud = pud_offset(p4d, vaddr);
  251. pmd = pmd_offset(__nocache_fix(pud), vaddr);
  252. pte = pte_offset_kernel(__nocache_fix(pmd), vaddr);
  253. pteval = ((paddr >> 4) | SRMMU_ET_PTE | SRMMU_PRIV);
  254. if (srmmu_cache_pagetables)
  255. pteval |= SRMMU_CACHE;
  256. set_pte(__nocache_fix(pte), __pte(pteval));
  257. vaddr += PAGE_SIZE;
  258. paddr += PAGE_SIZE;
  259. }
  260. flush_cache_all();
  261. flush_tlb_all();
  262. }
  263. pgd_t *get_pgd_fast(void)
  264. {
  265. pgd_t *pgd = NULL;
  266. pgd = __srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE);
  267. if (pgd) {
  268. pgd_t *init = pgd_offset_k(0);
  269. memset(pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t));
  270. memcpy(pgd + USER_PTRS_PER_PGD, init + USER_PTRS_PER_PGD,
  271. (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t));
  272. }
  273. return pgd;
  274. }
  275. /*
  276. * Hardware needs alignment to 256 only, but we align to whole page size
  277. * to reduce fragmentation problems due to the buddy principle.
  278. * XXX Provide actual fragmentation statistics in /proc.
  279. *
  280. * Alignments up to the page size are the same for physical and virtual
  281. * addresses of the nocache area.
  282. */
  283. pgtable_t pte_alloc_one(struct mm_struct *mm)
  284. {
  285. pte_t *ptep;
  286. struct page *page;
  287. if ((ptep = pte_alloc_one_kernel(mm)) == 0)
  288. return NULL;
  289. page = pfn_to_page(__nocache_pa((unsigned long)ptep) >> PAGE_SHIFT);
  290. spin_lock(&mm->page_table_lock);
  291. if (page_ref_inc_return(page) == 2 && !pgtable_pte_page_ctor(page)) {
  292. page_ref_dec(page);
  293. ptep = NULL;
  294. }
  295. spin_unlock(&mm->page_table_lock);
  296. return ptep;
  297. }
  298. void pte_free(struct mm_struct *mm, pgtable_t ptep)
  299. {
  300. struct page *page;
  301. page = pfn_to_page(__nocache_pa((unsigned long)ptep) >> PAGE_SHIFT);
  302. spin_lock(&mm->page_table_lock);
  303. if (page_ref_dec_return(page) == 1)
  304. pgtable_pte_page_dtor(page);
  305. spin_unlock(&mm->page_table_lock);
  306. srmmu_free_nocache(ptep, SRMMU_PTE_TABLE_SIZE);
  307. }
  308. /* context handling - a dynamically sized pool is used */
  309. #define NO_CONTEXT -1
  310. struct ctx_list {
  311. struct ctx_list *next;
  312. struct ctx_list *prev;
  313. unsigned int ctx_number;
  314. struct mm_struct *ctx_mm;
  315. };
  316. static struct ctx_list *ctx_list_pool;
  317. static struct ctx_list ctx_free;
  318. static struct ctx_list ctx_used;
  319. /* At boot time we determine the number of contexts */
  320. static int num_contexts;
  321. static inline void remove_from_ctx_list(struct ctx_list *entry)
  322. {
  323. entry->next->prev = entry->prev;
  324. entry->prev->next = entry->next;
  325. }
  326. static inline void add_to_ctx_list(struct ctx_list *head, struct ctx_list *entry)
  327. {
  328. entry->next = head;
  329. (entry->prev = head->prev)->next = entry;
  330. head->prev = entry;
  331. }
  332. #define add_to_free_ctxlist(entry) add_to_ctx_list(&ctx_free, entry)
  333. #define add_to_used_ctxlist(entry) add_to_ctx_list(&ctx_used, entry)
  334. static inline void alloc_context(struct mm_struct *old_mm, struct mm_struct *mm)
  335. {
  336. struct ctx_list *ctxp;
  337. ctxp = ctx_free.next;
  338. if (ctxp != &ctx_free) {
  339. remove_from_ctx_list(ctxp);
  340. add_to_used_ctxlist(ctxp);
  341. mm->context = ctxp->ctx_number;
  342. ctxp->ctx_mm = mm;
  343. return;
  344. }
  345. ctxp = ctx_used.next;
  346. if (ctxp->ctx_mm == old_mm)
  347. ctxp = ctxp->next;
  348. if (ctxp == &ctx_used)
  349. panic("out of mmu contexts");
  350. flush_cache_mm(ctxp->ctx_mm);
  351. flush_tlb_mm(ctxp->ctx_mm);
  352. remove_from_ctx_list(ctxp);
  353. add_to_used_ctxlist(ctxp);
  354. ctxp->ctx_mm->context = NO_CONTEXT;
  355. ctxp->ctx_mm = mm;
  356. mm->context = ctxp->ctx_number;
  357. }
  358. static inline void free_context(int context)
  359. {
  360. struct ctx_list *ctx_old;
  361. ctx_old = ctx_list_pool + context;
  362. remove_from_ctx_list(ctx_old);
  363. add_to_free_ctxlist(ctx_old);
  364. }
  365. static void __init sparc_context_init(int numctx)
  366. {
  367. int ctx;
  368. unsigned long size;
  369. size = numctx * sizeof(struct ctx_list);
  370. ctx_list_pool = memblock_alloc(size, SMP_CACHE_BYTES);
  371. if (!ctx_list_pool)
  372. panic("%s: Failed to allocate %lu bytes\n", __func__, size);
  373. for (ctx = 0; ctx < numctx; ctx++) {
  374. struct ctx_list *clist;
  375. clist = (ctx_list_pool + ctx);
  376. clist->ctx_number = ctx;
  377. clist->ctx_mm = NULL;
  378. }
  379. ctx_free.next = ctx_free.prev = &ctx_free;
  380. ctx_used.next = ctx_used.prev = &ctx_used;
  381. for (ctx = 0; ctx < numctx; ctx++)
  382. add_to_free_ctxlist(ctx_list_pool + ctx);
  383. }
  384. void switch_mm(struct mm_struct *old_mm, struct mm_struct *mm,
  385. struct task_struct *tsk)
  386. {
  387. unsigned long flags;
  388. if (mm->context == NO_CONTEXT) {
  389. spin_lock_irqsave(&srmmu_context_spinlock, flags);
  390. alloc_context(old_mm, mm);
  391. spin_unlock_irqrestore(&srmmu_context_spinlock, flags);
  392. srmmu_ctxd_set(&srmmu_context_table[mm->context], mm->pgd);
  393. }
  394. if (sparc_cpu_model == sparc_leon)
  395. leon_switch_mm();
  396. if (is_hypersparc)
  397. hyper_flush_whole_icache();
  398. srmmu_set_context(mm->context);
  399. }
  400. /* Low level IO area allocation on the SRMMU. */
  401. static inline void srmmu_mapioaddr(unsigned long physaddr,
  402. unsigned long virt_addr, int bus_type)
  403. {
  404. pgd_t *pgdp;
  405. p4d_t *p4dp;
  406. pud_t *pudp;
  407. pmd_t *pmdp;
  408. pte_t *ptep;
  409. unsigned long tmp;
  410. physaddr &= PAGE_MASK;
  411. pgdp = pgd_offset_k(virt_addr);
  412. p4dp = p4d_offset(pgdp, virt_addr);
  413. pudp = pud_offset(p4dp, virt_addr);
  414. pmdp = pmd_offset(pudp, virt_addr);
  415. ptep = pte_offset_kernel(pmdp, virt_addr);
  416. tmp = (physaddr >> 4) | SRMMU_ET_PTE;
  417. /* I need to test whether this is consistent over all
  418. * sun4m's. The bus_type represents the upper 4 bits of
  419. * 36-bit physical address on the I/O space lines...
  420. */
  421. tmp |= (bus_type << 28);
  422. tmp |= SRMMU_PRIV;
  423. __flush_page_to_ram(virt_addr);
  424. set_pte(ptep, __pte(tmp));
  425. }
  426. void srmmu_mapiorange(unsigned int bus, unsigned long xpa,
  427. unsigned long xva, unsigned int len)
  428. {
  429. while (len != 0) {
  430. len -= PAGE_SIZE;
  431. srmmu_mapioaddr(xpa, xva, bus);
  432. xva += PAGE_SIZE;
  433. xpa += PAGE_SIZE;
  434. }
  435. flush_tlb_all();
  436. }
  437. static inline void srmmu_unmapioaddr(unsigned long virt_addr)
  438. {
  439. pgd_t *pgdp;
  440. p4d_t *p4dp;
  441. pud_t *pudp;
  442. pmd_t *pmdp;
  443. pte_t *ptep;
  444. pgdp = pgd_offset_k(virt_addr);
  445. p4dp = p4d_offset(pgdp, virt_addr);
  446. pudp = pud_offset(p4dp, virt_addr);
  447. pmdp = pmd_offset(pudp, virt_addr);
  448. ptep = pte_offset_kernel(pmdp, virt_addr);
  449. /* No need to flush uncacheable page. */
  450. __pte_clear(ptep);
  451. }
  452. void srmmu_unmapiorange(unsigned long virt_addr, unsigned int len)
  453. {
  454. while (len != 0) {
  455. len -= PAGE_SIZE;
  456. srmmu_unmapioaddr(virt_addr);
  457. virt_addr += PAGE_SIZE;
  458. }
  459. flush_tlb_all();
  460. }
  461. /* tsunami.S */
  462. extern void tsunami_flush_cache_all(void);
  463. extern void tsunami_flush_cache_mm(struct mm_struct *mm);
  464. extern void tsunami_flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end);
  465. extern void tsunami_flush_cache_page(struct vm_area_struct *vma, unsigned long page);
  466. extern void tsunami_flush_page_to_ram(unsigned long page);
  467. extern void tsunami_flush_page_for_dma(unsigned long page);
  468. extern void tsunami_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr);
  469. extern void tsunami_flush_tlb_all(void);
  470. extern void tsunami_flush_tlb_mm(struct mm_struct *mm);
  471. extern void tsunami_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end);
  472. extern void tsunami_flush_tlb_page(struct vm_area_struct *vma, unsigned long page);
  473. extern void tsunami_setup_blockops(void);
  474. /* swift.S */
  475. extern void swift_flush_cache_all(void);
  476. extern void swift_flush_cache_mm(struct mm_struct *mm);
  477. extern void swift_flush_cache_range(struct vm_area_struct *vma,
  478. unsigned long start, unsigned long end);
  479. extern void swift_flush_cache_page(struct vm_area_struct *vma, unsigned long page);
  480. extern void swift_flush_page_to_ram(unsigned long page);
  481. extern void swift_flush_page_for_dma(unsigned long page);
  482. extern void swift_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr);
  483. extern void swift_flush_tlb_all(void);
  484. extern void swift_flush_tlb_mm(struct mm_struct *mm);
  485. extern void swift_flush_tlb_range(struct vm_area_struct *vma,
  486. unsigned long start, unsigned long end);
  487. extern void swift_flush_tlb_page(struct vm_area_struct *vma, unsigned long page);
  488. #if 0 /* P3: deadwood to debug precise flushes on Swift. */
  489. void swift_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
  490. {
  491. int cctx, ctx1;
  492. page &= PAGE_MASK;
  493. if ((ctx1 = vma->vm_mm->context) != -1) {
  494. cctx = srmmu_get_context();
  495. /* Is context # ever different from current context? P3 */
  496. if (cctx != ctx1) {
  497. printk("flush ctx %02x curr %02x\n", ctx1, cctx);
  498. srmmu_set_context(ctx1);
  499. swift_flush_page(page);
  500. __asm__ __volatile__("sta %%g0, [%0] %1\n\t" : :
  501. "r" (page), "i" (ASI_M_FLUSH_PROBE));
  502. srmmu_set_context(cctx);
  503. } else {
  504. /* Rm. prot. bits from virt. c. */
  505. /* swift_flush_cache_all(); */
  506. /* swift_flush_cache_page(vma, page); */
  507. swift_flush_page(page);
  508. __asm__ __volatile__("sta %%g0, [%0] %1\n\t" : :
  509. "r" (page), "i" (ASI_M_FLUSH_PROBE));
  510. /* same as above: srmmu_flush_tlb_page() */
  511. }
  512. }
  513. }
  514. #endif
  515. /*
  516. * The following are all MBUS based SRMMU modules, and therefore could
  517. * be found in a multiprocessor configuration. On the whole, these
  518. * chips seems to be much more touchy about DVMA and page tables
  519. * with respect to cache coherency.
  520. */
  521. /* viking.S */
  522. extern void viking_flush_cache_all(void);
  523. extern void viking_flush_cache_mm(struct mm_struct *mm);
  524. extern void viking_flush_cache_range(struct vm_area_struct *vma, unsigned long start,
  525. unsigned long end);
  526. extern void viking_flush_cache_page(struct vm_area_struct *vma, unsigned long page);
  527. extern void viking_flush_page_to_ram(unsigned long page);
  528. extern void viking_flush_page_for_dma(unsigned long page);
  529. extern void viking_flush_sig_insns(struct mm_struct *mm, unsigned long addr);
  530. extern void viking_flush_page(unsigned long page);
  531. extern void viking_mxcc_flush_page(unsigned long page);
  532. extern void viking_flush_tlb_all(void);
  533. extern void viking_flush_tlb_mm(struct mm_struct *mm);
  534. extern void viking_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
  535. unsigned long end);
  536. extern void viking_flush_tlb_page(struct vm_area_struct *vma,
  537. unsigned long page);
  538. extern void sun4dsmp_flush_tlb_all(void);
  539. extern void sun4dsmp_flush_tlb_mm(struct mm_struct *mm);
  540. extern void sun4dsmp_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
  541. unsigned long end);
  542. extern void sun4dsmp_flush_tlb_page(struct vm_area_struct *vma,
  543. unsigned long page);
  544. /* hypersparc.S */
  545. extern void hypersparc_flush_cache_all(void);
  546. extern void hypersparc_flush_cache_mm(struct mm_struct *mm);
  547. extern void hypersparc_flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end);
  548. extern void hypersparc_flush_cache_page(struct vm_area_struct *vma, unsigned long page);
  549. extern void hypersparc_flush_page_to_ram(unsigned long page);
  550. extern void hypersparc_flush_page_for_dma(unsigned long page);
  551. extern void hypersparc_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr);
  552. extern void hypersparc_flush_tlb_all(void);
  553. extern void hypersparc_flush_tlb_mm(struct mm_struct *mm);
  554. extern void hypersparc_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end);
  555. extern void hypersparc_flush_tlb_page(struct vm_area_struct *vma, unsigned long page);
  556. extern void hypersparc_setup_blockops(void);
  557. /*
  558. * NOTE: All of this startup code assumes the low 16mb (approx.) of
  559. * kernel mappings are done with one single contiguous chunk of
  560. * ram. On small ram machines (classics mainly) we only get
  561. * around 8mb mapped for us.
  562. */
  563. static void __init early_pgtable_allocfail(char *type)
  564. {
  565. prom_printf("inherit_prom_mappings: Cannot alloc kernel %s.\n", type);
  566. prom_halt();
  567. }
  568. static void __init srmmu_early_allocate_ptable_skeleton(unsigned long start,
  569. unsigned long end)
  570. {
  571. pgd_t *pgdp;
  572. p4d_t *p4dp;
  573. pud_t *pudp;
  574. pmd_t *pmdp;
  575. pte_t *ptep;
  576. while (start < end) {
  577. pgdp = pgd_offset_k(start);
  578. p4dp = p4d_offset(pgdp, start);
  579. pudp = pud_offset(p4dp, start);
  580. if (pud_none(*(pud_t *)__nocache_fix(pudp))) {
  581. pmdp = __srmmu_get_nocache(
  582. SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE);
  583. if (pmdp == NULL)
  584. early_pgtable_allocfail("pmd");
  585. memset(__nocache_fix(pmdp), 0, SRMMU_PMD_TABLE_SIZE);
  586. pud_set(__nocache_fix(pudp), pmdp);
  587. }
  588. pmdp = pmd_offset(__nocache_fix(pudp), start);
  589. if (srmmu_pmd_none(*(pmd_t *)__nocache_fix(pmdp))) {
  590. ptep = __srmmu_get_nocache(PTE_SIZE, PTE_SIZE);
  591. if (ptep == NULL)
  592. early_pgtable_allocfail("pte");
  593. memset(__nocache_fix(ptep), 0, PTE_SIZE);
  594. pmd_set(__nocache_fix(pmdp), ptep);
  595. }
  596. if (start > (0xffffffffUL - PMD_SIZE))
  597. break;
  598. start = (start + PMD_SIZE) & PMD_MASK;
  599. }
  600. }
  601. static void __init srmmu_allocate_ptable_skeleton(unsigned long start,
  602. unsigned long end)
  603. {
  604. pgd_t *pgdp;
  605. p4d_t *p4dp;
  606. pud_t *pudp;
  607. pmd_t *pmdp;
  608. pte_t *ptep;
  609. while (start < end) {
  610. pgdp = pgd_offset_k(start);
  611. p4dp = p4d_offset(pgdp, start);
  612. pudp = pud_offset(p4dp, start);
  613. if (pud_none(*pudp)) {
  614. pmdp = __srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE);
  615. if (pmdp == NULL)
  616. early_pgtable_allocfail("pmd");
  617. memset(pmdp, 0, SRMMU_PMD_TABLE_SIZE);
  618. pud_set((pud_t *)pgdp, pmdp);
  619. }
  620. pmdp = pmd_offset(pudp, start);
  621. if (srmmu_pmd_none(*pmdp)) {
  622. ptep = __srmmu_get_nocache(PTE_SIZE,
  623. PTE_SIZE);
  624. if (ptep == NULL)
  625. early_pgtable_allocfail("pte");
  626. memset(ptep, 0, PTE_SIZE);
  627. pmd_set(pmdp, ptep);
  628. }
  629. if (start > (0xffffffffUL - PMD_SIZE))
  630. break;
  631. start = (start + PMD_SIZE) & PMD_MASK;
  632. }
  633. }
  634. /* These flush types are not available on all chips... */
  635. static inline unsigned long srmmu_probe(unsigned long vaddr)
  636. {
  637. unsigned long retval;
  638. if (sparc_cpu_model != sparc_leon) {
  639. vaddr &= PAGE_MASK;
  640. __asm__ __volatile__("lda [%1] %2, %0\n\t" :
  641. "=r" (retval) :
  642. "r" (vaddr | 0x400), "i" (ASI_M_FLUSH_PROBE));
  643. } else {
  644. retval = leon_swprobe(vaddr, NULL);
  645. }
  646. return retval;
  647. }
  648. /*
  649. * This is much cleaner than poking around physical address space
  650. * looking at the prom's page table directly which is what most
  651. * other OS's do. Yuck... this is much better.
  652. */
  653. static void __init srmmu_inherit_prom_mappings(unsigned long start,
  654. unsigned long end)
  655. {
  656. unsigned long probed;
  657. unsigned long addr;
  658. pgd_t *pgdp;
  659. p4d_t *p4dp;
  660. pud_t *pudp;
  661. pmd_t *pmdp;
  662. pte_t *ptep;
  663. int what; /* 0 = normal-pte, 1 = pmd-level pte, 2 = pgd-level pte */
  664. while (start <= end) {
  665. if (start == 0)
  666. break; /* probably wrap around */
  667. if (start == 0xfef00000)
  668. start = KADB_DEBUGGER_BEGVM;
  669. probed = srmmu_probe(start);
  670. if (!probed) {
  671. /* continue probing until we find an entry */
  672. start += PAGE_SIZE;
  673. continue;
  674. }
  675. /* A red snapper, see what it really is. */
  676. what = 0;
  677. addr = start - PAGE_SIZE;
  678. if (!(start & ~(PMD_MASK))) {
  679. if (srmmu_probe(addr + PMD_SIZE) == probed)
  680. what = 1;
  681. }
  682. if (!(start & ~(PGDIR_MASK))) {
  683. if (srmmu_probe(addr + PGDIR_SIZE) == probed)
  684. what = 2;
  685. }
  686. pgdp = pgd_offset_k(start);
  687. p4dp = p4d_offset(pgdp, start);
  688. pudp = pud_offset(p4dp, start);
  689. if (what == 2) {
  690. *(pgd_t *)__nocache_fix(pgdp) = __pgd(probed);
  691. start += PGDIR_SIZE;
  692. continue;
  693. }
  694. if (pud_none(*(pud_t *)__nocache_fix(pudp))) {
  695. pmdp = __srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE,
  696. SRMMU_PMD_TABLE_SIZE);
  697. if (pmdp == NULL)
  698. early_pgtable_allocfail("pmd");
  699. memset(__nocache_fix(pmdp), 0, SRMMU_PMD_TABLE_SIZE);
  700. pud_set(__nocache_fix(pudp), pmdp);
  701. }
  702. pmdp = pmd_offset(__nocache_fix(pgdp), start);
  703. if (what == 1) {
  704. *(pmd_t *)__nocache_fix(pmdp) = __pmd(probed);
  705. start += PMD_SIZE;
  706. continue;
  707. }
  708. if (srmmu_pmd_none(*(pmd_t *)__nocache_fix(pmdp))) {
  709. ptep = __srmmu_get_nocache(PTE_SIZE, PTE_SIZE);
  710. if (ptep == NULL)
  711. early_pgtable_allocfail("pte");
  712. memset(__nocache_fix(ptep), 0, PTE_SIZE);
  713. pmd_set(__nocache_fix(pmdp), ptep);
  714. }
  715. ptep = pte_offset_kernel(__nocache_fix(pmdp), start);
  716. *(pte_t *)__nocache_fix(ptep) = __pte(probed);
  717. start += PAGE_SIZE;
  718. }
  719. }
  720. #define KERNEL_PTE(page_shifted) ((page_shifted)|SRMMU_CACHE|SRMMU_PRIV|SRMMU_VALID)
  721. /* Create a third-level SRMMU 16MB page mapping. */
  722. static void __init do_large_mapping(unsigned long vaddr, unsigned long phys_base)
  723. {
  724. pgd_t *pgdp = pgd_offset_k(vaddr);
  725. unsigned long big_pte;
  726. big_pte = KERNEL_PTE(phys_base >> 4);
  727. *(pgd_t *)__nocache_fix(pgdp) = __pgd(big_pte);
  728. }
  729. /* Map sp_bank entry SP_ENTRY, starting at virtual address VBASE. */
  730. static unsigned long __init map_spbank(unsigned long vbase, int sp_entry)
  731. {
  732. unsigned long pstart = (sp_banks[sp_entry].base_addr & PGDIR_MASK);
  733. unsigned long vstart = (vbase & PGDIR_MASK);
  734. unsigned long vend = PGDIR_ALIGN(vbase + sp_banks[sp_entry].num_bytes);
  735. /* Map "low" memory only */
  736. const unsigned long min_vaddr = PAGE_OFFSET;
  737. const unsigned long max_vaddr = PAGE_OFFSET + SRMMU_MAXMEM;
  738. if (vstart < min_vaddr || vstart >= max_vaddr)
  739. return vstart;
  740. if (vend > max_vaddr || vend < min_vaddr)
  741. vend = max_vaddr;
  742. while (vstart < vend) {
  743. do_large_mapping(vstart, pstart);
  744. vstart += PGDIR_SIZE; pstart += PGDIR_SIZE;
  745. }
  746. return vstart;
  747. }
  748. static void __init map_kernel(void)
  749. {
  750. int i;
  751. if (phys_base > 0) {
  752. do_large_mapping(PAGE_OFFSET, phys_base);
  753. }
  754. for (i = 0; sp_banks[i].num_bytes != 0; i++) {
  755. map_spbank((unsigned long)__va(sp_banks[i].base_addr), i);
  756. }
  757. }
  758. void (*poke_srmmu)(void) = NULL;
  759. void __init srmmu_paging_init(void)
  760. {
  761. int i;
  762. phandle cpunode;
  763. char node_str[128];
  764. pgd_t *pgd;
  765. p4d_t *p4d;
  766. pud_t *pud;
  767. pmd_t *pmd;
  768. pte_t *pte;
  769. unsigned long pages_avail;
  770. init_mm.context = (unsigned long) NO_CONTEXT;
  771. sparc_iomap.start = SUN4M_IOBASE_VADDR; /* 16MB of IOSPACE on all sun4m's. */
  772. if (sparc_cpu_model == sun4d)
  773. num_contexts = 65536; /* We know it is Viking */
  774. else {
  775. /* Find the number of contexts on the srmmu. */
  776. cpunode = prom_getchild(prom_root_node);
  777. num_contexts = 0;
  778. while (cpunode != 0) {
  779. prom_getstring(cpunode, "device_type", node_str, sizeof(node_str));
  780. if (!strcmp(node_str, "cpu")) {
  781. num_contexts = prom_getintdefault(cpunode, "mmu-nctx", 0x8);
  782. break;
  783. }
  784. cpunode = prom_getsibling(cpunode);
  785. }
  786. }
  787. if (!num_contexts) {
  788. prom_printf("Something wrong, can't find cpu node in paging_init.\n");
  789. prom_halt();
  790. }
  791. pages_avail = 0;
  792. last_valid_pfn = bootmem_init(&pages_avail);
  793. srmmu_nocache_calcsize();
  794. srmmu_nocache_init();
  795. srmmu_inherit_prom_mappings(0xfe400000, (LINUX_OPPROM_ENDVM - PAGE_SIZE));
  796. map_kernel();
  797. /* ctx table has to be physically aligned to its size */
  798. srmmu_context_table = __srmmu_get_nocache(num_contexts * sizeof(ctxd_t), num_contexts * sizeof(ctxd_t));
  799. srmmu_ctx_table_phys = (ctxd_t *)__nocache_pa(srmmu_context_table);
  800. for (i = 0; i < num_contexts; i++)
  801. srmmu_ctxd_set((ctxd_t *)__nocache_fix(&srmmu_context_table[i]), srmmu_swapper_pg_dir);
  802. flush_cache_all();
  803. srmmu_set_ctable_ptr((unsigned long)srmmu_ctx_table_phys);
  804. #ifdef CONFIG_SMP
  805. /* Stop from hanging here... */
  806. local_ops->tlb_all();
  807. #else
  808. flush_tlb_all();
  809. #endif
  810. poke_srmmu();
  811. srmmu_allocate_ptable_skeleton(sparc_iomap.start, IOBASE_END);
  812. srmmu_allocate_ptable_skeleton(DVMA_VADDR, DVMA_END);
  813. srmmu_allocate_ptable_skeleton(
  814. __fix_to_virt(__end_of_fixed_addresses - 1), FIXADDR_TOP);
  815. srmmu_allocate_ptable_skeleton(PKMAP_BASE, PKMAP_END);
  816. pgd = pgd_offset_k(PKMAP_BASE);
  817. p4d = p4d_offset(pgd, PKMAP_BASE);
  818. pud = pud_offset(p4d, PKMAP_BASE);
  819. pmd = pmd_offset(pud, PKMAP_BASE);
  820. pte = pte_offset_kernel(pmd, PKMAP_BASE);
  821. pkmap_page_table = pte;
  822. flush_cache_all();
  823. flush_tlb_all();
  824. sparc_context_init(num_contexts);
  825. kmap_init();
  826. {
  827. unsigned long max_zone_pfn[MAX_NR_ZONES] = { 0 };
  828. max_zone_pfn[ZONE_DMA] = max_low_pfn;
  829. max_zone_pfn[ZONE_NORMAL] = max_low_pfn;
  830. max_zone_pfn[ZONE_HIGHMEM] = highend_pfn;
  831. free_area_init(max_zone_pfn);
  832. }
  833. }
  834. void mmu_info(struct seq_file *m)
  835. {
  836. seq_printf(m,
  837. "MMU type\t: %s\n"
  838. "contexts\t: %d\n"
  839. "nocache total\t: %ld\n"
  840. "nocache used\t: %d\n",
  841. srmmu_name,
  842. num_contexts,
  843. srmmu_nocache_size,
  844. srmmu_nocache_map.used << SRMMU_NOCACHE_BITMAP_SHIFT);
  845. }
  846. int init_new_context(struct task_struct *tsk, struct mm_struct *mm)
  847. {
  848. mm->context = NO_CONTEXT;
  849. return 0;
  850. }
  851. void destroy_context(struct mm_struct *mm)
  852. {
  853. unsigned long flags;
  854. if (mm->context != NO_CONTEXT) {
  855. flush_cache_mm(mm);
  856. srmmu_ctxd_set(&srmmu_context_table[mm->context], srmmu_swapper_pg_dir);
  857. flush_tlb_mm(mm);
  858. spin_lock_irqsave(&srmmu_context_spinlock, flags);
  859. free_context(mm->context);
  860. spin_unlock_irqrestore(&srmmu_context_spinlock, flags);
  861. mm->context = NO_CONTEXT;
  862. }
  863. }
  864. /* Init various srmmu chip types. */
  865. static void __init srmmu_is_bad(void)
  866. {
  867. prom_printf("Could not determine SRMMU chip type.\n");
  868. prom_halt();
  869. }
  870. static void __init init_vac_layout(void)
  871. {
  872. phandle nd;
  873. int cache_lines;
  874. char node_str[128];
  875. #ifdef CONFIG_SMP
  876. int cpu = 0;
  877. unsigned long max_size = 0;
  878. unsigned long min_line_size = 0x10000000;
  879. #endif
  880. nd = prom_getchild(prom_root_node);
  881. while ((nd = prom_getsibling(nd)) != 0) {
  882. prom_getstring(nd, "device_type", node_str, sizeof(node_str));
  883. if (!strcmp(node_str, "cpu")) {
  884. vac_line_size = prom_getint(nd, "cache-line-size");
  885. if (vac_line_size == -1) {
  886. prom_printf("can't determine cache-line-size, halting.\n");
  887. prom_halt();
  888. }
  889. cache_lines = prom_getint(nd, "cache-nlines");
  890. if (cache_lines == -1) {
  891. prom_printf("can't determine cache-nlines, halting.\n");
  892. prom_halt();
  893. }
  894. vac_cache_size = cache_lines * vac_line_size;
  895. #ifdef CONFIG_SMP
  896. if (vac_cache_size > max_size)
  897. max_size = vac_cache_size;
  898. if (vac_line_size < min_line_size)
  899. min_line_size = vac_line_size;
  900. //FIXME: cpus not contiguous!!
  901. cpu++;
  902. if (cpu >= nr_cpu_ids || !cpu_online(cpu))
  903. break;
  904. #else
  905. break;
  906. #endif
  907. }
  908. }
  909. if (nd == 0) {
  910. prom_printf("No CPU nodes found, halting.\n");
  911. prom_halt();
  912. }
  913. #ifdef CONFIG_SMP
  914. vac_cache_size = max_size;
  915. vac_line_size = min_line_size;
  916. #endif
  917. printk("SRMMU: Using VAC size of %d bytes, line size %d bytes.\n",
  918. (int)vac_cache_size, (int)vac_line_size);
  919. }
  920. static void poke_hypersparc(void)
  921. {
  922. volatile unsigned long clear;
  923. unsigned long mreg = srmmu_get_mmureg();
  924. hyper_flush_unconditional_combined();
  925. mreg &= ~(HYPERSPARC_CWENABLE);
  926. mreg |= (HYPERSPARC_CENABLE | HYPERSPARC_WBENABLE);
  927. mreg |= (HYPERSPARC_CMODE);
  928. srmmu_set_mmureg(mreg);
  929. #if 0 /* XXX I think this is bad news... -DaveM */
  930. hyper_clear_all_tags();
  931. #endif
  932. put_ross_icr(HYPERSPARC_ICCR_FTD | HYPERSPARC_ICCR_ICE);
  933. hyper_flush_whole_icache();
  934. clear = srmmu_get_faddr();
  935. clear = srmmu_get_fstatus();
  936. }
  937. static const struct sparc32_cachetlb_ops hypersparc_ops = {
  938. .cache_all = hypersparc_flush_cache_all,
  939. .cache_mm = hypersparc_flush_cache_mm,
  940. .cache_page = hypersparc_flush_cache_page,
  941. .cache_range = hypersparc_flush_cache_range,
  942. .tlb_all = hypersparc_flush_tlb_all,
  943. .tlb_mm = hypersparc_flush_tlb_mm,
  944. .tlb_page = hypersparc_flush_tlb_page,
  945. .tlb_range = hypersparc_flush_tlb_range,
  946. .page_to_ram = hypersparc_flush_page_to_ram,
  947. .sig_insns = hypersparc_flush_sig_insns,
  948. .page_for_dma = hypersparc_flush_page_for_dma,
  949. };
  950. static void __init init_hypersparc(void)
  951. {
  952. srmmu_name = "ROSS HyperSparc";
  953. srmmu_modtype = HyperSparc;
  954. init_vac_layout();
  955. is_hypersparc = 1;
  956. sparc32_cachetlb_ops = &hypersparc_ops;
  957. poke_srmmu = poke_hypersparc;
  958. hypersparc_setup_blockops();
  959. }
  960. static void poke_swift(void)
  961. {
  962. unsigned long mreg;
  963. /* Clear any crap from the cache or else... */
  964. swift_flush_cache_all();
  965. /* Enable I & D caches */
  966. mreg = srmmu_get_mmureg();
  967. mreg |= (SWIFT_IE | SWIFT_DE);
  968. /*
  969. * The Swift branch folding logic is completely broken. At
  970. * trap time, if things are just right, if can mistakenly
  971. * think that a trap is coming from kernel mode when in fact
  972. * it is coming from user mode (it mis-executes the branch in
  973. * the trap code). So you see things like crashme completely
  974. * hosing your machine which is completely unacceptable. Turn
  975. * this shit off... nice job Fujitsu.
  976. */
  977. mreg &= ~(SWIFT_BF);
  978. srmmu_set_mmureg(mreg);
  979. }
  980. static const struct sparc32_cachetlb_ops swift_ops = {
  981. .cache_all = swift_flush_cache_all,
  982. .cache_mm = swift_flush_cache_mm,
  983. .cache_page = swift_flush_cache_page,
  984. .cache_range = swift_flush_cache_range,
  985. .tlb_all = swift_flush_tlb_all,
  986. .tlb_mm = swift_flush_tlb_mm,
  987. .tlb_page = swift_flush_tlb_page,
  988. .tlb_range = swift_flush_tlb_range,
  989. .page_to_ram = swift_flush_page_to_ram,
  990. .sig_insns = swift_flush_sig_insns,
  991. .page_for_dma = swift_flush_page_for_dma,
  992. };
  993. #define SWIFT_MASKID_ADDR 0x10003018
  994. static void __init init_swift(void)
  995. {
  996. unsigned long swift_rev;
  997. __asm__ __volatile__("lda [%1] %2, %0\n\t"
  998. "srl %0, 0x18, %0\n\t" :
  999. "=r" (swift_rev) :
  1000. "r" (SWIFT_MASKID_ADDR), "i" (ASI_M_BYPASS));
  1001. srmmu_name = "Fujitsu Swift";
  1002. switch (swift_rev) {
  1003. case 0x11:
  1004. case 0x20:
  1005. case 0x23:
  1006. case 0x30:
  1007. srmmu_modtype = Swift_lots_o_bugs;
  1008. hwbug_bitmask |= (HWBUG_KERN_ACCBROKEN | HWBUG_KERN_CBITBROKEN);
  1009. /*
  1010. * Gee george, I wonder why Sun is so hush hush about
  1011. * this hardware bug... really braindamage stuff going
  1012. * on here. However I think we can find a way to avoid
  1013. * all of the workaround overhead under Linux. Basically,
  1014. * any page fault can cause kernel pages to become user
  1015. * accessible (the mmu gets confused and clears some of
  1016. * the ACC bits in kernel ptes). Aha, sounds pretty
  1017. * horrible eh? But wait, after extensive testing it appears
  1018. * that if you use pgd_t level large kernel pte's (like the
  1019. * 4MB pages on the Pentium) the bug does not get tripped
  1020. * at all. This avoids almost all of the major overhead.
  1021. * Welcome to a world where your vendor tells you to,
  1022. * "apply this kernel patch" instead of "sorry for the
  1023. * broken hardware, send it back and we'll give you
  1024. * properly functioning parts"
  1025. */
  1026. break;
  1027. case 0x25:
  1028. case 0x31:
  1029. srmmu_modtype = Swift_bad_c;
  1030. hwbug_bitmask |= HWBUG_KERN_CBITBROKEN;
  1031. /*
  1032. * You see Sun allude to this hardware bug but never
  1033. * admit things directly, they'll say things like,
  1034. * "the Swift chip cache problems" or similar.
  1035. */
  1036. break;
  1037. default:
  1038. srmmu_modtype = Swift_ok;
  1039. break;
  1040. }
  1041. sparc32_cachetlb_ops = &swift_ops;
  1042. flush_page_for_dma_global = 0;
  1043. /*
  1044. * Are you now convinced that the Swift is one of the
  1045. * biggest VLSI abortions of all time? Bravo Fujitsu!
  1046. * Fujitsu, the !#?!%$'d up processor people. I bet if
  1047. * you examined the microcode of the Swift you'd find
  1048. * XXX's all over the place.
  1049. */
  1050. poke_srmmu = poke_swift;
  1051. }
  1052. static void turbosparc_flush_cache_all(void)
  1053. {
  1054. flush_user_windows();
  1055. turbosparc_idflash_clear();
  1056. }
  1057. static void turbosparc_flush_cache_mm(struct mm_struct *mm)
  1058. {
  1059. FLUSH_BEGIN(mm)
  1060. flush_user_windows();
  1061. turbosparc_idflash_clear();
  1062. FLUSH_END
  1063. }
  1064. static void turbosparc_flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
  1065. {
  1066. FLUSH_BEGIN(vma->vm_mm)
  1067. flush_user_windows();
  1068. turbosparc_idflash_clear();
  1069. FLUSH_END
  1070. }
  1071. static void turbosparc_flush_cache_page(struct vm_area_struct *vma, unsigned long page)
  1072. {
  1073. FLUSH_BEGIN(vma->vm_mm)
  1074. flush_user_windows();
  1075. if (vma->vm_flags & VM_EXEC)
  1076. turbosparc_flush_icache();
  1077. turbosparc_flush_dcache();
  1078. FLUSH_END
  1079. }
  1080. /* TurboSparc is copy-back, if we turn it on, but this does not work. */
  1081. static void turbosparc_flush_page_to_ram(unsigned long page)
  1082. {
  1083. #ifdef TURBOSPARC_WRITEBACK
  1084. volatile unsigned long clear;
  1085. if (srmmu_probe(page))
  1086. turbosparc_flush_page_cache(page);
  1087. clear = srmmu_get_fstatus();
  1088. #endif
  1089. }
  1090. static void turbosparc_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr)
  1091. {
  1092. }
  1093. static void turbosparc_flush_page_for_dma(unsigned long page)
  1094. {
  1095. turbosparc_flush_dcache();
  1096. }
  1097. static void turbosparc_flush_tlb_all(void)
  1098. {
  1099. srmmu_flush_whole_tlb();
  1100. }
  1101. static void turbosparc_flush_tlb_mm(struct mm_struct *mm)
  1102. {
  1103. FLUSH_BEGIN(mm)
  1104. srmmu_flush_whole_tlb();
  1105. FLUSH_END
  1106. }
  1107. static void turbosparc_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
  1108. {
  1109. FLUSH_BEGIN(vma->vm_mm)
  1110. srmmu_flush_whole_tlb();
  1111. FLUSH_END
  1112. }
  1113. static void turbosparc_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
  1114. {
  1115. FLUSH_BEGIN(vma->vm_mm)
  1116. srmmu_flush_whole_tlb();
  1117. FLUSH_END
  1118. }
  1119. static void poke_turbosparc(void)
  1120. {
  1121. unsigned long mreg = srmmu_get_mmureg();
  1122. unsigned long ccreg;
  1123. /* Clear any crap from the cache or else... */
  1124. turbosparc_flush_cache_all();
  1125. /* Temporarily disable I & D caches */
  1126. mreg &= ~(TURBOSPARC_ICENABLE | TURBOSPARC_DCENABLE);
  1127. mreg &= ~(TURBOSPARC_PCENABLE); /* Don't check parity */
  1128. srmmu_set_mmureg(mreg);
  1129. ccreg = turbosparc_get_ccreg();
  1130. #ifdef TURBOSPARC_WRITEBACK
  1131. ccreg |= (TURBOSPARC_SNENABLE); /* Do DVMA snooping in Dcache */
  1132. ccreg &= ~(TURBOSPARC_uS2 | TURBOSPARC_WTENABLE);
  1133. /* Write-back D-cache, emulate VLSI
  1134. * abortion number three, not number one */
  1135. #else
  1136. /* For now let's play safe, optimize later */
  1137. ccreg |= (TURBOSPARC_SNENABLE | TURBOSPARC_WTENABLE);
  1138. /* Do DVMA snooping in Dcache, Write-thru D-cache */
  1139. ccreg &= ~(TURBOSPARC_uS2);
  1140. /* Emulate VLSI abortion number three, not number one */
  1141. #endif
  1142. switch (ccreg & 7) {
  1143. case 0: /* No SE cache */
  1144. case 7: /* Test mode */
  1145. break;
  1146. default:
  1147. ccreg |= (TURBOSPARC_SCENABLE);
  1148. }
  1149. turbosparc_set_ccreg(ccreg);
  1150. mreg |= (TURBOSPARC_ICENABLE | TURBOSPARC_DCENABLE); /* I & D caches on */
  1151. mreg |= (TURBOSPARC_ICSNOOP); /* Icache snooping on */
  1152. srmmu_set_mmureg(mreg);
  1153. }
  1154. static const struct sparc32_cachetlb_ops turbosparc_ops = {
  1155. .cache_all = turbosparc_flush_cache_all,
  1156. .cache_mm = turbosparc_flush_cache_mm,
  1157. .cache_page = turbosparc_flush_cache_page,
  1158. .cache_range = turbosparc_flush_cache_range,
  1159. .tlb_all = turbosparc_flush_tlb_all,
  1160. .tlb_mm = turbosparc_flush_tlb_mm,
  1161. .tlb_page = turbosparc_flush_tlb_page,
  1162. .tlb_range = turbosparc_flush_tlb_range,
  1163. .page_to_ram = turbosparc_flush_page_to_ram,
  1164. .sig_insns = turbosparc_flush_sig_insns,
  1165. .page_for_dma = turbosparc_flush_page_for_dma,
  1166. };
  1167. static void __init init_turbosparc(void)
  1168. {
  1169. srmmu_name = "Fujitsu TurboSparc";
  1170. srmmu_modtype = TurboSparc;
  1171. sparc32_cachetlb_ops = &turbosparc_ops;
  1172. poke_srmmu = poke_turbosparc;
  1173. }
  1174. static void poke_tsunami(void)
  1175. {
  1176. unsigned long mreg = srmmu_get_mmureg();
  1177. tsunami_flush_icache();
  1178. tsunami_flush_dcache();
  1179. mreg &= ~TSUNAMI_ITD;
  1180. mreg |= (TSUNAMI_IENAB | TSUNAMI_DENAB);
  1181. srmmu_set_mmureg(mreg);
  1182. }
  1183. static const struct sparc32_cachetlb_ops tsunami_ops = {
  1184. .cache_all = tsunami_flush_cache_all,
  1185. .cache_mm = tsunami_flush_cache_mm,
  1186. .cache_page = tsunami_flush_cache_page,
  1187. .cache_range = tsunami_flush_cache_range,
  1188. .tlb_all = tsunami_flush_tlb_all,
  1189. .tlb_mm = tsunami_flush_tlb_mm,
  1190. .tlb_page = tsunami_flush_tlb_page,
  1191. .tlb_range = tsunami_flush_tlb_range,
  1192. .page_to_ram = tsunami_flush_page_to_ram,
  1193. .sig_insns = tsunami_flush_sig_insns,
  1194. .page_for_dma = tsunami_flush_page_for_dma,
  1195. };
  1196. static void __init init_tsunami(void)
  1197. {
  1198. /*
  1199. * Tsunami's pretty sane, Sun and TI actually got it
  1200. * somewhat right this time. Fujitsu should have
  1201. * taken some lessons from them.
  1202. */
  1203. srmmu_name = "TI Tsunami";
  1204. srmmu_modtype = Tsunami;
  1205. sparc32_cachetlb_ops = &tsunami_ops;
  1206. poke_srmmu = poke_tsunami;
  1207. tsunami_setup_blockops();
  1208. }
  1209. static void poke_viking(void)
  1210. {
  1211. unsigned long mreg = srmmu_get_mmureg();
  1212. static int smp_catch;
  1213. if (viking_mxcc_present) {
  1214. unsigned long mxcc_control = mxcc_get_creg();
  1215. mxcc_control |= (MXCC_CTL_ECE | MXCC_CTL_PRE | MXCC_CTL_MCE);
  1216. mxcc_control &= ~(MXCC_CTL_RRC);
  1217. mxcc_set_creg(mxcc_control);
  1218. /*
  1219. * We don't need memory parity checks.
  1220. * XXX This is a mess, have to dig out later. ecd.
  1221. viking_mxcc_turn_off_parity(&mreg, &mxcc_control);
  1222. */
  1223. /* We do cache ptables on MXCC. */
  1224. mreg |= VIKING_TCENABLE;
  1225. } else {
  1226. unsigned long bpreg;
  1227. mreg &= ~(VIKING_TCENABLE);
  1228. if (smp_catch++) {
  1229. /* Must disable mixed-cmd mode here for other cpu's. */
  1230. bpreg = viking_get_bpreg();
  1231. bpreg &= ~(VIKING_ACTION_MIX);
  1232. viking_set_bpreg(bpreg);
  1233. /* Just in case PROM does something funny. */
  1234. msi_set_sync();
  1235. }
  1236. }
  1237. mreg |= VIKING_SPENABLE;
  1238. mreg |= (VIKING_ICENABLE | VIKING_DCENABLE);
  1239. mreg |= VIKING_SBENABLE;
  1240. mreg &= ~(VIKING_ACENABLE);
  1241. srmmu_set_mmureg(mreg);
  1242. }
  1243. static struct sparc32_cachetlb_ops viking_ops __ro_after_init = {
  1244. .cache_all = viking_flush_cache_all,
  1245. .cache_mm = viking_flush_cache_mm,
  1246. .cache_page = viking_flush_cache_page,
  1247. .cache_range = viking_flush_cache_range,
  1248. .tlb_all = viking_flush_tlb_all,
  1249. .tlb_mm = viking_flush_tlb_mm,
  1250. .tlb_page = viking_flush_tlb_page,
  1251. .tlb_range = viking_flush_tlb_range,
  1252. .page_to_ram = viking_flush_page_to_ram,
  1253. .sig_insns = viking_flush_sig_insns,
  1254. .page_for_dma = viking_flush_page_for_dma,
  1255. };
  1256. #ifdef CONFIG_SMP
  1257. /* On sun4d the cpu broadcasts local TLB flushes, so we can just
  1258. * perform the local TLB flush and all the other cpus will see it.
  1259. * But, unfortunately, there is a bug in the sun4d XBUS backplane
  1260. * that requires that we add some synchronization to these flushes.
  1261. *
  1262. * The bug is that the fifo which keeps track of all the pending TLB
  1263. * broadcasts in the system is an entry or two too small, so if we
  1264. * have too many going at once we'll overflow that fifo and lose a TLB
  1265. * flush resulting in corruption.
  1266. *
  1267. * Our workaround is to take a global spinlock around the TLB flushes,
  1268. * which guarentees we won't ever have too many pending. It's a big
  1269. * hammer, but a semaphore like system to make sure we only have N TLB
  1270. * flushes going at once will require SMP locking anyways so there's
  1271. * no real value in trying any harder than this.
  1272. */
  1273. static struct sparc32_cachetlb_ops viking_sun4d_smp_ops __ro_after_init = {
  1274. .cache_all = viking_flush_cache_all,
  1275. .cache_mm = viking_flush_cache_mm,
  1276. .cache_page = viking_flush_cache_page,
  1277. .cache_range = viking_flush_cache_range,
  1278. .tlb_all = sun4dsmp_flush_tlb_all,
  1279. .tlb_mm = sun4dsmp_flush_tlb_mm,
  1280. .tlb_page = sun4dsmp_flush_tlb_page,
  1281. .tlb_range = sun4dsmp_flush_tlb_range,
  1282. .page_to_ram = viking_flush_page_to_ram,
  1283. .sig_insns = viking_flush_sig_insns,
  1284. .page_for_dma = viking_flush_page_for_dma,
  1285. };
  1286. #endif
  1287. static void __init init_viking(void)
  1288. {
  1289. unsigned long mreg = srmmu_get_mmureg();
  1290. /* Ahhh, the viking. SRMMU VLSI abortion number two... */
  1291. if (mreg & VIKING_MMODE) {
  1292. srmmu_name = "TI Viking";
  1293. viking_mxcc_present = 0;
  1294. msi_set_sync();
  1295. /*
  1296. * We need this to make sure old viking takes no hits
  1297. * on it's cache for dma snoops to workaround the
  1298. * "load from non-cacheable memory" interrupt bug.
  1299. * This is only necessary because of the new way in
  1300. * which we use the IOMMU.
  1301. */
  1302. viking_ops.page_for_dma = viking_flush_page;
  1303. #ifdef CONFIG_SMP
  1304. viking_sun4d_smp_ops.page_for_dma = viking_flush_page;
  1305. #endif
  1306. flush_page_for_dma_global = 0;
  1307. } else {
  1308. srmmu_name = "TI Viking/MXCC";
  1309. viking_mxcc_present = 1;
  1310. srmmu_cache_pagetables = 1;
  1311. }
  1312. sparc32_cachetlb_ops = (const struct sparc32_cachetlb_ops *)
  1313. &viking_ops;
  1314. #ifdef CONFIG_SMP
  1315. if (sparc_cpu_model == sun4d)
  1316. sparc32_cachetlb_ops = (const struct sparc32_cachetlb_ops *)
  1317. &viking_sun4d_smp_ops;
  1318. #endif
  1319. poke_srmmu = poke_viking;
  1320. }
  1321. /* Probe for the srmmu chip version. */
  1322. static void __init get_srmmu_type(void)
  1323. {
  1324. unsigned long mreg, psr;
  1325. unsigned long mod_typ, mod_rev, psr_typ, psr_vers;
  1326. srmmu_modtype = SRMMU_INVAL_MOD;
  1327. hwbug_bitmask = 0;
  1328. mreg = srmmu_get_mmureg(); psr = get_psr();
  1329. mod_typ = (mreg & 0xf0000000) >> 28;
  1330. mod_rev = (mreg & 0x0f000000) >> 24;
  1331. psr_typ = (psr >> 28) & 0xf;
  1332. psr_vers = (psr >> 24) & 0xf;
  1333. /* First, check for sparc-leon. */
  1334. if (sparc_cpu_model == sparc_leon) {
  1335. init_leon();
  1336. return;
  1337. }
  1338. /* Second, check for HyperSparc or Cypress. */
  1339. if (mod_typ == 1) {
  1340. switch (mod_rev) {
  1341. case 7:
  1342. /* UP or MP Hypersparc */
  1343. init_hypersparc();
  1344. break;
  1345. case 0:
  1346. case 2:
  1347. case 10:
  1348. case 11:
  1349. case 12:
  1350. case 13:
  1351. case 14:
  1352. case 15:
  1353. default:
  1354. prom_printf("Sparc-Linux Cypress support does not longer exit.\n");
  1355. prom_halt();
  1356. break;
  1357. }
  1358. return;
  1359. }
  1360. /* Now Fujitsu TurboSparc. It might happen that it is
  1361. * in Swift emulation mode, so we will check later...
  1362. */
  1363. if (psr_typ == 0 && psr_vers == 5) {
  1364. init_turbosparc();
  1365. return;
  1366. }
  1367. /* Next check for Fujitsu Swift. */
  1368. if (psr_typ == 0 && psr_vers == 4) {
  1369. phandle cpunode;
  1370. char node_str[128];
  1371. /* Look if it is not a TurboSparc emulating Swift... */
  1372. cpunode = prom_getchild(prom_root_node);
  1373. while ((cpunode = prom_getsibling(cpunode)) != 0) {
  1374. prom_getstring(cpunode, "device_type", node_str, sizeof(node_str));
  1375. if (!strcmp(node_str, "cpu")) {
  1376. if (!prom_getintdefault(cpunode, "psr-implementation", 1) &&
  1377. prom_getintdefault(cpunode, "psr-version", 1) == 5) {
  1378. init_turbosparc();
  1379. return;
  1380. }
  1381. break;
  1382. }
  1383. }
  1384. init_swift();
  1385. return;
  1386. }
  1387. /* Now the Viking family of srmmu. */
  1388. if (psr_typ == 4 &&
  1389. ((psr_vers == 0) ||
  1390. ((psr_vers == 1) && (mod_typ == 0) && (mod_rev == 0)))) {
  1391. init_viking();
  1392. return;
  1393. }
  1394. /* Finally the Tsunami. */
  1395. if (psr_typ == 4 && psr_vers == 1 && (mod_typ || mod_rev)) {
  1396. init_tsunami();
  1397. return;
  1398. }
  1399. /* Oh well */
  1400. srmmu_is_bad();
  1401. }
  1402. #ifdef CONFIG_SMP
  1403. /* Local cross-calls. */
  1404. static void smp_flush_page_for_dma(unsigned long page)
  1405. {
  1406. xc1((smpfunc_t) local_ops->page_for_dma, page);
  1407. local_ops->page_for_dma(page);
  1408. }
  1409. static void smp_flush_cache_all(void)
  1410. {
  1411. xc0((smpfunc_t) local_ops->cache_all);
  1412. local_ops->cache_all();
  1413. }
  1414. static void smp_flush_tlb_all(void)
  1415. {
  1416. xc0((smpfunc_t) local_ops->tlb_all);
  1417. local_ops->tlb_all();
  1418. }
  1419. static void smp_flush_cache_mm(struct mm_struct *mm)
  1420. {
  1421. if (mm->context != NO_CONTEXT) {
  1422. cpumask_t cpu_mask;
  1423. cpumask_copy(&cpu_mask, mm_cpumask(mm));
  1424. cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
  1425. if (!cpumask_empty(&cpu_mask))
  1426. xc1((smpfunc_t) local_ops->cache_mm, (unsigned long) mm);
  1427. local_ops->cache_mm(mm);
  1428. }
  1429. }
  1430. static void smp_flush_tlb_mm(struct mm_struct *mm)
  1431. {
  1432. if (mm->context != NO_CONTEXT) {
  1433. cpumask_t cpu_mask;
  1434. cpumask_copy(&cpu_mask, mm_cpumask(mm));
  1435. cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
  1436. if (!cpumask_empty(&cpu_mask)) {
  1437. xc1((smpfunc_t) local_ops->tlb_mm, (unsigned long) mm);
  1438. if (atomic_read(&mm->mm_users) == 1 && current->active_mm == mm)
  1439. cpumask_copy(mm_cpumask(mm),
  1440. cpumask_of(smp_processor_id()));
  1441. }
  1442. local_ops->tlb_mm(mm);
  1443. }
  1444. }
  1445. static void smp_flush_cache_range(struct vm_area_struct *vma,
  1446. unsigned long start,
  1447. unsigned long end)
  1448. {
  1449. struct mm_struct *mm = vma->vm_mm;
  1450. if (mm->context != NO_CONTEXT) {
  1451. cpumask_t cpu_mask;
  1452. cpumask_copy(&cpu_mask, mm_cpumask(mm));
  1453. cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
  1454. if (!cpumask_empty(&cpu_mask))
  1455. xc3((smpfunc_t) local_ops->cache_range,
  1456. (unsigned long) vma, start, end);
  1457. local_ops->cache_range(vma, start, end);
  1458. }
  1459. }
  1460. static void smp_flush_tlb_range(struct vm_area_struct *vma,
  1461. unsigned long start,
  1462. unsigned long end)
  1463. {
  1464. struct mm_struct *mm = vma->vm_mm;
  1465. if (mm->context != NO_CONTEXT) {
  1466. cpumask_t cpu_mask;
  1467. cpumask_copy(&cpu_mask, mm_cpumask(mm));
  1468. cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
  1469. if (!cpumask_empty(&cpu_mask))
  1470. xc3((smpfunc_t) local_ops->tlb_range,
  1471. (unsigned long) vma, start, end);
  1472. local_ops->tlb_range(vma, start, end);
  1473. }
  1474. }
  1475. static void smp_flush_cache_page(struct vm_area_struct *vma, unsigned long page)
  1476. {
  1477. struct mm_struct *mm = vma->vm_mm;
  1478. if (mm->context != NO_CONTEXT) {
  1479. cpumask_t cpu_mask;
  1480. cpumask_copy(&cpu_mask, mm_cpumask(mm));
  1481. cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
  1482. if (!cpumask_empty(&cpu_mask))
  1483. xc2((smpfunc_t) local_ops->cache_page,
  1484. (unsigned long) vma, page);
  1485. local_ops->cache_page(vma, page);
  1486. }
  1487. }
  1488. static void smp_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
  1489. {
  1490. struct mm_struct *mm = vma->vm_mm;
  1491. if (mm->context != NO_CONTEXT) {
  1492. cpumask_t cpu_mask;
  1493. cpumask_copy(&cpu_mask, mm_cpumask(mm));
  1494. cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
  1495. if (!cpumask_empty(&cpu_mask))
  1496. xc2((smpfunc_t) local_ops->tlb_page,
  1497. (unsigned long) vma, page);
  1498. local_ops->tlb_page(vma, page);
  1499. }
  1500. }
  1501. static void smp_flush_page_to_ram(unsigned long page)
  1502. {
  1503. /* Current theory is that those who call this are the one's
  1504. * who have just dirtied their cache with the pages contents
  1505. * in kernel space, therefore we only run this on local cpu.
  1506. *
  1507. * XXX This experiment failed, research further... -DaveM
  1508. */
  1509. #if 1
  1510. xc1((smpfunc_t) local_ops->page_to_ram, page);
  1511. #endif
  1512. local_ops->page_to_ram(page);
  1513. }
  1514. static void smp_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr)
  1515. {
  1516. cpumask_t cpu_mask;
  1517. cpumask_copy(&cpu_mask, mm_cpumask(mm));
  1518. cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
  1519. if (!cpumask_empty(&cpu_mask))
  1520. xc2((smpfunc_t) local_ops->sig_insns,
  1521. (unsigned long) mm, insn_addr);
  1522. local_ops->sig_insns(mm, insn_addr);
  1523. }
  1524. static struct sparc32_cachetlb_ops smp_cachetlb_ops __ro_after_init = {
  1525. .cache_all = smp_flush_cache_all,
  1526. .cache_mm = smp_flush_cache_mm,
  1527. .cache_page = smp_flush_cache_page,
  1528. .cache_range = smp_flush_cache_range,
  1529. .tlb_all = smp_flush_tlb_all,
  1530. .tlb_mm = smp_flush_tlb_mm,
  1531. .tlb_page = smp_flush_tlb_page,
  1532. .tlb_range = smp_flush_tlb_range,
  1533. .page_to_ram = smp_flush_page_to_ram,
  1534. .sig_insns = smp_flush_sig_insns,
  1535. .page_for_dma = smp_flush_page_for_dma,
  1536. };
  1537. #endif
  1538. /* Load up routines and constants for sun4m and sun4d mmu */
  1539. void __init load_mmu(void)
  1540. {
  1541. /* Functions */
  1542. get_srmmu_type();
  1543. #ifdef CONFIG_SMP
  1544. /* El switcheroo... */
  1545. local_ops = sparc32_cachetlb_ops;
  1546. if (sparc_cpu_model == sun4d || sparc_cpu_model == sparc_leon) {
  1547. smp_cachetlb_ops.tlb_all = local_ops->tlb_all;
  1548. smp_cachetlb_ops.tlb_mm = local_ops->tlb_mm;
  1549. smp_cachetlb_ops.tlb_range = local_ops->tlb_range;
  1550. smp_cachetlb_ops.tlb_page = local_ops->tlb_page;
  1551. }
  1552. if (poke_srmmu == poke_viking) {
  1553. /* Avoid unnecessary cross calls. */
  1554. smp_cachetlb_ops.cache_all = local_ops->cache_all;
  1555. smp_cachetlb_ops.cache_mm = local_ops->cache_mm;
  1556. smp_cachetlb_ops.cache_range = local_ops->cache_range;
  1557. smp_cachetlb_ops.cache_page = local_ops->cache_page;
  1558. smp_cachetlb_ops.page_to_ram = local_ops->page_to_ram;
  1559. smp_cachetlb_ops.sig_insns = local_ops->sig_insns;
  1560. smp_cachetlb_ops.page_for_dma = local_ops->page_for_dma;
  1561. }
  1562. /* It really is const after this point. */
  1563. sparc32_cachetlb_ops = (const struct sparc32_cachetlb_ops *)
  1564. &smp_cachetlb_ops;
  1565. #endif
  1566. if (sparc_cpu_model != sun4d)
  1567. ld_mmu_iommu();
  1568. #ifdef CONFIG_SMP
  1569. if (sparc_cpu_model == sun4d)
  1570. sun4d_init_smp();
  1571. else if (sparc_cpu_model == sparc_leon)
  1572. leon_init_smp();
  1573. else
  1574. sun4m_init_smp();
  1575. #endif
  1576. }