cache.c 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720
  1. // SPDX-License-Identifier: GPL-2.0+
  2. /*
  3. * Copyright (C) 2013-2014 Synopsys, Inc. All rights reserved.
  4. */
  5. #include <config.h>
  6. #include <common.h>
  7. #include <cpu_func.h>
  8. #include <linux/compiler.h>
  9. #include <linux/kernel.h>
  10. #include <linux/log2.h>
  11. #include <asm/arcregs.h>
  12. #include <asm/arc-bcr.h>
  13. #include <asm/cache.h>
  14. /*
  15. * [ NOTE 1 ]:
  16. * Data cache (L1 D$ or SL$) entire invalidate operation or data cache disable
  17. * operation may result in unexpected behavior and data loss even if we flush
  18. * data cache right before invalidation. That may happens if we store any context
  19. * on stack (like we store BLINK register on stack before function call).
  20. * BLINK register is the register where return address is automatically saved
  21. * when we do function call with instructions like 'bl'.
  22. *
  23. * There is the real example:
  24. * We may hang in the next code as we store any BLINK register on stack in
  25. * invalidate_dcache_all() function.
  26. *
  27. * void flush_dcache_all() {
  28. * __dc_entire_op(OP_FLUSH);
  29. * // Other code //
  30. * }
  31. *
  32. * void invalidate_dcache_all() {
  33. * __dc_entire_op(OP_INV);
  34. * // Other code //
  35. * }
  36. *
  37. * void foo(void) {
  38. * flush_dcache_all();
  39. * invalidate_dcache_all();
  40. * }
  41. *
  42. * Now let's see what really happens during that code execution:
  43. *
  44. * foo()
  45. * |->> call flush_dcache_all
  46. * [return address is saved to BLINK register]
  47. * [push BLINK] (save to stack) ![point 1]
  48. * |->> call __dc_entire_op(OP_FLUSH)
  49. * [return address is saved to BLINK register]
  50. * [flush L1 D$]
  51. * return [jump to BLINK]
  52. * <<------
  53. * [other flush_dcache_all code]
  54. * [pop BLINK] (get from stack)
  55. * return [jump to BLINK]
  56. * <<------
  57. * |->> call invalidate_dcache_all
  58. * [return address is saved to BLINK register]
  59. * [push BLINK] (save to stack) ![point 2]
  60. * |->> call __dc_entire_op(OP_FLUSH)
  61. * [return address is saved to BLINK register]
  62. * [invalidate L1 D$] ![point 3]
  63. * // Oops!!!
  64. * // We lose return address from invalidate_dcache_all function:
  65. * // we save it to stack and invalidate L1 D$ after that!
  66. * return [jump to BLINK]
  67. * <<------
  68. * [other invalidate_dcache_all code]
  69. * [pop BLINK] (get from stack)
  70. * // we don't have this data in L1 dcache as we invalidated it in [point 3]
  71. * // so we get it from next memory level (for example DDR memory)
  72. * // but in the memory we have value which we save in [point 1], which
  73. * // is return address from flush_dcache_all function (instead of
  74. * // address from current invalidate_dcache_all function which we
  75. * // saved in [point 2] !)
  76. * return [jump to BLINK]
  77. * <<------
  78. * // As BLINK points to invalidate_dcache_all, we call it again and
  79. * // loop forever.
  80. *
  81. * Fortunately we may fix that by using flush & invalidation of D$ with a single
  82. * one instruction (instead of flush and invalidation instructions pair) and
  83. * enabling force function inline with '__attribute__((always_inline))' gcc
  84. * attribute to avoid any function call (and BLINK store) between cache flush
  85. * and disable.
  86. *
  87. *
  88. * [ NOTE 2 ]:
  89. * As of today we only support the following cache configurations on ARC.
  90. * Other configurations may exist in HW (for example, since version 3.0 HS
  91. * supports SL$ (L2 system level cache) disable) but we don't support it in SW.
  92. * Configuration 1:
  93. * ______________________
  94. * | |
  95. * | ARC CPU |
  96. * |______________________|
  97. * ___|___ ___|___
  98. * | | | |
  99. * | L1 I$ | | L1 D$ |
  100. * |_______| |_______|
  101. * on/off on/off
  102. * ___|______________|____
  103. * | |
  104. * | main memory |
  105. * |______________________|
  106. *
  107. * Configuration 2:
  108. * ______________________
  109. * | |
  110. * | ARC CPU |
  111. * |______________________|
  112. * ___|___ ___|___
  113. * | | | |
  114. * | L1 I$ | | L1 D$ |
  115. * |_______| |_______|
  116. * on/off on/off
  117. * ___|______________|____
  118. * | |
  119. * | L2 (SL$) |
  120. * |______________________|
  121. * always must be on
  122. * ___|______________|____
  123. * | |
  124. * | main memory |
  125. * |______________________|
  126. *
  127. * Configuration 3:
  128. * ______________________
  129. * | |
  130. * | ARC CPU |
  131. * |______________________|
  132. * ___|___ ___|___
  133. * | | | |
  134. * | L1 I$ | | L1 D$ |
  135. * |_______| |_______|
  136. * on/off must be on
  137. * ___|______________|____ _______
  138. * | | | |
  139. * | L2 (SL$) |-----| IOC |
  140. * |______________________| |_______|
  141. * always must be on on/off
  142. * ___|______________|____
  143. * | |
  144. * | main memory |
  145. * |______________________|
  146. */
  147. DECLARE_GLOBAL_DATA_PTR;
  148. /* Bit values in IC_CTRL */
  149. #define IC_CTRL_CACHE_DISABLE BIT(0)
  150. /* Bit values in DC_CTRL */
  151. #define DC_CTRL_CACHE_DISABLE BIT(0)
  152. #define DC_CTRL_INV_MODE_FLUSH BIT(6)
  153. #define DC_CTRL_FLUSH_STATUS BIT(8)
  154. #define OP_INV BIT(0)
  155. #define OP_FLUSH BIT(1)
  156. #define OP_FLUSH_N_INV (OP_FLUSH | OP_INV)
  157. /* Bit val in SLC_CONTROL */
  158. #define SLC_CTRL_DIS 0x001
  159. #define SLC_CTRL_IM 0x040
  160. #define SLC_CTRL_BUSY 0x100
  161. #define SLC_CTRL_RGN_OP_INV 0x200
  162. #define CACHE_LINE_MASK (~(gd->arch.l1_line_sz - 1))
  163. /*
  164. * We don't want to use '__always_inline' macro here as it can be redefined
  165. * to simple 'inline' in some cases which breaks stuff. See [ NOTE 1 ] for more
  166. * details about the reasons we need to use always_inline functions.
  167. */
  168. #define inlined_cachefunc inline __attribute__((always_inline))
  169. static inlined_cachefunc void __ic_entire_invalidate(void);
  170. static inlined_cachefunc void __dc_entire_op(const int cacheop);
  171. static inline bool pae_exists(void)
  172. {
  173. /* TODO: should we compare mmu version from BCR and from CONFIG? */
  174. #if (CONFIG_ARC_MMU_VER >= 4)
  175. union bcr_mmu_4 mmu4;
  176. mmu4.word = read_aux_reg(ARC_AUX_MMU_BCR);
  177. if (mmu4.fields.pae)
  178. return true;
  179. #endif /* (CONFIG_ARC_MMU_VER >= 4) */
  180. return false;
  181. }
  182. static inlined_cachefunc bool icache_exists(void)
  183. {
  184. union bcr_di_cache ibcr;
  185. ibcr.word = read_aux_reg(ARC_BCR_IC_BUILD);
  186. return !!ibcr.fields.ver;
  187. }
  188. static inlined_cachefunc bool icache_enabled(void)
  189. {
  190. if (!icache_exists())
  191. return false;
  192. return !(read_aux_reg(ARC_AUX_IC_CTRL) & IC_CTRL_CACHE_DISABLE);
  193. }
  194. static inlined_cachefunc bool dcache_exists(void)
  195. {
  196. union bcr_di_cache dbcr;
  197. dbcr.word = read_aux_reg(ARC_BCR_DC_BUILD);
  198. return !!dbcr.fields.ver;
  199. }
  200. static inlined_cachefunc bool dcache_enabled(void)
  201. {
  202. if (!dcache_exists())
  203. return false;
  204. return !(read_aux_reg(ARC_AUX_DC_CTRL) & DC_CTRL_CACHE_DISABLE);
  205. }
  206. static inlined_cachefunc bool slc_exists(void)
  207. {
  208. if (is_isa_arcv2()) {
  209. union bcr_generic sbcr;
  210. sbcr.word = read_aux_reg(ARC_BCR_SLC);
  211. return !!sbcr.fields.ver;
  212. }
  213. return false;
  214. }
  215. static inlined_cachefunc bool slc_data_bypass(void)
  216. {
  217. /*
  218. * If L1 data cache is disabled SL$ is bypassed and all load/store
  219. * requests are sent directly to main memory.
  220. */
  221. return !dcache_enabled();
  222. }
  223. static inline bool ioc_exists(void)
  224. {
  225. if (is_isa_arcv2()) {
  226. union bcr_clust_cfg cbcr;
  227. cbcr.word = read_aux_reg(ARC_BCR_CLUSTER);
  228. return cbcr.fields.c;
  229. }
  230. return false;
  231. }
  232. static inline bool ioc_enabled(void)
  233. {
  234. /*
  235. * We check only CONFIG option instead of IOC HW state check as IOC
  236. * must be disabled by default.
  237. */
  238. if (is_ioc_enabled())
  239. return ioc_exists();
  240. return false;
  241. }
  242. static inlined_cachefunc void __slc_entire_op(const int op)
  243. {
  244. unsigned int ctrl;
  245. if (!slc_exists())
  246. return;
  247. ctrl = read_aux_reg(ARC_AUX_SLC_CTRL);
  248. if (!(op & OP_FLUSH)) /* i.e. OP_INV */
  249. ctrl &= ~SLC_CTRL_IM; /* clear IM: Disable flush before Inv */
  250. else
  251. ctrl |= SLC_CTRL_IM;
  252. write_aux_reg(ARC_AUX_SLC_CTRL, ctrl);
  253. if (op & OP_INV) /* Inv or flush-n-inv use same cmd reg */
  254. write_aux_reg(ARC_AUX_SLC_INVALIDATE, 0x1);
  255. else
  256. write_aux_reg(ARC_AUX_SLC_FLUSH, 0x1);
  257. /* Make sure "busy" bit reports correct stataus, see STAR 9001165532 */
  258. read_aux_reg(ARC_AUX_SLC_CTRL);
  259. /* Important to wait for flush to complete */
  260. while (read_aux_reg(ARC_AUX_SLC_CTRL) & SLC_CTRL_BUSY);
  261. }
  262. static void slc_upper_region_init(void)
  263. {
  264. /*
  265. * ARC_AUX_SLC_RGN_START1 and ARC_AUX_SLC_RGN_END1 register exist
  266. * only if PAE exists in current HW. So we had to check pae_exist
  267. * before using them.
  268. */
  269. if (!pae_exists())
  270. return;
  271. /*
  272. * ARC_AUX_SLC_RGN_END1 and ARC_AUX_SLC_RGN_START1 are always == 0
  273. * as we don't use PAE40.
  274. */
  275. write_aux_reg(ARC_AUX_SLC_RGN_END1, 0);
  276. write_aux_reg(ARC_AUX_SLC_RGN_START1, 0);
  277. }
  278. static void __slc_rgn_op(unsigned long paddr, unsigned long sz, const int op)
  279. {
  280. #ifdef CONFIG_ISA_ARCV2
  281. unsigned int ctrl;
  282. unsigned long end;
  283. if (!slc_exists())
  284. return;
  285. /*
  286. * The Region Flush operation is specified by CTRL.RGN_OP[11..9]
  287. * - b'000 (default) is Flush,
  288. * - b'001 is Invalidate if CTRL.IM == 0
  289. * - b'001 is Flush-n-Invalidate if CTRL.IM == 1
  290. */
  291. ctrl = read_aux_reg(ARC_AUX_SLC_CTRL);
  292. /* Don't rely on default value of IM bit */
  293. if (!(op & OP_FLUSH)) /* i.e. OP_INV */
  294. ctrl &= ~SLC_CTRL_IM; /* clear IM: Disable flush before Inv */
  295. else
  296. ctrl |= SLC_CTRL_IM;
  297. if (op & OP_INV)
  298. ctrl |= SLC_CTRL_RGN_OP_INV; /* Inv or flush-n-inv */
  299. else
  300. ctrl &= ~SLC_CTRL_RGN_OP_INV;
  301. write_aux_reg(ARC_AUX_SLC_CTRL, ctrl);
  302. /*
  303. * Lower bits are ignored, no need to clip
  304. * END needs to be setup before START (latter triggers the operation)
  305. * END can't be same as START, so add (l2_line_sz - 1) to sz
  306. */
  307. end = paddr + sz + gd->arch.slc_line_sz - 1;
  308. /*
  309. * Upper addresses (ARC_AUX_SLC_RGN_END1 and ARC_AUX_SLC_RGN_START1)
  310. * are always == 0 as we don't use PAE40, so we only setup lower ones
  311. * (ARC_AUX_SLC_RGN_END and ARC_AUX_SLC_RGN_START)
  312. */
  313. write_aux_reg(ARC_AUX_SLC_RGN_END, end);
  314. write_aux_reg(ARC_AUX_SLC_RGN_START, paddr);
  315. /* Make sure "busy" bit reports correct stataus, see STAR 9001165532 */
  316. read_aux_reg(ARC_AUX_SLC_CTRL);
  317. while (read_aux_reg(ARC_AUX_SLC_CTRL) & SLC_CTRL_BUSY);
  318. #endif /* CONFIG_ISA_ARCV2 */
  319. }
  320. static void arc_ioc_setup(void)
  321. {
  322. /* IOC Aperture start is equal to DDR start */
  323. unsigned int ap_base = CONFIG_SYS_SDRAM_BASE;
  324. /* IOC Aperture size is equal to DDR size */
  325. long ap_size = CONFIG_SYS_SDRAM_SIZE;
  326. /* Unsupported configuration. See [ NOTE 2 ] for more details. */
  327. if (!slc_exists())
  328. panic("Try to enable IOC but SLC is not present");
  329. /* Unsupported configuration. See [ NOTE 2 ] for more details. */
  330. if (!dcache_enabled())
  331. panic("Try to enable IOC but L1 D$ is disabled");
  332. if (!is_power_of_2(ap_size) || ap_size < 4096)
  333. panic("IOC Aperture size must be power of 2 and bigger 4Kib");
  334. /* IOC Aperture start must be aligned to the size of the aperture */
  335. if (ap_base % ap_size != 0)
  336. panic("IOC Aperture start must be aligned to the size of the aperture");
  337. flush_n_invalidate_dcache_all();
  338. /*
  339. * IOC Aperture size decoded as 2 ^ (SIZE + 2) KB,
  340. * so setting 0x11 implies 512M, 0x12 implies 1G...
  341. */
  342. write_aux_reg(ARC_AUX_IO_COH_AP0_SIZE,
  343. order_base_2(ap_size / 1024) - 2);
  344. write_aux_reg(ARC_AUX_IO_COH_AP0_BASE, ap_base >> 12);
  345. write_aux_reg(ARC_AUX_IO_COH_PARTIAL, 1);
  346. write_aux_reg(ARC_AUX_IO_COH_ENABLE, 1);
  347. }
  348. static void read_decode_cache_bcr_arcv2(void)
  349. {
  350. #ifdef CONFIG_ISA_ARCV2
  351. union bcr_slc_cfg slc_cfg;
  352. if (slc_exists()) {
  353. slc_cfg.word = read_aux_reg(ARC_AUX_SLC_CONFIG);
  354. gd->arch.slc_line_sz = (slc_cfg.fields.lsz == 0) ? 128 : 64;
  355. /*
  356. * We don't support configuration where L1 I$ or L1 D$ is
  357. * absent but SL$ exists. See [ NOTE 2 ] for more details.
  358. */
  359. if (!icache_exists() || !dcache_exists())
  360. panic("Unsupported cache configuration: SLC exists but one of L1 caches is absent");
  361. }
  362. #endif /* CONFIG_ISA_ARCV2 */
  363. }
  364. void read_decode_cache_bcr(void)
  365. {
  366. int dc_line_sz = 0, ic_line_sz = 0;
  367. union bcr_di_cache ibcr, dbcr;
  368. /*
  369. * We don't care much about I$ line length really as there're
  370. * no per-line ops on I$ instead we only do full invalidation of it
  371. * on occasion of relocation and right before jumping to the OS.
  372. * Still we check insane config with zero-encoded line length in
  373. * presense of version field in I$ BCR. Just in case.
  374. */
  375. ibcr.word = read_aux_reg(ARC_BCR_IC_BUILD);
  376. if (ibcr.fields.ver) {
  377. ic_line_sz = 8 << ibcr.fields.line_len;
  378. if (!ic_line_sz)
  379. panic("Instruction exists but line length is 0\n");
  380. }
  381. dbcr.word = read_aux_reg(ARC_BCR_DC_BUILD);
  382. if (dbcr.fields.ver) {
  383. gd->arch.l1_line_sz = dc_line_sz = 16 << dbcr.fields.line_len;
  384. if (!dc_line_sz)
  385. panic("Data cache exists but line length is 0\n");
  386. }
  387. }
  388. void cache_init(void)
  389. {
  390. read_decode_cache_bcr();
  391. if (is_isa_arcv2())
  392. read_decode_cache_bcr_arcv2();
  393. if (is_isa_arcv2() && ioc_enabled())
  394. arc_ioc_setup();
  395. if (is_isa_arcv2() && slc_exists())
  396. slc_upper_region_init();
  397. }
  398. int icache_status(void)
  399. {
  400. return icache_enabled();
  401. }
  402. void icache_enable(void)
  403. {
  404. if (icache_exists())
  405. write_aux_reg(ARC_AUX_IC_CTRL, read_aux_reg(ARC_AUX_IC_CTRL) &
  406. ~IC_CTRL_CACHE_DISABLE);
  407. }
  408. void icache_disable(void)
  409. {
  410. if (!icache_exists())
  411. return;
  412. __ic_entire_invalidate();
  413. write_aux_reg(ARC_AUX_IC_CTRL, read_aux_reg(ARC_AUX_IC_CTRL) |
  414. IC_CTRL_CACHE_DISABLE);
  415. }
  416. /* IC supports only invalidation */
  417. static inlined_cachefunc void __ic_entire_invalidate(void)
  418. {
  419. if (!icache_enabled())
  420. return;
  421. /* Any write to IC_IVIC register triggers invalidation of entire I$ */
  422. write_aux_reg(ARC_AUX_IC_IVIC, 1);
  423. /*
  424. * As per ARC HS databook (see chapter 5.3.3.2)
  425. * it is required to add 3 NOPs after each write to IC_IVIC.
  426. */
  427. __builtin_arc_nop();
  428. __builtin_arc_nop();
  429. __builtin_arc_nop();
  430. read_aux_reg(ARC_AUX_IC_CTRL); /* blocks */
  431. }
  432. void invalidate_icache_all(void)
  433. {
  434. __ic_entire_invalidate();
  435. /*
  436. * If SL$ is bypassed for data it is used only for instructions,
  437. * so we need to invalidate it too.
  438. * TODO: HS 3.0 supports SLC disable so we need to check slc
  439. * enable/disable status here.
  440. */
  441. if (is_isa_arcv2() && slc_data_bypass())
  442. __slc_entire_op(OP_INV);
  443. }
  444. int dcache_status(void)
  445. {
  446. return dcache_enabled();
  447. }
  448. void dcache_enable(void)
  449. {
  450. if (!dcache_exists())
  451. return;
  452. write_aux_reg(ARC_AUX_DC_CTRL, read_aux_reg(ARC_AUX_DC_CTRL) &
  453. ~(DC_CTRL_INV_MODE_FLUSH | DC_CTRL_CACHE_DISABLE));
  454. }
  455. void dcache_disable(void)
  456. {
  457. if (!dcache_exists())
  458. return;
  459. __dc_entire_op(OP_FLUSH_N_INV);
  460. /*
  461. * As SLC will be bypassed for data after L1 D$ disable we need to
  462. * flush it first before L1 D$ disable. Also we invalidate SLC to
  463. * avoid any inconsistent data problems after enabling L1 D$ again with
  464. * dcache_enable function.
  465. */
  466. if (is_isa_arcv2())
  467. __slc_entire_op(OP_FLUSH_N_INV);
  468. write_aux_reg(ARC_AUX_DC_CTRL, read_aux_reg(ARC_AUX_DC_CTRL) |
  469. DC_CTRL_CACHE_DISABLE);
  470. }
  471. /* Common Helper for Line Operations on D-cache */
  472. static inline void __dcache_line_loop(unsigned long paddr, unsigned long sz,
  473. const int cacheop)
  474. {
  475. unsigned int aux_cmd;
  476. int num_lines;
  477. /* d$ cmd: INV (discard or wback-n-discard) OR FLUSH (wback) */
  478. aux_cmd = cacheop & OP_INV ? ARC_AUX_DC_IVDL : ARC_AUX_DC_FLDL;
  479. sz += paddr & ~CACHE_LINE_MASK;
  480. paddr &= CACHE_LINE_MASK;
  481. num_lines = DIV_ROUND_UP(sz, gd->arch.l1_line_sz);
  482. while (num_lines-- > 0) {
  483. #if (CONFIG_ARC_MMU_VER == 3)
  484. write_aux_reg(ARC_AUX_DC_PTAG, paddr);
  485. #endif
  486. write_aux_reg(aux_cmd, paddr);
  487. paddr += gd->arch.l1_line_sz;
  488. }
  489. }
  490. static inlined_cachefunc void __before_dc_op(const int op)
  491. {
  492. unsigned int ctrl;
  493. ctrl = read_aux_reg(ARC_AUX_DC_CTRL);
  494. /* IM bit implies flush-n-inv, instead of vanilla inv */
  495. if (op == OP_INV)
  496. ctrl &= ~DC_CTRL_INV_MODE_FLUSH;
  497. else
  498. ctrl |= DC_CTRL_INV_MODE_FLUSH;
  499. write_aux_reg(ARC_AUX_DC_CTRL, ctrl);
  500. }
  501. static inlined_cachefunc void __after_dc_op(const int op)
  502. {
  503. if (op & OP_FLUSH) /* flush / flush-n-inv both wait */
  504. while (read_aux_reg(ARC_AUX_DC_CTRL) & DC_CTRL_FLUSH_STATUS);
  505. }
  506. static inlined_cachefunc void __dc_entire_op(const int cacheop)
  507. {
  508. int aux;
  509. if (!dcache_enabled())
  510. return;
  511. __before_dc_op(cacheop);
  512. if (cacheop & OP_INV) /* Inv or flush-n-inv use same cmd reg */
  513. aux = ARC_AUX_DC_IVDC;
  514. else
  515. aux = ARC_AUX_DC_FLSH;
  516. write_aux_reg(aux, 0x1);
  517. __after_dc_op(cacheop);
  518. }
  519. static inline void __dc_line_op(unsigned long paddr, unsigned long sz,
  520. const int cacheop)
  521. {
  522. if (!dcache_enabled())
  523. return;
  524. __before_dc_op(cacheop);
  525. __dcache_line_loop(paddr, sz, cacheop);
  526. __after_dc_op(cacheop);
  527. }
  528. void invalidate_dcache_range(unsigned long start, unsigned long end)
  529. {
  530. if (start >= end)
  531. return;
  532. /*
  533. * ARCv1 -> call __dc_line_op
  534. * ARCv2 && L1 D$ disabled -> nothing
  535. * ARCv2 && L1 D$ enabled && IOC enabled -> nothing
  536. * ARCv2 && L1 D$ enabled && no IOC -> call __dc_line_op; call __slc_rgn_op
  537. */
  538. if (!is_isa_arcv2() || !ioc_enabled())
  539. __dc_line_op(start, end - start, OP_INV);
  540. if (is_isa_arcv2() && !ioc_enabled() && !slc_data_bypass())
  541. __slc_rgn_op(start, end - start, OP_INV);
  542. }
  543. void flush_dcache_range(unsigned long start, unsigned long end)
  544. {
  545. if (start >= end)
  546. return;
  547. /*
  548. * ARCv1 -> call __dc_line_op
  549. * ARCv2 && L1 D$ disabled -> nothing
  550. * ARCv2 && L1 D$ enabled && IOC enabled -> nothing
  551. * ARCv2 && L1 D$ enabled && no IOC -> call __dc_line_op; call __slc_rgn_op
  552. */
  553. if (!is_isa_arcv2() || !ioc_enabled())
  554. __dc_line_op(start, end - start, OP_FLUSH);
  555. if (is_isa_arcv2() && !ioc_enabled() && !slc_data_bypass())
  556. __slc_rgn_op(start, end - start, OP_FLUSH);
  557. }
  558. void flush_cache(unsigned long start, unsigned long size)
  559. {
  560. flush_dcache_range(start, start + size);
  561. }
  562. /*
  563. * As invalidate_dcache_all() is not used in generic U-Boot code and as we
  564. * don't need it in arch/arc code alone (invalidate without flush) we implement
  565. * flush_n_invalidate_dcache_all (flush and invalidate in 1 operation) because
  566. * it's much safer. See [ NOTE 1 ] for more details.
  567. */
  568. void flush_n_invalidate_dcache_all(void)
  569. {
  570. __dc_entire_op(OP_FLUSH_N_INV);
  571. if (is_isa_arcv2() && !slc_data_bypass())
  572. __slc_entire_op(OP_FLUSH_N_INV);
  573. }
  574. void flush_dcache_all(void)
  575. {
  576. __dc_entire_op(OP_FLUSH);
  577. if (is_isa_arcv2() && !slc_data_bypass())
  578. __slc_entire_op(OP_FLUSH);
  579. }
  580. /*
  581. * This is function to cleanup all caches (and therefore sync I/D caches) which
  582. * can be used for cleanup before linux launch or to sync caches during
  583. * relocation.
  584. */
  585. void sync_n_cleanup_cache_all(void)
  586. {
  587. __dc_entire_op(OP_FLUSH_N_INV);
  588. /*
  589. * If SL$ is bypassed for data it is used only for instructions,
  590. * and we shouldn't flush it. So invalidate it instead of flush_n_inv.
  591. */
  592. if (is_isa_arcv2()) {
  593. if (slc_data_bypass())
  594. __slc_entire_op(OP_INV);
  595. else
  596. __slc_entire_op(OP_FLUSH_N_INV);
  597. }
  598. __ic_entire_invalidate();
  599. }