ocmem.c 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * The On Chip Memory (OCMEM) allocator allows various clients to allocate
  4. * memory from OCMEM based on performance, latency and power requirements.
  5. * This is typically used by the GPU, camera/video, and audio components on
  6. * some Snapdragon SoCs.
  7. *
  8. * Copyright (C) 2019 Brian Masney <masneyb@onstation.org>
  9. * Copyright (C) 2015 Red Hat. Author: Rob Clark <robdclark@gmail.com>
  10. */
  11. #include <linux/bitfield.h>
  12. #include <linux/clk.h>
  13. #include <linux/io.h>
  14. #include <linux/kernel.h>
  15. #include <linux/module.h>
  16. #include <linux/of_device.h>
  17. #include <linux/platform_device.h>
  18. #include <linux/qcom_scm.h>
  19. #include <linux/sizes.h>
  20. #include <linux/slab.h>
  21. #include <linux/types.h>
  22. #include <soc/qcom/ocmem.h>
  23. enum region_mode {
  24. WIDE_MODE = 0x0,
  25. THIN_MODE,
  26. MODE_DEFAULT = WIDE_MODE,
  27. };
  28. enum ocmem_macro_state {
  29. PASSTHROUGH = 0,
  30. PERI_ON = 1,
  31. CORE_ON = 2,
  32. CLK_OFF = 4,
  33. };
  34. struct ocmem_region {
  35. bool interleaved;
  36. enum region_mode mode;
  37. unsigned int num_macros;
  38. enum ocmem_macro_state macro_state[4];
  39. unsigned long macro_size;
  40. unsigned long region_size;
  41. };
  42. struct ocmem_config {
  43. uint8_t num_regions;
  44. unsigned long macro_size;
  45. };
  46. struct ocmem {
  47. struct device *dev;
  48. const struct ocmem_config *config;
  49. struct resource *memory;
  50. void __iomem *mmio;
  51. unsigned int num_ports;
  52. unsigned int num_macros;
  53. bool interleaved;
  54. struct ocmem_region *regions;
  55. unsigned long active_allocations;
  56. };
  57. #define OCMEM_MIN_ALIGN SZ_64K
  58. #define OCMEM_MIN_ALLOC SZ_64K
  59. #define OCMEM_REG_HW_VERSION 0x00000000
  60. #define OCMEM_REG_HW_PROFILE 0x00000004
  61. #define OCMEM_REG_REGION_MODE_CTL 0x00001000
  62. #define OCMEM_REGION_MODE_CTL_REG0_THIN 0x00000001
  63. #define OCMEM_REGION_MODE_CTL_REG1_THIN 0x00000002
  64. #define OCMEM_REGION_MODE_CTL_REG2_THIN 0x00000004
  65. #define OCMEM_REGION_MODE_CTL_REG3_THIN 0x00000008
  66. #define OCMEM_REG_GFX_MPU_START 0x00001004
  67. #define OCMEM_REG_GFX_MPU_END 0x00001008
  68. #define OCMEM_HW_PROFILE_NUM_PORTS(val) FIELD_PREP(0x0000000f, (val))
  69. #define OCMEM_HW_PROFILE_NUM_MACROS(val) FIELD_PREP(0x00003f00, (val))
  70. #define OCMEM_HW_PROFILE_LAST_REGN_HALFSIZE 0x00010000
  71. #define OCMEM_HW_PROFILE_INTERLEAVING 0x00020000
  72. #define OCMEM_REG_GEN_STATUS 0x0000000c
  73. #define OCMEM_REG_PSGSC_STATUS 0x00000038
  74. #define OCMEM_REG_PSGSC_CTL(i0) (0x0000003c + 0x1*(i0))
  75. #define OCMEM_PSGSC_CTL_MACRO0_MODE(val) FIELD_PREP(0x00000007, (val))
  76. #define OCMEM_PSGSC_CTL_MACRO1_MODE(val) FIELD_PREP(0x00000070, (val))
  77. #define OCMEM_PSGSC_CTL_MACRO2_MODE(val) FIELD_PREP(0x00000700, (val))
  78. #define OCMEM_PSGSC_CTL_MACRO3_MODE(val) FIELD_PREP(0x00007000, (val))
  79. #define OCMEM_CLK_CORE_IDX 0
  80. static struct clk_bulk_data ocmem_clks[] = {
  81. {
  82. .id = "core",
  83. },
  84. {
  85. .id = "iface",
  86. },
  87. };
  88. static inline void ocmem_write(struct ocmem *ocmem, u32 reg, u32 data)
  89. {
  90. writel(data, ocmem->mmio + reg);
  91. }
  92. static inline u32 ocmem_read(struct ocmem *ocmem, u32 reg)
  93. {
  94. return readl(ocmem->mmio + reg);
  95. }
  96. static void update_ocmem(struct ocmem *ocmem)
  97. {
  98. uint32_t region_mode_ctrl = 0x0;
  99. int i;
  100. if (!qcom_scm_ocmem_lock_available()) {
  101. for (i = 0; i < ocmem->config->num_regions; i++) {
  102. struct ocmem_region *region = &ocmem->regions[i];
  103. if (region->mode == THIN_MODE)
  104. region_mode_ctrl |= BIT(i);
  105. }
  106. dev_dbg(ocmem->dev, "ocmem_region_mode_control %x\n",
  107. region_mode_ctrl);
  108. ocmem_write(ocmem, OCMEM_REG_REGION_MODE_CTL, region_mode_ctrl);
  109. }
  110. for (i = 0; i < ocmem->config->num_regions; i++) {
  111. struct ocmem_region *region = &ocmem->regions[i];
  112. u32 data;
  113. data = OCMEM_PSGSC_CTL_MACRO0_MODE(region->macro_state[0]) |
  114. OCMEM_PSGSC_CTL_MACRO1_MODE(region->macro_state[1]) |
  115. OCMEM_PSGSC_CTL_MACRO2_MODE(region->macro_state[2]) |
  116. OCMEM_PSGSC_CTL_MACRO3_MODE(region->macro_state[3]);
  117. ocmem_write(ocmem, OCMEM_REG_PSGSC_CTL(i), data);
  118. }
  119. }
  120. static unsigned long phys_to_offset(struct ocmem *ocmem,
  121. unsigned long addr)
  122. {
  123. if (addr < ocmem->memory->start || addr >= ocmem->memory->end)
  124. return 0;
  125. return addr - ocmem->memory->start;
  126. }
  127. static unsigned long device_address(struct ocmem *ocmem,
  128. enum ocmem_client client,
  129. unsigned long addr)
  130. {
  131. WARN_ON(client != OCMEM_GRAPHICS);
  132. /* TODO: gpu uses phys_to_offset, but others do not.. */
  133. return phys_to_offset(ocmem, addr);
  134. }
  135. static void update_range(struct ocmem *ocmem, struct ocmem_buf *buf,
  136. enum ocmem_macro_state mstate, enum region_mode rmode)
  137. {
  138. unsigned long offset = 0;
  139. int i, j;
  140. for (i = 0; i < ocmem->config->num_regions; i++) {
  141. struct ocmem_region *region = &ocmem->regions[i];
  142. if (buf->offset <= offset && offset < buf->offset + buf->len)
  143. region->mode = rmode;
  144. for (j = 0; j < region->num_macros; j++) {
  145. if (buf->offset <= offset &&
  146. offset < buf->offset + buf->len)
  147. region->macro_state[j] = mstate;
  148. offset += region->macro_size;
  149. }
  150. }
  151. update_ocmem(ocmem);
  152. }
  153. struct ocmem *of_get_ocmem(struct device *dev)
  154. {
  155. struct platform_device *pdev;
  156. struct device_node *devnode;
  157. struct ocmem *ocmem;
  158. devnode = of_parse_phandle(dev->of_node, "sram", 0);
  159. if (!devnode || !devnode->parent) {
  160. dev_err(dev, "Cannot look up sram phandle\n");
  161. return ERR_PTR(-ENODEV);
  162. }
  163. pdev = of_find_device_by_node(devnode->parent);
  164. if (!pdev) {
  165. dev_err(dev, "Cannot find device node %s\n", devnode->name);
  166. return ERR_PTR(-EPROBE_DEFER);
  167. }
  168. ocmem = platform_get_drvdata(pdev);
  169. if (!ocmem) {
  170. dev_err(dev, "Cannot get ocmem\n");
  171. put_device(&pdev->dev);
  172. return ERR_PTR(-ENODEV);
  173. }
  174. return ocmem;
  175. }
  176. EXPORT_SYMBOL(of_get_ocmem);
  177. struct ocmem_buf *ocmem_allocate(struct ocmem *ocmem, enum ocmem_client client,
  178. unsigned long size)
  179. {
  180. struct ocmem_buf *buf;
  181. int ret;
  182. /* TODO: add support for other clients... */
  183. if (WARN_ON(client != OCMEM_GRAPHICS))
  184. return ERR_PTR(-ENODEV);
  185. if (size < OCMEM_MIN_ALLOC || !IS_ALIGNED(size, OCMEM_MIN_ALIGN))
  186. return ERR_PTR(-EINVAL);
  187. if (test_and_set_bit_lock(BIT(client), &ocmem->active_allocations))
  188. return ERR_PTR(-EBUSY);
  189. buf = kzalloc(sizeof(*buf), GFP_KERNEL);
  190. if (!buf) {
  191. ret = -ENOMEM;
  192. goto err_unlock;
  193. }
  194. buf->offset = 0;
  195. buf->addr = device_address(ocmem, client, buf->offset);
  196. buf->len = size;
  197. update_range(ocmem, buf, CORE_ON, WIDE_MODE);
  198. if (qcom_scm_ocmem_lock_available()) {
  199. ret = qcom_scm_ocmem_lock(QCOM_SCM_OCMEM_GRAPHICS_ID,
  200. buf->offset, buf->len, WIDE_MODE);
  201. if (ret) {
  202. dev_err(ocmem->dev, "could not lock: %d\n", ret);
  203. ret = -EINVAL;
  204. goto err_kfree;
  205. }
  206. } else {
  207. ocmem_write(ocmem, OCMEM_REG_GFX_MPU_START, buf->offset);
  208. ocmem_write(ocmem, OCMEM_REG_GFX_MPU_END,
  209. buf->offset + buf->len);
  210. }
  211. dev_dbg(ocmem->dev, "using %ldK of OCMEM at 0x%08lx for client %d\n",
  212. size / 1024, buf->addr, client);
  213. return buf;
  214. err_kfree:
  215. kfree(buf);
  216. err_unlock:
  217. clear_bit_unlock(BIT(client), &ocmem->active_allocations);
  218. return ERR_PTR(ret);
  219. }
  220. EXPORT_SYMBOL(ocmem_allocate);
  221. void ocmem_free(struct ocmem *ocmem, enum ocmem_client client,
  222. struct ocmem_buf *buf)
  223. {
  224. /* TODO: add support for other clients... */
  225. if (WARN_ON(client != OCMEM_GRAPHICS))
  226. return;
  227. update_range(ocmem, buf, CLK_OFF, MODE_DEFAULT);
  228. if (qcom_scm_ocmem_lock_available()) {
  229. int ret;
  230. ret = qcom_scm_ocmem_unlock(QCOM_SCM_OCMEM_GRAPHICS_ID,
  231. buf->offset, buf->len);
  232. if (ret)
  233. dev_err(ocmem->dev, "could not unlock: %d\n", ret);
  234. } else {
  235. ocmem_write(ocmem, OCMEM_REG_GFX_MPU_START, 0x0);
  236. ocmem_write(ocmem, OCMEM_REG_GFX_MPU_END, 0x0);
  237. }
  238. kfree(buf);
  239. clear_bit_unlock(BIT(client), &ocmem->active_allocations);
  240. }
  241. EXPORT_SYMBOL(ocmem_free);
  242. static int ocmem_dev_probe(struct platform_device *pdev)
  243. {
  244. struct device *dev = &pdev->dev;
  245. unsigned long reg, region_size;
  246. int i, j, ret, num_banks;
  247. struct resource *res;
  248. struct ocmem *ocmem;
  249. if (!qcom_scm_is_available())
  250. return -EPROBE_DEFER;
  251. ocmem = devm_kzalloc(dev, sizeof(*ocmem), GFP_KERNEL);
  252. if (!ocmem)
  253. return -ENOMEM;
  254. ocmem->dev = dev;
  255. ocmem->config = device_get_match_data(dev);
  256. ret = devm_clk_bulk_get(dev, ARRAY_SIZE(ocmem_clks), ocmem_clks);
  257. if (ret) {
  258. if (ret != -EPROBE_DEFER)
  259. dev_err(dev, "Unable to get clocks\n");
  260. return ret;
  261. }
  262. res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ctrl");
  263. ocmem->mmio = devm_ioremap_resource(&pdev->dev, res);
  264. if (IS_ERR(ocmem->mmio)) {
  265. dev_err(&pdev->dev, "Failed to ioremap ocmem_ctrl resource\n");
  266. return PTR_ERR(ocmem->mmio);
  267. }
  268. ocmem->memory = platform_get_resource_byname(pdev, IORESOURCE_MEM,
  269. "mem");
  270. if (!ocmem->memory) {
  271. dev_err(dev, "Could not get mem region\n");
  272. return -ENXIO;
  273. }
  274. /* The core clock is synchronous with graphics */
  275. WARN_ON(clk_set_rate(ocmem_clks[OCMEM_CLK_CORE_IDX].clk, 1000) < 0);
  276. ret = clk_bulk_prepare_enable(ARRAY_SIZE(ocmem_clks), ocmem_clks);
  277. if (ret) {
  278. dev_info(ocmem->dev, "Failed to enable clocks\n");
  279. return ret;
  280. }
  281. if (qcom_scm_restore_sec_cfg_available()) {
  282. dev_dbg(dev, "configuring scm\n");
  283. ret = qcom_scm_restore_sec_cfg(QCOM_SCM_OCMEM_DEV_ID, 0);
  284. if (ret) {
  285. dev_err(dev, "Could not enable secure configuration\n");
  286. goto err_clk_disable;
  287. }
  288. }
  289. reg = ocmem_read(ocmem, OCMEM_REG_HW_PROFILE);
  290. ocmem->num_ports = OCMEM_HW_PROFILE_NUM_PORTS(reg);
  291. ocmem->num_macros = OCMEM_HW_PROFILE_NUM_MACROS(reg);
  292. ocmem->interleaved = !!(reg & OCMEM_HW_PROFILE_INTERLEAVING);
  293. num_banks = ocmem->num_ports / 2;
  294. region_size = ocmem->config->macro_size * num_banks;
  295. dev_info(dev, "%u ports, %u regions, %u macros, %sinterleaved\n",
  296. ocmem->num_ports, ocmem->config->num_regions,
  297. ocmem->num_macros, ocmem->interleaved ? "" : "not ");
  298. ocmem->regions = devm_kcalloc(dev, ocmem->config->num_regions,
  299. sizeof(struct ocmem_region), GFP_KERNEL);
  300. if (!ocmem->regions) {
  301. ret = -ENOMEM;
  302. goto err_clk_disable;
  303. }
  304. for (i = 0; i < ocmem->config->num_regions; i++) {
  305. struct ocmem_region *region = &ocmem->regions[i];
  306. if (WARN_ON(num_banks > ARRAY_SIZE(region->macro_state))) {
  307. ret = -EINVAL;
  308. goto err_clk_disable;
  309. }
  310. region->mode = MODE_DEFAULT;
  311. region->num_macros = num_banks;
  312. if (i == (ocmem->config->num_regions - 1) &&
  313. reg & OCMEM_HW_PROFILE_LAST_REGN_HALFSIZE) {
  314. region->macro_size = ocmem->config->macro_size / 2;
  315. region->region_size = region_size / 2;
  316. } else {
  317. region->macro_size = ocmem->config->macro_size;
  318. region->region_size = region_size;
  319. }
  320. for (j = 0; j < ARRAY_SIZE(region->macro_state); j++)
  321. region->macro_state[j] = CLK_OFF;
  322. }
  323. platform_set_drvdata(pdev, ocmem);
  324. return 0;
  325. err_clk_disable:
  326. clk_bulk_disable_unprepare(ARRAY_SIZE(ocmem_clks), ocmem_clks);
  327. return ret;
  328. }
  329. static int ocmem_dev_remove(struct platform_device *pdev)
  330. {
  331. clk_bulk_disable_unprepare(ARRAY_SIZE(ocmem_clks), ocmem_clks);
  332. return 0;
  333. }
  334. static const struct ocmem_config ocmem_8974_config = {
  335. .num_regions = 3,
  336. .macro_size = SZ_128K,
  337. };
  338. static const struct of_device_id ocmem_of_match[] = {
  339. { .compatible = "qcom,msm8974-ocmem", .data = &ocmem_8974_config },
  340. { }
  341. };
  342. MODULE_DEVICE_TABLE(of, ocmem_of_match);
  343. static struct platform_driver ocmem_driver = {
  344. .probe = ocmem_dev_probe,
  345. .remove = ocmem_dev_remove,
  346. .driver = {
  347. .name = "ocmem",
  348. .of_match_table = ocmem_of_match,
  349. },
  350. };
  351. module_platform_driver(ocmem_driver);
  352. MODULE_DESCRIPTION("On Chip Memory (OCMEM) allocator for some Snapdragon SoCs");
  353. MODULE_LICENSE("GPL v2");