imx8m-ddrc.c 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Copyright 2019 NXP
  4. */
  5. #include <linux/module.h>
  6. #include <linux/device.h>
  7. #include <linux/of_device.h>
  8. #include <linux/platform_device.h>
  9. #include <linux/devfreq.h>
  10. #include <linux/pm_opp.h>
  11. #include <linux/clk.h>
  12. #include <linux/clk-provider.h>
  13. #include <linux/arm-smccc.h>
  14. #define IMX_SIP_DDR_DVFS 0xc2000004
  15. /* Query available frequencies. */
  16. #define IMX_SIP_DDR_DVFS_GET_FREQ_COUNT 0x10
  17. #define IMX_SIP_DDR_DVFS_GET_FREQ_INFO 0x11
  18. /*
  19. * This should be in a 1:1 mapping with devicetree OPPs but
  20. * firmware provides additional info.
  21. */
  22. struct imx8m_ddrc_freq {
  23. unsigned long rate;
  24. unsigned long smcarg;
  25. int dram_core_parent_index;
  26. int dram_alt_parent_index;
  27. int dram_apb_parent_index;
  28. };
  29. /* Hardware limitation */
  30. #define IMX8M_DDRC_MAX_FREQ_COUNT 4
  31. /*
  32. * i.MX8M DRAM Controller clocks have the following structure (abridged):
  33. *
  34. * +----------+ |\ +------+
  35. * | dram_pll |-------|M| dram_core | |
  36. * +----------+ |U|---------->| D |
  37. * /--|X| | D |
  38. * dram_alt_root | |/ | R |
  39. * | | C |
  40. * +---------+ | |
  41. * |FIX DIV/4| | |
  42. * +---------+ | |
  43. * composite: | | |
  44. * +----------+ | | |
  45. * | dram_alt |----/ | |
  46. * +----------+ | |
  47. * | dram_apb |-------------------->| |
  48. * +----------+ +------+
  49. *
  50. * The dram_pll is used for higher rates and dram_alt is used for lower rates.
  51. *
  52. * Frequency switching is implemented in TF-A (via SMC call) and can change the
  53. * configuration of the clocks, including mux parents. The dram_alt and
  54. * dram_apb clocks are "imx composite" and their parent can change too.
  55. *
  56. * We need to prepare/enable the new mux parents head of switching and update
  57. * their information afterwards.
  58. */
  59. struct imx8m_ddrc {
  60. struct devfreq_dev_profile profile;
  61. struct devfreq *devfreq;
  62. /* For frequency switching: */
  63. struct clk *dram_core;
  64. struct clk *dram_pll;
  65. struct clk *dram_alt;
  66. struct clk *dram_apb;
  67. int freq_count;
  68. struct imx8m_ddrc_freq freq_table[IMX8M_DDRC_MAX_FREQ_COUNT];
  69. };
  70. static struct imx8m_ddrc_freq *imx8m_ddrc_find_freq(struct imx8m_ddrc *priv,
  71. unsigned long rate)
  72. {
  73. struct imx8m_ddrc_freq *freq;
  74. int i;
  75. /*
  76. * Firmware reports values in MT/s, so we round-down from Hz
  77. * Rounding is extra generous to ensure a match.
  78. */
  79. rate = DIV_ROUND_CLOSEST(rate, 250000);
  80. for (i = 0; i < priv->freq_count; ++i) {
  81. freq = &priv->freq_table[i];
  82. if (freq->rate == rate ||
  83. freq->rate + 1 == rate ||
  84. freq->rate - 1 == rate)
  85. return freq;
  86. }
  87. return NULL;
  88. }
  89. static void imx8m_ddrc_smc_set_freq(int target_freq)
  90. {
  91. struct arm_smccc_res res;
  92. u32 online_cpus = 0;
  93. int cpu;
  94. local_irq_disable();
  95. for_each_online_cpu(cpu)
  96. online_cpus |= (1 << (cpu * 8));
  97. /* change the ddr freqency */
  98. arm_smccc_smc(IMX_SIP_DDR_DVFS, target_freq, online_cpus,
  99. 0, 0, 0, 0, 0, &res);
  100. local_irq_enable();
  101. }
  102. static struct clk *clk_get_parent_by_index(struct clk *clk, int index)
  103. {
  104. struct clk_hw *hw;
  105. hw = clk_hw_get_parent_by_index(__clk_get_hw(clk), index);
  106. return hw ? hw->clk : NULL;
  107. }
  108. static int imx8m_ddrc_set_freq(struct device *dev, struct imx8m_ddrc_freq *freq)
  109. {
  110. struct imx8m_ddrc *priv = dev_get_drvdata(dev);
  111. struct clk *new_dram_core_parent;
  112. struct clk *new_dram_alt_parent;
  113. struct clk *new_dram_apb_parent;
  114. int ret;
  115. /*
  116. * Fetch new parents
  117. *
  118. * new_dram_alt_parent and new_dram_apb_parent are optional but
  119. * new_dram_core_parent is not.
  120. */
  121. new_dram_core_parent = clk_get_parent_by_index(
  122. priv->dram_core, freq->dram_core_parent_index - 1);
  123. if (!new_dram_core_parent) {
  124. dev_err(dev, "failed to fetch new dram_core parent\n");
  125. return -EINVAL;
  126. }
  127. if (freq->dram_alt_parent_index) {
  128. new_dram_alt_parent = clk_get_parent_by_index(
  129. priv->dram_alt,
  130. freq->dram_alt_parent_index - 1);
  131. if (!new_dram_alt_parent) {
  132. dev_err(dev, "failed to fetch new dram_alt parent\n");
  133. return -EINVAL;
  134. }
  135. } else
  136. new_dram_alt_parent = NULL;
  137. if (freq->dram_apb_parent_index) {
  138. new_dram_apb_parent = clk_get_parent_by_index(
  139. priv->dram_apb,
  140. freq->dram_apb_parent_index - 1);
  141. if (!new_dram_apb_parent) {
  142. dev_err(dev, "failed to fetch new dram_apb parent\n");
  143. return -EINVAL;
  144. }
  145. } else
  146. new_dram_apb_parent = NULL;
  147. /* increase reference counts and ensure clks are ON before switch */
  148. ret = clk_prepare_enable(new_dram_core_parent);
  149. if (ret) {
  150. dev_err(dev, "failed to enable new dram_core parent: %d\n",
  151. ret);
  152. goto out;
  153. }
  154. ret = clk_prepare_enable(new_dram_alt_parent);
  155. if (ret) {
  156. dev_err(dev, "failed to enable new dram_alt parent: %d\n",
  157. ret);
  158. goto out_disable_core_parent;
  159. }
  160. ret = clk_prepare_enable(new_dram_apb_parent);
  161. if (ret) {
  162. dev_err(dev, "failed to enable new dram_apb parent: %d\n",
  163. ret);
  164. goto out_disable_alt_parent;
  165. }
  166. imx8m_ddrc_smc_set_freq(freq->smcarg);
  167. /* update parents in clk tree after switch. */
  168. ret = clk_set_parent(priv->dram_core, new_dram_core_parent);
  169. if (ret)
  170. dev_warn(dev, "failed to set dram_core parent: %d\n", ret);
  171. if (new_dram_alt_parent) {
  172. ret = clk_set_parent(priv->dram_alt, new_dram_alt_parent);
  173. if (ret)
  174. dev_warn(dev, "failed to set dram_alt parent: %d\n",
  175. ret);
  176. }
  177. if (new_dram_apb_parent) {
  178. ret = clk_set_parent(priv->dram_apb, new_dram_apb_parent);
  179. if (ret)
  180. dev_warn(dev, "failed to set dram_apb parent: %d\n",
  181. ret);
  182. }
  183. /*
  184. * Explicitly refresh dram PLL rate.
  185. *
  186. * Even if it's marked with CLK_GET_RATE_NOCACHE the rate will not be
  187. * automatically refreshed when clk_get_rate is called on children.
  188. */
  189. clk_get_rate(priv->dram_pll);
  190. /*
  191. * clk_set_parent transfer the reference count from old parent.
  192. * now we drop extra reference counts used during the switch
  193. */
  194. clk_disable_unprepare(new_dram_apb_parent);
  195. out_disable_alt_parent:
  196. clk_disable_unprepare(new_dram_alt_parent);
  197. out_disable_core_parent:
  198. clk_disable_unprepare(new_dram_core_parent);
  199. out:
  200. return ret;
  201. }
  202. static int imx8m_ddrc_target(struct device *dev, unsigned long *freq, u32 flags)
  203. {
  204. struct imx8m_ddrc *priv = dev_get_drvdata(dev);
  205. struct imx8m_ddrc_freq *freq_info;
  206. struct dev_pm_opp *new_opp;
  207. unsigned long old_freq, new_freq;
  208. int ret;
  209. new_opp = devfreq_recommended_opp(dev, freq, flags);
  210. if (IS_ERR(new_opp)) {
  211. ret = PTR_ERR(new_opp);
  212. dev_err(dev, "failed to get recommended opp: %d\n", ret);
  213. return ret;
  214. }
  215. dev_pm_opp_put(new_opp);
  216. old_freq = clk_get_rate(priv->dram_core);
  217. if (*freq == old_freq)
  218. return 0;
  219. freq_info = imx8m_ddrc_find_freq(priv, *freq);
  220. if (!freq_info)
  221. return -EINVAL;
  222. /*
  223. * Read back the clk rate to verify switch was correct and so that
  224. * we can report it on all error paths.
  225. */
  226. ret = imx8m_ddrc_set_freq(dev, freq_info);
  227. new_freq = clk_get_rate(priv->dram_core);
  228. if (ret)
  229. dev_err(dev, "ddrc failed freq switch to %lu from %lu: error %d. now at %lu\n",
  230. *freq, old_freq, ret, new_freq);
  231. else if (*freq != new_freq)
  232. dev_err(dev, "ddrc failed freq update to %lu from %lu, now at %lu\n",
  233. *freq, old_freq, new_freq);
  234. else
  235. dev_dbg(dev, "ddrc freq set to %lu (was %lu)\n",
  236. *freq, old_freq);
  237. return ret;
  238. }
  239. static int imx8m_ddrc_get_cur_freq(struct device *dev, unsigned long *freq)
  240. {
  241. struct imx8m_ddrc *priv = dev_get_drvdata(dev);
  242. *freq = clk_get_rate(priv->dram_core);
  243. return 0;
  244. }
  245. static int imx8m_ddrc_get_dev_status(struct device *dev,
  246. struct devfreq_dev_status *stat)
  247. {
  248. struct imx8m_ddrc *priv = dev_get_drvdata(dev);
  249. stat->busy_time = 0;
  250. stat->total_time = 0;
  251. stat->current_frequency = clk_get_rate(priv->dram_core);
  252. return 0;
  253. }
  254. static int imx8m_ddrc_init_freq_info(struct device *dev)
  255. {
  256. struct imx8m_ddrc *priv = dev_get_drvdata(dev);
  257. struct arm_smccc_res res;
  258. int index;
  259. /* An error here means DDR DVFS API not supported by firmware */
  260. arm_smccc_smc(IMX_SIP_DDR_DVFS, IMX_SIP_DDR_DVFS_GET_FREQ_COUNT,
  261. 0, 0, 0, 0, 0, 0, &res);
  262. priv->freq_count = res.a0;
  263. if (priv->freq_count <= 0 ||
  264. priv->freq_count > IMX8M_DDRC_MAX_FREQ_COUNT)
  265. return -ENODEV;
  266. for (index = 0; index < priv->freq_count; ++index) {
  267. struct imx8m_ddrc_freq *freq = &priv->freq_table[index];
  268. arm_smccc_smc(IMX_SIP_DDR_DVFS, IMX_SIP_DDR_DVFS_GET_FREQ_INFO,
  269. index, 0, 0, 0, 0, 0, &res);
  270. /* Result should be strictly positive */
  271. if ((long)res.a0 <= 0)
  272. return -ENODEV;
  273. freq->rate = res.a0;
  274. freq->smcarg = index;
  275. freq->dram_core_parent_index = res.a1;
  276. freq->dram_alt_parent_index = res.a2;
  277. freq->dram_apb_parent_index = res.a3;
  278. /* dram_core has 2 options: dram_pll or dram_alt_root */
  279. if (freq->dram_core_parent_index != 1 &&
  280. freq->dram_core_parent_index != 2)
  281. return -ENODEV;
  282. /* dram_apb and dram_alt have exactly 8 possible parents */
  283. if (freq->dram_alt_parent_index > 8 ||
  284. freq->dram_apb_parent_index > 8)
  285. return -ENODEV;
  286. /* dram_core from alt requires explicit dram_alt parent */
  287. if (freq->dram_core_parent_index == 2 &&
  288. freq->dram_alt_parent_index == 0)
  289. return -ENODEV;
  290. }
  291. return 0;
  292. }
  293. static int imx8m_ddrc_check_opps(struct device *dev)
  294. {
  295. struct imx8m_ddrc *priv = dev_get_drvdata(dev);
  296. struct imx8m_ddrc_freq *freq_info;
  297. struct dev_pm_opp *opp;
  298. unsigned long freq;
  299. int i, opp_count;
  300. /* Enumerate DT OPPs and disable those not supported by firmware */
  301. opp_count = dev_pm_opp_get_opp_count(dev);
  302. if (opp_count < 0)
  303. return opp_count;
  304. for (i = 0, freq = 0; i < opp_count; ++i, ++freq) {
  305. opp = dev_pm_opp_find_freq_ceil(dev, &freq);
  306. if (IS_ERR(opp)) {
  307. dev_err(dev, "Failed enumerating OPPs: %ld\n",
  308. PTR_ERR(opp));
  309. return PTR_ERR(opp);
  310. }
  311. dev_pm_opp_put(opp);
  312. freq_info = imx8m_ddrc_find_freq(priv, freq);
  313. if (!freq_info) {
  314. dev_info(dev, "Disable unsupported OPP %luHz %luMT/s\n",
  315. freq, DIV_ROUND_CLOSEST(freq, 250000));
  316. dev_pm_opp_disable(dev, freq);
  317. }
  318. }
  319. return 0;
  320. }
  321. static void imx8m_ddrc_exit(struct device *dev)
  322. {
  323. dev_pm_opp_of_remove_table(dev);
  324. }
  325. static int imx8m_ddrc_probe(struct platform_device *pdev)
  326. {
  327. struct device *dev = &pdev->dev;
  328. struct imx8m_ddrc *priv;
  329. const char *gov = DEVFREQ_GOV_USERSPACE;
  330. int ret;
  331. priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
  332. if (!priv)
  333. return -ENOMEM;
  334. platform_set_drvdata(pdev, priv);
  335. ret = imx8m_ddrc_init_freq_info(dev);
  336. if (ret) {
  337. dev_err(dev, "failed to init firmware freq info: %d\n", ret);
  338. return ret;
  339. }
  340. priv->dram_core = devm_clk_get(dev, "core");
  341. if (IS_ERR(priv->dram_core)) {
  342. ret = PTR_ERR(priv->dram_core);
  343. dev_err(dev, "failed to fetch core clock: %d\n", ret);
  344. return ret;
  345. }
  346. priv->dram_pll = devm_clk_get(dev, "pll");
  347. if (IS_ERR(priv->dram_pll)) {
  348. ret = PTR_ERR(priv->dram_pll);
  349. dev_err(dev, "failed to fetch pll clock: %d\n", ret);
  350. return ret;
  351. }
  352. priv->dram_alt = devm_clk_get(dev, "alt");
  353. if (IS_ERR(priv->dram_alt)) {
  354. ret = PTR_ERR(priv->dram_alt);
  355. dev_err(dev, "failed to fetch alt clock: %d\n", ret);
  356. return ret;
  357. }
  358. priv->dram_apb = devm_clk_get(dev, "apb");
  359. if (IS_ERR(priv->dram_apb)) {
  360. ret = PTR_ERR(priv->dram_apb);
  361. dev_err(dev, "failed to fetch apb clock: %d\n", ret);
  362. return ret;
  363. }
  364. ret = dev_pm_opp_of_add_table(dev);
  365. if (ret < 0) {
  366. dev_err(dev, "failed to get OPP table\n");
  367. return ret;
  368. }
  369. ret = imx8m_ddrc_check_opps(dev);
  370. if (ret < 0)
  371. goto err;
  372. priv->profile.polling_ms = 1000;
  373. priv->profile.target = imx8m_ddrc_target;
  374. priv->profile.get_dev_status = imx8m_ddrc_get_dev_status;
  375. priv->profile.exit = imx8m_ddrc_exit;
  376. priv->profile.get_cur_freq = imx8m_ddrc_get_cur_freq;
  377. priv->profile.initial_freq = clk_get_rate(priv->dram_core);
  378. priv->devfreq = devm_devfreq_add_device(dev, &priv->profile,
  379. gov, NULL);
  380. if (IS_ERR(priv->devfreq)) {
  381. ret = PTR_ERR(priv->devfreq);
  382. dev_err(dev, "failed to add devfreq device: %d\n", ret);
  383. goto err;
  384. }
  385. return 0;
  386. err:
  387. dev_pm_opp_of_remove_table(dev);
  388. return ret;
  389. }
  390. static const struct of_device_id imx8m_ddrc_of_match[] = {
  391. { .compatible = "fsl,imx8m-ddrc", },
  392. { /* sentinel */ },
  393. };
  394. MODULE_DEVICE_TABLE(of, imx8m_ddrc_of_match);
  395. static struct platform_driver imx8m_ddrc_platdrv = {
  396. .probe = imx8m_ddrc_probe,
  397. .driver = {
  398. .name = "imx8m-ddrc-devfreq",
  399. .of_match_table = of_match_ptr(imx8m_ddrc_of_match),
  400. },
  401. };
  402. module_platform_driver(imx8m_ddrc_platdrv);
  403. MODULE_DESCRIPTION("i.MX8M DDR Controller frequency driver");
  404. MODULE_AUTHOR("Leonard Crestez <leonard.crestez@nxp.com>");
  405. MODULE_LICENSE("GPL v2");