numa.c 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * NUMA support, based on the x86 implementation.
  4. *
  5. * Copyright (C) 2015 Cavium Inc.
  6. * Author: Ganapatrao Kulkarni <gkulkarni@cavium.com>
  7. */
  8. #define pr_fmt(fmt) "NUMA: " fmt
  9. #include <linux/acpi.h>
  10. #include <linux/memblock.h>
  11. #include <linux/module.h>
  12. #include <linux/of.h>
  13. #include <asm/acpi.h>
  14. #include <asm/sections.h>
  15. struct pglist_data *node_data[MAX_NUMNODES] __read_mostly;
  16. EXPORT_SYMBOL(node_data);
  17. nodemask_t numa_nodes_parsed __initdata;
  18. static int cpu_to_node_map[NR_CPUS] = { [0 ... NR_CPUS-1] = NUMA_NO_NODE };
  19. static int numa_distance_cnt;
  20. static u8 *numa_distance;
  21. bool numa_off;
  22. static __init int numa_parse_early_param(char *opt)
  23. {
  24. if (!opt)
  25. return -EINVAL;
  26. if (str_has_prefix(opt, "off"))
  27. numa_off = true;
  28. return 0;
  29. }
  30. early_param("numa", numa_parse_early_param);
  31. cpumask_var_t node_to_cpumask_map[MAX_NUMNODES];
  32. EXPORT_SYMBOL(node_to_cpumask_map);
  33. #ifdef CONFIG_DEBUG_PER_CPU_MAPS
  34. /*
  35. * Returns a pointer to the bitmask of CPUs on Node 'node'.
  36. */
  37. const struct cpumask *cpumask_of_node(int node)
  38. {
  39. if (node == NUMA_NO_NODE)
  40. return cpu_all_mask;
  41. if (WARN_ON(node < 0 || node >= nr_node_ids))
  42. return cpu_none_mask;
  43. if (WARN_ON(node_to_cpumask_map[node] == NULL))
  44. return cpu_online_mask;
  45. return node_to_cpumask_map[node];
  46. }
  47. EXPORT_SYMBOL(cpumask_of_node);
  48. #endif
  49. static void numa_update_cpu(unsigned int cpu, bool remove)
  50. {
  51. int nid = cpu_to_node(cpu);
  52. if (nid == NUMA_NO_NODE)
  53. return;
  54. if (remove)
  55. cpumask_clear_cpu(cpu, node_to_cpumask_map[nid]);
  56. else
  57. cpumask_set_cpu(cpu, node_to_cpumask_map[nid]);
  58. }
  59. void numa_add_cpu(unsigned int cpu)
  60. {
  61. numa_update_cpu(cpu, false);
  62. }
  63. void numa_remove_cpu(unsigned int cpu)
  64. {
  65. numa_update_cpu(cpu, true);
  66. }
  67. void numa_clear_node(unsigned int cpu)
  68. {
  69. numa_remove_cpu(cpu);
  70. set_cpu_numa_node(cpu, NUMA_NO_NODE);
  71. }
  72. /*
  73. * Allocate node_to_cpumask_map based on number of available nodes
  74. * Requires node_possible_map to be valid.
  75. *
  76. * Note: cpumask_of_node() is not valid until after this is done.
  77. * (Use CONFIG_DEBUG_PER_CPU_MAPS to check this.)
  78. */
  79. static void __init setup_node_to_cpumask_map(void)
  80. {
  81. int node;
  82. /* setup nr_node_ids if not done yet */
  83. if (nr_node_ids == MAX_NUMNODES)
  84. setup_nr_node_ids();
  85. /* allocate and clear the mapping */
  86. for (node = 0; node < nr_node_ids; node++) {
  87. alloc_bootmem_cpumask_var(&node_to_cpumask_map[node]);
  88. cpumask_clear(node_to_cpumask_map[node]);
  89. }
  90. /* cpumask_of_node() will now work */
  91. pr_debug("Node to cpumask map for %u nodes\n", nr_node_ids);
  92. }
  93. /*
  94. * Set the cpu to node and mem mapping
  95. */
  96. void numa_store_cpu_info(unsigned int cpu)
  97. {
  98. set_cpu_numa_node(cpu, cpu_to_node_map[cpu]);
  99. }
  100. void __init early_map_cpu_to_node(unsigned int cpu, int nid)
  101. {
  102. /* fallback to node 0 */
  103. if (nid < 0 || nid >= MAX_NUMNODES || numa_off)
  104. nid = 0;
  105. cpu_to_node_map[cpu] = nid;
  106. /*
  107. * We should set the numa node of cpu0 as soon as possible, because it
  108. * has already been set up online before. cpu_to_node(0) will soon be
  109. * called.
  110. */
  111. if (!cpu)
  112. set_cpu_numa_node(cpu, nid);
  113. }
  114. #ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
  115. unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;
  116. EXPORT_SYMBOL(__per_cpu_offset);
  117. static int __init early_cpu_to_node(int cpu)
  118. {
  119. return cpu_to_node_map[cpu];
  120. }
  121. static int __init pcpu_cpu_distance(unsigned int from, unsigned int to)
  122. {
  123. return node_distance(early_cpu_to_node(from), early_cpu_to_node(to));
  124. }
  125. static void * __init pcpu_fc_alloc(unsigned int cpu, size_t size,
  126. size_t align)
  127. {
  128. int nid = early_cpu_to_node(cpu);
  129. return memblock_alloc_try_nid(size, align,
  130. __pa(MAX_DMA_ADDRESS), MEMBLOCK_ALLOC_ACCESSIBLE, nid);
  131. }
  132. static void __init pcpu_fc_free(void *ptr, size_t size)
  133. {
  134. memblock_free_early(__pa(ptr), size);
  135. }
  136. void __init setup_per_cpu_areas(void)
  137. {
  138. unsigned long delta;
  139. unsigned int cpu;
  140. int rc;
  141. /*
  142. * Always reserve area for module percpu variables. That's
  143. * what the legacy allocator did.
  144. */
  145. rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE,
  146. PERCPU_DYNAMIC_RESERVE, PAGE_SIZE,
  147. pcpu_cpu_distance,
  148. pcpu_fc_alloc, pcpu_fc_free);
  149. if (rc < 0)
  150. panic("Failed to initialize percpu areas.");
  151. delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start;
  152. for_each_possible_cpu(cpu)
  153. __per_cpu_offset[cpu] = delta + pcpu_unit_offsets[cpu];
  154. }
  155. #endif
  156. /**
  157. * numa_add_memblk() - Set node id to memblk
  158. * @nid: NUMA node ID of the new memblk
  159. * @start: Start address of the new memblk
  160. * @end: End address of the new memblk
  161. *
  162. * RETURNS:
  163. * 0 on success, -errno on failure.
  164. */
  165. int __init numa_add_memblk(int nid, u64 start, u64 end)
  166. {
  167. int ret;
  168. ret = memblock_set_node(start, (end - start), &memblock.memory, nid);
  169. if (ret < 0) {
  170. pr_err("memblock [0x%llx - 0x%llx] failed to add on node %d\n",
  171. start, (end - 1), nid);
  172. return ret;
  173. }
  174. node_set(nid, numa_nodes_parsed);
  175. return ret;
  176. }
  177. /*
  178. * Initialize NODE_DATA for a node on the local memory
  179. */
  180. static void __init setup_node_data(int nid, u64 start_pfn, u64 end_pfn)
  181. {
  182. const size_t nd_size = roundup(sizeof(pg_data_t), SMP_CACHE_BYTES);
  183. u64 nd_pa;
  184. void *nd;
  185. int tnid;
  186. if (start_pfn >= end_pfn)
  187. pr_info("Initmem setup node %d [<memory-less node>]\n", nid);
  188. nd_pa = memblock_phys_alloc_try_nid(nd_size, SMP_CACHE_BYTES, nid);
  189. if (!nd_pa)
  190. panic("Cannot allocate %zu bytes for node %d data\n",
  191. nd_size, nid);
  192. nd = __va(nd_pa);
  193. /* report and initialize */
  194. pr_info("NODE_DATA [mem %#010Lx-%#010Lx]\n",
  195. nd_pa, nd_pa + nd_size - 1);
  196. tnid = early_pfn_to_nid(nd_pa >> PAGE_SHIFT);
  197. if (tnid != nid)
  198. pr_info("NODE_DATA(%d) on node %d\n", nid, tnid);
  199. node_data[nid] = nd;
  200. memset(NODE_DATA(nid), 0, sizeof(pg_data_t));
  201. NODE_DATA(nid)->node_id = nid;
  202. NODE_DATA(nid)->node_start_pfn = start_pfn;
  203. NODE_DATA(nid)->node_spanned_pages = end_pfn - start_pfn;
  204. }
  205. /*
  206. * numa_free_distance
  207. *
  208. * The current table is freed.
  209. */
  210. void __init numa_free_distance(void)
  211. {
  212. size_t size;
  213. if (!numa_distance)
  214. return;
  215. size = numa_distance_cnt * numa_distance_cnt *
  216. sizeof(numa_distance[0]);
  217. memblock_free(__pa(numa_distance), size);
  218. numa_distance_cnt = 0;
  219. numa_distance = NULL;
  220. }
  221. /*
  222. * Create a new NUMA distance table.
  223. */
  224. static int __init numa_alloc_distance(void)
  225. {
  226. size_t size;
  227. u64 phys;
  228. int i, j;
  229. size = nr_node_ids * nr_node_ids * sizeof(numa_distance[0]);
  230. phys = memblock_find_in_range(0, PFN_PHYS(max_pfn),
  231. size, PAGE_SIZE);
  232. if (WARN_ON(!phys))
  233. return -ENOMEM;
  234. memblock_reserve(phys, size);
  235. numa_distance = __va(phys);
  236. numa_distance_cnt = nr_node_ids;
  237. /* fill with the default distances */
  238. for (i = 0; i < numa_distance_cnt; i++)
  239. for (j = 0; j < numa_distance_cnt; j++)
  240. numa_distance[i * numa_distance_cnt + j] = i == j ?
  241. LOCAL_DISTANCE : REMOTE_DISTANCE;
  242. pr_debug("Initialized distance table, cnt=%d\n", numa_distance_cnt);
  243. return 0;
  244. }
  245. /**
  246. * numa_set_distance() - Set inter node NUMA distance from node to node.
  247. * @from: the 'from' node to set distance
  248. * @to: the 'to' node to set distance
  249. * @distance: NUMA distance
  250. *
  251. * Set the distance from node @from to @to to @distance.
  252. * If distance table doesn't exist, a warning is printed.
  253. *
  254. * If @from or @to is higher than the highest known node or lower than zero
  255. * or @distance doesn't make sense, the call is ignored.
  256. */
  257. void __init numa_set_distance(int from, int to, int distance)
  258. {
  259. if (!numa_distance) {
  260. pr_warn_once("Warning: distance table not allocated yet\n");
  261. return;
  262. }
  263. if (from >= numa_distance_cnt || to >= numa_distance_cnt ||
  264. from < 0 || to < 0) {
  265. pr_warn_once("Warning: node ids are out of bound, from=%d to=%d distance=%d\n",
  266. from, to, distance);
  267. return;
  268. }
  269. if ((u8)distance != distance ||
  270. (from == to && distance != LOCAL_DISTANCE)) {
  271. pr_warn_once("Warning: invalid distance parameter, from=%d to=%d distance=%d\n",
  272. from, to, distance);
  273. return;
  274. }
  275. numa_distance[from * numa_distance_cnt + to] = distance;
  276. }
  277. /*
  278. * Return NUMA distance @from to @to
  279. */
  280. int __node_distance(int from, int to)
  281. {
  282. if (from >= numa_distance_cnt || to >= numa_distance_cnt)
  283. return from == to ? LOCAL_DISTANCE : REMOTE_DISTANCE;
  284. return numa_distance[from * numa_distance_cnt + to];
  285. }
  286. EXPORT_SYMBOL(__node_distance);
  287. static int __init numa_register_nodes(void)
  288. {
  289. int nid;
  290. struct memblock_region *mblk;
  291. /* Check that valid nid is set to memblks */
  292. for_each_mem_region(mblk) {
  293. int mblk_nid = memblock_get_region_node(mblk);
  294. if (mblk_nid == NUMA_NO_NODE || mblk_nid >= MAX_NUMNODES) {
  295. pr_warn("Warning: invalid memblk node %d [mem %#010Lx-%#010Lx]\n",
  296. mblk_nid, mblk->base,
  297. mblk->base + mblk->size - 1);
  298. return -EINVAL;
  299. }
  300. }
  301. /* Finally register nodes. */
  302. for_each_node_mask(nid, numa_nodes_parsed) {
  303. unsigned long start_pfn, end_pfn;
  304. get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);
  305. setup_node_data(nid, start_pfn, end_pfn);
  306. node_set_online(nid);
  307. }
  308. /* Setup online nodes to actual nodes*/
  309. node_possible_map = numa_nodes_parsed;
  310. return 0;
  311. }
  312. static int __init numa_init(int (*init_func)(void))
  313. {
  314. int ret;
  315. nodes_clear(numa_nodes_parsed);
  316. nodes_clear(node_possible_map);
  317. nodes_clear(node_online_map);
  318. ret = numa_alloc_distance();
  319. if (ret < 0)
  320. return ret;
  321. ret = init_func();
  322. if (ret < 0)
  323. goto out_free_distance;
  324. if (nodes_empty(numa_nodes_parsed)) {
  325. pr_info("No NUMA configuration found\n");
  326. ret = -EINVAL;
  327. goto out_free_distance;
  328. }
  329. ret = numa_register_nodes();
  330. if (ret < 0)
  331. goto out_free_distance;
  332. setup_node_to_cpumask_map();
  333. return 0;
  334. out_free_distance:
  335. numa_free_distance();
  336. return ret;
  337. }
  338. /**
  339. * dummy_numa_init() - Fallback dummy NUMA init
  340. *
  341. * Used if there's no underlying NUMA architecture, NUMA initialization
  342. * fails, or NUMA is disabled on the command line.
  343. *
  344. * Must online at least one node (node 0) and add memory blocks that cover all
  345. * allowed memory. It is unlikely that this function fails.
  346. *
  347. * Return: 0 on success, -errno on failure.
  348. */
  349. static int __init dummy_numa_init(void)
  350. {
  351. phys_addr_t start = memblock_start_of_DRAM();
  352. phys_addr_t end = memblock_end_of_DRAM();
  353. int ret;
  354. if (numa_off)
  355. pr_info("NUMA disabled\n"); /* Forced off on command line. */
  356. pr_info("Faking a node at [mem %#018Lx-%#018Lx]\n", start, end - 1);
  357. ret = numa_add_memblk(0, start, end);
  358. if (ret) {
  359. pr_err("NUMA init failed\n");
  360. return ret;
  361. }
  362. numa_off = true;
  363. return 0;
  364. }
  365. /**
  366. * arm64_numa_init() - Initialize NUMA
  367. *
  368. * Try each configured NUMA initialization method until one succeeds. The
  369. * last fallback is dummy single node config encompassing whole memory.
  370. */
  371. void __init arm64_numa_init(void)
  372. {
  373. if (!numa_off) {
  374. if (!acpi_disabled && !numa_init(arm64_acpi_numa_init))
  375. return;
  376. if (acpi_disabled && !numa_init(of_numa_init))
  377. return;
  378. }
  379. numa_init(dummy_numa_init);
  380. }