node.c 28 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Basic Node interface support
  4. */
  5. #include <linux/module.h>
  6. #include <linux/init.h>
  7. #include <linux/mm.h>
  8. #include <linux/memory.h>
  9. #include <linux/vmstat.h>
  10. #include <linux/notifier.h>
  11. #include <linux/node.h>
  12. #include <linux/hugetlb.h>
  13. #include <linux/compaction.h>
  14. #include <linux/cpumask.h>
  15. #include <linux/topology.h>
  16. #include <linux/nodemask.h>
  17. #include <linux/cpu.h>
  18. #include <linux/device.h>
  19. #include <linux/pm_runtime.h>
  20. #include <linux/swap.h>
  21. #include <linux/slab.h>
  22. static struct bus_type node_subsys = {
  23. .name = "node",
  24. .dev_name = "node",
  25. };
  26. static ssize_t node_read_cpumap(struct device *dev, bool list, char *buf)
  27. {
  28. ssize_t n;
  29. cpumask_var_t mask;
  30. struct node *node_dev = to_node(dev);
  31. /* 2008/04/07: buf currently PAGE_SIZE, need 9 chars per 32 bits. */
  32. BUILD_BUG_ON((NR_CPUS/32 * 9) > (PAGE_SIZE-1));
  33. if (!alloc_cpumask_var(&mask, GFP_KERNEL))
  34. return 0;
  35. cpumask_and(mask, cpumask_of_node(node_dev->dev.id), cpu_online_mask);
  36. n = cpumap_print_to_pagebuf(list, buf, mask);
  37. free_cpumask_var(mask);
  38. return n;
  39. }
  40. static inline ssize_t cpumap_show(struct device *dev,
  41. struct device_attribute *attr,
  42. char *buf)
  43. {
  44. return node_read_cpumap(dev, false, buf);
  45. }
  46. static DEVICE_ATTR_RO(cpumap);
  47. static inline ssize_t cpulist_show(struct device *dev,
  48. struct device_attribute *attr,
  49. char *buf)
  50. {
  51. return node_read_cpumap(dev, true, buf);
  52. }
  53. static DEVICE_ATTR_RO(cpulist);
  54. /**
  55. * struct node_access_nodes - Access class device to hold user visible
  56. * relationships to other nodes.
  57. * @dev: Device for this memory access class
  58. * @list_node: List element in the node's access list
  59. * @access: The access class rank
  60. * @hmem_attrs: Heterogeneous memory performance attributes
  61. */
  62. struct node_access_nodes {
  63. struct device dev;
  64. struct list_head list_node;
  65. unsigned access;
  66. #ifdef CONFIG_HMEM_REPORTING
  67. struct node_hmem_attrs hmem_attrs;
  68. #endif
  69. };
  70. #define to_access_nodes(dev) container_of(dev, struct node_access_nodes, dev)
  71. static struct attribute *node_init_access_node_attrs[] = {
  72. NULL,
  73. };
  74. static struct attribute *node_targ_access_node_attrs[] = {
  75. NULL,
  76. };
  77. static const struct attribute_group initiators = {
  78. .name = "initiators",
  79. .attrs = node_init_access_node_attrs,
  80. };
  81. static const struct attribute_group targets = {
  82. .name = "targets",
  83. .attrs = node_targ_access_node_attrs,
  84. };
  85. static const struct attribute_group *node_access_node_groups[] = {
  86. &initiators,
  87. &targets,
  88. NULL,
  89. };
  90. static void node_remove_accesses(struct node *node)
  91. {
  92. struct node_access_nodes *c, *cnext;
  93. list_for_each_entry_safe(c, cnext, &node->access_list, list_node) {
  94. list_del(&c->list_node);
  95. device_unregister(&c->dev);
  96. }
  97. }
  98. static void node_access_release(struct device *dev)
  99. {
  100. kfree(to_access_nodes(dev));
  101. }
  102. static struct node_access_nodes *node_init_node_access(struct node *node,
  103. unsigned access)
  104. {
  105. struct node_access_nodes *access_node;
  106. struct device *dev;
  107. list_for_each_entry(access_node, &node->access_list, list_node)
  108. if (access_node->access == access)
  109. return access_node;
  110. access_node = kzalloc(sizeof(*access_node), GFP_KERNEL);
  111. if (!access_node)
  112. return NULL;
  113. access_node->access = access;
  114. dev = &access_node->dev;
  115. dev->parent = &node->dev;
  116. dev->release = node_access_release;
  117. dev->groups = node_access_node_groups;
  118. if (dev_set_name(dev, "access%u", access))
  119. goto free;
  120. if (device_register(dev))
  121. goto free_name;
  122. pm_runtime_no_callbacks(dev);
  123. list_add_tail(&access_node->list_node, &node->access_list);
  124. return access_node;
  125. free_name:
  126. kfree_const(dev->kobj.name);
  127. free:
  128. kfree(access_node);
  129. return NULL;
  130. }
  131. #ifdef CONFIG_HMEM_REPORTING
  132. #define ACCESS_ATTR(name) \
  133. static ssize_t name##_show(struct device *dev, \
  134. struct device_attribute *attr, \
  135. char *buf) \
  136. { \
  137. return sysfs_emit(buf, "%u\n", \
  138. to_access_nodes(dev)->hmem_attrs.name); \
  139. } \
  140. static DEVICE_ATTR_RO(name)
  141. ACCESS_ATTR(read_bandwidth);
  142. ACCESS_ATTR(read_latency);
  143. ACCESS_ATTR(write_bandwidth);
  144. ACCESS_ATTR(write_latency);
  145. static struct attribute *access_attrs[] = {
  146. &dev_attr_read_bandwidth.attr,
  147. &dev_attr_read_latency.attr,
  148. &dev_attr_write_bandwidth.attr,
  149. &dev_attr_write_latency.attr,
  150. NULL,
  151. };
  152. /**
  153. * node_set_perf_attrs - Set the performance values for given access class
  154. * @nid: Node identifier to be set
  155. * @hmem_attrs: Heterogeneous memory performance attributes
  156. * @access: The access class the for the given attributes
  157. */
  158. void node_set_perf_attrs(unsigned int nid, struct node_hmem_attrs *hmem_attrs,
  159. unsigned access)
  160. {
  161. struct node_access_nodes *c;
  162. struct node *node;
  163. int i;
  164. if (WARN_ON_ONCE(!node_online(nid)))
  165. return;
  166. node = node_devices[nid];
  167. c = node_init_node_access(node, access);
  168. if (!c)
  169. return;
  170. c->hmem_attrs = *hmem_attrs;
  171. for (i = 0; access_attrs[i] != NULL; i++) {
  172. if (sysfs_add_file_to_group(&c->dev.kobj, access_attrs[i],
  173. "initiators")) {
  174. pr_info("failed to add performance attribute to node %d\n",
  175. nid);
  176. break;
  177. }
  178. }
  179. }
  180. /**
  181. * struct node_cache_info - Internal tracking for memory node caches
  182. * @dev: Device represeting the cache level
  183. * @node: List element for tracking in the node
  184. * @cache_attrs:Attributes for this cache level
  185. */
  186. struct node_cache_info {
  187. struct device dev;
  188. struct list_head node;
  189. struct node_cache_attrs cache_attrs;
  190. };
  191. #define to_cache_info(device) container_of(device, struct node_cache_info, dev)
  192. #define CACHE_ATTR(name, fmt) \
  193. static ssize_t name##_show(struct device *dev, \
  194. struct device_attribute *attr, \
  195. char *buf) \
  196. { \
  197. return sysfs_emit(buf, fmt "\n", \
  198. to_cache_info(dev)->cache_attrs.name); \
  199. } \
  200. DEVICE_ATTR_RO(name);
  201. CACHE_ATTR(size, "%llu")
  202. CACHE_ATTR(line_size, "%u")
  203. CACHE_ATTR(indexing, "%u")
  204. CACHE_ATTR(write_policy, "%u")
  205. static struct attribute *cache_attrs[] = {
  206. &dev_attr_indexing.attr,
  207. &dev_attr_size.attr,
  208. &dev_attr_line_size.attr,
  209. &dev_attr_write_policy.attr,
  210. NULL,
  211. };
  212. ATTRIBUTE_GROUPS(cache);
  213. static void node_cache_release(struct device *dev)
  214. {
  215. kfree(dev);
  216. }
  217. static void node_cacheinfo_release(struct device *dev)
  218. {
  219. struct node_cache_info *info = to_cache_info(dev);
  220. kfree(info);
  221. }
  222. static void node_init_cache_dev(struct node *node)
  223. {
  224. struct device *dev;
  225. dev = kzalloc(sizeof(*dev), GFP_KERNEL);
  226. if (!dev)
  227. return;
  228. device_initialize(dev);
  229. dev->parent = &node->dev;
  230. dev->release = node_cache_release;
  231. if (dev_set_name(dev, "memory_side_cache"))
  232. goto put_device;
  233. if (device_add(dev))
  234. goto put_device;
  235. pm_runtime_no_callbacks(dev);
  236. node->cache_dev = dev;
  237. return;
  238. put_device:
  239. put_device(dev);
  240. }
  241. /**
  242. * node_add_cache() - add cache attribute to a memory node
  243. * @nid: Node identifier that has new cache attributes
  244. * @cache_attrs: Attributes for the cache being added
  245. */
  246. void node_add_cache(unsigned int nid, struct node_cache_attrs *cache_attrs)
  247. {
  248. struct node_cache_info *info;
  249. struct device *dev;
  250. struct node *node;
  251. if (!node_online(nid) || !node_devices[nid])
  252. return;
  253. node = node_devices[nid];
  254. list_for_each_entry(info, &node->cache_attrs, node) {
  255. if (info->cache_attrs.level == cache_attrs->level) {
  256. dev_warn(&node->dev,
  257. "attempt to add duplicate cache level:%d\n",
  258. cache_attrs->level);
  259. return;
  260. }
  261. }
  262. if (!node->cache_dev)
  263. node_init_cache_dev(node);
  264. if (!node->cache_dev)
  265. return;
  266. info = kzalloc(sizeof(*info), GFP_KERNEL);
  267. if (!info)
  268. return;
  269. dev = &info->dev;
  270. device_initialize(dev);
  271. dev->parent = node->cache_dev;
  272. dev->release = node_cacheinfo_release;
  273. dev->groups = cache_groups;
  274. if (dev_set_name(dev, "index%d", cache_attrs->level))
  275. goto put_device;
  276. info->cache_attrs = *cache_attrs;
  277. if (device_add(dev)) {
  278. dev_warn(&node->dev, "failed to add cache level:%d\n",
  279. cache_attrs->level);
  280. goto put_device;
  281. }
  282. pm_runtime_no_callbacks(dev);
  283. list_add_tail(&info->node, &node->cache_attrs);
  284. return;
  285. put_device:
  286. put_device(dev);
  287. }
  288. static void node_remove_caches(struct node *node)
  289. {
  290. struct node_cache_info *info, *next;
  291. if (!node->cache_dev)
  292. return;
  293. list_for_each_entry_safe(info, next, &node->cache_attrs, node) {
  294. list_del(&info->node);
  295. device_unregister(&info->dev);
  296. }
  297. device_unregister(node->cache_dev);
  298. }
  299. static void node_init_caches(unsigned int nid)
  300. {
  301. INIT_LIST_HEAD(&node_devices[nid]->cache_attrs);
  302. }
  303. #else
  304. static void node_init_caches(unsigned int nid) { }
  305. static void node_remove_caches(struct node *node) { }
  306. #endif
  307. #define K(x) ((x) << (PAGE_SHIFT - 10))
  308. static ssize_t node_read_meminfo(struct device *dev,
  309. struct device_attribute *attr, char *buf)
  310. {
  311. int len = 0;
  312. int nid = dev->id;
  313. struct pglist_data *pgdat = NODE_DATA(nid);
  314. struct sysinfo i;
  315. unsigned long sreclaimable, sunreclaimable;
  316. si_meminfo_node(&i, nid);
  317. sreclaimable = node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B);
  318. sunreclaimable = node_page_state_pages(pgdat, NR_SLAB_UNRECLAIMABLE_B);
  319. len = sysfs_emit_at(buf, len,
  320. "Node %d MemTotal: %8lu kB\n"
  321. "Node %d MemFree: %8lu kB\n"
  322. "Node %d MemUsed: %8lu kB\n"
  323. "Node %d Active: %8lu kB\n"
  324. "Node %d Inactive: %8lu kB\n"
  325. "Node %d Active(anon): %8lu kB\n"
  326. "Node %d Inactive(anon): %8lu kB\n"
  327. "Node %d Active(file): %8lu kB\n"
  328. "Node %d Inactive(file): %8lu kB\n"
  329. "Node %d Unevictable: %8lu kB\n"
  330. "Node %d Mlocked: %8lu kB\n",
  331. nid, K(i.totalram),
  332. nid, K(i.freeram),
  333. nid, K(i.totalram - i.freeram),
  334. nid, K(node_page_state(pgdat, NR_ACTIVE_ANON) +
  335. node_page_state(pgdat, NR_ACTIVE_FILE)),
  336. nid, K(node_page_state(pgdat, NR_INACTIVE_ANON) +
  337. node_page_state(pgdat, NR_INACTIVE_FILE)),
  338. nid, K(node_page_state(pgdat, NR_ACTIVE_ANON)),
  339. nid, K(node_page_state(pgdat, NR_INACTIVE_ANON)),
  340. nid, K(node_page_state(pgdat, NR_ACTIVE_FILE)),
  341. nid, K(node_page_state(pgdat, NR_INACTIVE_FILE)),
  342. nid, K(node_page_state(pgdat, NR_UNEVICTABLE)),
  343. nid, K(sum_zone_node_page_state(nid, NR_MLOCK)));
  344. #ifdef CONFIG_HIGHMEM
  345. len += sysfs_emit_at(buf, len,
  346. "Node %d HighTotal: %8lu kB\n"
  347. "Node %d HighFree: %8lu kB\n"
  348. "Node %d LowTotal: %8lu kB\n"
  349. "Node %d LowFree: %8lu kB\n",
  350. nid, K(i.totalhigh),
  351. nid, K(i.freehigh),
  352. nid, K(i.totalram - i.totalhigh),
  353. nid, K(i.freeram - i.freehigh));
  354. #endif
  355. len += sysfs_emit_at(buf, len,
  356. "Node %d Dirty: %8lu kB\n"
  357. "Node %d Writeback: %8lu kB\n"
  358. "Node %d FilePages: %8lu kB\n"
  359. "Node %d Mapped: %8lu kB\n"
  360. "Node %d AnonPages: %8lu kB\n"
  361. "Node %d Shmem: %8lu kB\n"
  362. "Node %d KernelStack: %8lu kB\n"
  363. #ifdef CONFIG_SHADOW_CALL_STACK
  364. "Node %d ShadowCallStack:%8lu kB\n"
  365. #endif
  366. "Node %d PageTables: %8lu kB\n"
  367. "Node %d NFS_Unstable: %8lu kB\n"
  368. "Node %d Bounce: %8lu kB\n"
  369. "Node %d WritebackTmp: %8lu kB\n"
  370. "Node %d KReclaimable: %8lu kB\n"
  371. "Node %d Slab: %8lu kB\n"
  372. "Node %d SReclaimable: %8lu kB\n"
  373. "Node %d SUnreclaim: %8lu kB\n"
  374. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  375. "Node %d AnonHugePages: %8lu kB\n"
  376. "Node %d ShmemHugePages: %8lu kB\n"
  377. "Node %d ShmemPmdMapped: %8lu kB\n"
  378. "Node %d FileHugePages: %8lu kB\n"
  379. "Node %d FilePmdMapped: %8lu kB\n"
  380. #endif
  381. ,
  382. nid, K(node_page_state(pgdat, NR_FILE_DIRTY)),
  383. nid, K(node_page_state(pgdat, NR_WRITEBACK)),
  384. nid, K(node_page_state(pgdat, NR_FILE_PAGES)),
  385. nid, K(node_page_state(pgdat, NR_FILE_MAPPED)),
  386. nid, K(node_page_state(pgdat, NR_ANON_MAPPED)),
  387. nid, K(i.sharedram),
  388. nid, node_page_state(pgdat, NR_KERNEL_STACK_KB),
  389. #ifdef CONFIG_SHADOW_CALL_STACK
  390. nid, node_page_state(pgdat, NR_KERNEL_SCS_KB),
  391. #endif
  392. nid, K(sum_zone_node_page_state(nid, NR_PAGETABLE)),
  393. nid, 0UL,
  394. nid, K(sum_zone_node_page_state(nid, NR_BOUNCE)),
  395. nid, K(node_page_state(pgdat, NR_WRITEBACK_TEMP)),
  396. nid, K(sreclaimable +
  397. node_page_state(pgdat, NR_KERNEL_MISC_RECLAIMABLE)),
  398. nid, K(sreclaimable + sunreclaimable),
  399. nid, K(sreclaimable),
  400. nid, K(sunreclaimable)
  401. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  402. ,
  403. nid, K(node_page_state(pgdat, NR_ANON_THPS) *
  404. HPAGE_PMD_NR),
  405. nid, K(node_page_state(pgdat, NR_SHMEM_THPS) *
  406. HPAGE_PMD_NR),
  407. nid, K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED) *
  408. HPAGE_PMD_NR),
  409. nid, K(node_page_state(pgdat, NR_FILE_THPS) *
  410. HPAGE_PMD_NR),
  411. nid, K(node_page_state(pgdat, NR_FILE_PMDMAPPED) *
  412. HPAGE_PMD_NR)
  413. #endif
  414. );
  415. len += hugetlb_report_node_meminfo(buf, len, nid);
  416. return len;
  417. }
  418. #undef K
  419. static DEVICE_ATTR(meminfo, 0444, node_read_meminfo, NULL);
  420. static ssize_t node_read_numastat(struct device *dev,
  421. struct device_attribute *attr, char *buf)
  422. {
  423. return sysfs_emit(buf,
  424. "numa_hit %lu\n"
  425. "numa_miss %lu\n"
  426. "numa_foreign %lu\n"
  427. "interleave_hit %lu\n"
  428. "local_node %lu\n"
  429. "other_node %lu\n",
  430. sum_zone_numa_state(dev->id, NUMA_HIT),
  431. sum_zone_numa_state(dev->id, NUMA_MISS),
  432. sum_zone_numa_state(dev->id, NUMA_FOREIGN),
  433. sum_zone_numa_state(dev->id, NUMA_INTERLEAVE_HIT),
  434. sum_zone_numa_state(dev->id, NUMA_LOCAL),
  435. sum_zone_numa_state(dev->id, NUMA_OTHER));
  436. }
  437. static DEVICE_ATTR(numastat, 0444, node_read_numastat, NULL);
  438. static ssize_t node_read_vmstat(struct device *dev,
  439. struct device_attribute *attr, char *buf)
  440. {
  441. int nid = dev->id;
  442. struct pglist_data *pgdat = NODE_DATA(nid);
  443. int i;
  444. int len = 0;
  445. for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
  446. len += sysfs_emit_at(buf, len, "%s %lu\n",
  447. zone_stat_name(i),
  448. sum_zone_node_page_state(nid, i));
  449. #ifdef CONFIG_NUMA
  450. for (i = 0; i < NR_VM_NUMA_STAT_ITEMS; i++)
  451. len += sysfs_emit_at(buf, len, "%s %lu\n",
  452. numa_stat_name(i),
  453. sum_zone_numa_state(nid, i));
  454. #endif
  455. for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
  456. len += sysfs_emit_at(buf, len, "%s %lu\n",
  457. node_stat_name(i),
  458. node_page_state_pages(pgdat, i));
  459. return len;
  460. }
  461. static DEVICE_ATTR(vmstat, 0444, node_read_vmstat, NULL);
  462. static ssize_t node_read_distance(struct device *dev,
  463. struct device_attribute *attr, char *buf)
  464. {
  465. int nid = dev->id;
  466. int len = 0;
  467. int i;
  468. /*
  469. * buf is currently PAGE_SIZE in length and each node needs 4 chars
  470. * at the most (distance + space or newline).
  471. */
  472. BUILD_BUG_ON(MAX_NUMNODES * 4 > PAGE_SIZE);
  473. for_each_online_node(i) {
  474. len += sysfs_emit_at(buf, len, "%s%d",
  475. i ? " " : "", node_distance(nid, i));
  476. }
  477. len += sysfs_emit_at(buf, len, "\n");
  478. return len;
  479. }
  480. static DEVICE_ATTR(distance, 0444, node_read_distance, NULL);
  481. static struct attribute *node_dev_attrs[] = {
  482. &dev_attr_cpumap.attr,
  483. &dev_attr_cpulist.attr,
  484. &dev_attr_meminfo.attr,
  485. &dev_attr_numastat.attr,
  486. &dev_attr_distance.attr,
  487. &dev_attr_vmstat.attr,
  488. NULL
  489. };
  490. ATTRIBUTE_GROUPS(node_dev);
  491. #ifdef CONFIG_HUGETLBFS
  492. /*
  493. * hugetlbfs per node attributes registration interface:
  494. * When/if hugetlb[fs] subsystem initializes [sometime after this module],
  495. * it will register its per node attributes for all online nodes with
  496. * memory. It will also call register_hugetlbfs_with_node(), below, to
  497. * register its attribute registration functions with this node driver.
  498. * Once these hooks have been initialized, the node driver will call into
  499. * the hugetlb module to [un]register attributes for hot-plugged nodes.
  500. */
  501. static node_registration_func_t __hugetlb_register_node;
  502. static node_registration_func_t __hugetlb_unregister_node;
  503. static inline bool hugetlb_register_node(struct node *node)
  504. {
  505. if (__hugetlb_register_node &&
  506. node_state(node->dev.id, N_MEMORY)) {
  507. __hugetlb_register_node(node);
  508. return true;
  509. }
  510. return false;
  511. }
  512. static inline void hugetlb_unregister_node(struct node *node)
  513. {
  514. if (__hugetlb_unregister_node)
  515. __hugetlb_unregister_node(node);
  516. }
  517. void register_hugetlbfs_with_node(node_registration_func_t doregister,
  518. node_registration_func_t unregister)
  519. {
  520. __hugetlb_register_node = doregister;
  521. __hugetlb_unregister_node = unregister;
  522. }
  523. #else
  524. static inline void hugetlb_register_node(struct node *node) {}
  525. static inline void hugetlb_unregister_node(struct node *node) {}
  526. #endif
  527. static void node_device_release(struct device *dev)
  528. {
  529. struct node *node = to_node(dev);
  530. #if defined(CONFIG_MEMORY_HOTPLUG_SPARSE) && defined(CONFIG_HUGETLBFS)
  531. /*
  532. * We schedule the work only when a memory section is
  533. * onlined/offlined on this node. When we come here,
  534. * all the memory on this node has been offlined,
  535. * so we won't enqueue new work to this work.
  536. *
  537. * The work is using node->node_work, so we should
  538. * flush work before freeing the memory.
  539. */
  540. flush_work(&node->node_work);
  541. #endif
  542. kfree(node);
  543. }
  544. /*
  545. * register_node - Setup a sysfs device for a node.
  546. * @num - Node number to use when creating the device.
  547. *
  548. * Initialize and register the node device.
  549. */
  550. static int register_node(struct node *node, int num)
  551. {
  552. int error;
  553. node->dev.id = num;
  554. node->dev.bus = &node_subsys;
  555. node->dev.release = node_device_release;
  556. node->dev.groups = node_dev_groups;
  557. error = device_register(&node->dev);
  558. if (error)
  559. put_device(&node->dev);
  560. else {
  561. hugetlb_register_node(node);
  562. compaction_register_node(node);
  563. }
  564. return error;
  565. }
  566. /**
  567. * unregister_node - unregister a node device
  568. * @node: node going away
  569. *
  570. * Unregisters a node device @node. All the devices on the node must be
  571. * unregistered before calling this function.
  572. */
  573. void unregister_node(struct node *node)
  574. {
  575. hugetlb_unregister_node(node); /* no-op, if memoryless node */
  576. node_remove_accesses(node);
  577. node_remove_caches(node);
  578. device_unregister(&node->dev);
  579. }
  580. struct node *node_devices[MAX_NUMNODES];
  581. /*
  582. * register cpu under node
  583. */
  584. int register_cpu_under_node(unsigned int cpu, unsigned int nid)
  585. {
  586. int ret;
  587. struct device *obj;
  588. if (!node_online(nid))
  589. return 0;
  590. obj = get_cpu_device(cpu);
  591. if (!obj)
  592. return 0;
  593. ret = sysfs_create_link(&node_devices[nid]->dev.kobj,
  594. &obj->kobj,
  595. kobject_name(&obj->kobj));
  596. if (ret)
  597. return ret;
  598. return sysfs_create_link(&obj->kobj,
  599. &node_devices[nid]->dev.kobj,
  600. kobject_name(&node_devices[nid]->dev.kobj));
  601. }
  602. /**
  603. * register_memory_node_under_compute_node - link memory node to its compute
  604. * node for a given access class.
  605. * @mem_nid: Memory node number
  606. * @cpu_nid: Cpu node number
  607. * @access: Access class to register
  608. *
  609. * Description:
  610. * For use with platforms that may have separate memory and compute nodes.
  611. * This function will export node relationships linking which memory
  612. * initiator nodes can access memory targets at a given ranked access
  613. * class.
  614. */
  615. int register_memory_node_under_compute_node(unsigned int mem_nid,
  616. unsigned int cpu_nid,
  617. unsigned access)
  618. {
  619. struct node *init_node, *targ_node;
  620. struct node_access_nodes *initiator, *target;
  621. int ret;
  622. if (!node_online(cpu_nid) || !node_online(mem_nid))
  623. return -ENODEV;
  624. init_node = node_devices[cpu_nid];
  625. targ_node = node_devices[mem_nid];
  626. initiator = node_init_node_access(init_node, access);
  627. target = node_init_node_access(targ_node, access);
  628. if (!initiator || !target)
  629. return -ENOMEM;
  630. ret = sysfs_add_link_to_group(&initiator->dev.kobj, "targets",
  631. &targ_node->dev.kobj,
  632. dev_name(&targ_node->dev));
  633. if (ret)
  634. return ret;
  635. ret = sysfs_add_link_to_group(&target->dev.kobj, "initiators",
  636. &init_node->dev.kobj,
  637. dev_name(&init_node->dev));
  638. if (ret)
  639. goto err;
  640. return 0;
  641. err:
  642. sysfs_remove_link_from_group(&initiator->dev.kobj, "targets",
  643. dev_name(&targ_node->dev));
  644. return ret;
  645. }
  646. int unregister_cpu_under_node(unsigned int cpu, unsigned int nid)
  647. {
  648. struct device *obj;
  649. if (!node_online(nid))
  650. return 0;
  651. obj = get_cpu_device(cpu);
  652. if (!obj)
  653. return 0;
  654. sysfs_remove_link(&node_devices[nid]->dev.kobj,
  655. kobject_name(&obj->kobj));
  656. sysfs_remove_link(&obj->kobj,
  657. kobject_name(&node_devices[nid]->dev.kobj));
  658. return 0;
  659. }
  660. #ifdef CONFIG_MEMORY_HOTPLUG_SPARSE
  661. static int __ref get_nid_for_pfn(unsigned long pfn)
  662. {
  663. if (!pfn_valid_within(pfn))
  664. return -1;
  665. #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
  666. if (system_state < SYSTEM_RUNNING)
  667. return early_pfn_to_nid(pfn);
  668. #endif
  669. return pfn_to_nid(pfn);
  670. }
  671. static void do_register_memory_block_under_node(int nid,
  672. struct memory_block *mem_blk)
  673. {
  674. int ret;
  675. /*
  676. * If this memory block spans multiple nodes, we only indicate
  677. * the last processed node.
  678. */
  679. mem_blk->nid = nid;
  680. ret = sysfs_create_link_nowarn(&node_devices[nid]->dev.kobj,
  681. &mem_blk->dev.kobj,
  682. kobject_name(&mem_blk->dev.kobj));
  683. if (ret && ret != -EEXIST)
  684. dev_err_ratelimited(&node_devices[nid]->dev,
  685. "can't create link to %s in sysfs (%d)\n",
  686. kobject_name(&mem_blk->dev.kobj), ret);
  687. ret = sysfs_create_link_nowarn(&mem_blk->dev.kobj,
  688. &node_devices[nid]->dev.kobj,
  689. kobject_name(&node_devices[nid]->dev.kobj));
  690. if (ret && ret != -EEXIST)
  691. dev_err_ratelimited(&mem_blk->dev,
  692. "can't create link to %s in sysfs (%d)\n",
  693. kobject_name(&node_devices[nid]->dev.kobj),
  694. ret);
  695. }
  696. /* register memory section under specified node if it spans that node */
  697. static int register_mem_block_under_node_early(struct memory_block *mem_blk,
  698. void *arg)
  699. {
  700. unsigned long memory_block_pfns = memory_block_size_bytes() / PAGE_SIZE;
  701. unsigned long start_pfn = section_nr_to_pfn(mem_blk->start_section_nr);
  702. unsigned long end_pfn = start_pfn + memory_block_pfns - 1;
  703. int nid = *(int *)arg;
  704. unsigned long pfn;
  705. for (pfn = start_pfn; pfn <= end_pfn; pfn++) {
  706. int page_nid;
  707. /*
  708. * memory block could have several absent sections from start.
  709. * skip pfn range from absent section
  710. */
  711. if (!pfn_in_present_section(pfn)) {
  712. pfn = round_down(pfn + PAGES_PER_SECTION,
  713. PAGES_PER_SECTION) - 1;
  714. continue;
  715. }
  716. /*
  717. * We need to check if page belongs to nid only at the boot
  718. * case because node's ranges can be interleaved.
  719. */
  720. page_nid = get_nid_for_pfn(pfn);
  721. if (page_nid < 0)
  722. continue;
  723. if (page_nid != nid)
  724. continue;
  725. do_register_memory_block_under_node(nid, mem_blk);
  726. return 0;
  727. }
  728. /* mem section does not span the specified node */
  729. return 0;
  730. }
  731. /*
  732. * During hotplug we know that all pages in the memory block belong to the same
  733. * node.
  734. */
  735. static int register_mem_block_under_node_hotplug(struct memory_block *mem_blk,
  736. void *arg)
  737. {
  738. int nid = *(int *)arg;
  739. do_register_memory_block_under_node(nid, mem_blk);
  740. return 0;
  741. }
  742. /*
  743. * Unregister a memory block device under the node it spans. Memory blocks
  744. * with multiple nodes cannot be offlined and therefore also never be removed.
  745. */
  746. void unregister_memory_block_under_nodes(struct memory_block *mem_blk)
  747. {
  748. if (mem_blk->nid == NUMA_NO_NODE)
  749. return;
  750. sysfs_remove_link(&node_devices[mem_blk->nid]->dev.kobj,
  751. kobject_name(&mem_blk->dev.kobj));
  752. sysfs_remove_link(&mem_blk->dev.kobj,
  753. kobject_name(&node_devices[mem_blk->nid]->dev.kobj));
  754. }
  755. void link_mem_sections(int nid, unsigned long start_pfn, unsigned long end_pfn,
  756. enum meminit_context context)
  757. {
  758. walk_memory_blocks_func_t func;
  759. if (context == MEMINIT_HOTPLUG)
  760. func = register_mem_block_under_node_hotplug;
  761. else
  762. func = register_mem_block_under_node_early;
  763. walk_memory_blocks(PFN_PHYS(start_pfn), PFN_PHYS(end_pfn - start_pfn),
  764. (void *)&nid, func);
  765. return;
  766. }
  767. #ifdef CONFIG_HUGETLBFS
  768. /*
  769. * Handle per node hstate attribute [un]registration on transistions
  770. * to/from memoryless state.
  771. */
  772. static void node_hugetlb_work(struct work_struct *work)
  773. {
  774. struct node *node = container_of(work, struct node, node_work);
  775. /*
  776. * We only get here when a node transitions to/from memoryless state.
  777. * We can detect which transition occurred by examining whether the
  778. * node has memory now. hugetlb_register_node() already check this
  779. * so we try to register the attributes. If that fails, then the
  780. * node has transitioned to memoryless, try to unregister the
  781. * attributes.
  782. */
  783. if (!hugetlb_register_node(node))
  784. hugetlb_unregister_node(node);
  785. }
  786. static void init_node_hugetlb_work(int nid)
  787. {
  788. INIT_WORK(&node_devices[nid]->node_work, node_hugetlb_work);
  789. }
  790. static int node_memory_callback(struct notifier_block *self,
  791. unsigned long action, void *arg)
  792. {
  793. struct memory_notify *mnb = arg;
  794. int nid = mnb->status_change_nid;
  795. switch (action) {
  796. case MEM_ONLINE:
  797. case MEM_OFFLINE:
  798. /*
  799. * offload per node hstate [un]registration to a work thread
  800. * when transitioning to/from memoryless state.
  801. */
  802. if (nid != NUMA_NO_NODE)
  803. schedule_work(&node_devices[nid]->node_work);
  804. break;
  805. case MEM_GOING_ONLINE:
  806. case MEM_GOING_OFFLINE:
  807. case MEM_CANCEL_ONLINE:
  808. case MEM_CANCEL_OFFLINE:
  809. default:
  810. break;
  811. }
  812. return NOTIFY_OK;
  813. }
  814. #endif /* CONFIG_HUGETLBFS */
  815. #endif /* CONFIG_MEMORY_HOTPLUG_SPARSE */
  816. #if !defined(CONFIG_MEMORY_HOTPLUG_SPARSE) || \
  817. !defined(CONFIG_HUGETLBFS)
  818. static inline int node_memory_callback(struct notifier_block *self,
  819. unsigned long action, void *arg)
  820. {
  821. return NOTIFY_OK;
  822. }
  823. static void init_node_hugetlb_work(int nid) { }
  824. #endif
  825. int __register_one_node(int nid)
  826. {
  827. int error;
  828. int cpu;
  829. node_devices[nid] = kzalloc(sizeof(struct node), GFP_KERNEL);
  830. if (!node_devices[nid])
  831. return -ENOMEM;
  832. error = register_node(node_devices[nid], nid);
  833. /* link cpu under this node */
  834. for_each_present_cpu(cpu) {
  835. if (cpu_to_node(cpu) == nid)
  836. register_cpu_under_node(cpu, nid);
  837. }
  838. INIT_LIST_HEAD(&node_devices[nid]->access_list);
  839. /* initialize work queue for memory hot plug */
  840. init_node_hugetlb_work(nid);
  841. node_init_caches(nid);
  842. return error;
  843. }
  844. void unregister_one_node(int nid)
  845. {
  846. if (!node_devices[nid])
  847. return;
  848. unregister_node(node_devices[nid]);
  849. node_devices[nid] = NULL;
  850. }
  851. /*
  852. * node states attributes
  853. */
  854. struct node_attr {
  855. struct device_attribute attr;
  856. enum node_states state;
  857. };
  858. static ssize_t show_node_state(struct device *dev,
  859. struct device_attribute *attr, char *buf)
  860. {
  861. struct node_attr *na = container_of(attr, struct node_attr, attr);
  862. return sysfs_emit(buf, "%*pbl\n",
  863. nodemask_pr_args(&node_states[na->state]));
  864. }
  865. #define _NODE_ATTR(name, state) \
  866. { __ATTR(name, 0444, show_node_state, NULL), state }
  867. static struct node_attr node_state_attr[] = {
  868. [N_POSSIBLE] = _NODE_ATTR(possible, N_POSSIBLE),
  869. [N_ONLINE] = _NODE_ATTR(online, N_ONLINE),
  870. [N_NORMAL_MEMORY] = _NODE_ATTR(has_normal_memory, N_NORMAL_MEMORY),
  871. #ifdef CONFIG_HIGHMEM
  872. [N_HIGH_MEMORY] = _NODE_ATTR(has_high_memory, N_HIGH_MEMORY),
  873. #endif
  874. [N_MEMORY] = _NODE_ATTR(has_memory, N_MEMORY),
  875. [N_CPU] = _NODE_ATTR(has_cpu, N_CPU),
  876. [N_GENERIC_INITIATOR] = _NODE_ATTR(has_generic_initiator,
  877. N_GENERIC_INITIATOR),
  878. };
  879. static struct attribute *node_state_attrs[] = {
  880. &node_state_attr[N_POSSIBLE].attr.attr,
  881. &node_state_attr[N_ONLINE].attr.attr,
  882. &node_state_attr[N_NORMAL_MEMORY].attr.attr,
  883. #ifdef CONFIG_HIGHMEM
  884. &node_state_attr[N_HIGH_MEMORY].attr.attr,
  885. #endif
  886. &node_state_attr[N_MEMORY].attr.attr,
  887. &node_state_attr[N_CPU].attr.attr,
  888. &node_state_attr[N_GENERIC_INITIATOR].attr.attr,
  889. NULL
  890. };
  891. static struct attribute_group memory_root_attr_group = {
  892. .attrs = node_state_attrs,
  893. };
  894. static const struct attribute_group *cpu_root_attr_groups[] = {
  895. &memory_root_attr_group,
  896. NULL,
  897. };
  898. #define NODE_CALLBACK_PRI 2 /* lower than SLAB */
  899. static int __init register_node_type(void)
  900. {
  901. int ret;
  902. BUILD_BUG_ON(ARRAY_SIZE(node_state_attr) != NR_NODE_STATES);
  903. BUILD_BUG_ON(ARRAY_SIZE(node_state_attrs)-1 != NR_NODE_STATES);
  904. ret = subsys_system_register(&node_subsys, cpu_root_attr_groups);
  905. if (!ret) {
  906. static struct notifier_block node_memory_callback_nb = {
  907. .notifier_call = node_memory_callback,
  908. .priority = NODE_CALLBACK_PRI,
  909. };
  910. register_hotmemory_notifier(&node_memory_callback_nb);
  911. }
  912. /*
  913. * Note: we're not going to unregister the node class if we fail
  914. * to register the node state class attribute files.
  915. */
  916. return ret;
  917. }
  918. postcore_initcall(register_node_type);