mmzone.h 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789
  1. #ifndef _LINUX_MMZONE_H
  2. #define _LINUX_MMZONE_H
  3. #ifdef __KERNEL__
  4. #ifndef __ASSEMBLY__
  5. #include <linux/spinlock.h>
  6. #include <linux/list.h>
  7. #include <linux/wait.h>
  8. #include <linux/cache.h>
  9. #include <linux/threads.h>
  10. #include <linux/numa.h>
  11. #include <linux/init.h>
  12. #include <linux/seqlock.h>
  13. #include <linux/nodemask.h>
  14. #include <asm/atomic.h>
  15. #include <asm/page.h>
  16. /* Free memory management - zoned buddy allocator. */
  17. #ifndef CONFIG_FORCE_MAX_ZONEORDER
  18. #define MAX_ORDER 11
  19. #else
  20. #define MAX_ORDER CONFIG_FORCE_MAX_ZONEORDER
  21. #endif
  22. #define MAX_ORDER_NR_PAGES (1 << (MAX_ORDER - 1))
  23. struct free_area {
  24. struct list_head free_list;
  25. unsigned long nr_free;
  26. };
  27. struct pglist_data;
  28. /*
  29. * zone->lock and zone->lru_lock are two of the hottest locks in the kernel.
  30. * So add a wild amount of padding here to ensure that they fall into separate
  31. * cachelines. There are very few zone structures in the machine, so space
  32. * consumption is not a concern here.
  33. */
  34. #if defined(CONFIG_SMP)
  35. struct zone_padding {
  36. char x[0];
  37. } ____cacheline_internodealigned_in_smp;
  38. #define ZONE_PADDING(name) struct zone_padding name;
  39. #else
  40. #define ZONE_PADDING(name)
  41. #endif
  42. enum zone_stat_item {
  43. /* First 128 byte cacheline (assuming 64 bit words) */
  44. NR_FREE_PAGES,
  45. NR_INACTIVE,
  46. NR_ACTIVE,
  47. NR_ANON_PAGES, /* Mapped anonymous pages */
  48. NR_FILE_MAPPED, /* pagecache pages mapped into pagetables.
  49. only modified from process context */
  50. NR_FILE_PAGES,
  51. NR_FILE_DIRTY,
  52. NR_WRITEBACK,
  53. /* Second 128 byte cacheline */
  54. NR_SLAB_RECLAIMABLE,
  55. NR_SLAB_UNRECLAIMABLE,
  56. NR_PAGETABLE, /* used for pagetables */
  57. NR_UNSTABLE_NFS, /* NFS unstable pages */
  58. NR_BOUNCE,
  59. NR_VMSCAN_WRITE,
  60. #ifdef CONFIG_NUMA
  61. NUMA_HIT, /* allocated in intended node */
  62. NUMA_MISS, /* allocated in non intended node */
  63. NUMA_FOREIGN, /* was intended here, hit elsewhere */
  64. NUMA_INTERLEAVE_HIT, /* interleaver preferred this zone */
  65. NUMA_LOCAL, /* allocation from local node */
  66. NUMA_OTHER, /* allocation from other node */
  67. #endif
  68. NR_VM_ZONE_STAT_ITEMS };
  69. struct per_cpu_pages {
  70. int count; /* number of pages in the list */
  71. int high; /* high watermark, emptying needed */
  72. int batch; /* chunk size for buddy add/remove */
  73. struct list_head list; /* the list of pages */
  74. };
  75. struct per_cpu_pageset {
  76. struct per_cpu_pages pcp[2]; /* 0: hot. 1: cold */
  77. #ifdef CONFIG_SMP
  78. s8 stat_threshold;
  79. s8 vm_stat_diff[NR_VM_ZONE_STAT_ITEMS];
  80. #endif
  81. } ____cacheline_aligned_in_smp;
  82. #ifdef CONFIG_NUMA
  83. #define zone_pcp(__z, __cpu) ((__z)->pageset[(__cpu)])
  84. #else
  85. #define zone_pcp(__z, __cpu) (&(__z)->pageset[(__cpu)])
  86. #endif
  87. enum zone_type {
  88. #ifdef CONFIG_ZONE_DMA
  89. /*
  90. * ZONE_DMA is used when there are devices that are not able
  91. * to do DMA to all of addressable memory (ZONE_NORMAL). Then we
  92. * carve out the portion of memory that is needed for these devices.
  93. * The range is arch specific.
  94. *
  95. * Some examples
  96. *
  97. * Architecture Limit
  98. * ---------------------------
  99. * parisc, ia64, sparc <4G
  100. * s390 <2G
  101. * arm26 <48M
  102. * arm Various
  103. * alpha Unlimited or 0-16MB.
  104. *
  105. * i386, x86_64 and multiple other arches
  106. * <16M.
  107. */
  108. ZONE_DMA,
  109. #endif
  110. #ifdef CONFIG_ZONE_DMA32
  111. /*
  112. * x86_64 needs two ZONE_DMAs because it supports devices that are
  113. * only able to do DMA to the lower 16M but also 32 bit devices that
  114. * can only do DMA areas below 4G.
  115. */
  116. ZONE_DMA32,
  117. #endif
  118. /*
  119. * Normal addressable memory is in ZONE_NORMAL. DMA operations can be
  120. * performed on pages in ZONE_NORMAL if the DMA devices support
  121. * transfers to all addressable memory.
  122. */
  123. ZONE_NORMAL,
  124. #ifdef CONFIG_HIGHMEM
  125. /*
  126. * A memory area that is only addressable by the kernel through
  127. * mapping portions into its own address space. This is for example
  128. * used by i386 to allow the kernel to address the memory beyond
  129. * 900MB. The kernel will set up special mappings (page
  130. * table entries on i386) for each page that the kernel needs to
  131. * access.
  132. */
  133. ZONE_HIGHMEM,
  134. #endif
  135. MAX_NR_ZONES
  136. };
  137. /*
  138. * When a memory allocation must conform to specific limitations (such
  139. * as being suitable for DMA) the caller will pass in hints to the
  140. * allocator in the gfp_mask, in the zone modifier bits. These bits
  141. * are used to select a priority ordered list of memory zones which
  142. * match the requested limits. See gfp_zone() in include/linux/gfp.h
  143. */
  144. /*
  145. * Count the active zones. Note that the use of defined(X) outside
  146. * #if and family is not necessarily defined so ensure we cannot use
  147. * it later. Use __ZONE_COUNT to work out how many shift bits we need.
  148. */
  149. #define __ZONE_COUNT ( \
  150. defined(CONFIG_ZONE_DMA) \
  151. + defined(CONFIG_ZONE_DMA32) \
  152. + 1 \
  153. + defined(CONFIG_HIGHMEM) \
  154. )
  155. #if __ZONE_COUNT < 2
  156. #define ZONES_SHIFT 0
  157. #elif __ZONE_COUNT <= 2
  158. #define ZONES_SHIFT 1
  159. #elif __ZONE_COUNT <= 4
  160. #define ZONES_SHIFT 2
  161. #else
  162. #error ZONES_SHIFT -- too many zones configured adjust calculation
  163. #endif
  164. #undef __ZONE_COUNT
  165. struct zone {
  166. /* Fields commonly accessed by the page allocator */
  167. unsigned long pages_min, pages_low, pages_high;
  168. /*
  169. * We don't know if the memory that we're going to allocate will be freeable
  170. * or/and it will be released eventually, so to avoid totally wasting several
  171. * GB of ram we must reserve some of the lower zone memory (otherwise we risk
  172. * to run OOM on the lower zones despite there's tons of freeable ram
  173. * on the higher zones). This array is recalculated at runtime if the
  174. * sysctl_lowmem_reserve_ratio sysctl changes.
  175. */
  176. unsigned long lowmem_reserve[MAX_NR_ZONES];
  177. #ifdef CONFIG_NUMA
  178. int node;
  179. /*
  180. * zone reclaim becomes active if more unmapped pages exist.
  181. */
  182. unsigned long min_unmapped_pages;
  183. unsigned long min_slab_pages;
  184. struct per_cpu_pageset *pageset[NR_CPUS];
  185. #else
  186. struct per_cpu_pageset pageset[NR_CPUS];
  187. #endif
  188. /*
  189. * free areas of different sizes
  190. */
  191. spinlock_t lock;
  192. #ifdef CONFIG_MEMORY_HOTPLUG
  193. /* see spanned/present_pages for more description */
  194. seqlock_t span_seqlock;
  195. #endif
  196. struct free_area free_area[MAX_ORDER];
  197. ZONE_PADDING(_pad1_)
  198. /* Fields commonly accessed by the page reclaim scanner */
  199. spinlock_t lru_lock;
  200. struct list_head active_list;
  201. struct list_head inactive_list;
  202. unsigned long nr_scan_active;
  203. unsigned long nr_scan_inactive;
  204. unsigned long pages_scanned; /* since last reclaim */
  205. int all_unreclaimable; /* All pages pinned */
  206. /* A count of how many reclaimers are scanning this zone */
  207. atomic_t reclaim_in_progress;
  208. /* Zone statistics */
  209. atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS];
  210. /*
  211. * prev_priority holds the scanning priority for this zone. It is
  212. * defined as the scanning priority at which we achieved our reclaim
  213. * target at the previous try_to_free_pages() or balance_pgdat()
  214. * invokation.
  215. *
  216. * We use prev_priority as a measure of how much stress page reclaim is
  217. * under - it drives the swappiness decision: whether to unmap mapped
  218. * pages.
  219. *
  220. * Access to both this field is quite racy even on uniprocessor. But
  221. * it is expected to average out OK.
  222. */
  223. int prev_priority;
  224. ZONE_PADDING(_pad2_)
  225. /* Rarely used or read-mostly fields */
  226. /*
  227. * wait_table -- the array holding the hash table
  228. * wait_table_hash_nr_entries -- the size of the hash table array
  229. * wait_table_bits -- wait_table_size == (1 << wait_table_bits)
  230. *
  231. * The purpose of all these is to keep track of the people
  232. * waiting for a page to become available and make them
  233. * runnable again when possible. The trouble is that this
  234. * consumes a lot of space, especially when so few things
  235. * wait on pages at a given time. So instead of using
  236. * per-page waitqueues, we use a waitqueue hash table.
  237. *
  238. * The bucket discipline is to sleep on the same queue when
  239. * colliding and wake all in that wait queue when removing.
  240. * When something wakes, it must check to be sure its page is
  241. * truly available, a la thundering herd. The cost of a
  242. * collision is great, but given the expected load of the
  243. * table, they should be so rare as to be outweighed by the
  244. * benefits from the saved space.
  245. *
  246. * __wait_on_page_locked() and unlock_page() in mm/filemap.c, are the
  247. * primary users of these fields, and in mm/page_alloc.c
  248. * free_area_init_core() performs the initialization of them.
  249. */
  250. wait_queue_head_t * wait_table;
  251. unsigned long wait_table_hash_nr_entries;
  252. unsigned long wait_table_bits;
  253. /*
  254. * Discontig memory support fields.
  255. */
  256. struct pglist_data *zone_pgdat;
  257. /* zone_start_pfn == zone_start_paddr >> PAGE_SHIFT */
  258. unsigned long zone_start_pfn;
  259. /*
  260. * zone_start_pfn, spanned_pages and present_pages are all
  261. * protected by span_seqlock. It is a seqlock because it has
  262. * to be read outside of zone->lock, and it is done in the main
  263. * allocator path. But, it is written quite infrequently.
  264. *
  265. * The lock is declared along with zone->lock because it is
  266. * frequently read in proximity to zone->lock. It's good to
  267. * give them a chance of being in the same cacheline.
  268. */
  269. unsigned long spanned_pages; /* total size, including holes */
  270. unsigned long present_pages; /* amount of memory (excluding holes) */
  271. /*
  272. * rarely used fields:
  273. */
  274. const char *name;
  275. } ____cacheline_internodealigned_in_smp;
  276. /*
  277. * The "priority" of VM scanning is how much of the queues we will scan in one
  278. * go. A value of 12 for DEF_PRIORITY implies that we will scan 1/4096th of the
  279. * queues ("queue_length >> 12") during an aging round.
  280. */
  281. #define DEF_PRIORITY 12
  282. /* Maximum number of zones on a zonelist */
  283. #define MAX_ZONES_PER_ZONELIST (MAX_NUMNODES * MAX_NR_ZONES)
  284. #ifdef CONFIG_NUMA
  285. /*
  286. * We cache key information from each zonelist for smaller cache
  287. * footprint when scanning for free pages in get_page_from_freelist().
  288. *
  289. * 1) The BITMAP fullzones tracks which zones in a zonelist have come
  290. * up short of free memory since the last time (last_fullzone_zap)
  291. * we zero'd fullzones.
  292. * 2) The array z_to_n[] maps each zone in the zonelist to its node
  293. * id, so that we can efficiently evaluate whether that node is
  294. * set in the current tasks mems_allowed.
  295. *
  296. * Both fullzones and z_to_n[] are one-to-one with the zonelist,
  297. * indexed by a zones offset in the zonelist zones[] array.
  298. *
  299. * The get_page_from_freelist() routine does two scans. During the
  300. * first scan, we skip zones whose corresponding bit in 'fullzones'
  301. * is set or whose corresponding node in current->mems_allowed (which
  302. * comes from cpusets) is not set. During the second scan, we bypass
  303. * this zonelist_cache, to ensure we look methodically at each zone.
  304. *
  305. * Once per second, we zero out (zap) fullzones, forcing us to
  306. * reconsider nodes that might have regained more free memory.
  307. * The field last_full_zap is the time we last zapped fullzones.
  308. *
  309. * This mechanism reduces the amount of time we waste repeatedly
  310. * reexaming zones for free memory when they just came up low on
  311. * memory momentarilly ago.
  312. *
  313. * The zonelist_cache struct members logically belong in struct
  314. * zonelist. However, the mempolicy zonelists constructed for
  315. * MPOL_BIND are intentionally variable length (and usually much
  316. * shorter). A general purpose mechanism for handling structs with
  317. * multiple variable length members is more mechanism than we want
  318. * here. We resort to some special case hackery instead.
  319. *
  320. * The MPOL_BIND zonelists don't need this zonelist_cache (in good
  321. * part because they are shorter), so we put the fixed length stuff
  322. * at the front of the zonelist struct, ending in a variable length
  323. * zones[], as is needed by MPOL_BIND.
  324. *
  325. * Then we put the optional zonelist cache on the end of the zonelist
  326. * struct. This optional stuff is found by a 'zlcache_ptr' pointer in
  327. * the fixed length portion at the front of the struct. This pointer
  328. * both enables us to find the zonelist cache, and in the case of
  329. * MPOL_BIND zonelists, (which will just set the zlcache_ptr to NULL)
  330. * to know that the zonelist cache is not there.
  331. *
  332. * The end result is that struct zonelists come in two flavors:
  333. * 1) The full, fixed length version, shown below, and
  334. * 2) The custom zonelists for MPOL_BIND.
  335. * The custom MPOL_BIND zonelists have a NULL zlcache_ptr and no zlcache.
  336. *
  337. * Even though there may be multiple CPU cores on a node modifying
  338. * fullzones or last_full_zap in the same zonelist_cache at the same
  339. * time, we don't lock it. This is just hint data - if it is wrong now
  340. * and then, the allocator will still function, perhaps a bit slower.
  341. */
  342. struct zonelist_cache {
  343. unsigned short z_to_n[MAX_ZONES_PER_ZONELIST]; /* zone->nid */
  344. DECLARE_BITMAP(fullzones, MAX_ZONES_PER_ZONELIST); /* zone full? */
  345. unsigned long last_full_zap; /* when last zap'd (jiffies) */
  346. };
  347. #else
  348. struct zonelist_cache;
  349. #endif
  350. /*
  351. * One allocation request operates on a zonelist. A zonelist
  352. * is a list of zones, the first one is the 'goal' of the
  353. * allocation, the other zones are fallback zones, in decreasing
  354. * priority.
  355. *
  356. * If zlcache_ptr is not NULL, then it is just the address of zlcache,
  357. * as explained above. If zlcache_ptr is NULL, there is no zlcache.
  358. */
  359. struct zonelist {
  360. struct zonelist_cache *zlcache_ptr; // NULL or &zlcache
  361. struct zone *zones[MAX_ZONES_PER_ZONELIST + 1]; // NULL delimited
  362. #ifdef CONFIG_NUMA
  363. struct zonelist_cache zlcache; // optional ...
  364. #endif
  365. };
  366. #ifdef CONFIG_ARCH_POPULATES_NODE_MAP
  367. struct node_active_region {
  368. unsigned long start_pfn;
  369. unsigned long end_pfn;
  370. int nid;
  371. };
  372. #endif /* CONFIG_ARCH_POPULATES_NODE_MAP */
  373. #ifndef CONFIG_DISCONTIGMEM
  374. /* The array of struct pages - for discontigmem use pgdat->lmem_map */
  375. extern struct page *mem_map;
  376. #endif
  377. /*
  378. * The pg_data_t structure is used in machines with CONFIG_DISCONTIGMEM
  379. * (mostly NUMA machines?) to denote a higher-level memory zone than the
  380. * zone denotes.
  381. *
  382. * On NUMA machines, each NUMA node would have a pg_data_t to describe
  383. * it's memory layout.
  384. *
  385. * Memory statistics and page replacement data structures are maintained on a
  386. * per-zone basis.
  387. */
  388. struct bootmem_data;
  389. typedef struct pglist_data {
  390. struct zone node_zones[MAX_NR_ZONES];
  391. struct zonelist node_zonelists[MAX_NR_ZONES];
  392. int nr_zones;
  393. #ifdef CONFIG_FLAT_NODE_MEM_MAP
  394. struct page *node_mem_map;
  395. #endif
  396. struct bootmem_data *bdata;
  397. #ifdef CONFIG_MEMORY_HOTPLUG
  398. /*
  399. * Must be held any time you expect node_start_pfn, node_present_pages
  400. * or node_spanned_pages stay constant. Holding this will also
  401. * guarantee that any pfn_valid() stays that way.
  402. *
  403. * Nests above zone->lock and zone->size_seqlock.
  404. */
  405. spinlock_t node_size_lock;
  406. #endif
  407. unsigned long node_start_pfn;
  408. unsigned long node_present_pages; /* total number of physical pages */
  409. unsigned long node_spanned_pages; /* total size of physical page
  410. range, including holes */
  411. int node_id;
  412. wait_queue_head_t kswapd_wait;
  413. struct task_struct *kswapd;
  414. int kswapd_max_order;
  415. } pg_data_t;
  416. #define node_present_pages(nid) (NODE_DATA(nid)->node_present_pages)
  417. #define node_spanned_pages(nid) (NODE_DATA(nid)->node_spanned_pages)
  418. #ifdef CONFIG_FLAT_NODE_MEM_MAP
  419. #define pgdat_page_nr(pgdat, pagenr) ((pgdat)->node_mem_map + (pagenr))
  420. #else
  421. #define pgdat_page_nr(pgdat, pagenr) pfn_to_page((pgdat)->node_start_pfn + (pagenr))
  422. #endif
  423. #define nid_page_nr(nid, pagenr) pgdat_page_nr(NODE_DATA(nid),(pagenr))
  424. #include <linux/memory_hotplug.h>
  425. void get_zone_counts(unsigned long *active, unsigned long *inactive,
  426. unsigned long *free);
  427. void build_all_zonelists(void);
  428. void wakeup_kswapd(struct zone *zone, int order);
  429. int zone_watermark_ok(struct zone *z, int order, unsigned long mark,
  430. int classzone_idx, int alloc_flags);
  431. enum memmap_context {
  432. MEMMAP_EARLY,
  433. MEMMAP_HOTPLUG,
  434. };
  435. extern int init_currently_empty_zone(struct zone *zone, unsigned long start_pfn,
  436. unsigned long size,
  437. enum memmap_context context);
  438. #ifdef CONFIG_HAVE_MEMORY_PRESENT
  439. void memory_present(int nid, unsigned long start, unsigned long end);
  440. #else
  441. static inline void memory_present(int nid, unsigned long start, unsigned long end) {}
  442. #endif
  443. #ifdef CONFIG_NEED_NODE_MEMMAP_SIZE
  444. unsigned long __init node_memmap_size_bytes(int, unsigned long, unsigned long);
  445. #endif
  446. /*
  447. * zone_idx() returns 0 for the ZONE_DMA zone, 1 for the ZONE_NORMAL zone, etc.
  448. */
  449. #define zone_idx(zone) ((zone) - (zone)->zone_pgdat->node_zones)
  450. static inline int populated_zone(struct zone *zone)
  451. {
  452. return (!!zone->present_pages);
  453. }
  454. static inline int is_highmem_idx(enum zone_type idx)
  455. {
  456. #ifdef CONFIG_HIGHMEM
  457. return (idx == ZONE_HIGHMEM);
  458. #else
  459. return 0;
  460. #endif
  461. }
  462. static inline int is_normal_idx(enum zone_type idx)
  463. {
  464. return (idx == ZONE_NORMAL);
  465. }
  466. /**
  467. * is_highmem - helper function to quickly check if a struct zone is a
  468. * highmem zone or not. This is an attempt to keep references
  469. * to ZONE_{DMA/NORMAL/HIGHMEM/etc} in general code to a minimum.
  470. * @zone - pointer to struct zone variable
  471. */
  472. static inline int is_highmem(struct zone *zone)
  473. {
  474. #ifdef CONFIG_HIGHMEM
  475. return zone == zone->zone_pgdat->node_zones + ZONE_HIGHMEM;
  476. #else
  477. return 0;
  478. #endif
  479. }
  480. static inline int is_normal(struct zone *zone)
  481. {
  482. return zone == zone->zone_pgdat->node_zones + ZONE_NORMAL;
  483. }
  484. static inline int is_dma32(struct zone *zone)
  485. {
  486. #ifdef CONFIG_ZONE_DMA32
  487. return zone == zone->zone_pgdat->node_zones + ZONE_DMA32;
  488. #else
  489. return 0;
  490. #endif
  491. }
  492. static inline int is_dma(struct zone *zone)
  493. {
  494. #ifdef CONFIG_ZONE_DMA
  495. return zone == zone->zone_pgdat->node_zones + ZONE_DMA;
  496. #else
  497. return 0;
  498. #endif
  499. }
  500. /* These two functions are used to setup the per zone pages min values */
  501. struct ctl_table;
  502. struct file;
  503. int min_free_kbytes_sysctl_handler(struct ctl_table *, int, struct file *,
  504. void __user *, size_t *, loff_t *);
  505. extern int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES-1];
  506. int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *, int, struct file *,
  507. void __user *, size_t *, loff_t *);
  508. int percpu_pagelist_fraction_sysctl_handler(struct ctl_table *, int, struct file *,
  509. void __user *, size_t *, loff_t *);
  510. int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *, int,
  511. struct file *, void __user *, size_t *, loff_t *);
  512. int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *, int,
  513. struct file *, void __user *, size_t *, loff_t *);
  514. #include <linux/topology.h>
  515. /* Returns the number of the current Node. */
  516. #ifndef numa_node_id
  517. #define numa_node_id() (cpu_to_node(raw_smp_processor_id()))
  518. #endif
  519. #ifndef CONFIG_NEED_MULTIPLE_NODES
  520. extern struct pglist_data contig_page_data;
  521. #define NODE_DATA(nid) (&contig_page_data)
  522. #define NODE_MEM_MAP(nid) mem_map
  523. #define MAX_NODES_SHIFT 1
  524. #else /* CONFIG_NEED_MULTIPLE_NODES */
  525. #include <asm/mmzone.h>
  526. #endif /* !CONFIG_NEED_MULTIPLE_NODES */
  527. extern struct pglist_data *first_online_pgdat(void);
  528. extern struct pglist_data *next_online_pgdat(struct pglist_data *pgdat);
  529. extern struct zone *next_zone(struct zone *zone);
  530. /**
  531. * for_each_pgdat - helper macro to iterate over all nodes
  532. * @pgdat - pointer to a pg_data_t variable
  533. */
  534. #define for_each_online_pgdat(pgdat) \
  535. for (pgdat = first_online_pgdat(); \
  536. pgdat; \
  537. pgdat = next_online_pgdat(pgdat))
  538. /**
  539. * for_each_zone - helper macro to iterate over all memory zones
  540. * @zone - pointer to struct zone variable
  541. *
  542. * The user only needs to declare the zone variable, for_each_zone
  543. * fills it in.
  544. */
  545. #define for_each_zone(zone) \
  546. for (zone = (first_online_pgdat())->node_zones; \
  547. zone; \
  548. zone = next_zone(zone))
  549. #ifdef CONFIG_SPARSEMEM
  550. #include <asm/sparsemem.h>
  551. #endif
  552. #if BITS_PER_LONG == 32
  553. /*
  554. * with 32 bit page->flags field, we reserve 9 bits for node/zone info.
  555. * there are 4 zones (3 bits) and this leaves 9-3=6 bits for nodes.
  556. */
  557. #define FLAGS_RESERVED 9
  558. #elif BITS_PER_LONG == 64
  559. /*
  560. * with 64 bit flags field, there's plenty of room.
  561. */
  562. #define FLAGS_RESERVED 32
  563. #else
  564. #error BITS_PER_LONG not defined
  565. #endif
  566. #if !defined(CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID) && \
  567. !defined(CONFIG_ARCH_POPULATES_NODE_MAP)
  568. #define early_pfn_to_nid(nid) (0UL)
  569. #endif
  570. #ifdef CONFIG_FLATMEM
  571. #define pfn_to_nid(pfn) (0)
  572. #endif
  573. #define pfn_to_section_nr(pfn) ((pfn) >> PFN_SECTION_SHIFT)
  574. #define section_nr_to_pfn(sec) ((sec) << PFN_SECTION_SHIFT)
  575. #ifdef CONFIG_SPARSEMEM
  576. /*
  577. * SECTION_SHIFT #bits space required to store a section #
  578. *
  579. * PA_SECTION_SHIFT physical address to/from section number
  580. * PFN_SECTION_SHIFT pfn to/from section number
  581. */
  582. #define SECTIONS_SHIFT (MAX_PHYSMEM_BITS - SECTION_SIZE_BITS)
  583. #define PA_SECTION_SHIFT (SECTION_SIZE_BITS)
  584. #define PFN_SECTION_SHIFT (SECTION_SIZE_BITS - PAGE_SHIFT)
  585. #define NR_MEM_SECTIONS (1UL << SECTIONS_SHIFT)
  586. #define PAGES_PER_SECTION (1UL << PFN_SECTION_SHIFT)
  587. #define PAGE_SECTION_MASK (~(PAGES_PER_SECTION-1))
  588. #if (MAX_ORDER - 1 + PAGE_SHIFT) > SECTION_SIZE_BITS
  589. #error Allocator MAX_ORDER exceeds SECTION_SIZE
  590. #endif
  591. struct page;
  592. struct mem_section {
  593. /*
  594. * This is, logically, a pointer to an array of struct
  595. * pages. However, it is stored with some other magic.
  596. * (see sparse.c::sparse_init_one_section())
  597. *
  598. * Additionally during early boot we encode node id of
  599. * the location of the section here to guide allocation.
  600. * (see sparse.c::memory_present())
  601. *
  602. * Making it a UL at least makes someone do a cast
  603. * before using it wrong.
  604. */
  605. unsigned long section_mem_map;
  606. };
  607. #ifdef CONFIG_SPARSEMEM_EXTREME
  608. #define SECTIONS_PER_ROOT (PAGE_SIZE / sizeof (struct mem_section))
  609. #else
  610. #define SECTIONS_PER_ROOT 1
  611. #endif
  612. #define SECTION_NR_TO_ROOT(sec) ((sec) / SECTIONS_PER_ROOT)
  613. #define NR_SECTION_ROOTS (NR_MEM_SECTIONS / SECTIONS_PER_ROOT)
  614. #define SECTION_ROOT_MASK (SECTIONS_PER_ROOT - 1)
  615. #ifdef CONFIG_SPARSEMEM_EXTREME
  616. extern struct mem_section *mem_section[NR_SECTION_ROOTS];
  617. #else
  618. extern struct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT];
  619. #endif
  620. static inline struct mem_section *__nr_to_section(unsigned long nr)
  621. {
  622. if (!mem_section[SECTION_NR_TO_ROOT(nr)])
  623. return NULL;
  624. return &mem_section[SECTION_NR_TO_ROOT(nr)][nr & SECTION_ROOT_MASK];
  625. }
  626. extern int __section_nr(struct mem_section* ms);
  627. /*
  628. * We use the lower bits of the mem_map pointer to store
  629. * a little bit of information. There should be at least
  630. * 3 bits here due to 32-bit alignment.
  631. */
  632. #define SECTION_MARKED_PRESENT (1UL<<0)
  633. #define SECTION_HAS_MEM_MAP (1UL<<1)
  634. #define SECTION_MAP_LAST_BIT (1UL<<2)
  635. #define SECTION_MAP_MASK (~(SECTION_MAP_LAST_BIT-1))
  636. #define SECTION_NID_SHIFT 2
  637. static inline struct page *__section_mem_map_addr(struct mem_section *section)
  638. {
  639. unsigned long map = section->section_mem_map;
  640. map &= SECTION_MAP_MASK;
  641. return (struct page *)map;
  642. }
  643. static inline int valid_section(struct mem_section *section)
  644. {
  645. return (section && (section->section_mem_map & SECTION_MARKED_PRESENT));
  646. }
  647. static inline int section_has_mem_map(struct mem_section *section)
  648. {
  649. return (section && (section->section_mem_map & SECTION_HAS_MEM_MAP));
  650. }
  651. static inline int valid_section_nr(unsigned long nr)
  652. {
  653. return valid_section(__nr_to_section(nr));
  654. }
  655. static inline struct mem_section *__pfn_to_section(unsigned long pfn)
  656. {
  657. return __nr_to_section(pfn_to_section_nr(pfn));
  658. }
  659. static inline int pfn_valid(unsigned long pfn)
  660. {
  661. if (pfn_to_section_nr(pfn) >= NR_MEM_SECTIONS)
  662. return 0;
  663. return valid_section(__nr_to_section(pfn_to_section_nr(pfn)));
  664. }
  665. /*
  666. * These are _only_ used during initialisation, therefore they
  667. * can use __initdata ... They could have names to indicate
  668. * this restriction.
  669. */
  670. #ifdef CONFIG_NUMA
  671. #define pfn_to_nid(pfn) \
  672. ({ \
  673. unsigned long __pfn_to_nid_pfn = (pfn); \
  674. page_to_nid(pfn_to_page(__pfn_to_nid_pfn)); \
  675. })
  676. #else
  677. #define pfn_to_nid(pfn) (0)
  678. #endif
  679. #define early_pfn_valid(pfn) pfn_valid(pfn)
  680. void sparse_init(void);
  681. #else
  682. #define sparse_init() do {} while (0)
  683. #define sparse_index_init(_sec, _nid) do {} while (0)
  684. #endif /* CONFIG_SPARSEMEM */
  685. #ifdef CONFIG_NODES_SPAN_OTHER_NODES
  686. #define early_pfn_in_nid(pfn, nid) (early_pfn_to_nid(pfn) == (nid))
  687. #else
  688. #define early_pfn_in_nid(pfn, nid) (1)
  689. #endif
  690. #ifndef early_pfn_valid
  691. #define early_pfn_valid(pfn) (1)
  692. #endif
  693. void memory_present(int nid, unsigned long start, unsigned long end);
  694. unsigned long __init node_memmap_size_bytes(int, unsigned long, unsigned long);
  695. #endif /* !__ASSEMBLY__ */
  696. #endif /* __KERNEL__ */
  697. #endif /* _LINUX_MMZONE_H */