ksm.c 89 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Memory merging support.
  4. *
  5. * This code enables dynamic sharing of identical pages found in different
  6. * memory areas, even if they are not shared by fork()
  7. *
  8. * Copyright (C) 2008-2009 Red Hat, Inc.
  9. * Authors:
  10. * Izik Eidus
  11. * Andrea Arcangeli
  12. * Chris Wright
  13. * Hugh Dickins
  14. */
  15. #include <linux/errno.h>
  16. #include <linux/mm.h>
  17. #include <linux/fs.h>
  18. #include <linux/mman.h>
  19. #include <linux/sched.h>
  20. #include <linux/sched/mm.h>
  21. #include <linux/sched/coredump.h>
  22. #include <linux/rwsem.h>
  23. #include <linux/pagemap.h>
  24. #include <linux/rmap.h>
  25. #include <linux/spinlock.h>
  26. #include <linux/xxhash.h>
  27. #include <linux/delay.h>
  28. #include <linux/kthread.h>
  29. #include <linux/wait.h>
  30. #include <linux/slab.h>
  31. #include <linux/rbtree.h>
  32. #include <linux/memory.h>
  33. #include <linux/mmu_notifier.h>
  34. #include <linux/swap.h>
  35. #include <linux/ksm.h>
  36. #include <linux/hashtable.h>
  37. #include <linux/freezer.h>
  38. #include <linux/oom.h>
  39. #include <linux/numa.h>
  40. #include <asm/tlbflush.h>
  41. #include "internal.h"
  42. #ifdef CONFIG_NUMA
  43. #define NUMA(x) (x)
  44. #define DO_NUMA(x) do { (x); } while (0)
  45. #else
  46. #define NUMA(x) (0)
  47. #define DO_NUMA(x) do { } while (0)
  48. #endif
  49. /**
  50. * DOC: Overview
  51. *
  52. * A few notes about the KSM scanning process,
  53. * to make it easier to understand the data structures below:
  54. *
  55. * In order to reduce excessive scanning, KSM sorts the memory pages by their
  56. * contents into a data structure that holds pointers to the pages' locations.
  57. *
  58. * Since the contents of the pages may change at any moment, KSM cannot just
  59. * insert the pages into a normal sorted tree and expect it to find anything.
  60. * Therefore KSM uses two data structures - the stable and the unstable tree.
  61. *
  62. * The stable tree holds pointers to all the merged pages (ksm pages), sorted
  63. * by their contents. Because each such page is write-protected, searching on
  64. * this tree is fully assured to be working (except when pages are unmapped),
  65. * and therefore this tree is called the stable tree.
  66. *
  67. * The stable tree node includes information required for reverse
  68. * mapping from a KSM page to virtual addresses that map this page.
  69. *
  70. * In order to avoid large latencies of the rmap walks on KSM pages,
  71. * KSM maintains two types of nodes in the stable tree:
  72. *
  73. * * the regular nodes that keep the reverse mapping structures in a
  74. * linked list
  75. * * the "chains" that link nodes ("dups") that represent the same
  76. * write protected memory content, but each "dup" corresponds to a
  77. * different KSM page copy of that content
  78. *
  79. * Internally, the regular nodes, "dups" and "chains" are represented
  80. * using the same struct stable_node structure.
  81. *
  82. * In addition to the stable tree, KSM uses a second data structure called the
  83. * unstable tree: this tree holds pointers to pages which have been found to
  84. * be "unchanged for a period of time". The unstable tree sorts these pages
  85. * by their contents, but since they are not write-protected, KSM cannot rely
  86. * upon the unstable tree to work correctly - the unstable tree is liable to
  87. * be corrupted as its contents are modified, and so it is called unstable.
  88. *
  89. * KSM solves this problem by several techniques:
  90. *
  91. * 1) The unstable tree is flushed every time KSM completes scanning all
  92. * memory areas, and then the tree is rebuilt again from the beginning.
  93. * 2) KSM will only insert into the unstable tree, pages whose hash value
  94. * has not changed since the previous scan of all memory areas.
  95. * 3) The unstable tree is a RedBlack Tree - so its balancing is based on the
  96. * colors of the nodes and not on their contents, assuring that even when
  97. * the tree gets "corrupted" it won't get out of balance, so scanning time
  98. * remains the same (also, searching and inserting nodes in an rbtree uses
  99. * the same algorithm, so we have no overhead when we flush and rebuild).
  100. * 4) KSM never flushes the stable tree, which means that even if it were to
  101. * take 10 attempts to find a page in the unstable tree, once it is found,
  102. * it is secured in the stable tree. (When we scan a new page, we first
  103. * compare it against the stable tree, and then against the unstable tree.)
  104. *
  105. * If the merge_across_nodes tunable is unset, then KSM maintains multiple
  106. * stable trees and multiple unstable trees: one of each for each NUMA node.
  107. */
  108. /**
  109. * struct mm_slot - ksm information per mm that is being scanned
  110. * @link: link to the mm_slots hash list
  111. * @mm_list: link into the mm_slots list, rooted in ksm_mm_head
  112. * @rmap_list: head for this mm_slot's singly-linked list of rmap_items
  113. * @mm: the mm that this information is valid for
  114. */
  115. struct mm_slot {
  116. struct hlist_node link;
  117. struct list_head mm_list;
  118. struct rmap_item *rmap_list;
  119. struct mm_struct *mm;
  120. };
  121. /**
  122. * struct ksm_scan - cursor for scanning
  123. * @mm_slot: the current mm_slot we are scanning
  124. * @address: the next address inside that to be scanned
  125. * @rmap_list: link to the next rmap to be scanned in the rmap_list
  126. * @seqnr: count of completed full scans (needed when removing unstable node)
  127. *
  128. * There is only the one ksm_scan instance of this cursor structure.
  129. */
  130. struct ksm_scan {
  131. struct mm_slot *mm_slot;
  132. unsigned long address;
  133. struct rmap_item **rmap_list;
  134. unsigned long seqnr;
  135. };
  136. /**
  137. * struct stable_node - node of the stable rbtree
  138. * @node: rb node of this ksm page in the stable tree
  139. * @head: (overlaying parent) &migrate_nodes indicates temporarily on that list
  140. * @hlist_dup: linked into the stable_node->hlist with a stable_node chain
  141. * @list: linked into migrate_nodes, pending placement in the proper node tree
  142. * @hlist: hlist head of rmap_items using this ksm page
  143. * @kpfn: page frame number of this ksm page (perhaps temporarily on wrong nid)
  144. * @chain_prune_time: time of the last full garbage collection
  145. * @rmap_hlist_len: number of rmap_item entries in hlist or STABLE_NODE_CHAIN
  146. * @nid: NUMA node id of stable tree in which linked (may not match kpfn)
  147. */
  148. struct stable_node {
  149. union {
  150. struct rb_node node; /* when node of stable tree */
  151. struct { /* when listed for migration */
  152. struct list_head *head;
  153. struct {
  154. struct hlist_node hlist_dup;
  155. struct list_head list;
  156. };
  157. };
  158. };
  159. struct hlist_head hlist;
  160. union {
  161. unsigned long kpfn;
  162. unsigned long chain_prune_time;
  163. };
  164. /*
  165. * STABLE_NODE_CHAIN can be any negative number in
  166. * rmap_hlist_len negative range, but better not -1 to be able
  167. * to reliably detect underflows.
  168. */
  169. #define STABLE_NODE_CHAIN -1024
  170. int rmap_hlist_len;
  171. #ifdef CONFIG_NUMA
  172. int nid;
  173. #endif
  174. };
  175. /**
  176. * struct rmap_item - reverse mapping item for virtual addresses
  177. * @rmap_list: next rmap_item in mm_slot's singly-linked rmap_list
  178. * @anon_vma: pointer to anon_vma for this mm,address, when in stable tree
  179. * @nid: NUMA node id of unstable tree in which linked (may not match page)
  180. * @mm: the memory structure this rmap_item is pointing into
  181. * @address: the virtual address this rmap_item tracks (+ flags in low bits)
  182. * @oldchecksum: previous checksum of the page at that virtual address
  183. * @node: rb node of this rmap_item in the unstable tree
  184. * @head: pointer to stable_node heading this list in the stable tree
  185. * @hlist: link into hlist of rmap_items hanging off that stable_node
  186. */
  187. struct rmap_item {
  188. struct rmap_item *rmap_list;
  189. union {
  190. struct anon_vma *anon_vma; /* when stable */
  191. #ifdef CONFIG_NUMA
  192. int nid; /* when node of unstable tree */
  193. #endif
  194. };
  195. struct mm_struct *mm;
  196. unsigned long address; /* + low bits used for flags below */
  197. unsigned int oldchecksum; /* when unstable */
  198. union {
  199. struct rb_node node; /* when node of unstable tree */
  200. struct { /* when listed from stable tree */
  201. struct stable_node *head;
  202. struct hlist_node hlist;
  203. };
  204. };
  205. };
  206. #define SEQNR_MASK 0x0ff /* low bits of unstable tree seqnr */
  207. #define UNSTABLE_FLAG 0x100 /* is a node of the unstable tree */
  208. #define STABLE_FLAG 0x200 /* is listed from the stable tree */
  209. #define KSM_FLAG_MASK (SEQNR_MASK|UNSTABLE_FLAG|STABLE_FLAG)
  210. /* to mask all the flags */
  211. /* The stable and unstable tree heads */
  212. static struct rb_root one_stable_tree[1] = { RB_ROOT };
  213. static struct rb_root one_unstable_tree[1] = { RB_ROOT };
  214. static struct rb_root *root_stable_tree = one_stable_tree;
  215. static struct rb_root *root_unstable_tree = one_unstable_tree;
  216. /* Recently migrated nodes of stable tree, pending proper placement */
  217. static LIST_HEAD(migrate_nodes);
  218. #define STABLE_NODE_DUP_HEAD ((struct list_head *)&migrate_nodes.prev)
  219. #define MM_SLOTS_HASH_BITS 10
  220. static DEFINE_HASHTABLE(mm_slots_hash, MM_SLOTS_HASH_BITS);
  221. static struct mm_slot ksm_mm_head = {
  222. .mm_list = LIST_HEAD_INIT(ksm_mm_head.mm_list),
  223. };
  224. static struct ksm_scan ksm_scan = {
  225. .mm_slot = &ksm_mm_head,
  226. };
  227. static struct kmem_cache *rmap_item_cache;
  228. static struct kmem_cache *stable_node_cache;
  229. static struct kmem_cache *mm_slot_cache;
  230. /* The number of nodes in the stable tree */
  231. static unsigned long ksm_pages_shared;
  232. /* The number of page slots additionally sharing those nodes */
  233. static unsigned long ksm_pages_sharing;
  234. /* The number of nodes in the unstable tree */
  235. static unsigned long ksm_pages_unshared;
  236. /* The number of rmap_items in use: to calculate pages_volatile */
  237. static unsigned long ksm_rmap_items;
  238. /* The number of stable_node chains */
  239. static unsigned long ksm_stable_node_chains;
  240. /* The number of stable_node dups linked to the stable_node chains */
  241. static unsigned long ksm_stable_node_dups;
  242. /* Delay in pruning stale stable_node_dups in the stable_node_chains */
  243. static int ksm_stable_node_chains_prune_millisecs = 2000;
  244. /* Maximum number of page slots sharing a stable node */
  245. static int ksm_max_page_sharing = 256;
  246. /* Number of pages ksmd should scan in one batch */
  247. static unsigned int ksm_thread_pages_to_scan = 100;
  248. /* Milliseconds ksmd should sleep between batches */
  249. static unsigned int ksm_thread_sleep_millisecs = 20;
  250. /* Checksum of an empty (zeroed) page */
  251. static unsigned int zero_checksum __read_mostly;
  252. /* Whether to merge empty (zeroed) pages with actual zero pages */
  253. static bool ksm_use_zero_pages __read_mostly;
  254. #ifdef CONFIG_NUMA
  255. /* Zeroed when merging across nodes is not allowed */
  256. static unsigned int ksm_merge_across_nodes = 1;
  257. static int ksm_nr_node_ids = 1;
  258. #else
  259. #define ksm_merge_across_nodes 1U
  260. #define ksm_nr_node_ids 1
  261. #endif
  262. #define KSM_RUN_STOP 0
  263. #define KSM_RUN_MERGE 1
  264. #define KSM_RUN_UNMERGE 2
  265. #define KSM_RUN_OFFLINE 4
  266. static unsigned long ksm_run = KSM_RUN_STOP;
  267. static void wait_while_offlining(void);
  268. static DECLARE_WAIT_QUEUE_HEAD(ksm_thread_wait);
  269. static DECLARE_WAIT_QUEUE_HEAD(ksm_iter_wait);
  270. static DEFINE_MUTEX(ksm_thread_mutex);
  271. static DEFINE_SPINLOCK(ksm_mmlist_lock);
  272. #define KSM_KMEM_CACHE(__struct, __flags) kmem_cache_create("ksm_"#__struct,\
  273. sizeof(struct __struct), __alignof__(struct __struct),\
  274. (__flags), NULL)
  275. static int __init ksm_slab_init(void)
  276. {
  277. rmap_item_cache = KSM_KMEM_CACHE(rmap_item, 0);
  278. if (!rmap_item_cache)
  279. goto out;
  280. stable_node_cache = KSM_KMEM_CACHE(stable_node, 0);
  281. if (!stable_node_cache)
  282. goto out_free1;
  283. mm_slot_cache = KSM_KMEM_CACHE(mm_slot, 0);
  284. if (!mm_slot_cache)
  285. goto out_free2;
  286. return 0;
  287. out_free2:
  288. kmem_cache_destroy(stable_node_cache);
  289. out_free1:
  290. kmem_cache_destroy(rmap_item_cache);
  291. out:
  292. return -ENOMEM;
  293. }
  294. static void __init ksm_slab_free(void)
  295. {
  296. kmem_cache_destroy(mm_slot_cache);
  297. kmem_cache_destroy(stable_node_cache);
  298. kmem_cache_destroy(rmap_item_cache);
  299. mm_slot_cache = NULL;
  300. }
  301. static __always_inline bool is_stable_node_chain(struct stable_node *chain)
  302. {
  303. return chain->rmap_hlist_len == STABLE_NODE_CHAIN;
  304. }
  305. static __always_inline bool is_stable_node_dup(struct stable_node *dup)
  306. {
  307. return dup->head == STABLE_NODE_DUP_HEAD;
  308. }
  309. static inline void stable_node_chain_add_dup(struct stable_node *dup,
  310. struct stable_node *chain)
  311. {
  312. VM_BUG_ON(is_stable_node_dup(dup));
  313. dup->head = STABLE_NODE_DUP_HEAD;
  314. VM_BUG_ON(!is_stable_node_chain(chain));
  315. hlist_add_head(&dup->hlist_dup, &chain->hlist);
  316. ksm_stable_node_dups++;
  317. }
  318. static inline void __stable_node_dup_del(struct stable_node *dup)
  319. {
  320. VM_BUG_ON(!is_stable_node_dup(dup));
  321. hlist_del(&dup->hlist_dup);
  322. ksm_stable_node_dups--;
  323. }
  324. static inline void stable_node_dup_del(struct stable_node *dup)
  325. {
  326. VM_BUG_ON(is_stable_node_chain(dup));
  327. if (is_stable_node_dup(dup))
  328. __stable_node_dup_del(dup);
  329. else
  330. rb_erase(&dup->node, root_stable_tree + NUMA(dup->nid));
  331. #ifdef CONFIG_DEBUG_VM
  332. dup->head = NULL;
  333. #endif
  334. }
  335. static inline struct rmap_item *alloc_rmap_item(void)
  336. {
  337. struct rmap_item *rmap_item;
  338. rmap_item = kmem_cache_zalloc(rmap_item_cache, GFP_KERNEL |
  339. __GFP_NORETRY | __GFP_NOWARN);
  340. if (rmap_item)
  341. ksm_rmap_items++;
  342. return rmap_item;
  343. }
  344. static inline void free_rmap_item(struct rmap_item *rmap_item)
  345. {
  346. ksm_rmap_items--;
  347. rmap_item->mm = NULL; /* debug safety */
  348. kmem_cache_free(rmap_item_cache, rmap_item);
  349. }
  350. static inline struct stable_node *alloc_stable_node(void)
  351. {
  352. /*
  353. * The allocation can take too long with GFP_KERNEL when memory is under
  354. * pressure, which may lead to hung task warnings. Adding __GFP_HIGH
  355. * grants access to memory reserves, helping to avoid this problem.
  356. */
  357. return kmem_cache_alloc(stable_node_cache, GFP_KERNEL | __GFP_HIGH);
  358. }
  359. static inline void free_stable_node(struct stable_node *stable_node)
  360. {
  361. VM_BUG_ON(stable_node->rmap_hlist_len &&
  362. !is_stable_node_chain(stable_node));
  363. kmem_cache_free(stable_node_cache, stable_node);
  364. }
  365. static inline struct mm_slot *alloc_mm_slot(void)
  366. {
  367. if (!mm_slot_cache) /* initialization failed */
  368. return NULL;
  369. return kmem_cache_zalloc(mm_slot_cache, GFP_KERNEL);
  370. }
  371. static inline void free_mm_slot(struct mm_slot *mm_slot)
  372. {
  373. kmem_cache_free(mm_slot_cache, mm_slot);
  374. }
  375. static struct mm_slot *get_mm_slot(struct mm_struct *mm)
  376. {
  377. struct mm_slot *slot;
  378. hash_for_each_possible(mm_slots_hash, slot, link, (unsigned long)mm)
  379. if (slot->mm == mm)
  380. return slot;
  381. return NULL;
  382. }
  383. static void insert_to_mm_slots_hash(struct mm_struct *mm,
  384. struct mm_slot *mm_slot)
  385. {
  386. mm_slot->mm = mm;
  387. hash_add(mm_slots_hash, &mm_slot->link, (unsigned long)mm);
  388. }
  389. /*
  390. * ksmd, and unmerge_and_remove_all_rmap_items(), must not touch an mm's
  391. * page tables after it has passed through ksm_exit() - which, if necessary,
  392. * takes mmap_lock briefly to serialize against them. ksm_exit() does not set
  393. * a special flag: they can just back out as soon as mm_users goes to zero.
  394. * ksm_test_exit() is used throughout to make this test for exit: in some
  395. * places for correctness, in some places just to avoid unnecessary work.
  396. */
  397. static inline bool ksm_test_exit(struct mm_struct *mm)
  398. {
  399. return atomic_read(&mm->mm_users) == 0;
  400. }
  401. /*
  402. * We use break_ksm to break COW on a ksm page: it's a stripped down
  403. *
  404. * if (get_user_pages(addr, 1, FOLL_WRITE, &page, NULL) == 1)
  405. * put_page(page);
  406. *
  407. * but taking great care only to touch a ksm page, in a VM_MERGEABLE vma,
  408. * in case the application has unmapped and remapped mm,addr meanwhile.
  409. * Could a ksm page appear anywhere else? Actually yes, in a VM_PFNMAP
  410. * mmap of /dev/mem or /dev/kmem, where we would not want to touch it.
  411. *
  412. * FAULT_FLAG/FOLL_REMOTE are because we do this outside the context
  413. * of the process that owns 'vma'. We also do not want to enforce
  414. * protection keys here anyway.
  415. */
  416. static int break_ksm(struct vm_area_struct *vma, unsigned long addr)
  417. {
  418. struct page *page;
  419. vm_fault_t ret = 0;
  420. do {
  421. cond_resched();
  422. page = follow_page(vma, addr,
  423. FOLL_GET | FOLL_MIGRATION | FOLL_REMOTE);
  424. if (IS_ERR_OR_NULL(page))
  425. break;
  426. if (PageKsm(page))
  427. ret = handle_mm_fault(vma, addr,
  428. FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE,
  429. NULL);
  430. else
  431. ret = VM_FAULT_WRITE;
  432. put_user_page(page);
  433. } while (!(ret & (VM_FAULT_WRITE | VM_FAULT_SIGBUS | VM_FAULT_SIGSEGV | VM_FAULT_OOM)));
  434. /*
  435. * We must loop because handle_mm_fault() may back out if there's
  436. * any difficulty e.g. if pte accessed bit gets updated concurrently.
  437. *
  438. * VM_FAULT_WRITE is what we have been hoping for: it indicates that
  439. * COW has been broken, even if the vma does not permit VM_WRITE;
  440. * but note that a concurrent fault might break PageKsm for us.
  441. *
  442. * VM_FAULT_SIGBUS could occur if we race with truncation of the
  443. * backing file, which also invalidates anonymous pages: that's
  444. * okay, that truncation will have unmapped the PageKsm for us.
  445. *
  446. * VM_FAULT_OOM: at the time of writing (late July 2009), setting
  447. * aside mem_cgroup limits, VM_FAULT_OOM would only be set if the
  448. * current task has TIF_MEMDIE set, and will be OOM killed on return
  449. * to user; and ksmd, having no mm, would never be chosen for that.
  450. *
  451. * But if the mm is in a limited mem_cgroup, then the fault may fail
  452. * with VM_FAULT_OOM even if the current task is not TIF_MEMDIE; and
  453. * even ksmd can fail in this way - though it's usually breaking ksm
  454. * just to undo a merge it made a moment before, so unlikely to oom.
  455. *
  456. * That's a pity: we might therefore have more kernel pages allocated
  457. * than we're counting as nodes in the stable tree; but ksm_do_scan
  458. * will retry to break_cow on each pass, so should recover the page
  459. * in due course. The important thing is to not let VM_MERGEABLE
  460. * be cleared while any such pages might remain in the area.
  461. */
  462. return (ret & VM_FAULT_OOM) ? -ENOMEM : 0;
  463. }
  464. static struct vm_area_struct *find_mergeable_vma(struct mm_struct *mm,
  465. unsigned long addr)
  466. {
  467. struct vm_area_struct *vma;
  468. if (ksm_test_exit(mm))
  469. return NULL;
  470. vma = find_vma(mm, addr);
  471. if (!vma || vma->vm_start > addr)
  472. return NULL;
  473. if (!(vma->vm_flags & VM_MERGEABLE) || !vma->anon_vma)
  474. return NULL;
  475. return vma;
  476. }
  477. static void break_cow(struct rmap_item *rmap_item)
  478. {
  479. struct mm_struct *mm = rmap_item->mm;
  480. unsigned long addr = rmap_item->address;
  481. struct vm_area_struct *vma;
  482. /*
  483. * It is not an accident that whenever we want to break COW
  484. * to undo, we also need to drop a reference to the anon_vma.
  485. */
  486. put_anon_vma(rmap_item->anon_vma);
  487. mmap_read_lock(mm);
  488. vma = find_mergeable_vma(mm, addr);
  489. if (vma)
  490. break_ksm(vma, addr);
  491. mmap_read_unlock(mm);
  492. }
  493. static struct page *get_mergeable_page(struct rmap_item *rmap_item)
  494. {
  495. struct mm_struct *mm = rmap_item->mm;
  496. unsigned long addr = rmap_item->address;
  497. struct vm_area_struct *vma;
  498. struct page *page;
  499. mmap_read_lock(mm);
  500. vma = find_mergeable_vma(mm, addr);
  501. if (!vma)
  502. goto out;
  503. page = follow_page(vma, addr, FOLL_GET);
  504. if (IS_ERR_OR_NULL(page))
  505. goto out;
  506. if (PageAnon(page)) {
  507. flush_anon_page(vma, page, addr);
  508. flush_dcache_page(page);
  509. } else {
  510. put_user_page(page);
  511. out:
  512. page = NULL;
  513. }
  514. mmap_read_unlock(mm);
  515. return page;
  516. }
  517. /*
  518. * This helper is used for getting right index into array of tree roots.
  519. * When merge_across_nodes knob is set to 1, there are only two rb-trees for
  520. * stable and unstable pages from all nodes with roots in index 0. Otherwise,
  521. * every node has its own stable and unstable tree.
  522. */
  523. static inline int get_kpfn_nid(unsigned long kpfn)
  524. {
  525. return ksm_merge_across_nodes ? 0 : NUMA(pfn_to_nid(kpfn));
  526. }
  527. static struct stable_node *alloc_stable_node_chain(struct stable_node *dup,
  528. struct rb_root *root)
  529. {
  530. struct stable_node *chain = alloc_stable_node();
  531. VM_BUG_ON(is_stable_node_chain(dup));
  532. if (likely(chain)) {
  533. INIT_HLIST_HEAD(&chain->hlist);
  534. chain->chain_prune_time = jiffies;
  535. chain->rmap_hlist_len = STABLE_NODE_CHAIN;
  536. #if defined (CONFIG_DEBUG_VM) && defined(CONFIG_NUMA)
  537. chain->nid = NUMA_NO_NODE; /* debug */
  538. #endif
  539. ksm_stable_node_chains++;
  540. /*
  541. * Put the stable node chain in the first dimension of
  542. * the stable tree and at the same time remove the old
  543. * stable node.
  544. */
  545. rb_replace_node(&dup->node, &chain->node, root);
  546. /*
  547. * Move the old stable node to the second dimension
  548. * queued in the hlist_dup. The invariant is that all
  549. * dup stable_nodes in the chain->hlist point to pages
  550. * that are write protected and have the exact same
  551. * content.
  552. */
  553. stable_node_chain_add_dup(dup, chain);
  554. }
  555. return chain;
  556. }
  557. static inline void free_stable_node_chain(struct stable_node *chain,
  558. struct rb_root *root)
  559. {
  560. rb_erase(&chain->node, root);
  561. free_stable_node(chain);
  562. ksm_stable_node_chains--;
  563. }
  564. static void remove_node_from_stable_tree(struct stable_node *stable_node)
  565. {
  566. struct rmap_item *rmap_item;
  567. /* check it's not STABLE_NODE_CHAIN or negative */
  568. BUG_ON(stable_node->rmap_hlist_len < 0);
  569. hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) {
  570. if (rmap_item->hlist.next)
  571. ksm_pages_sharing--;
  572. else
  573. ksm_pages_shared--;
  574. VM_BUG_ON(stable_node->rmap_hlist_len <= 0);
  575. stable_node->rmap_hlist_len--;
  576. put_anon_vma(rmap_item->anon_vma);
  577. rmap_item->address &= PAGE_MASK;
  578. cond_resched();
  579. }
  580. /*
  581. * We need the second aligned pointer of the migrate_nodes
  582. * list_head to stay clear from the rb_parent_color union
  583. * (aligned and different than any node) and also different
  584. * from &migrate_nodes. This will verify that future list.h changes
  585. * don't break STABLE_NODE_DUP_HEAD. Only recent gcc can handle it.
  586. */
  587. #if defined(GCC_VERSION) && GCC_VERSION >= 40903
  588. BUILD_BUG_ON(STABLE_NODE_DUP_HEAD <= &migrate_nodes);
  589. BUILD_BUG_ON(STABLE_NODE_DUP_HEAD >= &migrate_nodes + 1);
  590. #endif
  591. if (stable_node->head == &migrate_nodes)
  592. list_del(&stable_node->list);
  593. else
  594. stable_node_dup_del(stable_node);
  595. free_stable_node(stable_node);
  596. }
  597. enum get_ksm_page_flags {
  598. GET_KSM_PAGE_NOLOCK,
  599. GET_KSM_PAGE_LOCK,
  600. GET_KSM_PAGE_TRYLOCK
  601. };
  602. /*
  603. * get_ksm_page: checks if the page indicated by the stable node
  604. * is still its ksm page, despite having held no reference to it.
  605. * In which case we can trust the content of the page, and it
  606. * returns the gotten page; but if the page has now been zapped,
  607. * remove the stale node from the stable tree and return NULL.
  608. * But beware, the stable node's page might be being migrated.
  609. *
  610. * You would expect the stable_node to hold a reference to the ksm page.
  611. * But if it increments the page's count, swapping out has to wait for
  612. * ksmd to come around again before it can free the page, which may take
  613. * seconds or even minutes: much too unresponsive. So instead we use a
  614. * "keyhole reference": access to the ksm page from the stable node peeps
  615. * out through its keyhole to see if that page still holds the right key,
  616. * pointing back to this stable node. This relies on freeing a PageAnon
  617. * page to reset its page->mapping to NULL, and relies on no other use of
  618. * a page to put something that might look like our key in page->mapping.
  619. * is on its way to being freed; but it is an anomaly to bear in mind.
  620. */
  621. static struct page *get_ksm_page(struct stable_node *stable_node,
  622. enum get_ksm_page_flags flags)
  623. {
  624. struct page *page;
  625. void *expected_mapping;
  626. unsigned long kpfn;
  627. expected_mapping = (void *)((unsigned long)stable_node |
  628. PAGE_MAPPING_KSM);
  629. again:
  630. kpfn = READ_ONCE(stable_node->kpfn); /* Address dependency. */
  631. page = pfn_to_page(kpfn);
  632. if (READ_ONCE(page->mapping) != expected_mapping)
  633. goto stale;
  634. /*
  635. * We cannot do anything with the page while its refcount is 0.
  636. * Usually 0 means free, or tail of a higher-order page: in which
  637. * case this node is no longer referenced, and should be freed;
  638. * however, it might mean that the page is under page_ref_freeze().
  639. * The __remove_mapping() case is easy, again the node is now stale;
  640. * the same is in reuse_ksm_page() case; but if page is swapcache
  641. * in migrate_page_move_mapping(), it might still be our page,
  642. * in which case it's essential to keep the node.
  643. */
  644. while (!get_page_unless_zero(page)) {
  645. /*
  646. * Another check for page->mapping != expected_mapping would
  647. * work here too. We have chosen the !PageSwapCache test to
  648. * optimize the common case, when the page is or is about to
  649. * be freed: PageSwapCache is cleared (under spin_lock_irq)
  650. * in the ref_freeze section of __remove_mapping(); but Anon
  651. * page->mapping reset to NULL later, in free_pages_prepare().
  652. */
  653. if (!PageSwapCache(page))
  654. goto stale;
  655. cpu_relax();
  656. }
  657. if (READ_ONCE(page->mapping) != expected_mapping) {
  658. put_page(page);
  659. goto stale;
  660. }
  661. if (flags == GET_KSM_PAGE_TRYLOCK) {
  662. if (!trylock_page(page)) {
  663. put_page(page);
  664. return ERR_PTR(-EBUSY);
  665. }
  666. } else if (flags == GET_KSM_PAGE_LOCK)
  667. lock_page(page);
  668. if (flags != GET_KSM_PAGE_NOLOCK) {
  669. if (READ_ONCE(page->mapping) != expected_mapping) {
  670. unlock_page(page);
  671. put_page(page);
  672. goto stale;
  673. }
  674. }
  675. return page;
  676. stale:
  677. /*
  678. * We come here from above when page->mapping or !PageSwapCache
  679. * suggests that the node is stale; but it might be under migration.
  680. * We need smp_rmb(), matching the smp_wmb() in ksm_migrate_page(),
  681. * before checking whether node->kpfn has been changed.
  682. */
  683. smp_rmb();
  684. if (READ_ONCE(stable_node->kpfn) != kpfn)
  685. goto again;
  686. remove_node_from_stable_tree(stable_node);
  687. return NULL;
  688. }
  689. /*
  690. * Removing rmap_item from stable or unstable tree.
  691. * This function will clean the information from the stable/unstable tree.
  692. */
  693. static void remove_rmap_item_from_tree(struct rmap_item *rmap_item)
  694. {
  695. if (rmap_item->address & STABLE_FLAG) {
  696. struct stable_node *stable_node;
  697. struct page *page;
  698. stable_node = rmap_item->head;
  699. page = get_ksm_page(stable_node, GET_KSM_PAGE_LOCK);
  700. if (!page)
  701. goto out;
  702. hlist_del(&rmap_item->hlist);
  703. unlock_page(page);
  704. put_page(page);
  705. if (!hlist_empty(&stable_node->hlist))
  706. ksm_pages_sharing--;
  707. else
  708. ksm_pages_shared--;
  709. VM_BUG_ON(stable_node->rmap_hlist_len <= 0);
  710. stable_node->rmap_hlist_len--;
  711. put_anon_vma(rmap_item->anon_vma);
  712. rmap_item->head = NULL;
  713. rmap_item->address &= PAGE_MASK;
  714. } else if (rmap_item->address & UNSTABLE_FLAG) {
  715. unsigned char age;
  716. /*
  717. * Usually ksmd can and must skip the rb_erase, because
  718. * root_unstable_tree was already reset to RB_ROOT.
  719. * But be careful when an mm is exiting: do the rb_erase
  720. * if this rmap_item was inserted by this scan, rather
  721. * than left over from before.
  722. */
  723. age = (unsigned char)(ksm_scan.seqnr - rmap_item->address);
  724. BUG_ON(age > 1);
  725. if (!age)
  726. rb_erase(&rmap_item->node,
  727. root_unstable_tree + NUMA(rmap_item->nid));
  728. ksm_pages_unshared--;
  729. rmap_item->address &= PAGE_MASK;
  730. }
  731. out:
  732. cond_resched(); /* we're called from many long loops */
  733. }
  734. static void remove_trailing_rmap_items(struct mm_slot *mm_slot,
  735. struct rmap_item **rmap_list)
  736. {
  737. while (*rmap_list) {
  738. struct rmap_item *rmap_item = *rmap_list;
  739. *rmap_list = rmap_item->rmap_list;
  740. remove_rmap_item_from_tree(rmap_item);
  741. free_rmap_item(rmap_item);
  742. }
  743. }
  744. /*
  745. * Though it's very tempting to unmerge rmap_items from stable tree rather
  746. * than check every pte of a given vma, the locking doesn't quite work for
  747. * that - an rmap_item is assigned to the stable tree after inserting ksm
  748. * page and upping mmap_lock. Nor does it fit with the way we skip dup'ing
  749. * rmap_items from parent to child at fork time (so as not to waste time
  750. * if exit comes before the next scan reaches it).
  751. *
  752. * Similarly, although we'd like to remove rmap_items (so updating counts
  753. * and freeing memory) when unmerging an area, it's easier to leave that
  754. * to the next pass of ksmd - consider, for example, how ksmd might be
  755. * in cmp_and_merge_page on one of the rmap_items we would be removing.
  756. */
  757. static int unmerge_ksm_pages(struct vm_area_struct *vma,
  758. unsigned long start, unsigned long end)
  759. {
  760. unsigned long addr;
  761. int err = 0;
  762. for (addr = start; addr < end && !err; addr += PAGE_SIZE) {
  763. if (ksm_test_exit(vma->vm_mm))
  764. break;
  765. if (signal_pending(current))
  766. err = -ERESTARTSYS;
  767. else
  768. err = break_ksm(vma, addr);
  769. }
  770. return err;
  771. }
  772. static inline struct stable_node *page_stable_node(struct page *page)
  773. {
  774. return PageKsm(page) ? page_rmapping(page) : NULL;
  775. }
  776. static inline void set_page_stable_node(struct page *page,
  777. struct stable_node *stable_node)
  778. {
  779. page->mapping = (void *)((unsigned long)stable_node | PAGE_MAPPING_KSM);
  780. }
  781. #ifdef CONFIG_SYSFS
  782. /*
  783. * Only called through the sysfs control interface:
  784. */
  785. static int remove_stable_node(struct stable_node *stable_node)
  786. {
  787. struct page *page;
  788. int err;
  789. page = get_ksm_page(stable_node, GET_KSM_PAGE_LOCK);
  790. if (!page) {
  791. /*
  792. * get_ksm_page did remove_node_from_stable_tree itself.
  793. */
  794. return 0;
  795. }
  796. /*
  797. * Page could be still mapped if this races with __mmput() running in
  798. * between ksm_exit() and exit_mmap(). Just refuse to let
  799. * merge_across_nodes/max_page_sharing be switched.
  800. */
  801. err = -EBUSY;
  802. if (!page_mapped(page)) {
  803. /*
  804. * The stable node did not yet appear stale to get_ksm_page(),
  805. * since that allows for an unmapped ksm page to be recognized
  806. * right up until it is freed; but the node is safe to remove.
  807. * This page might be in a pagevec waiting to be freed,
  808. * or it might be PageSwapCache (perhaps under writeback),
  809. * or it might have been removed from swapcache a moment ago.
  810. */
  811. set_page_stable_node(page, NULL);
  812. remove_node_from_stable_tree(stable_node);
  813. err = 0;
  814. }
  815. unlock_page(page);
  816. put_page(page);
  817. return err;
  818. }
  819. static int remove_stable_node_chain(struct stable_node *stable_node,
  820. struct rb_root *root)
  821. {
  822. struct stable_node *dup;
  823. struct hlist_node *hlist_safe;
  824. if (!is_stable_node_chain(stable_node)) {
  825. VM_BUG_ON(is_stable_node_dup(stable_node));
  826. if (remove_stable_node(stable_node))
  827. return true;
  828. else
  829. return false;
  830. }
  831. hlist_for_each_entry_safe(dup, hlist_safe,
  832. &stable_node->hlist, hlist_dup) {
  833. VM_BUG_ON(!is_stable_node_dup(dup));
  834. if (remove_stable_node(dup))
  835. return true;
  836. }
  837. BUG_ON(!hlist_empty(&stable_node->hlist));
  838. free_stable_node_chain(stable_node, root);
  839. return false;
  840. }
  841. static int remove_all_stable_nodes(void)
  842. {
  843. struct stable_node *stable_node, *next;
  844. int nid;
  845. int err = 0;
  846. for (nid = 0; nid < ksm_nr_node_ids; nid++) {
  847. while (root_stable_tree[nid].rb_node) {
  848. stable_node = rb_entry(root_stable_tree[nid].rb_node,
  849. struct stable_node, node);
  850. if (remove_stable_node_chain(stable_node,
  851. root_stable_tree + nid)) {
  852. err = -EBUSY;
  853. break; /* proceed to next nid */
  854. }
  855. cond_resched();
  856. }
  857. }
  858. list_for_each_entry_safe(stable_node, next, &migrate_nodes, list) {
  859. if (remove_stable_node(stable_node))
  860. err = -EBUSY;
  861. cond_resched();
  862. }
  863. return err;
  864. }
  865. static int unmerge_and_remove_all_rmap_items(void)
  866. {
  867. struct mm_slot *mm_slot;
  868. struct mm_struct *mm;
  869. struct vm_area_struct *vma;
  870. int err = 0;
  871. spin_lock(&ksm_mmlist_lock);
  872. ksm_scan.mm_slot = list_entry(ksm_mm_head.mm_list.next,
  873. struct mm_slot, mm_list);
  874. spin_unlock(&ksm_mmlist_lock);
  875. for (mm_slot = ksm_scan.mm_slot;
  876. mm_slot != &ksm_mm_head; mm_slot = ksm_scan.mm_slot) {
  877. mm = mm_slot->mm;
  878. mmap_read_lock(mm);
  879. for (vma = mm->mmap; vma; vma = vma->vm_next) {
  880. if (ksm_test_exit(mm))
  881. break;
  882. if (!(vma->vm_flags & VM_MERGEABLE) || !vma->anon_vma)
  883. continue;
  884. err = unmerge_ksm_pages(vma,
  885. vma->vm_start, vma->vm_end);
  886. if (err)
  887. goto error;
  888. }
  889. remove_trailing_rmap_items(mm_slot, &mm_slot->rmap_list);
  890. mmap_read_unlock(mm);
  891. spin_lock(&ksm_mmlist_lock);
  892. ksm_scan.mm_slot = list_entry(mm_slot->mm_list.next,
  893. struct mm_slot, mm_list);
  894. if (ksm_test_exit(mm)) {
  895. hash_del(&mm_slot->link);
  896. list_del(&mm_slot->mm_list);
  897. spin_unlock(&ksm_mmlist_lock);
  898. free_mm_slot(mm_slot);
  899. clear_bit(MMF_VM_MERGEABLE, &mm->flags);
  900. mmdrop(mm);
  901. } else
  902. spin_unlock(&ksm_mmlist_lock);
  903. }
  904. /* Clean up stable nodes, but don't worry if some are still busy */
  905. remove_all_stable_nodes();
  906. ksm_scan.seqnr = 0;
  907. return 0;
  908. error:
  909. mmap_read_unlock(mm);
  910. spin_lock(&ksm_mmlist_lock);
  911. ksm_scan.mm_slot = &ksm_mm_head;
  912. spin_unlock(&ksm_mmlist_lock);
  913. return err;
  914. }
  915. #endif /* CONFIG_SYSFS */
  916. static u32 calc_checksum(struct page *page)
  917. {
  918. u32 checksum;
  919. void *addr = kmap_atomic(page);
  920. checksum = xxhash(addr, PAGE_SIZE, 0);
  921. kunmap_atomic(addr);
  922. return checksum;
  923. }
  924. static int write_protect_page(struct vm_area_struct *vma, struct page *page,
  925. pte_t *orig_pte)
  926. {
  927. struct mm_struct *mm = vma->vm_mm;
  928. struct page_vma_mapped_walk pvmw = {
  929. .page = page,
  930. .vma = vma,
  931. };
  932. int swapped;
  933. int err = -EFAULT;
  934. struct mmu_notifier_range range;
  935. pvmw.address = page_address_in_vma(page, vma);
  936. if (pvmw.address == -EFAULT)
  937. goto out;
  938. BUG_ON(PageTransCompound(page));
  939. mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm,
  940. pvmw.address,
  941. pvmw.address + PAGE_SIZE);
  942. mmu_notifier_invalidate_range_start(&range);
  943. if (!page_vma_mapped_walk(&pvmw))
  944. goto out_mn;
  945. if (WARN_ONCE(!pvmw.pte, "Unexpected PMD mapping?"))
  946. goto out_unlock;
  947. if (pte_write(*pvmw.pte) || pte_dirty(*pvmw.pte) ||
  948. (pte_protnone(*pvmw.pte) && pte_savedwrite(*pvmw.pte)) ||
  949. mm_tlb_flush_pending(mm)) {
  950. pte_t entry;
  951. swapped = PageSwapCache(page);
  952. flush_cache_page(vma, pvmw.address, page_to_pfn(page));
  953. /*
  954. * Ok this is tricky, when get_user_pages_fast() run it doesn't
  955. * take any lock, therefore the check that we are going to make
  956. * with the pagecount against the mapcount is racey and
  957. * O_DIRECT can happen right after the check.
  958. * So we clear the pte and flush the tlb before the check
  959. * this assure us that no O_DIRECT can happen after the check
  960. * or in the middle of the check.
  961. *
  962. * No need to notify as we are downgrading page table to read
  963. * only not changing it to point to a new page.
  964. *
  965. * See Documentation/vm/mmu_notifier.rst
  966. */
  967. entry = ptep_clear_flush(vma, pvmw.address, pvmw.pte);
  968. /*
  969. * Check that no O_DIRECT or similar I/O is in progress on the
  970. * page
  971. */
  972. if (page_mapcount(page) + 1 + swapped != page_count(page)) {
  973. set_pte_at(mm, pvmw.address, pvmw.pte, entry);
  974. goto out_unlock;
  975. }
  976. if (pte_dirty(entry))
  977. set_page_dirty(page);
  978. if (pte_protnone(entry))
  979. entry = pte_mkclean(pte_clear_savedwrite(entry));
  980. else
  981. entry = pte_mkclean(pte_wrprotect(entry));
  982. set_pte_at_notify(mm, pvmw.address, pvmw.pte, entry);
  983. }
  984. *orig_pte = *pvmw.pte;
  985. err = 0;
  986. out_unlock:
  987. page_vma_mapped_walk_done(&pvmw);
  988. out_mn:
  989. mmu_notifier_invalidate_range_end(&range);
  990. out:
  991. return err;
  992. }
  993. /**
  994. * replace_page - replace page in vma by new ksm page
  995. * @vma: vma that holds the pte pointing to page
  996. * @page: the page we are replacing by kpage
  997. * @kpage: the ksm page we replace page by
  998. * @orig_pte: the original value of the pte
  999. *
  1000. * Returns 0 on success, -EFAULT on failure.
  1001. */
  1002. static int replace_page(struct vm_area_struct *vma, struct page *page,
  1003. struct page *kpage, pte_t orig_pte)
  1004. {
  1005. struct mm_struct *mm = vma->vm_mm;
  1006. pmd_t *pmd;
  1007. pte_t *ptep;
  1008. pte_t newpte;
  1009. spinlock_t *ptl;
  1010. unsigned long addr;
  1011. int err = -EFAULT;
  1012. struct mmu_notifier_range range;
  1013. addr = page_address_in_vma(page, vma);
  1014. if (addr == -EFAULT)
  1015. goto out;
  1016. pmd = mm_find_pmd(mm, addr);
  1017. if (!pmd)
  1018. goto out;
  1019. mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm, addr,
  1020. addr + PAGE_SIZE);
  1021. mmu_notifier_invalidate_range_start(&range);
  1022. ptep = pte_offset_map_lock(mm, pmd, addr, &ptl);
  1023. if (!pte_same(*ptep, orig_pte)) {
  1024. pte_unmap_unlock(ptep, ptl);
  1025. goto out_mn;
  1026. }
  1027. /*
  1028. * No need to check ksm_use_zero_pages here: we can only have a
  1029. * zero_page here if ksm_use_zero_pages was enabled already.
  1030. */
  1031. if (!is_zero_pfn(page_to_pfn(kpage))) {
  1032. get_page(kpage);
  1033. page_add_anon_rmap(kpage, vma, addr, false);
  1034. newpte = mk_pte(kpage, vma->vm_page_prot);
  1035. } else {
  1036. newpte = pte_mkspecial(pfn_pte(page_to_pfn(kpage),
  1037. vma->vm_page_prot));
  1038. /*
  1039. * We're replacing an anonymous page with a zero page, which is
  1040. * not anonymous. We need to do proper accounting otherwise we
  1041. * will get wrong values in /proc, and a BUG message in dmesg
  1042. * when tearing down the mm.
  1043. */
  1044. dec_mm_counter(mm, MM_ANONPAGES);
  1045. }
  1046. flush_cache_page(vma, addr, pte_pfn(*ptep));
  1047. /*
  1048. * No need to notify as we are replacing a read only page with another
  1049. * read only page with the same content.
  1050. *
  1051. * See Documentation/vm/mmu_notifier.rst
  1052. */
  1053. ptep_clear_flush(vma, addr, ptep);
  1054. set_pte_at_notify(mm, addr, ptep, newpte);
  1055. page_remove_rmap(page, false);
  1056. if (!page_mapped(page))
  1057. try_to_free_swap(page);
  1058. put_page(page);
  1059. pte_unmap_unlock(ptep, ptl);
  1060. err = 0;
  1061. out_mn:
  1062. mmu_notifier_invalidate_range_end(&range);
  1063. out:
  1064. return err;
  1065. }
  1066. /*
  1067. * try_to_merge_one_page - take two pages and merge them into one
  1068. * @vma: the vma that holds the pte pointing to page
  1069. * @page: the PageAnon page that we want to replace with kpage
  1070. * @kpage: the PageKsm page that we want to map instead of page,
  1071. * or NULL the first time when we want to use page as kpage.
  1072. *
  1073. * This function returns 0 if the pages were merged, -EFAULT otherwise.
  1074. */
  1075. static int try_to_merge_one_page(struct vm_area_struct *vma,
  1076. struct page *page, struct page *kpage)
  1077. {
  1078. pte_t orig_pte = __pte(0);
  1079. int err = -EFAULT;
  1080. if (page == kpage) /* ksm page forked */
  1081. return 0;
  1082. if (!PageAnon(page))
  1083. goto out;
  1084. /*
  1085. * We need the page lock to read a stable PageSwapCache in
  1086. * write_protect_page(). We use trylock_page() instead of
  1087. * lock_page() because we don't want to wait here - we
  1088. * prefer to continue scanning and merging different pages,
  1089. * then come back to this page when it is unlocked.
  1090. */
  1091. if (!trylock_page(page))
  1092. goto out;
  1093. if (PageTransCompound(page)) {
  1094. if (split_huge_page(page))
  1095. goto out_unlock;
  1096. }
  1097. /*
  1098. * If this anonymous page is mapped only here, its pte may need
  1099. * to be write-protected. If it's mapped elsewhere, all of its
  1100. * ptes are necessarily already write-protected. But in either
  1101. * case, we need to lock and check page_count is not raised.
  1102. */
  1103. if (write_protect_page(vma, page, &orig_pte) == 0) {
  1104. if (!kpage) {
  1105. /*
  1106. * While we hold page lock, upgrade page from
  1107. * PageAnon+anon_vma to PageKsm+NULL stable_node:
  1108. * stable_tree_insert() will update stable_node.
  1109. */
  1110. set_page_stable_node(page, NULL);
  1111. mark_page_accessed(page);
  1112. /*
  1113. * Page reclaim just frees a clean page with no dirty
  1114. * ptes: make sure that the ksm page would be swapped.
  1115. */
  1116. if (!PageDirty(page))
  1117. SetPageDirty(page);
  1118. err = 0;
  1119. } else if (pages_identical(page, kpage))
  1120. err = replace_page(vma, page, kpage, orig_pte);
  1121. }
  1122. if ((vma->vm_flags & VM_LOCKED) && kpage && !err) {
  1123. munlock_vma_page(page);
  1124. if (!PageMlocked(kpage)) {
  1125. unlock_page(page);
  1126. lock_page(kpage);
  1127. mlock_vma_page(kpage);
  1128. page = kpage; /* for final unlock */
  1129. }
  1130. }
  1131. out_unlock:
  1132. unlock_page(page);
  1133. out:
  1134. return err;
  1135. }
  1136. /*
  1137. * try_to_merge_with_ksm_page - like try_to_merge_two_pages,
  1138. * but no new kernel page is allocated: kpage must already be a ksm page.
  1139. *
  1140. * This function returns 0 if the pages were merged, -EFAULT otherwise.
  1141. */
  1142. static int try_to_merge_with_ksm_page(struct rmap_item *rmap_item,
  1143. struct page *page, struct page *kpage)
  1144. {
  1145. struct mm_struct *mm = rmap_item->mm;
  1146. struct vm_area_struct *vma;
  1147. int err = -EFAULT;
  1148. mmap_read_lock(mm);
  1149. vma = find_mergeable_vma(mm, rmap_item->address);
  1150. if (!vma)
  1151. goto out;
  1152. err = try_to_merge_one_page(vma, page, kpage);
  1153. if (err)
  1154. goto out;
  1155. /* Unstable nid is in union with stable anon_vma: remove first */
  1156. remove_rmap_item_from_tree(rmap_item);
  1157. /* Must get reference to anon_vma while still holding mmap_lock */
  1158. rmap_item->anon_vma = vma->anon_vma;
  1159. get_anon_vma(vma->anon_vma);
  1160. out:
  1161. mmap_read_unlock(mm);
  1162. return err;
  1163. }
  1164. /*
  1165. * try_to_merge_two_pages - take two identical pages and prepare them
  1166. * to be merged into one page.
  1167. *
  1168. * This function returns the kpage if we successfully merged two identical
  1169. * pages into one ksm page, NULL otherwise.
  1170. *
  1171. * Note that this function upgrades page to ksm page: if one of the pages
  1172. * is already a ksm page, try_to_merge_with_ksm_page should be used.
  1173. */
  1174. static struct page *try_to_merge_two_pages(struct rmap_item *rmap_item,
  1175. struct page *page,
  1176. struct rmap_item *tree_rmap_item,
  1177. struct page *tree_page)
  1178. {
  1179. int err;
  1180. err = try_to_merge_with_ksm_page(rmap_item, page, NULL);
  1181. if (!err) {
  1182. err = try_to_merge_with_ksm_page(tree_rmap_item,
  1183. tree_page, page);
  1184. /*
  1185. * If that fails, we have a ksm page with only one pte
  1186. * pointing to it: so break it.
  1187. */
  1188. if (err)
  1189. break_cow(rmap_item);
  1190. }
  1191. return err ? NULL : page;
  1192. }
  1193. static __always_inline
  1194. bool __is_page_sharing_candidate(struct stable_node *stable_node, int offset)
  1195. {
  1196. VM_BUG_ON(stable_node->rmap_hlist_len < 0);
  1197. /*
  1198. * Check that at least one mapping still exists, otherwise
  1199. * there's no much point to merge and share with this
  1200. * stable_node, as the underlying tree_page of the other
  1201. * sharer is going to be freed soon.
  1202. */
  1203. return stable_node->rmap_hlist_len &&
  1204. stable_node->rmap_hlist_len + offset < ksm_max_page_sharing;
  1205. }
  1206. static __always_inline
  1207. bool is_page_sharing_candidate(struct stable_node *stable_node)
  1208. {
  1209. return __is_page_sharing_candidate(stable_node, 0);
  1210. }
  1211. static struct page *stable_node_dup(struct stable_node **_stable_node_dup,
  1212. struct stable_node **_stable_node,
  1213. struct rb_root *root,
  1214. bool prune_stale_stable_nodes)
  1215. {
  1216. struct stable_node *dup, *found = NULL, *stable_node = *_stable_node;
  1217. struct hlist_node *hlist_safe;
  1218. struct page *_tree_page, *tree_page = NULL;
  1219. int nr = 0;
  1220. int found_rmap_hlist_len;
  1221. if (!prune_stale_stable_nodes ||
  1222. time_before(jiffies, stable_node->chain_prune_time +
  1223. msecs_to_jiffies(
  1224. ksm_stable_node_chains_prune_millisecs)))
  1225. prune_stale_stable_nodes = false;
  1226. else
  1227. stable_node->chain_prune_time = jiffies;
  1228. hlist_for_each_entry_safe(dup, hlist_safe,
  1229. &stable_node->hlist, hlist_dup) {
  1230. cond_resched();
  1231. /*
  1232. * We must walk all stable_node_dup to prune the stale
  1233. * stable nodes during lookup.
  1234. *
  1235. * get_ksm_page can drop the nodes from the
  1236. * stable_node->hlist if they point to freed pages
  1237. * (that's why we do a _safe walk). The "dup"
  1238. * stable_node parameter itself will be freed from
  1239. * under us if it returns NULL.
  1240. */
  1241. _tree_page = get_ksm_page(dup, GET_KSM_PAGE_NOLOCK);
  1242. if (!_tree_page)
  1243. continue;
  1244. nr += 1;
  1245. if (is_page_sharing_candidate(dup)) {
  1246. if (!found ||
  1247. dup->rmap_hlist_len > found_rmap_hlist_len) {
  1248. if (found)
  1249. put_page(tree_page);
  1250. found = dup;
  1251. found_rmap_hlist_len = found->rmap_hlist_len;
  1252. tree_page = _tree_page;
  1253. /* skip put_page for found dup */
  1254. if (!prune_stale_stable_nodes)
  1255. break;
  1256. continue;
  1257. }
  1258. }
  1259. put_page(_tree_page);
  1260. }
  1261. if (found) {
  1262. /*
  1263. * nr is counting all dups in the chain only if
  1264. * prune_stale_stable_nodes is true, otherwise we may
  1265. * break the loop at nr == 1 even if there are
  1266. * multiple entries.
  1267. */
  1268. if (prune_stale_stable_nodes && nr == 1) {
  1269. /*
  1270. * If there's not just one entry it would
  1271. * corrupt memory, better BUG_ON. In KSM
  1272. * context with no lock held it's not even
  1273. * fatal.
  1274. */
  1275. BUG_ON(stable_node->hlist.first->next);
  1276. /*
  1277. * There's just one entry and it is below the
  1278. * deduplication limit so drop the chain.
  1279. */
  1280. rb_replace_node(&stable_node->node, &found->node,
  1281. root);
  1282. free_stable_node(stable_node);
  1283. ksm_stable_node_chains--;
  1284. ksm_stable_node_dups--;
  1285. /*
  1286. * NOTE: the caller depends on the stable_node
  1287. * to be equal to stable_node_dup if the chain
  1288. * was collapsed.
  1289. */
  1290. *_stable_node = found;
  1291. /*
  1292. * Just for robustneess as stable_node is
  1293. * otherwise left as a stable pointer, the
  1294. * compiler shall optimize it away at build
  1295. * time.
  1296. */
  1297. stable_node = NULL;
  1298. } else if (stable_node->hlist.first != &found->hlist_dup &&
  1299. __is_page_sharing_candidate(found, 1)) {
  1300. /*
  1301. * If the found stable_node dup can accept one
  1302. * more future merge (in addition to the one
  1303. * that is underway) and is not at the head of
  1304. * the chain, put it there so next search will
  1305. * be quicker in the !prune_stale_stable_nodes
  1306. * case.
  1307. *
  1308. * NOTE: it would be inaccurate to use nr > 1
  1309. * instead of checking the hlist.first pointer
  1310. * directly, because in the
  1311. * prune_stale_stable_nodes case "nr" isn't
  1312. * the position of the found dup in the chain,
  1313. * but the total number of dups in the chain.
  1314. */
  1315. hlist_del(&found->hlist_dup);
  1316. hlist_add_head(&found->hlist_dup,
  1317. &stable_node->hlist);
  1318. }
  1319. }
  1320. *_stable_node_dup = found;
  1321. return tree_page;
  1322. }
  1323. static struct stable_node *stable_node_dup_any(struct stable_node *stable_node,
  1324. struct rb_root *root)
  1325. {
  1326. if (!is_stable_node_chain(stable_node))
  1327. return stable_node;
  1328. if (hlist_empty(&stable_node->hlist)) {
  1329. free_stable_node_chain(stable_node, root);
  1330. return NULL;
  1331. }
  1332. return hlist_entry(stable_node->hlist.first,
  1333. typeof(*stable_node), hlist_dup);
  1334. }
  1335. /*
  1336. * Like for get_ksm_page, this function can free the *_stable_node and
  1337. * *_stable_node_dup if the returned tree_page is NULL.
  1338. *
  1339. * It can also free and overwrite *_stable_node with the found
  1340. * stable_node_dup if the chain is collapsed (in which case
  1341. * *_stable_node will be equal to *_stable_node_dup like if the chain
  1342. * never existed). It's up to the caller to verify tree_page is not
  1343. * NULL before dereferencing *_stable_node or *_stable_node_dup.
  1344. *
  1345. * *_stable_node_dup is really a second output parameter of this
  1346. * function and will be overwritten in all cases, the caller doesn't
  1347. * need to initialize it.
  1348. */
  1349. static struct page *__stable_node_chain(struct stable_node **_stable_node_dup,
  1350. struct stable_node **_stable_node,
  1351. struct rb_root *root,
  1352. bool prune_stale_stable_nodes)
  1353. {
  1354. struct stable_node *stable_node = *_stable_node;
  1355. if (!is_stable_node_chain(stable_node)) {
  1356. if (is_page_sharing_candidate(stable_node)) {
  1357. *_stable_node_dup = stable_node;
  1358. return get_ksm_page(stable_node, GET_KSM_PAGE_NOLOCK);
  1359. }
  1360. /*
  1361. * _stable_node_dup set to NULL means the stable_node
  1362. * reached the ksm_max_page_sharing limit.
  1363. */
  1364. *_stable_node_dup = NULL;
  1365. return NULL;
  1366. }
  1367. return stable_node_dup(_stable_node_dup, _stable_node, root,
  1368. prune_stale_stable_nodes);
  1369. }
  1370. static __always_inline struct page *chain_prune(struct stable_node **s_n_d,
  1371. struct stable_node **s_n,
  1372. struct rb_root *root)
  1373. {
  1374. return __stable_node_chain(s_n_d, s_n, root, true);
  1375. }
  1376. static __always_inline struct page *chain(struct stable_node **s_n_d,
  1377. struct stable_node *s_n,
  1378. struct rb_root *root)
  1379. {
  1380. struct stable_node *old_stable_node = s_n;
  1381. struct page *tree_page;
  1382. tree_page = __stable_node_chain(s_n_d, &s_n, root, false);
  1383. /* not pruning dups so s_n cannot have changed */
  1384. VM_BUG_ON(s_n != old_stable_node);
  1385. return tree_page;
  1386. }
  1387. /*
  1388. * stable_tree_search - search for page inside the stable tree
  1389. *
  1390. * This function checks if there is a page inside the stable tree
  1391. * with identical content to the page that we are scanning right now.
  1392. *
  1393. * This function returns the stable tree node of identical content if found,
  1394. * NULL otherwise.
  1395. */
  1396. static struct page *stable_tree_search(struct page *page)
  1397. {
  1398. int nid;
  1399. struct rb_root *root;
  1400. struct rb_node **new;
  1401. struct rb_node *parent;
  1402. struct stable_node *stable_node, *stable_node_dup, *stable_node_any;
  1403. struct stable_node *page_node;
  1404. page_node = page_stable_node(page);
  1405. if (page_node && page_node->head != &migrate_nodes) {
  1406. /* ksm page forked */
  1407. get_page(page);
  1408. return page;
  1409. }
  1410. nid = get_kpfn_nid(page_to_pfn(page));
  1411. root = root_stable_tree + nid;
  1412. again:
  1413. new = &root->rb_node;
  1414. parent = NULL;
  1415. while (*new) {
  1416. struct page *tree_page;
  1417. int ret;
  1418. cond_resched();
  1419. stable_node = rb_entry(*new, struct stable_node, node);
  1420. stable_node_any = NULL;
  1421. tree_page = chain_prune(&stable_node_dup, &stable_node, root);
  1422. /*
  1423. * NOTE: stable_node may have been freed by
  1424. * chain_prune() if the returned stable_node_dup is
  1425. * not NULL. stable_node_dup may have been inserted in
  1426. * the rbtree instead as a regular stable_node (in
  1427. * order to collapse the stable_node chain if a single
  1428. * stable_node dup was found in it). In such case the
  1429. * stable_node is overwritten by the calleee to point
  1430. * to the stable_node_dup that was collapsed in the
  1431. * stable rbtree and stable_node will be equal to
  1432. * stable_node_dup like if the chain never existed.
  1433. */
  1434. if (!stable_node_dup) {
  1435. /*
  1436. * Either all stable_node dups were full in
  1437. * this stable_node chain, or this chain was
  1438. * empty and should be rb_erased.
  1439. */
  1440. stable_node_any = stable_node_dup_any(stable_node,
  1441. root);
  1442. if (!stable_node_any) {
  1443. /* rb_erase just run */
  1444. goto again;
  1445. }
  1446. /*
  1447. * Take any of the stable_node dups page of
  1448. * this stable_node chain to let the tree walk
  1449. * continue. All KSM pages belonging to the
  1450. * stable_node dups in a stable_node chain
  1451. * have the same content and they're
  1452. * write protected at all times. Any will work
  1453. * fine to continue the walk.
  1454. */
  1455. tree_page = get_ksm_page(stable_node_any,
  1456. GET_KSM_PAGE_NOLOCK);
  1457. }
  1458. VM_BUG_ON(!stable_node_dup ^ !!stable_node_any);
  1459. if (!tree_page) {
  1460. /*
  1461. * If we walked over a stale stable_node,
  1462. * get_ksm_page() will call rb_erase() and it
  1463. * may rebalance the tree from under us. So
  1464. * restart the search from scratch. Returning
  1465. * NULL would be safe too, but we'd generate
  1466. * false negative insertions just because some
  1467. * stable_node was stale.
  1468. */
  1469. goto again;
  1470. }
  1471. ret = memcmp_pages(page, tree_page);
  1472. put_page(tree_page);
  1473. parent = *new;
  1474. if (ret < 0)
  1475. new = &parent->rb_left;
  1476. else if (ret > 0)
  1477. new = &parent->rb_right;
  1478. else {
  1479. if (page_node) {
  1480. VM_BUG_ON(page_node->head != &migrate_nodes);
  1481. /*
  1482. * Test if the migrated page should be merged
  1483. * into a stable node dup. If the mapcount is
  1484. * 1 we can migrate it with another KSM page
  1485. * without adding it to the chain.
  1486. */
  1487. if (page_mapcount(page) > 1)
  1488. goto chain_append;
  1489. }
  1490. if (!stable_node_dup) {
  1491. /*
  1492. * If the stable_node is a chain and
  1493. * we got a payload match in memcmp
  1494. * but we cannot merge the scanned
  1495. * page in any of the existing
  1496. * stable_node dups because they're
  1497. * all full, we need to wait the
  1498. * scanned page to find itself a match
  1499. * in the unstable tree to create a
  1500. * brand new KSM page to add later to
  1501. * the dups of this stable_node.
  1502. */
  1503. return NULL;
  1504. }
  1505. /*
  1506. * Lock and unlock the stable_node's page (which
  1507. * might already have been migrated) so that page
  1508. * migration is sure to notice its raised count.
  1509. * It would be more elegant to return stable_node
  1510. * than kpage, but that involves more changes.
  1511. */
  1512. tree_page = get_ksm_page(stable_node_dup,
  1513. GET_KSM_PAGE_TRYLOCK);
  1514. if (PTR_ERR(tree_page) == -EBUSY)
  1515. return ERR_PTR(-EBUSY);
  1516. if (unlikely(!tree_page))
  1517. /*
  1518. * The tree may have been rebalanced,
  1519. * so re-evaluate parent and new.
  1520. */
  1521. goto again;
  1522. unlock_page(tree_page);
  1523. if (get_kpfn_nid(stable_node_dup->kpfn) !=
  1524. NUMA(stable_node_dup->nid)) {
  1525. put_page(tree_page);
  1526. goto replace;
  1527. }
  1528. return tree_page;
  1529. }
  1530. }
  1531. if (!page_node)
  1532. return NULL;
  1533. list_del(&page_node->list);
  1534. DO_NUMA(page_node->nid = nid);
  1535. rb_link_node(&page_node->node, parent, new);
  1536. rb_insert_color(&page_node->node, root);
  1537. out:
  1538. if (is_page_sharing_candidate(page_node)) {
  1539. get_page(page);
  1540. return page;
  1541. } else
  1542. return NULL;
  1543. replace:
  1544. /*
  1545. * If stable_node was a chain and chain_prune collapsed it,
  1546. * stable_node has been updated to be the new regular
  1547. * stable_node. A collapse of the chain is indistinguishable
  1548. * from the case there was no chain in the stable
  1549. * rbtree. Otherwise stable_node is the chain and
  1550. * stable_node_dup is the dup to replace.
  1551. */
  1552. if (stable_node_dup == stable_node) {
  1553. VM_BUG_ON(is_stable_node_chain(stable_node_dup));
  1554. VM_BUG_ON(is_stable_node_dup(stable_node_dup));
  1555. /* there is no chain */
  1556. if (page_node) {
  1557. VM_BUG_ON(page_node->head != &migrate_nodes);
  1558. list_del(&page_node->list);
  1559. DO_NUMA(page_node->nid = nid);
  1560. rb_replace_node(&stable_node_dup->node,
  1561. &page_node->node,
  1562. root);
  1563. if (is_page_sharing_candidate(page_node))
  1564. get_page(page);
  1565. else
  1566. page = NULL;
  1567. } else {
  1568. rb_erase(&stable_node_dup->node, root);
  1569. page = NULL;
  1570. }
  1571. } else {
  1572. VM_BUG_ON(!is_stable_node_chain(stable_node));
  1573. __stable_node_dup_del(stable_node_dup);
  1574. if (page_node) {
  1575. VM_BUG_ON(page_node->head != &migrate_nodes);
  1576. list_del(&page_node->list);
  1577. DO_NUMA(page_node->nid = nid);
  1578. stable_node_chain_add_dup(page_node, stable_node);
  1579. if (is_page_sharing_candidate(page_node))
  1580. get_page(page);
  1581. else
  1582. page = NULL;
  1583. } else {
  1584. page = NULL;
  1585. }
  1586. }
  1587. stable_node_dup->head = &migrate_nodes;
  1588. list_add(&stable_node_dup->list, stable_node_dup->head);
  1589. return page;
  1590. chain_append:
  1591. /* stable_node_dup could be null if it reached the limit */
  1592. if (!stable_node_dup)
  1593. stable_node_dup = stable_node_any;
  1594. /*
  1595. * If stable_node was a chain and chain_prune collapsed it,
  1596. * stable_node has been updated to be the new regular
  1597. * stable_node. A collapse of the chain is indistinguishable
  1598. * from the case there was no chain in the stable
  1599. * rbtree. Otherwise stable_node is the chain and
  1600. * stable_node_dup is the dup to replace.
  1601. */
  1602. if (stable_node_dup == stable_node) {
  1603. VM_BUG_ON(is_stable_node_chain(stable_node_dup));
  1604. VM_BUG_ON(is_stable_node_dup(stable_node_dup));
  1605. /* chain is missing so create it */
  1606. stable_node = alloc_stable_node_chain(stable_node_dup,
  1607. root);
  1608. if (!stable_node)
  1609. return NULL;
  1610. }
  1611. /*
  1612. * Add this stable_node dup that was
  1613. * migrated to the stable_node chain
  1614. * of the current nid for this page
  1615. * content.
  1616. */
  1617. VM_BUG_ON(!is_stable_node_chain(stable_node));
  1618. VM_BUG_ON(!is_stable_node_dup(stable_node_dup));
  1619. VM_BUG_ON(page_node->head != &migrate_nodes);
  1620. list_del(&page_node->list);
  1621. DO_NUMA(page_node->nid = nid);
  1622. stable_node_chain_add_dup(page_node, stable_node);
  1623. goto out;
  1624. }
  1625. /*
  1626. * stable_tree_insert - insert stable tree node pointing to new ksm page
  1627. * into the stable tree.
  1628. *
  1629. * This function returns the stable tree node just allocated on success,
  1630. * NULL otherwise.
  1631. */
  1632. static struct stable_node *stable_tree_insert(struct page *kpage)
  1633. {
  1634. int nid;
  1635. unsigned long kpfn;
  1636. struct rb_root *root;
  1637. struct rb_node **new;
  1638. struct rb_node *parent;
  1639. struct stable_node *stable_node, *stable_node_dup, *stable_node_any;
  1640. bool need_chain = false;
  1641. kpfn = page_to_pfn(kpage);
  1642. nid = get_kpfn_nid(kpfn);
  1643. root = root_stable_tree + nid;
  1644. again:
  1645. parent = NULL;
  1646. new = &root->rb_node;
  1647. while (*new) {
  1648. struct page *tree_page;
  1649. int ret;
  1650. cond_resched();
  1651. stable_node = rb_entry(*new, struct stable_node, node);
  1652. stable_node_any = NULL;
  1653. tree_page = chain(&stable_node_dup, stable_node, root);
  1654. if (!stable_node_dup) {
  1655. /*
  1656. * Either all stable_node dups were full in
  1657. * this stable_node chain, or this chain was
  1658. * empty and should be rb_erased.
  1659. */
  1660. stable_node_any = stable_node_dup_any(stable_node,
  1661. root);
  1662. if (!stable_node_any) {
  1663. /* rb_erase just run */
  1664. goto again;
  1665. }
  1666. /*
  1667. * Take any of the stable_node dups page of
  1668. * this stable_node chain to let the tree walk
  1669. * continue. All KSM pages belonging to the
  1670. * stable_node dups in a stable_node chain
  1671. * have the same content and they're
  1672. * write protected at all times. Any will work
  1673. * fine to continue the walk.
  1674. */
  1675. tree_page = get_ksm_page(stable_node_any,
  1676. GET_KSM_PAGE_NOLOCK);
  1677. }
  1678. VM_BUG_ON(!stable_node_dup ^ !!stable_node_any);
  1679. if (!tree_page) {
  1680. /*
  1681. * If we walked over a stale stable_node,
  1682. * get_ksm_page() will call rb_erase() and it
  1683. * may rebalance the tree from under us. So
  1684. * restart the search from scratch. Returning
  1685. * NULL would be safe too, but we'd generate
  1686. * false negative insertions just because some
  1687. * stable_node was stale.
  1688. */
  1689. goto again;
  1690. }
  1691. ret = memcmp_pages(kpage, tree_page);
  1692. put_page(tree_page);
  1693. parent = *new;
  1694. if (ret < 0)
  1695. new = &parent->rb_left;
  1696. else if (ret > 0)
  1697. new = &parent->rb_right;
  1698. else {
  1699. need_chain = true;
  1700. break;
  1701. }
  1702. }
  1703. stable_node_dup = alloc_stable_node();
  1704. if (!stable_node_dup)
  1705. return NULL;
  1706. INIT_HLIST_HEAD(&stable_node_dup->hlist);
  1707. stable_node_dup->kpfn = kpfn;
  1708. set_page_stable_node(kpage, stable_node_dup);
  1709. stable_node_dup->rmap_hlist_len = 0;
  1710. DO_NUMA(stable_node_dup->nid = nid);
  1711. if (!need_chain) {
  1712. rb_link_node(&stable_node_dup->node, parent, new);
  1713. rb_insert_color(&stable_node_dup->node, root);
  1714. } else {
  1715. if (!is_stable_node_chain(stable_node)) {
  1716. struct stable_node *orig = stable_node;
  1717. /* chain is missing so create it */
  1718. stable_node = alloc_stable_node_chain(orig, root);
  1719. if (!stable_node) {
  1720. free_stable_node(stable_node_dup);
  1721. return NULL;
  1722. }
  1723. }
  1724. stable_node_chain_add_dup(stable_node_dup, stable_node);
  1725. }
  1726. return stable_node_dup;
  1727. }
  1728. /*
  1729. * unstable_tree_search_insert - search for identical page,
  1730. * else insert rmap_item into the unstable tree.
  1731. *
  1732. * This function searches for a page in the unstable tree identical to the
  1733. * page currently being scanned; and if no identical page is found in the
  1734. * tree, we insert rmap_item as a new object into the unstable tree.
  1735. *
  1736. * This function returns pointer to rmap_item found to be identical
  1737. * to the currently scanned page, NULL otherwise.
  1738. *
  1739. * This function does both searching and inserting, because they share
  1740. * the same walking algorithm in an rbtree.
  1741. */
  1742. static
  1743. struct rmap_item *unstable_tree_search_insert(struct rmap_item *rmap_item,
  1744. struct page *page,
  1745. struct page **tree_pagep)
  1746. {
  1747. struct rb_node **new;
  1748. struct rb_root *root;
  1749. struct rb_node *parent = NULL;
  1750. int nid;
  1751. nid = get_kpfn_nid(page_to_pfn(page));
  1752. root = root_unstable_tree + nid;
  1753. new = &root->rb_node;
  1754. while (*new) {
  1755. struct rmap_item *tree_rmap_item;
  1756. struct page *tree_page;
  1757. int ret;
  1758. cond_resched();
  1759. tree_rmap_item = rb_entry(*new, struct rmap_item, node);
  1760. tree_page = get_mergeable_page(tree_rmap_item);
  1761. if (!tree_page)
  1762. return NULL;
  1763. /*
  1764. * Don't substitute a ksm page for a forked page.
  1765. */
  1766. if (page == tree_page) {
  1767. put_user_page(tree_page);
  1768. return NULL;
  1769. }
  1770. ret = memcmp_pages(page, tree_page);
  1771. parent = *new;
  1772. if (ret < 0) {
  1773. put_user_page(tree_page);
  1774. new = &parent->rb_left;
  1775. } else if (ret > 0) {
  1776. put_user_page(tree_page);
  1777. new = &parent->rb_right;
  1778. } else if (!ksm_merge_across_nodes &&
  1779. page_to_nid(tree_page) != nid) {
  1780. /*
  1781. * If tree_page has been migrated to another NUMA node,
  1782. * it will be flushed out and put in the right unstable
  1783. * tree next time: only merge with it when across_nodes.
  1784. */
  1785. put_user_page(tree_page);
  1786. return NULL;
  1787. } else {
  1788. *tree_pagep = tree_page;
  1789. return tree_rmap_item;
  1790. }
  1791. }
  1792. rmap_item->address |= UNSTABLE_FLAG;
  1793. rmap_item->address |= (ksm_scan.seqnr & SEQNR_MASK);
  1794. DO_NUMA(rmap_item->nid = nid);
  1795. rb_link_node(&rmap_item->node, parent, new);
  1796. rb_insert_color(&rmap_item->node, root);
  1797. ksm_pages_unshared++;
  1798. return NULL;
  1799. }
  1800. /*
  1801. * stable_tree_append - add another rmap_item to the linked list of
  1802. * rmap_items hanging off a given node of the stable tree, all sharing
  1803. * the same ksm page.
  1804. */
  1805. static void stable_tree_append(struct rmap_item *rmap_item,
  1806. struct stable_node *stable_node,
  1807. bool max_page_sharing_bypass)
  1808. {
  1809. /*
  1810. * rmap won't find this mapping if we don't insert the
  1811. * rmap_item in the right stable_node
  1812. * duplicate. page_migration could break later if rmap breaks,
  1813. * so we can as well crash here. We really need to check for
  1814. * rmap_hlist_len == STABLE_NODE_CHAIN, but we can as well check
  1815. * for other negative values as an underflow if detected here
  1816. * for the first time (and not when decreasing rmap_hlist_len)
  1817. * would be sign of memory corruption in the stable_node.
  1818. */
  1819. BUG_ON(stable_node->rmap_hlist_len < 0);
  1820. stable_node->rmap_hlist_len++;
  1821. if (!max_page_sharing_bypass)
  1822. /* possibly non fatal but unexpected overflow, only warn */
  1823. WARN_ON_ONCE(stable_node->rmap_hlist_len >
  1824. ksm_max_page_sharing);
  1825. rmap_item->head = stable_node;
  1826. rmap_item->address |= STABLE_FLAG;
  1827. hlist_add_head(&rmap_item->hlist, &stable_node->hlist);
  1828. if (rmap_item->hlist.next)
  1829. ksm_pages_sharing++;
  1830. else
  1831. ksm_pages_shared++;
  1832. }
  1833. /*
  1834. * cmp_and_merge_page - first see if page can be merged into the stable tree;
  1835. * if not, compare checksum to previous and if it's the same, see if page can
  1836. * be inserted into the unstable tree, or merged with a page already there and
  1837. * both transferred to the stable tree.
  1838. *
  1839. * @page: the page that we are searching identical page to.
  1840. * @rmap_item: the reverse mapping into the virtual address of this page
  1841. */
  1842. static void cmp_and_merge_page(struct page *page, struct rmap_item *rmap_item)
  1843. {
  1844. struct mm_struct *mm = rmap_item->mm;
  1845. struct rmap_item *tree_rmap_item;
  1846. struct page *tree_page = NULL;
  1847. struct stable_node *stable_node;
  1848. struct page *kpage;
  1849. unsigned int checksum;
  1850. int err;
  1851. bool max_page_sharing_bypass = false;
  1852. stable_node = page_stable_node(page);
  1853. if (stable_node) {
  1854. if (stable_node->head != &migrate_nodes &&
  1855. get_kpfn_nid(READ_ONCE(stable_node->kpfn)) !=
  1856. NUMA(stable_node->nid)) {
  1857. stable_node_dup_del(stable_node);
  1858. stable_node->head = &migrate_nodes;
  1859. list_add(&stable_node->list, stable_node->head);
  1860. }
  1861. if (stable_node->head != &migrate_nodes &&
  1862. rmap_item->head == stable_node)
  1863. return;
  1864. /*
  1865. * If it's a KSM fork, allow it to go over the sharing limit
  1866. * without warnings.
  1867. */
  1868. if (!is_page_sharing_candidate(stable_node))
  1869. max_page_sharing_bypass = true;
  1870. }
  1871. /* We first start with searching the page inside the stable tree */
  1872. kpage = stable_tree_search(page);
  1873. if (kpage == page && rmap_item->head == stable_node) {
  1874. put_page(kpage);
  1875. return;
  1876. }
  1877. remove_rmap_item_from_tree(rmap_item);
  1878. if (kpage) {
  1879. if (PTR_ERR(kpage) == -EBUSY)
  1880. return;
  1881. err = try_to_merge_with_ksm_page(rmap_item, page, kpage);
  1882. if (!err) {
  1883. /*
  1884. * The page was successfully merged:
  1885. * add its rmap_item to the stable tree.
  1886. */
  1887. lock_page(kpage);
  1888. stable_tree_append(rmap_item, page_stable_node(kpage),
  1889. max_page_sharing_bypass);
  1890. unlock_page(kpage);
  1891. }
  1892. put_page(kpage);
  1893. return;
  1894. }
  1895. /*
  1896. * If the hash value of the page has changed from the last time
  1897. * we calculated it, this page is changing frequently: therefore we
  1898. * don't want to insert it in the unstable tree, and we don't want
  1899. * to waste our time searching for something identical to it there.
  1900. */
  1901. checksum = calc_checksum(page);
  1902. if (rmap_item->oldchecksum != checksum) {
  1903. rmap_item->oldchecksum = checksum;
  1904. return;
  1905. }
  1906. /*
  1907. * Same checksum as an empty page. We attempt to merge it with the
  1908. * appropriate zero page if the user enabled this via sysfs.
  1909. */
  1910. if (ksm_use_zero_pages && (checksum == zero_checksum)) {
  1911. struct vm_area_struct *vma;
  1912. mmap_read_lock(mm);
  1913. vma = find_mergeable_vma(mm, rmap_item->address);
  1914. if (vma) {
  1915. err = try_to_merge_one_page(vma, page,
  1916. ZERO_PAGE(rmap_item->address));
  1917. } else {
  1918. /*
  1919. * If the vma is out of date, we do not need to
  1920. * continue.
  1921. */
  1922. err = 0;
  1923. }
  1924. mmap_read_unlock(mm);
  1925. /*
  1926. * In case of failure, the page was not really empty, so we
  1927. * need to continue. Otherwise we're done.
  1928. */
  1929. if (!err)
  1930. return;
  1931. }
  1932. tree_rmap_item =
  1933. unstable_tree_search_insert(rmap_item, page, &tree_page);
  1934. if (tree_rmap_item) {
  1935. bool split;
  1936. kpage = try_to_merge_two_pages(rmap_item, page,
  1937. tree_rmap_item, tree_page);
  1938. /*
  1939. * If both pages we tried to merge belong to the same compound
  1940. * page, then we actually ended up increasing the reference
  1941. * count of the same compound page twice, and split_huge_page
  1942. * failed.
  1943. * Here we set a flag if that happened, and we use it later to
  1944. * try split_huge_page again. Since we call put_page right
  1945. * afterwards, the reference count will be correct and
  1946. * split_huge_page should succeed.
  1947. */
  1948. split = PageTransCompound(page)
  1949. && compound_head(page) == compound_head(tree_page);
  1950. put_user_page(tree_page);
  1951. if (kpage) {
  1952. /*
  1953. * The pages were successfully merged: insert new
  1954. * node in the stable tree and add both rmap_items.
  1955. */
  1956. lock_page(kpage);
  1957. stable_node = stable_tree_insert(kpage);
  1958. if (stable_node) {
  1959. stable_tree_append(tree_rmap_item, stable_node,
  1960. false);
  1961. stable_tree_append(rmap_item, stable_node,
  1962. false);
  1963. }
  1964. unlock_page(kpage);
  1965. /*
  1966. * If we fail to insert the page into the stable tree,
  1967. * we will have 2 virtual addresses that are pointing
  1968. * to a ksm page left outside the stable tree,
  1969. * in which case we need to break_cow on both.
  1970. */
  1971. if (!stable_node) {
  1972. break_cow(tree_rmap_item);
  1973. break_cow(rmap_item);
  1974. }
  1975. } else if (split) {
  1976. /*
  1977. * We are here if we tried to merge two pages and
  1978. * failed because they both belonged to the same
  1979. * compound page. We will split the page now, but no
  1980. * merging will take place.
  1981. * We do not want to add the cost of a full lock; if
  1982. * the page is locked, it is better to skip it and
  1983. * perhaps try again later.
  1984. */
  1985. if (!trylock_page(page))
  1986. return;
  1987. split_huge_page(page);
  1988. unlock_page(page);
  1989. }
  1990. }
  1991. }
  1992. static struct rmap_item *get_next_rmap_item(struct mm_slot *mm_slot,
  1993. struct rmap_item **rmap_list,
  1994. unsigned long addr)
  1995. {
  1996. struct rmap_item *rmap_item;
  1997. while (*rmap_list) {
  1998. rmap_item = *rmap_list;
  1999. if ((rmap_item->address & PAGE_MASK) == addr)
  2000. return rmap_item;
  2001. if (rmap_item->address > addr)
  2002. break;
  2003. *rmap_list = rmap_item->rmap_list;
  2004. remove_rmap_item_from_tree(rmap_item);
  2005. free_rmap_item(rmap_item);
  2006. }
  2007. rmap_item = alloc_rmap_item();
  2008. if (rmap_item) {
  2009. /* It has already been zeroed */
  2010. rmap_item->mm = mm_slot->mm;
  2011. rmap_item->address = addr;
  2012. rmap_item->rmap_list = *rmap_list;
  2013. *rmap_list = rmap_item;
  2014. }
  2015. return rmap_item;
  2016. }
  2017. static struct rmap_item *scan_get_next_rmap_item(struct page **page)
  2018. {
  2019. struct mm_struct *mm;
  2020. struct mm_slot *slot;
  2021. struct vm_area_struct *vma;
  2022. struct rmap_item *rmap_item;
  2023. int nid;
  2024. if (list_empty(&ksm_mm_head.mm_list))
  2025. return NULL;
  2026. slot = ksm_scan.mm_slot;
  2027. if (slot == &ksm_mm_head) {
  2028. /*
  2029. * A number of pages can hang around indefinitely on per-cpu
  2030. * pagevecs, raised page count preventing write_protect_page
  2031. * from merging them. Though it doesn't really matter much,
  2032. * it is puzzling to see some stuck in pages_volatile until
  2033. * other activity jostles them out, and they also prevented
  2034. * LTP's KSM test from succeeding deterministically; so drain
  2035. * them here (here rather than on entry to ksm_do_scan(),
  2036. * so we don't IPI too often when pages_to_scan is set low).
  2037. */
  2038. lru_add_drain_all();
  2039. /*
  2040. * Whereas stale stable_nodes on the stable_tree itself
  2041. * get pruned in the regular course of stable_tree_search(),
  2042. * those moved out to the migrate_nodes list can accumulate:
  2043. * so prune them once before each full scan.
  2044. */
  2045. if (!ksm_merge_across_nodes) {
  2046. struct stable_node *stable_node, *next;
  2047. struct page *page;
  2048. list_for_each_entry_safe(stable_node, next,
  2049. &migrate_nodes, list) {
  2050. page = get_ksm_page(stable_node,
  2051. GET_KSM_PAGE_NOLOCK);
  2052. if (page)
  2053. put_page(page);
  2054. cond_resched();
  2055. }
  2056. }
  2057. for (nid = 0; nid < ksm_nr_node_ids; nid++)
  2058. root_unstable_tree[nid] = RB_ROOT;
  2059. spin_lock(&ksm_mmlist_lock);
  2060. slot = list_entry(slot->mm_list.next, struct mm_slot, mm_list);
  2061. ksm_scan.mm_slot = slot;
  2062. spin_unlock(&ksm_mmlist_lock);
  2063. /*
  2064. * Although we tested list_empty() above, a racing __ksm_exit
  2065. * of the last mm on the list may have removed it since then.
  2066. */
  2067. if (slot == &ksm_mm_head)
  2068. return NULL;
  2069. next_mm:
  2070. ksm_scan.address = 0;
  2071. ksm_scan.rmap_list = &slot->rmap_list;
  2072. }
  2073. mm = slot->mm;
  2074. mmap_read_lock(mm);
  2075. if (ksm_test_exit(mm))
  2076. vma = NULL;
  2077. else
  2078. vma = find_vma(mm, ksm_scan.address);
  2079. for (; vma; vma = vma->vm_next) {
  2080. if (!(vma->vm_flags & VM_MERGEABLE))
  2081. continue;
  2082. if (ksm_scan.address < vma->vm_start)
  2083. ksm_scan.address = vma->vm_start;
  2084. if (!vma->anon_vma)
  2085. ksm_scan.address = vma->vm_end;
  2086. while (ksm_scan.address < vma->vm_end) {
  2087. if (ksm_test_exit(mm))
  2088. break;
  2089. *page = follow_page(vma, ksm_scan.address, FOLL_GET);
  2090. if (IS_ERR_OR_NULL(*page)) {
  2091. ksm_scan.address += PAGE_SIZE;
  2092. cond_resched();
  2093. continue;
  2094. }
  2095. if (PageAnon(*page)) {
  2096. flush_anon_page(vma, *page, ksm_scan.address);
  2097. flush_dcache_page(*page);
  2098. rmap_item = get_next_rmap_item(slot,
  2099. ksm_scan.rmap_list, ksm_scan.address);
  2100. if (rmap_item) {
  2101. ksm_scan.rmap_list =
  2102. &rmap_item->rmap_list;
  2103. ksm_scan.address += PAGE_SIZE;
  2104. } else
  2105. put_user_page(*page);
  2106. mmap_read_unlock(mm);
  2107. return rmap_item;
  2108. }
  2109. put_user_page(*page);
  2110. ksm_scan.address += PAGE_SIZE;
  2111. cond_resched();
  2112. }
  2113. }
  2114. if (ksm_test_exit(mm)) {
  2115. ksm_scan.address = 0;
  2116. ksm_scan.rmap_list = &slot->rmap_list;
  2117. }
  2118. /*
  2119. * Nuke all the rmap_items that are above this current rmap:
  2120. * because there were no VM_MERGEABLE vmas with such addresses.
  2121. */
  2122. remove_trailing_rmap_items(slot, ksm_scan.rmap_list);
  2123. spin_lock(&ksm_mmlist_lock);
  2124. ksm_scan.mm_slot = list_entry(slot->mm_list.next,
  2125. struct mm_slot, mm_list);
  2126. if (ksm_scan.address == 0) {
  2127. /*
  2128. * We've completed a full scan of all vmas, holding mmap_lock
  2129. * throughout, and found no VM_MERGEABLE: so do the same as
  2130. * __ksm_exit does to remove this mm from all our lists now.
  2131. * This applies either when cleaning up after __ksm_exit
  2132. * (but beware: we can reach here even before __ksm_exit),
  2133. * or when all VM_MERGEABLE areas have been unmapped (and
  2134. * mmap_lock then protects against race with MADV_MERGEABLE).
  2135. */
  2136. hash_del(&slot->link);
  2137. list_del(&slot->mm_list);
  2138. spin_unlock(&ksm_mmlist_lock);
  2139. free_mm_slot(slot);
  2140. clear_bit(MMF_VM_MERGEABLE, &mm->flags);
  2141. mmap_read_unlock(mm);
  2142. mmdrop(mm);
  2143. } else {
  2144. mmap_read_unlock(mm);
  2145. /*
  2146. * mmap_read_unlock(mm) first because after
  2147. * spin_unlock(&ksm_mmlist_lock) run, the "mm" may
  2148. * already have been freed under us by __ksm_exit()
  2149. * because the "mm_slot" is still hashed and
  2150. * ksm_scan.mm_slot doesn't point to it anymore.
  2151. */
  2152. spin_unlock(&ksm_mmlist_lock);
  2153. }
  2154. /* Repeat until we've completed scanning the whole list */
  2155. slot = ksm_scan.mm_slot;
  2156. if (slot != &ksm_mm_head)
  2157. goto next_mm;
  2158. ksm_scan.seqnr++;
  2159. return NULL;
  2160. }
  2161. /**
  2162. * ksm_do_scan - the ksm scanner main worker function.
  2163. * @scan_npages: number of pages we want to scan before we return.
  2164. */
  2165. static void ksm_do_scan(unsigned int scan_npages)
  2166. {
  2167. struct rmap_item *rmap_item;
  2168. struct page *page;
  2169. while (scan_npages-- && likely(!freezing(current))) {
  2170. cond_resched();
  2171. rmap_item = scan_get_next_rmap_item(&page);
  2172. if (!rmap_item)
  2173. return;
  2174. cmp_and_merge_page(page, rmap_item);
  2175. put_page(page);
  2176. }
  2177. }
  2178. static int ksmd_should_run(void)
  2179. {
  2180. return (ksm_run & KSM_RUN_MERGE) && !list_empty(&ksm_mm_head.mm_list);
  2181. }
  2182. static int ksm_scan_thread(void *nothing)
  2183. {
  2184. unsigned int sleep_ms;
  2185. set_freezable();
  2186. set_user_nice(current, 5);
  2187. while (!kthread_should_stop()) {
  2188. mutex_lock(&ksm_thread_mutex);
  2189. wait_while_offlining();
  2190. if (ksmd_should_run())
  2191. ksm_do_scan(ksm_thread_pages_to_scan);
  2192. mutex_unlock(&ksm_thread_mutex);
  2193. try_to_freeze();
  2194. if (ksmd_should_run()) {
  2195. sleep_ms = READ_ONCE(ksm_thread_sleep_millisecs);
  2196. wait_event_interruptible_timeout(ksm_iter_wait,
  2197. sleep_ms != READ_ONCE(ksm_thread_sleep_millisecs),
  2198. msecs_to_jiffies(sleep_ms));
  2199. } else {
  2200. wait_event_freezable(ksm_thread_wait,
  2201. ksmd_should_run() || kthread_should_stop());
  2202. }
  2203. }
  2204. return 0;
  2205. }
  2206. int ksm_madvise(struct vm_area_struct *vma, unsigned long start,
  2207. unsigned long end, int advice, unsigned long *vm_flags)
  2208. {
  2209. struct mm_struct *mm = vma->vm_mm;
  2210. int err;
  2211. switch (advice) {
  2212. case MADV_MERGEABLE:
  2213. /*
  2214. * Be somewhat over-protective for now!
  2215. */
  2216. if (*vm_flags & (VM_MERGEABLE | VM_SHARED | VM_MAYSHARE |
  2217. VM_PFNMAP | VM_IO | VM_DONTEXPAND |
  2218. VM_HUGETLB | VM_MIXEDMAP))
  2219. return 0; /* just ignore the advice */
  2220. if (vma_is_dax(vma))
  2221. return 0;
  2222. #ifdef VM_SAO
  2223. if (*vm_flags & VM_SAO)
  2224. return 0;
  2225. #endif
  2226. #ifdef VM_SPARC_ADI
  2227. if (*vm_flags & VM_SPARC_ADI)
  2228. return 0;
  2229. #endif
  2230. if (!test_bit(MMF_VM_MERGEABLE, &mm->flags)) {
  2231. err = __ksm_enter(mm);
  2232. if (err)
  2233. return err;
  2234. }
  2235. *vm_flags |= VM_MERGEABLE;
  2236. break;
  2237. case MADV_UNMERGEABLE:
  2238. if (!(*vm_flags & VM_MERGEABLE))
  2239. return 0; /* just ignore the advice */
  2240. if (vma->anon_vma) {
  2241. err = unmerge_ksm_pages(vma, start, end);
  2242. if (err)
  2243. return err;
  2244. }
  2245. *vm_flags &= ~VM_MERGEABLE;
  2246. break;
  2247. }
  2248. return 0;
  2249. }
  2250. EXPORT_SYMBOL_GPL(ksm_madvise);
  2251. int __ksm_enter(struct mm_struct *mm)
  2252. {
  2253. struct mm_slot *mm_slot;
  2254. int needs_wakeup;
  2255. mm_slot = alloc_mm_slot();
  2256. if (!mm_slot)
  2257. return -ENOMEM;
  2258. /* Check ksm_run too? Would need tighter locking */
  2259. needs_wakeup = list_empty(&ksm_mm_head.mm_list);
  2260. spin_lock(&ksm_mmlist_lock);
  2261. insert_to_mm_slots_hash(mm, mm_slot);
  2262. /*
  2263. * When KSM_RUN_MERGE (or KSM_RUN_STOP),
  2264. * insert just behind the scanning cursor, to let the area settle
  2265. * down a little; when fork is followed by immediate exec, we don't
  2266. * want ksmd to waste time setting up and tearing down an rmap_list.
  2267. *
  2268. * But when KSM_RUN_UNMERGE, it's important to insert ahead of its
  2269. * scanning cursor, otherwise KSM pages in newly forked mms will be
  2270. * missed: then we might as well insert at the end of the list.
  2271. */
  2272. if (ksm_run & KSM_RUN_UNMERGE)
  2273. list_add_tail(&mm_slot->mm_list, &ksm_mm_head.mm_list);
  2274. else
  2275. list_add_tail(&mm_slot->mm_list, &ksm_scan.mm_slot->mm_list);
  2276. spin_unlock(&ksm_mmlist_lock);
  2277. set_bit(MMF_VM_MERGEABLE, &mm->flags);
  2278. mmgrab(mm);
  2279. if (needs_wakeup)
  2280. wake_up_interruptible(&ksm_thread_wait);
  2281. return 0;
  2282. }
  2283. void __ksm_exit(struct mm_struct *mm)
  2284. {
  2285. struct mm_slot *mm_slot;
  2286. int easy_to_free = 0;
  2287. /*
  2288. * This process is exiting: if it's straightforward (as is the
  2289. * case when ksmd was never running), free mm_slot immediately.
  2290. * But if it's at the cursor or has rmap_items linked to it, use
  2291. * mmap_lock to synchronize with any break_cows before pagetables
  2292. * are freed, and leave the mm_slot on the list for ksmd to free.
  2293. * Beware: ksm may already have noticed it exiting and freed the slot.
  2294. */
  2295. spin_lock(&ksm_mmlist_lock);
  2296. mm_slot = get_mm_slot(mm);
  2297. if (mm_slot && ksm_scan.mm_slot != mm_slot) {
  2298. if (!mm_slot->rmap_list) {
  2299. hash_del(&mm_slot->link);
  2300. list_del(&mm_slot->mm_list);
  2301. easy_to_free = 1;
  2302. } else {
  2303. list_move(&mm_slot->mm_list,
  2304. &ksm_scan.mm_slot->mm_list);
  2305. }
  2306. }
  2307. spin_unlock(&ksm_mmlist_lock);
  2308. if (easy_to_free) {
  2309. free_mm_slot(mm_slot);
  2310. clear_bit(MMF_VM_MERGEABLE, &mm->flags);
  2311. mmdrop(mm);
  2312. } else if (mm_slot) {
  2313. mmap_write_lock(mm);
  2314. mmap_write_unlock(mm);
  2315. }
  2316. }
  2317. struct page *ksm_might_need_to_copy(struct page *page,
  2318. struct vm_area_struct *vma, unsigned long address)
  2319. {
  2320. struct anon_vma *anon_vma = page_anon_vma(page);
  2321. struct page *new_page;
  2322. if (PageKsm(page)) {
  2323. if (page_stable_node(page) &&
  2324. !(ksm_run & KSM_RUN_UNMERGE))
  2325. return page; /* no need to copy it */
  2326. } else if (!anon_vma) {
  2327. return page; /* no need to copy it */
  2328. } else if (anon_vma->root == vma->anon_vma->root &&
  2329. page->index == linear_page_index(vma, address)) {
  2330. return page; /* still no need to copy it */
  2331. }
  2332. if (!PageUptodate(page))
  2333. return page; /* let do_swap_page report the error */
  2334. new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, address);
  2335. if (new_page && mem_cgroup_charge(new_page, vma->vm_mm, GFP_KERNEL)) {
  2336. put_page(new_page);
  2337. new_page = NULL;
  2338. }
  2339. if (new_page) {
  2340. copy_user_highpage(new_page, page, address, vma);
  2341. SetPageDirty(new_page);
  2342. __SetPageUptodate(new_page);
  2343. __SetPageLocked(new_page);
  2344. }
  2345. return new_page;
  2346. }
  2347. void rmap_walk_ksm(struct page *page, struct rmap_walk_control *rwc)
  2348. {
  2349. struct stable_node *stable_node;
  2350. struct rmap_item *rmap_item;
  2351. int search_new_forks = 0;
  2352. VM_BUG_ON_PAGE(!PageKsm(page), page);
  2353. /*
  2354. * Rely on the page lock to protect against concurrent modifications
  2355. * to that page's node of the stable tree.
  2356. */
  2357. VM_BUG_ON_PAGE(!PageLocked(page), page);
  2358. stable_node = page_stable_node(page);
  2359. if (!stable_node)
  2360. return;
  2361. again:
  2362. hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) {
  2363. struct anon_vma *anon_vma = rmap_item->anon_vma;
  2364. struct anon_vma_chain *vmac;
  2365. struct vm_area_struct *vma;
  2366. cond_resched();
  2367. anon_vma_lock_read(anon_vma);
  2368. anon_vma_interval_tree_foreach(vmac, &anon_vma->rb_root,
  2369. 0, ULONG_MAX) {
  2370. unsigned long addr;
  2371. cond_resched();
  2372. vma = vmac->vma;
  2373. /* Ignore the stable/unstable/sqnr flags */
  2374. addr = rmap_item->address & ~KSM_FLAG_MASK;
  2375. if (addr < vma->vm_start || addr >= vma->vm_end)
  2376. continue;
  2377. /*
  2378. * Initially we examine only the vma which covers this
  2379. * rmap_item; but later, if there is still work to do,
  2380. * we examine covering vmas in other mms: in case they
  2381. * were forked from the original since ksmd passed.
  2382. */
  2383. if ((rmap_item->mm == vma->vm_mm) == search_new_forks)
  2384. continue;
  2385. if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))
  2386. continue;
  2387. if (!rwc->rmap_one(page, vma, addr, rwc->arg)) {
  2388. anon_vma_unlock_read(anon_vma);
  2389. return;
  2390. }
  2391. if (rwc->done && rwc->done(page)) {
  2392. anon_vma_unlock_read(anon_vma);
  2393. return;
  2394. }
  2395. }
  2396. anon_vma_unlock_read(anon_vma);
  2397. }
  2398. if (!search_new_forks++)
  2399. goto again;
  2400. }
  2401. #ifdef CONFIG_MIGRATION
  2402. void ksm_migrate_page(struct page *newpage, struct page *oldpage)
  2403. {
  2404. struct stable_node *stable_node;
  2405. VM_BUG_ON_PAGE(!PageLocked(oldpage), oldpage);
  2406. VM_BUG_ON_PAGE(!PageLocked(newpage), newpage);
  2407. VM_BUG_ON_PAGE(newpage->mapping != oldpage->mapping, newpage);
  2408. stable_node = page_stable_node(newpage);
  2409. if (stable_node) {
  2410. VM_BUG_ON_PAGE(stable_node->kpfn != page_to_pfn(oldpage), oldpage);
  2411. stable_node->kpfn = page_to_pfn(newpage);
  2412. /*
  2413. * newpage->mapping was set in advance; now we need smp_wmb()
  2414. * to make sure that the new stable_node->kpfn is visible
  2415. * to get_ksm_page() before it can see that oldpage->mapping
  2416. * has gone stale (or that PageSwapCache has been cleared).
  2417. */
  2418. smp_wmb();
  2419. set_page_stable_node(oldpage, NULL);
  2420. }
  2421. }
  2422. #endif /* CONFIG_MIGRATION */
  2423. #ifdef CONFIG_MEMORY_HOTREMOVE
  2424. static void wait_while_offlining(void)
  2425. {
  2426. while (ksm_run & KSM_RUN_OFFLINE) {
  2427. mutex_unlock(&ksm_thread_mutex);
  2428. wait_on_bit(&ksm_run, ilog2(KSM_RUN_OFFLINE),
  2429. TASK_UNINTERRUPTIBLE);
  2430. mutex_lock(&ksm_thread_mutex);
  2431. }
  2432. }
  2433. static bool stable_node_dup_remove_range(struct stable_node *stable_node,
  2434. unsigned long start_pfn,
  2435. unsigned long end_pfn)
  2436. {
  2437. if (stable_node->kpfn >= start_pfn &&
  2438. stable_node->kpfn < end_pfn) {
  2439. /*
  2440. * Don't get_ksm_page, page has already gone:
  2441. * which is why we keep kpfn instead of page*
  2442. */
  2443. remove_node_from_stable_tree(stable_node);
  2444. return true;
  2445. }
  2446. return false;
  2447. }
  2448. static bool stable_node_chain_remove_range(struct stable_node *stable_node,
  2449. unsigned long start_pfn,
  2450. unsigned long end_pfn,
  2451. struct rb_root *root)
  2452. {
  2453. struct stable_node *dup;
  2454. struct hlist_node *hlist_safe;
  2455. if (!is_stable_node_chain(stable_node)) {
  2456. VM_BUG_ON(is_stable_node_dup(stable_node));
  2457. return stable_node_dup_remove_range(stable_node, start_pfn,
  2458. end_pfn);
  2459. }
  2460. hlist_for_each_entry_safe(dup, hlist_safe,
  2461. &stable_node->hlist, hlist_dup) {
  2462. VM_BUG_ON(!is_stable_node_dup(dup));
  2463. stable_node_dup_remove_range(dup, start_pfn, end_pfn);
  2464. }
  2465. if (hlist_empty(&stable_node->hlist)) {
  2466. free_stable_node_chain(stable_node, root);
  2467. return true; /* notify caller that tree was rebalanced */
  2468. } else
  2469. return false;
  2470. }
  2471. static void ksm_check_stable_tree(unsigned long start_pfn,
  2472. unsigned long end_pfn)
  2473. {
  2474. struct stable_node *stable_node, *next;
  2475. struct rb_node *node;
  2476. int nid;
  2477. for (nid = 0; nid < ksm_nr_node_ids; nid++) {
  2478. node = rb_first(root_stable_tree + nid);
  2479. while (node) {
  2480. stable_node = rb_entry(node, struct stable_node, node);
  2481. if (stable_node_chain_remove_range(stable_node,
  2482. start_pfn, end_pfn,
  2483. root_stable_tree +
  2484. nid))
  2485. node = rb_first(root_stable_tree + nid);
  2486. else
  2487. node = rb_next(node);
  2488. cond_resched();
  2489. }
  2490. }
  2491. list_for_each_entry_safe(stable_node, next, &migrate_nodes, list) {
  2492. if (stable_node->kpfn >= start_pfn &&
  2493. stable_node->kpfn < end_pfn)
  2494. remove_node_from_stable_tree(stable_node);
  2495. cond_resched();
  2496. }
  2497. }
  2498. static int ksm_memory_callback(struct notifier_block *self,
  2499. unsigned long action, void *arg)
  2500. {
  2501. struct memory_notify *mn = arg;
  2502. switch (action) {
  2503. case MEM_GOING_OFFLINE:
  2504. /*
  2505. * Prevent ksm_do_scan(), unmerge_and_remove_all_rmap_items()
  2506. * and remove_all_stable_nodes() while memory is going offline:
  2507. * it is unsafe for them to touch the stable tree at this time.
  2508. * But unmerge_ksm_pages(), rmap lookups and other entry points
  2509. * which do not need the ksm_thread_mutex are all safe.
  2510. */
  2511. mutex_lock(&ksm_thread_mutex);
  2512. ksm_run |= KSM_RUN_OFFLINE;
  2513. mutex_unlock(&ksm_thread_mutex);
  2514. break;
  2515. case MEM_OFFLINE:
  2516. /*
  2517. * Most of the work is done by page migration; but there might
  2518. * be a few stable_nodes left over, still pointing to struct
  2519. * pages which have been offlined: prune those from the tree,
  2520. * otherwise get_ksm_page() might later try to access a
  2521. * non-existent struct page.
  2522. */
  2523. ksm_check_stable_tree(mn->start_pfn,
  2524. mn->start_pfn + mn->nr_pages);
  2525. fallthrough;
  2526. case MEM_CANCEL_OFFLINE:
  2527. mutex_lock(&ksm_thread_mutex);
  2528. ksm_run &= ~KSM_RUN_OFFLINE;
  2529. mutex_unlock(&ksm_thread_mutex);
  2530. smp_mb(); /* wake_up_bit advises this */
  2531. wake_up_bit(&ksm_run, ilog2(KSM_RUN_OFFLINE));
  2532. break;
  2533. }
  2534. return NOTIFY_OK;
  2535. }
  2536. #else
  2537. static void wait_while_offlining(void)
  2538. {
  2539. }
  2540. #endif /* CONFIG_MEMORY_HOTREMOVE */
  2541. #ifdef CONFIG_SYSFS
  2542. /*
  2543. * This all compiles without CONFIG_SYSFS, but is a waste of space.
  2544. */
  2545. #define KSM_ATTR_RO(_name) \
  2546. static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
  2547. #define KSM_ATTR(_name) \
  2548. static struct kobj_attribute _name##_attr = \
  2549. __ATTR(_name, 0644, _name##_show, _name##_store)
  2550. static ssize_t sleep_millisecs_show(struct kobject *kobj,
  2551. struct kobj_attribute *attr, char *buf)
  2552. {
  2553. return sprintf(buf, "%u\n", ksm_thread_sleep_millisecs);
  2554. }
  2555. static ssize_t sleep_millisecs_store(struct kobject *kobj,
  2556. struct kobj_attribute *attr,
  2557. const char *buf, size_t count)
  2558. {
  2559. unsigned long msecs;
  2560. int err;
  2561. err = kstrtoul(buf, 10, &msecs);
  2562. if (err || msecs > UINT_MAX)
  2563. return -EINVAL;
  2564. ksm_thread_sleep_millisecs = msecs;
  2565. wake_up_interruptible(&ksm_iter_wait);
  2566. return count;
  2567. }
  2568. KSM_ATTR(sleep_millisecs);
  2569. static ssize_t pages_to_scan_show(struct kobject *kobj,
  2570. struct kobj_attribute *attr, char *buf)
  2571. {
  2572. return sprintf(buf, "%u\n", ksm_thread_pages_to_scan);
  2573. }
  2574. static ssize_t pages_to_scan_store(struct kobject *kobj,
  2575. struct kobj_attribute *attr,
  2576. const char *buf, size_t count)
  2577. {
  2578. int err;
  2579. unsigned long nr_pages;
  2580. err = kstrtoul(buf, 10, &nr_pages);
  2581. if (err || nr_pages > UINT_MAX)
  2582. return -EINVAL;
  2583. ksm_thread_pages_to_scan = nr_pages;
  2584. return count;
  2585. }
  2586. KSM_ATTR(pages_to_scan);
  2587. static ssize_t run_show(struct kobject *kobj, struct kobj_attribute *attr,
  2588. char *buf)
  2589. {
  2590. return sprintf(buf, "%lu\n", ksm_run);
  2591. }
  2592. static ssize_t run_store(struct kobject *kobj, struct kobj_attribute *attr,
  2593. const char *buf, size_t count)
  2594. {
  2595. int err;
  2596. unsigned long flags;
  2597. err = kstrtoul(buf, 10, &flags);
  2598. if (err || flags > UINT_MAX)
  2599. return -EINVAL;
  2600. if (flags > KSM_RUN_UNMERGE)
  2601. return -EINVAL;
  2602. /*
  2603. * KSM_RUN_MERGE sets ksmd running, and 0 stops it running.
  2604. * KSM_RUN_UNMERGE stops it running and unmerges all rmap_items,
  2605. * breaking COW to free the pages_shared (but leaves mm_slots
  2606. * on the list for when ksmd may be set running again).
  2607. */
  2608. mutex_lock(&ksm_thread_mutex);
  2609. wait_while_offlining();
  2610. if (ksm_run != flags) {
  2611. ksm_run = flags;
  2612. if (flags & KSM_RUN_UNMERGE) {
  2613. set_current_oom_origin();
  2614. err = unmerge_and_remove_all_rmap_items();
  2615. clear_current_oom_origin();
  2616. if (err) {
  2617. ksm_run = KSM_RUN_STOP;
  2618. count = err;
  2619. }
  2620. }
  2621. }
  2622. mutex_unlock(&ksm_thread_mutex);
  2623. if (flags & KSM_RUN_MERGE)
  2624. wake_up_interruptible(&ksm_thread_wait);
  2625. return count;
  2626. }
  2627. KSM_ATTR(run);
  2628. #ifdef CONFIG_NUMA
  2629. static ssize_t merge_across_nodes_show(struct kobject *kobj,
  2630. struct kobj_attribute *attr, char *buf)
  2631. {
  2632. return sprintf(buf, "%u\n", ksm_merge_across_nodes);
  2633. }
  2634. static ssize_t merge_across_nodes_store(struct kobject *kobj,
  2635. struct kobj_attribute *attr,
  2636. const char *buf, size_t count)
  2637. {
  2638. int err;
  2639. unsigned long knob;
  2640. err = kstrtoul(buf, 10, &knob);
  2641. if (err)
  2642. return err;
  2643. if (knob > 1)
  2644. return -EINVAL;
  2645. mutex_lock(&ksm_thread_mutex);
  2646. wait_while_offlining();
  2647. if (ksm_merge_across_nodes != knob) {
  2648. if (ksm_pages_shared || remove_all_stable_nodes())
  2649. err = -EBUSY;
  2650. else if (root_stable_tree == one_stable_tree) {
  2651. struct rb_root *buf;
  2652. /*
  2653. * This is the first time that we switch away from the
  2654. * default of merging across nodes: must now allocate
  2655. * a buffer to hold as many roots as may be needed.
  2656. * Allocate stable and unstable together:
  2657. * MAXSMP NODES_SHIFT 10 will use 16kB.
  2658. */
  2659. buf = kcalloc(nr_node_ids + nr_node_ids, sizeof(*buf),
  2660. GFP_KERNEL);
  2661. /* Let us assume that RB_ROOT is NULL is zero */
  2662. if (!buf)
  2663. err = -ENOMEM;
  2664. else {
  2665. root_stable_tree = buf;
  2666. root_unstable_tree = buf + nr_node_ids;
  2667. /* Stable tree is empty but not the unstable */
  2668. root_unstable_tree[0] = one_unstable_tree[0];
  2669. }
  2670. }
  2671. if (!err) {
  2672. ksm_merge_across_nodes = knob;
  2673. ksm_nr_node_ids = knob ? 1 : nr_node_ids;
  2674. }
  2675. }
  2676. mutex_unlock(&ksm_thread_mutex);
  2677. return err ? err : count;
  2678. }
  2679. KSM_ATTR(merge_across_nodes);
  2680. #endif
  2681. static ssize_t use_zero_pages_show(struct kobject *kobj,
  2682. struct kobj_attribute *attr, char *buf)
  2683. {
  2684. return sprintf(buf, "%u\n", ksm_use_zero_pages);
  2685. }
  2686. static ssize_t use_zero_pages_store(struct kobject *kobj,
  2687. struct kobj_attribute *attr,
  2688. const char *buf, size_t count)
  2689. {
  2690. int err;
  2691. bool value;
  2692. err = kstrtobool(buf, &value);
  2693. if (err)
  2694. return -EINVAL;
  2695. ksm_use_zero_pages = value;
  2696. return count;
  2697. }
  2698. KSM_ATTR(use_zero_pages);
  2699. static ssize_t max_page_sharing_show(struct kobject *kobj,
  2700. struct kobj_attribute *attr, char *buf)
  2701. {
  2702. return sprintf(buf, "%u\n", ksm_max_page_sharing);
  2703. }
  2704. static ssize_t max_page_sharing_store(struct kobject *kobj,
  2705. struct kobj_attribute *attr,
  2706. const char *buf, size_t count)
  2707. {
  2708. int err;
  2709. int knob;
  2710. err = kstrtoint(buf, 10, &knob);
  2711. if (err)
  2712. return err;
  2713. /*
  2714. * When a KSM page is created it is shared by 2 mappings. This
  2715. * being a signed comparison, it implicitly verifies it's not
  2716. * negative.
  2717. */
  2718. if (knob < 2)
  2719. return -EINVAL;
  2720. if (READ_ONCE(ksm_max_page_sharing) == knob)
  2721. return count;
  2722. mutex_lock(&ksm_thread_mutex);
  2723. wait_while_offlining();
  2724. if (ksm_max_page_sharing != knob) {
  2725. if (ksm_pages_shared || remove_all_stable_nodes())
  2726. err = -EBUSY;
  2727. else
  2728. ksm_max_page_sharing = knob;
  2729. }
  2730. mutex_unlock(&ksm_thread_mutex);
  2731. return err ? err : count;
  2732. }
  2733. KSM_ATTR(max_page_sharing);
  2734. static ssize_t pages_shared_show(struct kobject *kobj,
  2735. struct kobj_attribute *attr, char *buf)
  2736. {
  2737. return sprintf(buf, "%lu\n", ksm_pages_shared);
  2738. }
  2739. KSM_ATTR_RO(pages_shared);
  2740. static ssize_t pages_sharing_show(struct kobject *kobj,
  2741. struct kobj_attribute *attr, char *buf)
  2742. {
  2743. return sprintf(buf, "%lu\n", ksm_pages_sharing);
  2744. }
  2745. KSM_ATTR_RO(pages_sharing);
  2746. static ssize_t pages_unshared_show(struct kobject *kobj,
  2747. struct kobj_attribute *attr, char *buf)
  2748. {
  2749. return sprintf(buf, "%lu\n", ksm_pages_unshared);
  2750. }
  2751. KSM_ATTR_RO(pages_unshared);
  2752. static ssize_t pages_volatile_show(struct kobject *kobj,
  2753. struct kobj_attribute *attr, char *buf)
  2754. {
  2755. long ksm_pages_volatile;
  2756. ksm_pages_volatile = ksm_rmap_items - ksm_pages_shared
  2757. - ksm_pages_sharing - ksm_pages_unshared;
  2758. /*
  2759. * It was not worth any locking to calculate that statistic,
  2760. * but it might therefore sometimes be negative: conceal that.
  2761. */
  2762. if (ksm_pages_volatile < 0)
  2763. ksm_pages_volatile = 0;
  2764. return sprintf(buf, "%ld\n", ksm_pages_volatile);
  2765. }
  2766. KSM_ATTR_RO(pages_volatile);
  2767. static ssize_t stable_node_dups_show(struct kobject *kobj,
  2768. struct kobj_attribute *attr, char *buf)
  2769. {
  2770. return sprintf(buf, "%lu\n", ksm_stable_node_dups);
  2771. }
  2772. KSM_ATTR_RO(stable_node_dups);
  2773. static ssize_t stable_node_chains_show(struct kobject *kobj,
  2774. struct kobj_attribute *attr, char *buf)
  2775. {
  2776. return sprintf(buf, "%lu\n", ksm_stable_node_chains);
  2777. }
  2778. KSM_ATTR_RO(stable_node_chains);
  2779. static ssize_t
  2780. stable_node_chains_prune_millisecs_show(struct kobject *kobj,
  2781. struct kobj_attribute *attr,
  2782. char *buf)
  2783. {
  2784. return sprintf(buf, "%u\n", ksm_stable_node_chains_prune_millisecs);
  2785. }
  2786. static ssize_t
  2787. stable_node_chains_prune_millisecs_store(struct kobject *kobj,
  2788. struct kobj_attribute *attr,
  2789. const char *buf, size_t count)
  2790. {
  2791. unsigned long msecs;
  2792. int err;
  2793. err = kstrtoul(buf, 10, &msecs);
  2794. if (err || msecs > UINT_MAX)
  2795. return -EINVAL;
  2796. ksm_stable_node_chains_prune_millisecs = msecs;
  2797. return count;
  2798. }
  2799. KSM_ATTR(stable_node_chains_prune_millisecs);
  2800. static ssize_t full_scans_show(struct kobject *kobj,
  2801. struct kobj_attribute *attr, char *buf)
  2802. {
  2803. return sprintf(buf, "%lu\n", ksm_scan.seqnr);
  2804. }
  2805. KSM_ATTR_RO(full_scans);
  2806. static struct attribute *ksm_attrs[] = {
  2807. &sleep_millisecs_attr.attr,
  2808. &pages_to_scan_attr.attr,
  2809. &run_attr.attr,
  2810. &pages_shared_attr.attr,
  2811. &pages_sharing_attr.attr,
  2812. &pages_unshared_attr.attr,
  2813. &pages_volatile_attr.attr,
  2814. &full_scans_attr.attr,
  2815. #ifdef CONFIG_NUMA
  2816. &merge_across_nodes_attr.attr,
  2817. #endif
  2818. &max_page_sharing_attr.attr,
  2819. &stable_node_chains_attr.attr,
  2820. &stable_node_dups_attr.attr,
  2821. &stable_node_chains_prune_millisecs_attr.attr,
  2822. &use_zero_pages_attr.attr,
  2823. NULL,
  2824. };
  2825. static const struct attribute_group ksm_attr_group = {
  2826. .attrs = ksm_attrs,
  2827. .name = "ksm",
  2828. };
  2829. #endif /* CONFIG_SYSFS */
  2830. static int __init ksm_init(void)
  2831. {
  2832. struct task_struct *ksm_thread;
  2833. int err;
  2834. /* The correct value depends on page size and endianness */
  2835. zero_checksum = calc_checksum(ZERO_PAGE(0));
  2836. /* Default to false for backwards compatibility */
  2837. ksm_use_zero_pages = false;
  2838. err = ksm_slab_init();
  2839. if (err)
  2840. goto out;
  2841. ksm_thread = kthread_run(ksm_scan_thread, NULL, "ksmd");
  2842. if (IS_ERR(ksm_thread)) {
  2843. pr_err("ksm: creating kthread failed\n");
  2844. err = PTR_ERR(ksm_thread);
  2845. goto out_free;
  2846. }
  2847. #ifdef CONFIG_SYSFS
  2848. err = sysfs_create_group(mm_kobj, &ksm_attr_group);
  2849. if (err) {
  2850. pr_err("ksm: register sysfs failed\n");
  2851. kthread_stop(ksm_thread);
  2852. goto out_free;
  2853. }
  2854. #else
  2855. ksm_run = KSM_RUN_MERGE; /* no way for user to start it */
  2856. #endif /* CONFIG_SYSFS */
  2857. #ifdef CONFIG_MEMORY_HOTREMOVE
  2858. /* There is no significance to this priority 100 */
  2859. hotplug_memory_notifier(ksm_memory_callback, 100);
  2860. #endif
  2861. return 0;
  2862. out_free:
  2863. ksm_slab_free();
  2864. out:
  2865. return err;
  2866. }
  2867. subsys_initcall(ksm_init);