numa01.sh 9.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386
  1. #!/bin/sh
  2. # SPDX-License-Identifier: GPL-2.0-or-later
  3. # Copyright (c) International Business Machines Corp., 2007
  4. # Copyright (c) Linux Test Project, 2016-2020
  5. # Author: Sivakumar Chinnaiah <Sivakumar.C@in.ibm.com>
  6. #
  7. # Test Basic functionality of numactl command.
  8. # Test #1: Verifies cpunodebind and membind
  9. # Test #2: Verifies preferred node bind for memory allocation
  10. # Test #3: Verifies memory interleave on all nodes
  11. # Test #4: Verifies physcpubind
  12. # Test #5: Verifies localalloc
  13. # Test #6: Verifies memhog
  14. # Test #7: Verifies numa_node_size api
  15. # Test #8: Verifies hugepage alloacted on specified node
  16. # Test #9: Verifies THP memory allocated on preferred node
  17. TST_CNT=9
  18. TST_SETUP=setup
  19. TST_TESTFUNC=test
  20. TST_NEEDS_TMPDIR=1
  21. TST_NEEDS_ROOT=1
  22. TST_NEEDS_CMDS="awk bc numactl numastat"
  23. . tst_test.sh
  24. # Awk the field matching the node value for numastat
  25. # $1 - Pid number
  26. # $2 - Node number
  27. get_node_index()
  28. {
  29. local pid=$1
  30. local nid="Node $2"
  31. echo $(numastat -p $pid | sed '3q;d' | awk -F '[[:space:]][[:space:]]+' \
  32. -v node="$nid" '{ for (i = 1; i <= NF; ++i) if($i==node) print i; exit }')
  33. }
  34. # Convert the value of given numa node from the `numastat -p` output,
  35. # multiply by size.
  36. # $1 - Pid number
  37. # $2 - Node number
  38. # $3 - Size for multiplication (e.g. 1024, $MB)
  39. get_mem_cur()
  40. {
  41. local pid=$1
  42. local index=$(echo "$(get_node_index $pid $2)")
  43. local size=$3
  44. local numstat=$(numastat -p $pid |awk '/^Total/ {print $'$index'}')
  45. if [ -z "$numstat" ]; then
  46. echo 0
  47. return
  48. fi
  49. echo $(echo "$numstat * $size" | bc)
  50. }
  51. check_for_support_numa()
  52. {
  53. local pid=$1
  54. local state=$(awk '{print $3}' /proc/$pid/stat)
  55. if [ $state = 'T' ]; then
  56. return 0
  57. fi
  58. return 1
  59. }
  60. setup()
  61. {
  62. export MB=$((1024*1024))
  63. export PAGE_SIZE=$(tst_getconf PAGESIZE)
  64. export HPAGE_SIZE=$(awk '/Hugepagesize:/ {print $2}' /proc/meminfo)
  65. total_nodes=0
  66. nodes_list=$(numactl --show | grep nodebind | cut -d ':' -f 2)
  67. for node in $nodes_list; do
  68. total_nodes=$((total_nodes+1))
  69. done
  70. tst_res TINFO "The system contains $total_nodes nodes: $nodes_list"
  71. if [ $total_nodes -le 1 ]; then
  72. tst_brk TCONF "your machine does not support numa policy
  73. or your machine is not a NUMA machine"
  74. fi
  75. }
  76. # Verification of memory allocated on a node
  77. test1()
  78. {
  79. local mem_curr
  80. for node in $nodes_list; do
  81. numactl --cpunodebind=$node --membind=$node support_numa alloc_1MB &
  82. pid=$!
  83. TST_RETRY_FUNC "check_for_support_numa $pid" 0
  84. mem_curr=$(get_mem_cur $pid $node $MB)
  85. if [ $(echo "$mem_curr < $MB" | bc) -eq 1 ]; then
  86. tst_res TFAIL \
  87. "NUMA memory allocated in node$node is less than expected"
  88. kill -CONT $pid >/dev/null 2>&1
  89. return
  90. fi
  91. kill -CONT $pid >/dev/null 2>&1
  92. done
  93. tst_res TPASS "NUMA local node and memory affinity"
  94. }
  95. # Verification of memory allocated on preferred node
  96. test2()
  97. {
  98. local mem_curr
  99. local cnt=1
  100. for node in $nodes_list; do
  101. if [ $cnt -eq $total_nodes ]; then #wrap up for last node
  102. Preferred_node=$(echo $nodes_list | cut -d ' ' -f 1)
  103. else
  104. # always next node is preferred node
  105. Preferred_node=$(echo $nodes_list | cut -d ' ' -f $((cnt+1)))
  106. fi
  107. numactl --cpunodebind=$node --preferred=$Preferred_node support_numa alloc_1MB &
  108. pid=$!
  109. TST_RETRY_FUNC "check_for_support_numa $pid" 0
  110. mem_curr=$(get_mem_cur $pid $Preferred_node $MB)
  111. if [ $(echo "$mem_curr < $MB" |bc ) -eq 1 ]; then
  112. tst_res TFAIL \
  113. "NUMA memory allocated in node$Preferred_node is less than expected"
  114. kill -CONT $pid >/dev/null 2>&1
  115. return
  116. fi
  117. cnt=$((cnt+1))
  118. kill -CONT $pid >/dev/null 2>&1
  119. done
  120. tst_res TPASS "NUMA preferred node policy"
  121. }
  122. # Verification of memory interleaved on all nodes
  123. test3()
  124. {
  125. local mem_curr
  126. # Memory will be allocated using round robin on nodes.
  127. Exp_incr=$(echo "$MB / $total_nodes" |bc)
  128. numactl --interleave=all support_numa alloc_1MB &
  129. pid=$!
  130. TST_RETRY_FUNC "check_for_support_numa $pid" 0
  131. for node in $nodes_list; do
  132. mem_curr=$(get_mem_cur $pid $node $MB)
  133. if [ $(echo "$mem_curr < $Exp_incr" |bc ) -eq 1 ]; then
  134. tst_res TFAIL \
  135. "NUMA interleave memory allocated in node$node is less than expected"
  136. kill -CONT $pid >/dev/null 2>&1
  137. return
  138. fi
  139. done
  140. kill -CONT $pid >/dev/null 2>&1
  141. tst_res TPASS "NUMA interleave policy"
  142. }
  143. # Verification of physical cpu bind
  144. test4()
  145. {
  146. no_of_cpus=0 #no. of cpu's exist
  147. run_on_cpu=0
  148. running_on_cpu=0
  149. no_of_cpus=$(tst_ncpus)
  150. # not sure whether cpu's can't be in odd number
  151. run_on_cpu=$(($((no_of_cpus+1))/2))
  152. numactl --physcpubind=$run_on_cpu support_numa pause & #just waits for sigint
  153. pid=$!
  154. var=`awk '{ print $2 }' /proc/$pid/stat`
  155. while [ $var = '(numactl)' ]; do
  156. var=`awk '{ print $2 }' /proc/$pid/stat`
  157. tst_sleep 100ms
  158. done
  159. # Warning !! 39 represents cpu number, on which process pid is currently running and
  160. # this may change if Some more fields are added in the middle, may be in future
  161. running_on_cpu=$(awk '{ print $39; }' /proc/$pid/stat)
  162. if [ $running_on_cpu -ne $run_on_cpu ]; then
  163. tst_res TFAIL \
  164. "Process running on cpu$running_on_cpu but expected to run on cpu$run_on_cpu"
  165. ROD kill -INT $pid
  166. return
  167. fi
  168. ROD kill -INT $pid
  169. tst_res TPASS "NUMA phycpubind policy"
  170. }
  171. # Verification of local node allocation
  172. test5()
  173. {
  174. local mem_curr
  175. for node in $nodes_list; do
  176. numactl --cpunodebind=$node --localalloc support_numa alloc_1MB &
  177. pid=$!
  178. TST_RETRY_FUNC "check_for_support_numa $pid" 0
  179. mem_curr=$(get_mem_cur $pid $node $MB)
  180. if [ $(echo "$mem_curr < $MB" |bc ) -eq 1 ]; then
  181. tst_res TFAIL \
  182. "NUMA localnode memory allocated in node$node is less than expected"
  183. kill -CONT $pid >/dev/null 2>&1
  184. return
  185. fi
  186. kill -CONT $pid >/dev/null 2>&1
  187. done
  188. tst_res TPASS "NUMA local node allocation"
  189. }
  190. check_ltp_numa_test8_log()
  191. {
  192. grep -m1 -q '.' ltp_numa_test8.log
  193. }
  194. # Verification of memhog with interleave policy
  195. test6()
  196. {
  197. local mem_curr
  198. # Memory will be allocated using round robin on nodes.
  199. Exp_incr=$(echo "$MB / $total_nodes" |bc)
  200. numactl --interleave=all memhog -r1000000 1MB >ltp_numa_test8.log 2>&1 &
  201. pid=$!
  202. TST_RETRY_FUNC "check_ltp_numa_test8_log" 0
  203. for node in $nodes_list; do
  204. mem_curr=$(get_mem_cur $pid $node $MB)
  205. if [ $(echo "$mem_curr < $Exp_incr" |bc ) -eq 1 ]; then
  206. tst_res TFAIL \
  207. "NUMA interleave memhog in node$node is less than expected"
  208. kill -KILL $pid >/dev/null 2>&1
  209. return
  210. fi
  211. done
  212. kill -KILL $pid >/dev/null 2>&1
  213. tst_res TPASS "NUMA MEMHOG policy"
  214. }
  215. # Function: hardware cheking with numa_node_size api
  216. #
  217. # Description: - Returns the size of available nodes if success.
  218. #
  219. # Input: - o/p of numactl --hardware command which is expected in the format
  220. # shown below
  221. # available: 2 nodes (0-1)
  222. # node 0 size: 7808 MB
  223. # node 0 free: 7457 MB
  224. # node 1 size: 5807 MB
  225. # node 1 free: 5731 MB
  226. # node distances:
  227. # node 0 1
  228. # 0: 10 20
  229. # 1: 20 10
  230. #
  231. test7()
  232. {
  233. RC=0
  234. numactl --hardware > gavail_nodes
  235. RC=$(awk '{ if ( NR == 1 ) {print $1;} }' gavail_nodes)
  236. if [ $RC = "available:" ]; then
  237. RC=$(awk '{ if ( NR == 1 ) {print $3;} }' gavail_nodes)
  238. if [ $RC = "nodes" ]; then
  239. RC=$(awk '{ if ( NR == 1 ) {print $2;} }' gavail_nodes)
  240. tst_res TPASS "NUMA policy on lib NUMA_NODE_SIZE API"
  241. else
  242. tst_res TFAIL "Failed with numa policy"
  243. fi
  244. else
  245. tst_res TFAIL "Failed with numa policy"
  246. fi
  247. }
  248. # Verification of hugepage memory allocated on a node
  249. test8()
  250. {
  251. Mem_huge=0
  252. Sys_node=/sys/devices/system/node
  253. if [ ! -d "/sys/kernel/mm/hugepages/" ]; then
  254. tst_res TCONF "hugepage is not supported"
  255. return
  256. fi
  257. for node in $nodes_list; do
  258. Ori_hpgs=$(cat ${Sys_node}/node${node}/hugepages/hugepages-${HPAGE_SIZE}kB/nr_hugepages)
  259. New_hpgs=$((Ori_hpgs + 1))
  260. echo $New_hpgs >${Sys_node}/node${node}/hugepages/hugepages-${HPAGE_SIZE}kB/nr_hugepages
  261. Chk_hpgs=$(cat ${Sys_node}/node${node}/hugepages/hugepages-${HPAGE_SIZE}kB/nr_hugepages)
  262. if [ "$Chk_hpgs" -ne "$New_hpgs" ]; then
  263. tst_res TCONF "hugepage is not enough to test"
  264. return
  265. fi
  266. numactl --cpunodebind=$node --membind=$node support_numa alloc_1huge_page &
  267. pid=$!
  268. TST_RETRY_FUNC "check_for_support_numa $pid" 0
  269. local index=$(echo "$(get_node_index $pid $node)")
  270. Mem_huge=$(echo $(numastat -p $pid |awk '/^Huge/ {print $'$index'}'))
  271. Mem_huge=$((${Mem_huge%.*} * 1024))
  272. if [ "$Mem_huge" -lt "$HPAGE_SIZE" ]; then
  273. tst_res TFAIL \
  274. "NUMA memory allocated in node$node is less than expected"
  275. kill -CONT $pid >/dev/null 2>&1
  276. echo $Ori_hpgs >${Sys_node}/node${node}/hugepages/hugepages-${HPAGE_SIZE}kB/nr_hugepages
  277. return
  278. fi
  279. kill -CONT $pid >/dev/null 2>&1
  280. echo $Ori_hpgs >${Sys_node}/node${node}/hugepages/hugepages-${HPAGE_SIZE}kB/nr_hugepages
  281. done
  282. tst_res TPASS "NUMA local node hugepage memory allocated"
  283. }
  284. # Verification of THP memory allocated on preferred node
  285. test9()
  286. {
  287. local mem_curr
  288. local cnt=1
  289. if ! grep -q '\[always\]' /sys/kernel/mm/transparent_hugepage/enabled; then
  290. tst_res TCONF "THP is not supported/enabled"
  291. return
  292. fi
  293. for node in $nodes_list; do
  294. if [ $cnt -eq $total_nodes ]; then #wrap up for last node
  295. Preferred_node=$(echo $nodes_list | cut -d ' ' -f 1)
  296. else
  297. # always next node is preferred node
  298. Preferred_node=$(echo $nodes_list | cut -d ' ' -f $((cnt+1)))
  299. fi
  300. numactl --cpunodebind=$node --preferred=$Preferred_node support_numa alloc_2HPSZ_THP &
  301. pid=$!
  302. TST_RETRY_FUNC "check_for_support_numa $pid" 0
  303. mem_curr=$(get_mem_cur $pid $Preferred_node 1024)
  304. if [ $(echo "$mem_curr < $HPAGE_SIZE * 2" |bc ) -eq 1 ]; then
  305. tst_res TFAIL \
  306. "NUMA memory allocated in node$Preferred_node is less than expected"
  307. kill -CONT $pid >/dev/null 2>&1
  308. return
  309. fi
  310. cnt=$((cnt+1))
  311. kill -CONT $pid >/dev/null 2>&1
  312. done
  313. tst_res TPASS "NUMA preferred node policy verified with THP enabled"
  314. }
  315. tst_run