sstate-cache-management.sh 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458
  1. #!/bin/bash
  2. # Copyright (c) 2012 Wind River Systems, Inc.
  3. #
  4. # SPDX-License-Identifier: GPL-2.0-only
  5. #
  6. # Global vars
  7. cache_dir=
  8. confirm=
  9. fsym=
  10. total_deleted=0
  11. verbose=
  12. debug=0
  13. usage () {
  14. cat << EOF
  15. Welcome to sstate cache management utilities.
  16. sstate-cache-management.sh <OPTION>
  17. Options:
  18. -h, --help
  19. Display this help and exit.
  20. --cache-dir=<sstate cache dir>
  21. Specify sstate cache directory, will use the environment
  22. variable SSTATE_CACHE_DIR if it is not specified.
  23. --extra-archs=<arch1>,<arch2>...<archn>
  24. Specify list of architectures which should be tested, this list
  25. will be extended with native arch, allarch and empty arch. The
  26. script won't be trying to generate list of available archs from
  27. AVAILTUNES in tune files.
  28. --extra-layer=<layer1>,<layer2>...<layern>
  29. Specify the layer which will be used for searching the archs,
  30. it will search the meta and meta-* layers in the top dir by
  31. default, and will search meta, meta-*, <layer1>, <layer2>,
  32. ...<layern> when specified. Use "," as the separator.
  33. This is useless for --stamps-dir or when --extra-archs is used.
  34. -d, --remove-duplicated
  35. Remove the duplicated sstate cache files of one package, only
  36. the newest one will be kept. The duplicated sstate cache files
  37. of one package must have the same arch, which means sstate cache
  38. files with multiple archs are not considered duplicate.
  39. Conflicts with --stamps-dir.
  40. --stamps-dir=<dir1>,<dir2>...<dirn>
  41. Specify the build directory's stamps directories, the sstate
  42. cache file which IS USED by these build diretories will be KEPT,
  43. other sstate cache files in cache-dir will be removed. Use ","
  44. as the separator. For example:
  45. --stamps-dir=build1/tmp/stamps,build2/tmp/stamps
  46. Conflicts with --remove-duplicated.
  47. -L, --follow-symlink
  48. Remove both the symbol link and the destination file, default: no.
  49. -y, --yes
  50. Automatic yes to prompts; assume "yes" as answer to all prompts
  51. and run non-interactively.
  52. -v, --verbose
  53. Explain what is being done.
  54. -D, --debug
  55. Show debug info, repeat for more debug info.
  56. EOF
  57. }
  58. if [ $# -lt 1 ]; then
  59. usage
  60. exit 0
  61. fi
  62. # Echo no files to remove
  63. no_files () {
  64. echo No files to remove
  65. }
  66. # Echo nothing to do
  67. do_nothing () {
  68. echo Nothing to do
  69. }
  70. # Read the input "y"
  71. read_confirm () {
  72. echo "$total_deleted out of $total_files files will be removed! "
  73. if [ "$confirm" != "y" ]; then
  74. echo "Do you want to continue (y/n)? "
  75. while read confirm; do
  76. [ "$confirm" = "Y" -o "$confirm" = "y" -o "$confirm" = "n" \
  77. -o "$confirm" = "N" ] && break
  78. echo "Invalid input \"$confirm\", please input 'y' or 'n': "
  79. done
  80. else
  81. echo
  82. fi
  83. }
  84. # Print error information and exit.
  85. echo_error () {
  86. echo "ERROR: $1" >&2
  87. exit 1
  88. }
  89. # Generate the remove list:
  90. #
  91. # * Add .done/.siginfo to the remove list
  92. # * Add destination of symlink to the remove list
  93. #
  94. # $1: output file, others: sstate cache file (.tgz)
  95. gen_rmlist (){
  96. local rmlist_file="$1"
  97. shift
  98. local files="$@"
  99. for i in $files; do
  100. echo $i >> $rmlist_file
  101. # Add the ".siginfo"
  102. if [ -e $i.siginfo ]; then
  103. echo $i.siginfo >> $rmlist_file
  104. fi
  105. # Add the destination of symlink
  106. if [ -L "$i" ]; then
  107. if [ "$fsym" = "y" ]; then
  108. dest="`readlink -e $i`"
  109. if [ -n "$dest" ]; then
  110. echo $dest >> $rmlist_file
  111. # Remove the .siginfo when .tgz is removed
  112. if [ -f "$dest.siginfo" ]; then
  113. echo $dest.siginfo >> $rmlist_file
  114. fi
  115. fi
  116. fi
  117. # Add the ".tgz.done" and ".siginfo.done" (may exist in the future)
  118. base_fn="${i##/*/}"
  119. t_fn="$base_fn.done"
  120. s_fn="$base_fn.siginfo.done"
  121. for d in $t_fn $s_fn; do
  122. if [ -f $cache_dir/$d ]; then
  123. echo $cache_dir/$d >> $rmlist_file
  124. fi
  125. done
  126. fi
  127. done
  128. }
  129. # Remove the duplicated cache files for the pkg, keep the newest one
  130. remove_duplicated () {
  131. local topdir
  132. local oe_core_dir
  133. local tunedirs
  134. local all_archs
  135. local all_machines
  136. local ava_archs
  137. local arch
  138. local file_names
  139. local sstate_files_list
  140. local fn_tmp
  141. local list_suffix=`mktemp` || exit 1
  142. if [ -z "$extra_archs" ] ; then
  143. # Find out the archs in all the layers
  144. echo "Figuring out the archs in the layers ... "
  145. oe_core_dir=$(dirname $(dirname $(readlink -e $0)))
  146. topdir=$(dirname $oe_core_dir)
  147. tunedirs="`find $topdir/meta* ${oe_core_dir}/meta* $layers -path '*/meta*/conf/machine/include' 2>/dev/null`"
  148. [ -n "$tunedirs" ] || echo_error "Can't find the tune directory"
  149. all_machines="`find $topdir/meta* ${oe_core_dir}/meta* $layers -path '*/meta*/conf/machine/*' -name '*.conf' 2>/dev/null | sed -e 's/.*\///' -e 's/.conf$//'`"
  150. all_archs=`grep -r -h "^AVAILTUNES .*=" $tunedirs | sed -e 's/.*=//' -e 's/\"//g'`
  151. fi
  152. # Use the "_" to substitute "-", e.g., x86-64 to x86_64, but not for extra_archs which can be something like cortexa9t2-vfp-neon
  153. # Sort to remove the duplicated ones
  154. # Add allarch and builder arch (native)
  155. builder_arch=$(uname -m)
  156. all_archs="$(echo allarch $all_archs $all_machines $builder_arch \
  157. | sed -e 's/-/_/g' -e 's/ /\n/g' | sort -u) $extra_archs"
  158. echo "Done"
  159. # Total number of files including sstate-, .siginfo and .done files
  160. total_files=`find $cache_dir -name 'sstate*' | wc -l`
  161. # Save all the sstate files in a file
  162. sstate_files_list=`mktemp` || exit 1
  163. find $cache_dir -name 'sstate:*:*:*:*:*:*:*.tgz*' >$sstate_files_list
  164. echo "Figuring out the suffixes in the sstate cache dir ... "
  165. sstate_suffixes="`sed 's%.*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^_]*_\([^:]*\)\.tgz.*%\1%g' $sstate_files_list | sort -u`"
  166. echo "Done"
  167. echo "The following suffixes have been found in the cache dir:"
  168. echo $sstate_suffixes
  169. echo "Figuring out the archs in the sstate cache dir ... "
  170. # Using this SSTATE_PKGSPEC definition it's 6th colon separated field
  171. # SSTATE_PKGSPEC = "sstate:${PN}:${PACKAGE_ARCH}${TARGET_VENDOR}-${TARGET_OS}:${PV}:${PR}:${SSTATE_PKGARCH}:${SSTATE_VERSION}:"
  172. for arch in $all_archs; do
  173. grep -q ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:$arch:[^:]*:[^:]*\.tgz$" $sstate_files_list
  174. [ $? -eq 0 ] && ava_archs="$ava_archs $arch"
  175. # ${builder_arch}_$arch used by toolchain sstate
  176. grep -q ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:${builder_arch}_$arch:[^:]*:[^:]*\.tgz$" $sstate_files_list
  177. [ $? -eq 0 ] && ava_archs="$ava_archs ${builder_arch}_$arch"
  178. done
  179. echo "Done"
  180. echo "The following archs have been found in the cache dir:"
  181. echo $ava_archs
  182. echo ""
  183. # Save the file list which needs to be removed
  184. local remove_listdir=`mktemp -d` || exit 1
  185. for suffix in $sstate_suffixes; do
  186. if [ "$suffix" = "populate_lic" ] ; then
  187. echo "Skipping populate_lic, because removing duplicates doesn't work correctly for them (use --stamps-dir instead)"
  188. continue
  189. fi
  190. # Total number of files including .siginfo and .done files
  191. total_files_suffix=`grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:_]*_$suffix\.tgz.*" $sstate_files_list | wc -l 2>/dev/null`
  192. total_tgz_suffix=`grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:_]*_$suffix\.tgz$" $sstate_files_list | wc -l 2>/dev/null`
  193. # Save the file list to a file, some suffix's file may not exist
  194. grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:_]*_$suffix\.tgz.*" $sstate_files_list >$list_suffix 2>/dev/null
  195. local deleted_tgz=0
  196. local deleted_files=0
  197. for ext in tgz tgz.siginfo tgz.done; do
  198. echo "Figuring out the sstate:xxx_$suffix.$ext ... "
  199. # Uniq BPNs
  200. file_names=`for arch in $ava_archs ""; do
  201. sed -ne "s%.*/sstate:\([^:]*\):[^:]*:[^:]*:[^:]*:$arch:[^:]*:[^:]*\.${ext}$%\1%p" $list_suffix
  202. done | sort -u`
  203. fn_tmp=`mktemp` || exit 1
  204. rm_list="$remove_listdir/sstate:xxx_$suffix"
  205. for fn in $file_names; do
  206. [ -z "$verbose" ] || echo "Analyzing sstate:$fn-xxx_$suffix.${ext}"
  207. for arch in $ava_archs ""; do
  208. grep -h ".*/sstate:$fn:[^:]*:[^:]*:[^:]*:$arch:[^:]*:[^:]*\.${ext}$" $list_suffix >$fn_tmp
  209. if [ -s $fn_tmp ] ; then
  210. [ $debug -gt 1 ] && echo "Available files for $fn-$arch- with suffix $suffix.${ext}:" && cat $fn_tmp
  211. # Use the modification time
  212. to_del=$(ls -t $(cat $fn_tmp) | sed -n '1!p')
  213. [ $debug -gt 2 ] && echo "Considering to delete: $to_del"
  214. # The sstate file which is downloaded from the SSTATE_MIRROR is
  215. # put in SSTATE_DIR, and there is a symlink in SSTATE_DIR/??/ to
  216. # it, so filter it out from the remove list if it should not be
  217. # removed.
  218. to_keep=$(ls -t $(cat $fn_tmp) | sed -n '1p')
  219. [ $debug -gt 2 ] && echo "Considering to keep: $to_keep"
  220. for k in $to_keep; do
  221. if [ -L "$k" ]; then
  222. # The symlink's destination
  223. k_dest="`readlink -e $k`"
  224. # Maybe it is the one in cache_dir
  225. k_maybe="$cache_dir/${k##/*/}"
  226. # Remove it from the remove list if they are the same.
  227. if [ "$k_dest" = "$k_maybe" ]; then
  228. to_del="`echo $to_del | sed 's#'\"$k_maybe\"'##g'`"
  229. fi
  230. fi
  231. done
  232. rm -f $fn_tmp
  233. [ $debug -gt 2 ] && echo "Decided to delete: $to_del"
  234. gen_rmlist $rm_list.$ext "$to_del"
  235. fi
  236. done
  237. done
  238. done
  239. deleted_tgz=`cat $rm_list.* 2>/dev/null | grep ".tgz$" | wc -l`
  240. deleted_files=`cat $rm_list.* 2>/dev/null | wc -l`
  241. [ "$deleted_files" -gt 0 -a $debug -gt 0 ] && cat $rm_list.*
  242. echo "($deleted_tgz out of $total_tgz_suffix .tgz files for $suffix suffix will be removed or $deleted_files out of $total_files_suffix when counting also .siginfo and .done files)"
  243. let total_deleted=$total_deleted+$deleted_files
  244. done
  245. deleted_tgz=0
  246. rm_old_list=$remove_listdir/sstate-old-filenames
  247. find $cache_dir -name 'sstate-*.tgz' >$rm_old_list
  248. [ -s "$rm_old_list" ] && deleted_tgz=`cat $rm_old_list | grep ".tgz$" | wc -l`
  249. [ -s "$rm_old_list" ] && deleted_files=`cat $rm_old_list | wc -l`
  250. [ -s "$rm_old_list" -a $debug -gt 0 ] && cat $rm_old_list
  251. echo "($deleted_tgz .tgz files with old sstate-* filenames will be removed or $deleted_files when counting also .siginfo and .done files)"
  252. let total_deleted=$total_deleted+$deleted_files
  253. rm -f $list_suffix
  254. rm -f $sstate_files_list
  255. if [ $total_deleted -gt 0 ]; then
  256. read_confirm
  257. if [ "$confirm" = "y" -o "$confirm" = "Y" ]; then
  258. for list in `ls $remove_listdir/`; do
  259. echo "Removing $list.tgz (`cat $remove_listdir/$list | wc -w` files) ... "
  260. # Remove them one by one to avoid the argument list too long error
  261. for i in `cat $remove_listdir/$list`; do
  262. rm -f $verbose $i
  263. done
  264. echo "Done"
  265. done
  266. echo "$total_deleted files have been removed!"
  267. else
  268. do_nothing
  269. fi
  270. else
  271. no_files
  272. fi
  273. [ -d $remove_listdir ] && rm -fr $remove_listdir
  274. }
  275. # Remove the sstate file by stamps dir, the file not used by the stamps dir
  276. # will be removed.
  277. rm_by_stamps (){
  278. local cache_list=`mktemp` || exit 1
  279. local keep_list=`mktemp` || exit 1
  280. local rm_list=`mktemp` || exit 1
  281. local sums
  282. local all_sums
  283. # Total number of files including sstate-, .siginfo and .done files
  284. total_files=`find $cache_dir -type f -name 'sstate*' | wc -l`
  285. # Save all the state file list to a file
  286. find $cache_dir -type f -name 'sstate*' | sort -u -o $cache_list
  287. echo "Figuring out the suffixes in the sstate cache dir ... "
  288. local sstate_suffixes="`sed 's%.*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^_]*_\([^:]*\)\.tgz.*%\1%g' $cache_list | sort -u`"
  289. echo "Done"
  290. echo "The following suffixes have been found in the cache dir:"
  291. echo $sstate_suffixes
  292. # Figure out all the md5sums in the stamps dir.
  293. echo "Figuring out all the md5sums in stamps dir ... "
  294. for i in $sstate_suffixes; do
  295. # There is no "\.sigdata" but "_setcene" when it is mirrored
  296. # from the SSTATE_MIRRORS, use them to figure out the sum.
  297. sums=`find $stamps -maxdepth 3 -name "*.do_$i.*" \
  298. -o -name "*.do_${i}_setscene.*" | \
  299. sed -ne 's#.*_setscene\.##p' -e 's#.*\.sigdata\.##p' | \
  300. sed -e 's#\..*##' | sort -u`
  301. all_sums="$all_sums $sums"
  302. done
  303. echo "Done"
  304. echo "Figuring out the files which will be removed ... "
  305. for i in $all_sums; do
  306. grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:${i}_.*" $cache_list >>$keep_list
  307. done
  308. echo "Done"
  309. if [ -s $keep_list ]; then
  310. sort -u $keep_list -o $keep_list
  311. to_del=`comm -1 -3 $keep_list $cache_list`
  312. gen_rmlist $rm_list "$to_del"
  313. let total_deleted=`cat $rm_list | sort -u | wc -w`
  314. if [ $total_deleted -gt 0 ]; then
  315. [ $debug -gt 0 ] && cat $rm_list | sort -u
  316. read_confirm
  317. if [ "$confirm" = "y" -o "$confirm" = "Y" ]; then
  318. echo "Removing sstate cache files ... ($total_deleted files)"
  319. # Remove them one by one to avoid the argument list too long error
  320. for i in `cat $rm_list | sort -u`; do
  321. rm -f $verbose $i
  322. done
  323. echo "$total_deleted files have been removed"
  324. else
  325. do_nothing
  326. fi
  327. else
  328. no_files
  329. fi
  330. else
  331. echo_error "All files in cache dir will be removed! Abort!"
  332. fi
  333. rm -f $cache_list
  334. rm -f $keep_list
  335. rm -f $rm_list
  336. }
  337. # Parse arguments
  338. while [ -n "$1" ]; do
  339. case $1 in
  340. --cache-dir=*)
  341. cache_dir=`echo $1 | sed -e 's#^--cache-dir=##' | xargs readlink -e`
  342. [ -d "$cache_dir" ] || echo_error "Invalid argument to --cache-dir"
  343. shift
  344. ;;
  345. --remove-duplicated|-d)
  346. rm_duplicated="y"
  347. shift
  348. ;;
  349. --yes|-y)
  350. confirm="y"
  351. shift
  352. ;;
  353. --follow-symlink|-L)
  354. fsym="y"
  355. shift
  356. ;;
  357. --extra-archs=*)
  358. extra_archs=`echo $1 | sed -e 's#^--extra-archs=##' -e 's#,# #g'`
  359. [ -n "$extra_archs" ] || echo_error "Invalid extra arch parameter"
  360. shift
  361. ;;
  362. --extra-layer=*)
  363. extra_layers=`echo $1 | sed -e 's#^--extra-layer=##' -e 's#,# #g'`
  364. [ -n "$extra_layers" ] || echo_error "Invalid extra layer parameter"
  365. for i in $extra_layers; do
  366. l=`readlink -e $i`
  367. if [ -d "$l" ]; then
  368. layers="$layers $l"
  369. else
  370. echo_error "Can't find layer $i"
  371. fi
  372. done
  373. shift
  374. ;;
  375. --stamps-dir=*)
  376. stamps=`echo $1 | sed -e 's#^--stamps-dir=##' -e 's#,# #g'`
  377. [ -n "$stamps" ] || echo_error "Invalid stamps dir $i"
  378. for i in $stamps; do
  379. [ -d "$i" ] || echo_error "Invalid stamps dir $i"
  380. done
  381. shift
  382. ;;
  383. --verbose|-v)
  384. verbose="-v"
  385. shift
  386. ;;
  387. --debug|-D)
  388. debug=`expr $debug + 1`
  389. echo "Debug level $debug"
  390. shift
  391. ;;
  392. --help|-h)
  393. usage
  394. exit 0
  395. ;;
  396. *)
  397. echo "Invalid arguments $*"
  398. echo_error "Try 'sstate-cache-management.sh -h' for more information."
  399. ;;
  400. esac
  401. done
  402. # sstate cache directory, use environment variable SSTATE_CACHE_DIR
  403. # if it was not specified, otherwise, error.
  404. [ -n "$cache_dir" ] || cache_dir=$SSTATE_CACHE_DIR
  405. [ -n "$cache_dir" ] || echo_error "No cache dir found!"
  406. [ -d "$cache_dir" ] || echo_error "Invalid cache directory \"$cache_dir\""
  407. [ -n "$rm_duplicated" -a -n "$stamps" ] && \
  408. echo_error "Can not use both --remove-duplicated and --stamps-dir"
  409. [ "$rm_duplicated" = "y" ] && remove_duplicated
  410. [ -n "$stamps" ] && rm_by_stamps
  411. [ -z "$rm_duplicated" -a -z "$stamps" ] && \
  412. echo "What do you want to do?"
  413. exit 0