sstate-cache-management.sh 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469
  1. #!/bin/bash
  2. # Copyright (c) 2012 Wind River Systems, Inc.
  3. #
  4. # This program is free software; you can redistribute it and/or modify
  5. # it under the terms of the GNU General Public License version 2 as
  6. # published by the Free Software Foundation.
  7. #
  8. # This program is distributed in the hope that it will be useful,
  9. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  10. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
  11. # See the GNU General Public License for more details.
  12. #
  13. # You should have received a copy of the GNU General Public License
  14. # along with this program; if not, write to the Free Software
  15. # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  16. #
  17. # Global vars
  18. cache_dir=
  19. confirm=
  20. fsym=
  21. total_deleted=0
  22. verbose=
  23. debug=0
  24. usage () {
  25. cat << EOF
  26. Welcome to sstate cache management utilities.
  27. sstate-cache-management.sh <OPTION>
  28. Options:
  29. -h, --help
  30. Display this help and exit.
  31. --cache-dir=<sstate cache dir>
  32. Specify sstate cache directory, will use the environment
  33. variable SSTATE_CACHE_DIR if it is not specified.
  34. --extra-archs=<arch1>,<arch2>...<archn>
  35. Specify list of architectures which should be tested, this list
  36. will be extended with native arch, allarch and empty arch. The
  37. script won't be trying to generate list of available archs from
  38. AVAILTUNES in tune files.
  39. --extra-layer=<layer1>,<layer2>...<layern>
  40. Specify the layer which will be used for searching the archs,
  41. it will search the meta and meta-* layers in the top dir by
  42. default, and will search meta, meta-*, <layer1>, <layer2>,
  43. ...<layern> when specified. Use "," as the separator.
  44. This is useless for --stamps-dir or when --extra-archs is used.
  45. -d, --remove-duplicated
  46. Remove the duplicated sstate cache files of one package, only
  47. the newest one will be kept. The duplicated sstate cache files
  48. of one package must have the same arch, which means sstate cache
  49. files with multiple archs are not considered duplicate.
  50. Conflicts with --stamps-dir.
  51. --stamps-dir=<dir1>,<dir2>...<dirn>
  52. Specify the build directory's stamps directories, the sstate
  53. cache file which IS USED by these build diretories will be KEPT,
  54. other sstate cache files in cache-dir will be removed. Use ","
  55. as the separator. For example:
  56. --stamps-dir=build1/tmp/stamps,build2/tmp/stamps
  57. Conflicts with --remove-duplicated.
  58. -L, --follow-symlink
  59. Remove both the symbol link and the destination file, default: no.
  60. -y, --yes
  61. Automatic yes to prompts; assume "yes" as answer to all prompts
  62. and run non-interactively.
  63. -v, --verbose
  64. Explain what is being done.
  65. -D, --debug
  66. Show debug info, repeat for more debug info.
  67. EOF
  68. }
  69. if [ $# -lt 1 ]; then
  70. usage
  71. exit 0
  72. fi
  73. # Echo no files to remove
  74. no_files () {
  75. echo No files to remove
  76. }
  77. # Echo nothing to do
  78. do_nothing () {
  79. echo Nothing to do
  80. }
  81. # Read the input "y"
  82. read_confirm () {
  83. echo "$total_deleted out of $total_files files will be removed! "
  84. if [ "$confirm" != "y" ]; then
  85. echo "Do you want to continue (y/n)? "
  86. while read confirm; do
  87. [ "$confirm" = "Y" -o "$confirm" = "y" -o "$confirm" = "n" \
  88. -o "$confirm" = "N" ] && break
  89. echo "Invalid input \"$confirm\", please input 'y' or 'n': "
  90. done
  91. else
  92. echo
  93. fi
  94. }
  95. # Print error information and exit.
  96. echo_error () {
  97. echo "ERROR: $1" >&2
  98. exit 1
  99. }
  100. # Generate the remove list:
  101. #
  102. # * Add .done/.siginfo to the remove list
  103. # * Add destination of symlink to the remove list
  104. #
  105. # $1: output file, others: sstate cache file (.tgz)
  106. gen_rmlist (){
  107. local rmlist_file="$1"
  108. shift
  109. local files="$@"
  110. for i in $files; do
  111. echo $i >> $rmlist_file
  112. # Add the ".siginfo"
  113. if [ -e $i.siginfo ]; then
  114. echo $i.siginfo >> $rmlist_file
  115. fi
  116. # Add the destination of symlink
  117. if [ -L "$i" ]; then
  118. if [ "$fsym" = "y" ]; then
  119. dest="`readlink -e $i`"
  120. if [ -n "$dest" ]; then
  121. echo $dest >> $rmlist_file
  122. # Remove the .siginfo when .tgz is removed
  123. if [ -f "$dest.siginfo" ]; then
  124. echo $dest.siginfo >> $rmlist_file
  125. fi
  126. fi
  127. fi
  128. # Add the ".tgz.done" and ".siginfo.done" (may exist in the future)
  129. base_fn="${i##/*/}"
  130. t_fn="$base_fn.done"
  131. s_fn="$base_fn.siginfo.done"
  132. for d in $t_fn $s_fn; do
  133. if [ -f $cache_dir/$d ]; then
  134. echo $cache_dir/$d >> $rmlist_file
  135. fi
  136. done
  137. fi
  138. done
  139. }
  140. # Remove the duplicated cache files for the pkg, keep the newest one
  141. remove_duplicated () {
  142. local topdir
  143. local oe_core_dir
  144. local tunedirs
  145. local all_archs
  146. local all_machines
  147. local ava_archs
  148. local arch
  149. local file_names
  150. local sstate_files_list
  151. local fn_tmp
  152. local list_suffix=`mktemp` || exit 1
  153. if [ -z "$extra_archs" ] ; then
  154. # Find out the archs in all the layers
  155. echo "Figuring out the archs in the layers ... "
  156. oe_core_dir=$(dirname $(dirname $(readlink -e $0)))
  157. topdir=$(dirname $oe_core_dir)
  158. tunedirs="`find $topdir/meta* ${oe_core_dir}/meta* $layers -path '*/meta*/conf/machine/include' 2>/dev/null`"
  159. [ -n "$tunedirs" ] || echo_error "Can't find the tune directory"
  160. all_machines="`find $topdir/meta* ${oe_core_dir}/meta* $layers -path '*/meta*/conf/machine/*' -name '*.conf' 2>/dev/null | sed -e 's/.*\///' -e 's/.conf$//'`"
  161. all_archs=`grep -r -h "^AVAILTUNES .*=" $tunedirs | sed -e 's/.*=//' -e 's/\"//g'`
  162. fi
  163. # Use the "_" to substitute "-", e.g., x86-64 to x86_64, but not for extra_archs which can be something like cortexa9t2-vfp-neon
  164. # Sort to remove the duplicated ones
  165. # Add allarch and builder arch (native)
  166. builder_arch=$(uname -m)
  167. all_archs="$(echo allarch $all_archs $all_machines $builder_arch \
  168. | sed -e 's/-/_/g' -e 's/ /\n/g' | sort -u) $extra_archs"
  169. echo "Done"
  170. # Total number of files including sstate-, .siginfo and .done files
  171. total_files=`find $cache_dir -name 'sstate*' | wc -l`
  172. # Save all the sstate files in a file
  173. sstate_files_list=`mktemp` || exit 1
  174. find $cache_dir -name 'sstate:*:*:*:*:*:*:*.tgz*' >$sstate_files_list
  175. echo "Figuring out the suffixes in the sstate cache dir ... "
  176. sstate_suffixes="`sed 's%.*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^_]*_\([^:]*\)\.tgz.*%\1%g' $sstate_files_list | sort -u`"
  177. echo "Done"
  178. echo "The following suffixes have been found in the cache dir:"
  179. echo $sstate_suffixes
  180. echo "Figuring out the archs in the sstate cache dir ... "
  181. # Using this SSTATE_PKGSPEC definition it's 6th colon separated field
  182. # SSTATE_PKGSPEC = "sstate:${PN}:${PACKAGE_ARCH}${TARGET_VENDOR}-${TARGET_OS}:${PV}:${PR}:${SSTATE_PKGARCH}:${SSTATE_VERSION}:"
  183. for arch in $all_archs; do
  184. grep -q ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:$arch:[^:]*:[^:]*\.tgz$" $sstate_files_list
  185. [ $? -eq 0 ] && ava_archs="$ava_archs $arch"
  186. # ${builder_arch}_$arch used by toolchain sstate
  187. grep -q ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:${builder_arch}_$arch:[^:]*:[^:]*\.tgz$" $sstate_files_list
  188. [ $? -eq 0 ] && ava_archs="$ava_archs ${builder_arch}_$arch"
  189. done
  190. echo "Done"
  191. echo "The following archs have been found in the cache dir:"
  192. echo $ava_archs
  193. echo ""
  194. # Save the file list which needs to be removed
  195. local remove_listdir=`mktemp -d` || exit 1
  196. for suffix in $sstate_suffixes; do
  197. if [ "$suffix" = "populate_lic" ] ; then
  198. echo "Skipping populate_lic, because removing duplicates doesn't work correctly for them (use --stamps-dir instead)"
  199. continue
  200. fi
  201. # Total number of files including .siginfo and .done files
  202. total_files_suffix=`grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:_]*_$suffix\.tgz.*" $sstate_files_list | wc -l 2>/dev/null`
  203. total_tgz_suffix=`grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:_]*_$suffix\.tgz$" $sstate_files_list | wc -l 2>/dev/null`
  204. # Save the file list to a file, some suffix's file may not exist
  205. grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:_]*_$suffix\.tgz.*" $sstate_files_list >$list_suffix 2>/dev/null
  206. local deleted_tgz=0
  207. local deleted_files=0
  208. for ext in tgz tgz.siginfo tgz.done; do
  209. echo "Figuring out the sstate:xxx_$suffix.$ext ... "
  210. # Uniq BPNs
  211. file_names=`for arch in $ava_archs ""; do
  212. sed -ne "s%.*/sstate:\([^:]*\):[^:]*:[^:]*:[^:]*:$arch:[^:]*:[^:]*\.${ext}$%\1%p" $list_suffix
  213. done | sort -u`
  214. fn_tmp=`mktemp` || exit 1
  215. rm_list="$remove_listdir/sstate:xxx_$suffix"
  216. for fn in $file_names; do
  217. [ -z "$verbose" ] || echo "Analyzing sstate:$fn-xxx_$suffix.${ext}"
  218. for arch in $ava_archs ""; do
  219. grep -h ".*/sstate:$fn:[^:]*:[^:]*:[^:]*:$arch:[^:]*:[^:]*\.${ext}$" $list_suffix >$fn_tmp
  220. if [ -s $fn_tmp ] ; then
  221. [ $debug -gt 1 ] && echo "Available files for $fn-$arch- with suffix $suffix.${ext}:" && cat $fn_tmp
  222. # Use the modification time
  223. to_del=$(ls -t $(cat $fn_tmp) | sed -n '1!p')
  224. [ $debug -gt 2 ] && echo "Considering to delete: $to_del"
  225. # The sstate file which is downloaded from the SSTATE_MIRROR is
  226. # put in SSTATE_DIR, and there is a symlink in SSTATE_DIR/??/ to
  227. # it, so filter it out from the remove list if it should not be
  228. # removed.
  229. to_keep=$(ls -t $(cat $fn_tmp) | sed -n '1p')
  230. [ $debug -gt 2 ] && echo "Considering to keep: $to_keep"
  231. for k in $to_keep; do
  232. if [ -L "$k" ]; then
  233. # The symlink's destination
  234. k_dest="`readlink -e $k`"
  235. # Maybe it is the one in cache_dir
  236. k_maybe="$cache_dir/${k##/*/}"
  237. # Remove it from the remove list if they are the same.
  238. if [ "$k_dest" = "$k_maybe" ]; then
  239. to_del="`echo $to_del | sed 's#'\"$k_maybe\"'##g'`"
  240. fi
  241. fi
  242. done
  243. rm -f $fn_tmp
  244. [ $debug -gt 2 ] && echo "Decided to delete: $to_del"
  245. gen_rmlist $rm_list.$ext "$to_del"
  246. fi
  247. done
  248. done
  249. done
  250. deleted_tgz=`cat $rm_list.* 2>/dev/null | grep ".tgz$" | wc -l`
  251. deleted_files=`cat $rm_list.* 2>/dev/null | wc -l`
  252. [ "$deleted_files" -gt 0 -a $debug -gt 0 ] && cat $rm_list.*
  253. echo "($deleted_tgz out of $total_tgz_suffix .tgz files for $suffix suffix will be removed or $deleted_files out of $total_files_suffix when counting also .siginfo and .done files)"
  254. let total_deleted=$total_deleted+$deleted_files
  255. done
  256. deleted_tgz=0
  257. rm_old_list=$remove_listdir/sstate-old-filenames
  258. find $cache_dir -name 'sstate-*.tgz' >$rm_old_list
  259. [ -s "$rm_old_list" ] && deleted_tgz=`cat $rm_old_list | grep ".tgz$" | wc -l`
  260. [ -s "$rm_old_list" ] && deleted_files=`cat $rm_old_list | wc -l`
  261. [ -s "$rm_old_list" -a $debug -gt 0 ] && cat $rm_old_list
  262. echo "($deleted_tgz .tgz files with old sstate-* filenames will be removed or $deleted_files when counting also .siginfo and .done files)"
  263. let total_deleted=$total_deleted+$deleted_files
  264. rm -f $list_suffix
  265. rm -f $sstate_files_list
  266. if [ $total_deleted -gt 0 ]; then
  267. read_confirm
  268. if [ "$confirm" = "y" -o "$confirm" = "Y" ]; then
  269. for list in `ls $remove_listdir/`; do
  270. echo "Removing $list.tgz (`cat $remove_listdir/$list | wc -w` files) ... "
  271. # Remove them one by one to avoid the argument list too long error
  272. for i in `cat $remove_listdir/$list`; do
  273. rm -f $verbose $i
  274. done
  275. echo "Done"
  276. done
  277. echo "$total_deleted files have been removed!"
  278. else
  279. do_nothing
  280. fi
  281. else
  282. no_files
  283. fi
  284. [ -d $remove_listdir ] && rm -fr $remove_listdir
  285. }
  286. # Remove the sstate file by stamps dir, the file not used by the stamps dir
  287. # will be removed.
  288. rm_by_stamps (){
  289. local cache_list=`mktemp` || exit 1
  290. local keep_list=`mktemp` || exit 1
  291. local rm_list=`mktemp` || exit 1
  292. local sums
  293. local all_sums
  294. # Total number of files including sstate-, .siginfo and .done files
  295. total_files=`find $cache_dir -type f -name 'sstate*' | wc -l`
  296. # Save all the state file list to a file
  297. find $cache_dir -type f -name 'sstate*' | sort -u -o $cache_list
  298. echo "Figuring out the suffixes in the sstate cache dir ... "
  299. local sstate_suffixes="`sed 's%.*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^_]*_\([^:]*\)\.tgz.*%\1%g' $cache_list | sort -u`"
  300. echo "Done"
  301. echo "The following suffixes have been found in the cache dir:"
  302. echo $sstate_suffixes
  303. # Figure out all the md5sums in the stamps dir.
  304. echo "Figuring out all the md5sums in stamps dir ... "
  305. for i in $sstate_suffixes; do
  306. # There is no "\.sigdata" but "_setcene" when it is mirrored
  307. # from the SSTATE_MIRRORS, use them to figure out the sum.
  308. sums=`find $stamps -maxdepth 3 -name "*.do_$i.*" \
  309. -o -name "*.do_${i}_setscene.*" | \
  310. sed -ne 's#.*_setscene\.##p' -e 's#.*\.sigdata\.##p' | \
  311. sed -e 's#\..*##' | sort -u`
  312. all_sums="$all_sums $sums"
  313. done
  314. echo "Done"
  315. echo "Figuring out the files which will be removed ... "
  316. for i in $all_sums; do
  317. grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:${i}_.*" $cache_list >>$keep_list
  318. done
  319. echo "Done"
  320. if [ -s $keep_list ]; then
  321. sort -u $keep_list -o $keep_list
  322. to_del=`comm -1 -3 $keep_list $cache_list`
  323. gen_rmlist $rm_list "$to_del"
  324. let total_deleted=`cat $rm_list | sort -u | wc -w`
  325. if [ $total_deleted -gt 0 ]; then
  326. [ $debug -gt 0 ] && cat $rm_list | sort -u
  327. read_confirm
  328. if [ "$confirm" = "y" -o "$confirm" = "Y" ]; then
  329. echo "Removing sstate cache files ... ($total_deleted files)"
  330. # Remove them one by one to avoid the argument list too long error
  331. for i in `cat $rm_list | sort -u`; do
  332. rm -f $verbose $i
  333. done
  334. echo "$total_deleted files have been removed"
  335. else
  336. do_nothing
  337. fi
  338. else
  339. no_files
  340. fi
  341. else
  342. echo_error "All files in cache dir will be removed! Abort!"
  343. fi
  344. rm -f $cache_list
  345. rm -f $keep_list
  346. rm -f $rm_list
  347. }
  348. # Parse arguments
  349. while [ -n "$1" ]; do
  350. case $1 in
  351. --cache-dir=*)
  352. cache_dir=`echo $1 | sed -e 's#^--cache-dir=##' | xargs readlink -e`
  353. [ -d "$cache_dir" ] || echo_error "Invalid argument to --cache-dir"
  354. shift
  355. ;;
  356. --remove-duplicated|-d)
  357. rm_duplicated="y"
  358. shift
  359. ;;
  360. --yes|-y)
  361. confirm="y"
  362. shift
  363. ;;
  364. --follow-symlink|-L)
  365. fsym="y"
  366. shift
  367. ;;
  368. --extra-archs=*)
  369. extra_archs=`echo $1 | sed -e 's#^--extra-archs=##' -e 's#,# #g'`
  370. [ -n "$extra_archs" ] || echo_error "Invalid extra arch parameter"
  371. shift
  372. ;;
  373. --extra-layer=*)
  374. extra_layers=`echo $1 | sed -e 's#^--extra-layer=##' -e 's#,# #g'`
  375. [ -n "$extra_layers" ] || echo_error "Invalid extra layer parameter"
  376. for i in $extra_layers; do
  377. l=`readlink -e $i`
  378. if [ -d "$l" ]; then
  379. layers="$layers $l"
  380. else
  381. echo_error "Can't find layer $i"
  382. fi
  383. done
  384. shift
  385. ;;
  386. --stamps-dir=*)
  387. stamps=`echo $1 | sed -e 's#^--stamps-dir=##' -e 's#,# #g'`
  388. [ -n "$stamps" ] || echo_error "Invalid stamps dir $i"
  389. for i in $stamps; do
  390. [ -d "$i" ] || echo_error "Invalid stamps dir $i"
  391. done
  392. shift
  393. ;;
  394. --verbose|-v)
  395. verbose="-v"
  396. shift
  397. ;;
  398. --debug|-D)
  399. debug=`expr $debug + 1`
  400. echo "Debug level $debug"
  401. shift
  402. ;;
  403. --help|-h)
  404. usage
  405. exit 0
  406. ;;
  407. *)
  408. echo "Invalid arguments $*"
  409. echo_error "Try 'sstate-cache-management.sh -h' for more information."
  410. ;;
  411. esac
  412. done
  413. # sstate cache directory, use environment variable SSTATE_CACHE_DIR
  414. # if it was not specified, otherwise, error.
  415. [ -n "$cache_dir" ] || cache_dir=$SSTATE_CACHE_DIR
  416. [ -n "$cache_dir" ] || echo_error "No cache dir found!"
  417. [ -d "$cache_dir" ] || echo_error "Invalid cache directory \"$cache_dir\""
  418. [ -n "$rm_duplicated" -a -n "$stamps" ] && \
  419. echo_error "Can not use both --remove-duplicated and --stamps-dir"
  420. [ "$rm_duplicated" = "y" ] && remove_duplicated
  421. [ -n "$stamps" ] && rm_by_stamps
  422. [ -z "$rm_duplicated" -a -z "$stamps" ] && \
  423. echo "What do you want to do?"
  424. exit 0