sstate-sysroot-cruft.sh 5.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201
  1. #!/bin/sh
  2. #
  3. # SPDX-License-Identifier: GPL-2.0-only
  4. #
  5. # Used to find files installed in sysroot which are not tracked by sstate manifest
  6. # Global vars
  7. tmpdir=
  8. usage () {
  9. cat << EOF
  10. Welcome to sysroot cruft finding utility.
  11. $0 <OPTION>
  12. Options:
  13. -h, --help
  14. Display this help and exit.
  15. --tmpdir=<tmpdir>
  16. Specify tmpdir, will use the environment variable TMPDIR if it is not specified.
  17. Something like /OE/oe-core/tmp-eglibc (no / at the end).
  18. --whitelist=<whitelist-file>
  19. Text file, each line is regular expression for paths we want to ignore in resulting diff.
  20. You can use diff file from the script output, if it contains only expected exceptions.
  21. '#' is used as regexp delimiter, so you don't need to prefix forward slashes in paths.
  22. ^ and $ is automatically added, so provide only the middle part.
  23. Lines starting with '#' are ignored as comments.
  24. All paths are relative to "sysroots" directory.
  25. Directories don't end with forward slash.
  26. EOF
  27. }
  28. # Print error information and exit.
  29. echo_error () {
  30. echo "ERROR: $1" >&2
  31. exit 1
  32. }
  33. while [ -n "$1" ]; do
  34. case $1 in
  35. --tmpdir=*)
  36. tmpdir=`echo $1 | sed -e 's#^--tmpdir=##' | xargs readlink -e`
  37. [ -d "$tmpdir" ] || echo_error "Invalid argument to --tmpdir"
  38. shift
  39. ;;
  40. --whitelist=*)
  41. fwhitelist=`echo $1 | sed -e 's#^--whitelist=##' | xargs readlink -e`
  42. [ -f "$fwhitelist" ] || echo_error "Invalid argument to --whitelist"
  43. shift
  44. ;;
  45. --help|-h)
  46. usage
  47. exit 0
  48. ;;
  49. *)
  50. echo "Invalid arguments $*"
  51. echo_error "Try '$0 -h' for more information."
  52. ;;
  53. esac
  54. done
  55. # sstate cache directory, use environment variable TMPDIR
  56. # if it was not specified, otherwise, error.
  57. [ -n "$tmpdir" ] || tmpdir=$TMPDIR
  58. [ -n "$tmpdir" ] || echo_error "No tmpdir found!"
  59. [ -d "$tmpdir" ] || echo_error "Invalid tmpdir \"$tmpdir\""
  60. OUTPUT=${tmpdir}/sysroot.cruft.`date "+%s"`
  61. # top level directories
  62. WHITELIST="[^/]*"
  63. # generated by base-passwd recipe
  64. WHITELIST="${WHITELIST} \
  65. .*/etc/group-\? \
  66. .*/etc/passwd-\? \
  67. "
  68. # generated by pseudo-native
  69. WHITELIST="${WHITELIST} \
  70. .*/var/pseudo \
  71. .*/var/pseudo/[^/]* \
  72. "
  73. # generated by package.bbclass:SHLIBSDIRS = "${PKGDATA_DIR}/${MLPREFIX}shlibs"
  74. WHITELIST="${WHITELIST} \
  75. .*/shlibs \
  76. .*/pkgdata \
  77. "
  78. # generated by python
  79. WHITELIST="${WHITELIST} \
  80. .*\.pyc \
  81. .*\.pyo \
  82. .*/__pycache__ \
  83. "
  84. # generated by lua
  85. WHITELIST="${WHITELIST} \
  86. .*\.luac \
  87. "
  88. # generated by sgml-common-native
  89. WHITELIST="${WHITELIST} \
  90. .*/etc/sgml/sgml-docbook.bak \
  91. "
  92. # generated by php
  93. WHITELIST="${WHITELIST} \
  94. .*/usr/lib/php5/php/.channels \
  95. .*/usr/lib/php5/php/.channels/.* \
  96. .*/usr/lib/php5/php/.registry \
  97. .*/usr/lib/php5/php/.registry/.* \
  98. .*/usr/lib/php5/php/.depdb \
  99. .*/usr/lib/php5/php/.depdblock \
  100. .*/usr/lib/php5/php/.filemap \
  101. .*/usr/lib/php5/php/.lock \
  102. "
  103. # generated by toolchain
  104. WHITELIST="${WHITELIST} \
  105. [^/]*-tcbootstrap/lib \
  106. "
  107. # generated by useradd.bbclass
  108. WHITELIST="${WHITELIST} \
  109. [^/]*/home \
  110. [^/]*/home/xuser \
  111. [^/]*/home/xuser/.bashrc \
  112. [^/]*/home/xuser/.profile \
  113. [^/]*/home/builder \
  114. [^/]*/home/builder/.bashrc \
  115. [^/]*/home/builder/.profile \
  116. "
  117. # generated by image.py for WIC
  118. # introduced in oe-core commit 861ce6c5d4836df1a783be3b01d2de56117c9863
  119. WHITELIST="${WHITELIST} \
  120. [^/]*/imgdata \
  121. [^/]*/imgdata/[^/]*\.env \
  122. "
  123. # generated by fontcache.bbclass
  124. WHITELIST="${WHITELIST} \
  125. .*/var/cache/fontconfig/ \
  126. "
  127. # created by oe.utils.write_ld_so_conf which is used from few bbclasses and recipes:
  128. # meta/classes/image-prelink.bbclass: oe.utils.write_ld_so_conf(d)
  129. # meta/classes/insane.bbclass: oe.utils.write_ld_so_conf(d)
  130. # meta/classes/insane.bbclass: oe.utils.write_ld_so_conf(d)
  131. # meta/recipes-gnome/gobject-introspection/gobject-introspection_1.48.0.bb: oe.utils.write_ld_so_conf(d)
  132. # meta/recipes-gnome/gobject-introspection/gobject-introspection_1.48.0.bb: oe.utils.write_ld_so_conf(d)
  133. # introduced in oe-core commit 7fd1d7e639c2ed7e0699937a5cb245c187b7c811
  134. # and more visible since added to gobject-introspection in 10e0c1a3a452baa05d160a92a54b2e33cf0fd061
  135. WHITELIST="${WHITELIST} \
  136. [^/]*/etc/ld.so.conf \
  137. "
  138. SYSROOTS="`readlink -f ${tmpdir}`/sysroots/"
  139. mkdir ${OUTPUT}
  140. find ${tmpdir}/sstate-control -name \*.populate-sysroot\* -o -name \*.populate_sysroot\* -o -name \*.package\* | xargs cat | grep sysroots | \
  141. sed 's#/$##g; s#///*#/#g' | \
  142. # work around for paths ending with / for directories and multiplied // (e.g. paths to native sysroot)
  143. sort | sed "s#^${SYSROOTS}##g" > ${OUTPUT}/master.list.all.txt
  144. sort -u ${OUTPUT}/master.list.all.txt > ${OUTPUT}/master.list.txt # -u because some directories are listed for more recipes
  145. find ${tmpdir}/sysroots/ | \
  146. sort | sed "s#^${SYSROOTS}##g" > ${OUTPUT}/sysroot.list.txt
  147. diff ${OUTPUT}/master.list.all.txt ${OUTPUT}/master.list.txt > ${OUTPUT}/duplicates.txt
  148. diff ${OUTPUT}/master.list.txt ${OUTPUT}/sysroot.list.txt > ${OUTPUT}/diff.all.txt
  149. grep "^> ." ${OUTPUT}/diff.all.txt | sed 's/^> //g' > ${OUTPUT}/diff.txt
  150. for item in ${WHITELIST}; do
  151. sed -i "\\#^${item}\$#d" ${OUTPUT}/diff.txt;
  152. echo "${item}" >> ${OUTPUT}/used.whitelist.txt
  153. done
  154. if [ -s "$fwhitelist" ] ; then
  155. cat $fwhitelist >> ${OUTPUT}/used.whitelist.txt
  156. cat $fwhitelist | grep -v '^#' | while read item; do
  157. sed -i "\\#^${item}\$#d" ${OUTPUT}/diff.txt;
  158. done
  159. fi
  160. # too many false positives for directories
  161. # echo "Following files are installed in sysroot at least twice"
  162. # cat ${OUTPUT}/duplicates
  163. RESULT=`cat ${OUTPUT}/diff.txt | wc -l`
  164. if [ "${RESULT}" != "0" ] ; then
  165. echo "ERROR: ${RESULT} issues were found."
  166. echo "ERROR: Following files are installed in sysroot, but not tracked by sstate:"
  167. cat ${OUTPUT}/diff.txt
  168. else
  169. echo "INFO: All files are tracked by sstate or were explicitly ignored by this script"
  170. fi
  171. echo "INFO: Output written in: ${OUTPUT}"
  172. exit ${RESULT}