test-dependencies.sh 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286
  1. #!/bin/bash
  2. # Author: Martin Jansa <martin.jansa@gmail.com>
  3. #
  4. # Copyright (c) 2013 Martin Jansa <Martin.Jansa@gmail.com>
  5. # Used to detect missing dependencies or automagically
  6. # enabled dependencies which aren't explicitly enabled
  7. # or disabled. Using bash to have PIPESTATUS variable.
  8. # It does 3 builds of <target>
  9. # 1st to populate sstate-cache directory and sysroot
  10. # 2nd to rebuild each recipe with every possible
  11. # dependency found in sysroot (which stays populated
  12. # from 1st build
  13. # 3rd to rebuild each recipe only with dependencies defined
  14. # in DEPENDS
  15. # 4th (optional) repeat build like 3rd to make sure that
  16. # minimal versions of dependencies defined in DEPENDS
  17. # is also enough
  18. # Global vars
  19. tmpdir=
  20. targets=
  21. recipes=
  22. buildhistory=
  23. buildtype=
  24. default_targets="world"
  25. default_buildhistory="buildhistory"
  26. default_buildtype="1 2 3 c"
  27. usage () {
  28. cat << EOF
  29. Welcome to utility to detect missing or autoenabled dependencies.
  30. WARNING: this utility will completely remove your tmpdir (make sure
  31. you don't have important buildhistory or persistent dir there).
  32. $0 <OPTION>
  33. Options:
  34. -h, --help
  35. Display this help and exit.
  36. --tmpdir=<tmpdir>
  37. Specify tmpdir, will use the environment variable TMPDIR if it is not specified.
  38. Something like /OE/oe-core/tmp-eglibc (no / at the end).
  39. --targets=<targets>
  40. List of targets separated by space, will use the environment variable TARGETS if it is not specified.
  41. It will run "bitbake <targets>" to populate sysroots.
  42. Default value is "world".
  43. --recipes=<recipes>
  44. File with list of recipes we want to rebuild with minimal and maximal sysroot.
  45. Will use the environment variable RECIPES if it is not specified.
  46. Default value will use all packages ever recorded in buildhistory directory.
  47. --buildhistory=<buildhistory>
  48. Path to buildhistory directory, it needs to be enabled in your config,
  49. because it's used to detect different dependencies and to create list
  50. of recipes to rebuild when it's not specified.
  51. Will use the environment variable BUILDHISTORY if it is not specified.
  52. Default value is "buildhistory"
  53. --buildtype=<buildtype>
  54. There are 4 types of build:
  55. 1: build to populate sstate-cache directory and sysroot
  56. 2: build to rebuild each recipe with every possible dep
  57. 3: build to rebuild each recipe with minimal dependencies
  58. 4: build to rebuild each recipe again with minimal dependencies
  59. c: compare buildhistory directories from build 2 and 3
  60. Will use the environment variable BUILDTYPE if it is not specified.
  61. Default value is "1 2 3 c", order is important, type 4 is optional.
  62. EOF
  63. }
  64. # Print error information and exit.
  65. echo_error () {
  66. echo "ERROR: $1" >&2
  67. exit 1
  68. }
  69. while [ -n "$1" ]; do
  70. case $1 in
  71. --tmpdir=*)
  72. tmpdir=`echo $1 | sed -e 's#^--tmpdir=##' | xargs readlink -e`
  73. [ -d "$tmpdir" ] || echo_error "Invalid argument to --tmpdir"
  74. shift
  75. ;;
  76. --targets=*)
  77. targets=`echo $1 | sed -e 's#^--targets="*\([^"]*\)"*#\1#'`
  78. shift
  79. ;;
  80. --recipes=*)
  81. recipes=`echo $1 | sed -e 's#^--recipes="*\([^"]*\)"*#\1#'`
  82. shift
  83. ;;
  84. --buildhistory=*)
  85. buildhistory=`echo $1 | sed -e 's#^--buildhistory="*\([^"]*\)"*#\1#'`
  86. shift
  87. ;;
  88. --buildtype=*)
  89. buildtype=`echo $1 | sed -e 's#^--buildtype="*\([^"]*\)"*#\1#'`
  90. shift
  91. ;;
  92. --help|-h)
  93. usage
  94. exit 0
  95. ;;
  96. *)
  97. echo "Invalid arguments $*"
  98. echo_error "Try '$0 -h' for more information."
  99. ;;
  100. esac
  101. done
  102. # tmpdir directory, use environment variable TMPDIR
  103. # if it was not specified, otherwise, error.
  104. [ -n "$tmpdir" ] || tmpdir=$TMPDIR
  105. [ -n "$tmpdir" ] || echo_error "No tmpdir found!"
  106. [ -d "$tmpdir" ] || echo_error "Invalid tmpdir \"$tmpdir\""
  107. [ -n "$targets" ] || targets=$TARGETS
  108. [ -n "$targets" ] || targets=$default_targets
  109. [ -n "$recipes" ] || recipes=$RECIPES
  110. [ -n "$recipes" -a ! -f "$recipes" ] && echo_error "Invalid file with list of recipes to rebuild"
  111. [ -n "$recipes" ] || echo "All packages ever recorded in buildhistory directory will be rebuilt"
  112. [ -n "$buildhistory" ] || buildhistory=$BUILDHISTORY
  113. [ -n "$buildhistory" ] || buildhistory=$default_buildhistory
  114. [ -d "$buildhistory" ] || echo_error "Invalid buildhistory directory \"$buildhistory\""
  115. [ -n "$buildtype" ] || buildtype=$BUILDTYPE
  116. [ -n "$buildtype" ] || buildtype=$default_buildtype
  117. echo "$buildtype" | grep -v '^[1234c ]*$' && echo_error "Invalid buildtype \"$buildtype\", only some combination of 1, 2, 3, 4, c separated by space is allowed"
  118. OUTPUT_BASE=test-dependencies/`date "+%s"`
  119. declare -i RESULT=0
  120. build_all() {
  121. echo "===== 1st build to populate sstate-cache directory and sysroot ====="
  122. OUTPUT1=${OUTPUT_BASE}/${TYPE}_all
  123. mkdir -p ${OUTPUT1}
  124. echo "Logs will be stored in ${OUTPUT1} directory"
  125. bitbake -k $targets 2>&1 | tee -a ${OUTPUT1}/complete.log
  126. RESULT+=${PIPESTATUS[0]}
  127. grep "ERROR: Task.*failed" ${OUTPUT1}/complete.log > ${OUTPUT1}/failed-tasks.log
  128. cat ${OUTPUT1}/failed-tasks.log | sed 's@.*/@@g; s@_.*@@g; s@\.bb, .*@@g; s@\.bb:.*@@g' | sort -u > ${OUTPUT1}/failed-recipes.log
  129. }
  130. build_every_recipe() {
  131. if [ "${TYPE}" = "2" ] ; then
  132. echo "===== 2nd build to rebuild each recipe with every possible dep ====="
  133. OUTPUT_MAX=${OUTPUT_BASE}/${TYPE}_max
  134. OUTPUTB=${OUTPUT_MAX}
  135. else
  136. echo "===== 3rd or 4th build to rebuild each recipe with minimal dependencies ====="
  137. OUTPUT_MIN=${OUTPUT_BASE}/${TYPE}_min
  138. OUTPUTB=${OUTPUT_MIN}
  139. fi
  140. mkdir -p ${OUTPUTB} ${OUTPUTB}/failed ${OUTPUTB}/ok
  141. echo "Logs will be stored in ${OUTPUTB} directory"
  142. if [ -z "$recipes" ]; then
  143. ls -d $buildhistory/packages/*/* | xargs -n 1 basename | sort -u > ${OUTPUTB}/recipe.list
  144. recipes=${OUTPUTB}/recipe.list
  145. fi
  146. if [ "${TYPE}" != "2" ] ; then
  147. echo "!!!Removing tmpdir \"$tmpdir\"!!!"
  148. rm -rf $tmpdir/deploy $tmpdir/pkgdata $tmpdir/sstate-control $tmpdir/stamps $tmpdir/sysroots $tmpdir/work $tmpdir/work-shared 2>/dev/null
  149. fi
  150. i=1
  151. count=`cat $recipes ${OUTPUT1}/failed-recipes.log | sort -u | wc -l`
  152. for recipe in `cat $recipes ${OUTPUT1}/failed-recipes.log | sort -u`; do
  153. echo "Building recipe: ${recipe} ($i/$count)"
  154. declare -i RECIPE_RESULT=0
  155. bitbake -c cleansstate ${recipe} > ${OUTPUTB}/${recipe}.log 2>&1;
  156. RECIPE_RESULT+=$?
  157. bitbake ${recipe} >> ${OUTPUTB}/${recipe}.log 2>&1;
  158. RECIPE_RESULT+=$?
  159. if [ "${RECIPE_RESULT}" != "0" ] ; then
  160. RESULT+=${RECIPE_RESULT}
  161. mv ${OUTPUTB}/${recipe}.log ${OUTPUTB}/failed/
  162. grep "ERROR: Task.*failed" ${OUTPUTB}/failed/${recipe}.log | tee -a ${OUTPUTB}/failed-tasks.log
  163. grep "ERROR: Task.*failed" ${OUTPUTB}/failed/${recipe}.log | sed 's@.*/@@g; s@_.*@@g; s@\.bb, .*@@g; s@\.bb:.*@@g' >> ${OUTPUTB}/failed-recipes.log
  164. # and append also ${recipe} in case the failed task was from some dependency
  165. echo ${recipe} >> ${OUTPUTB}/failed-recipes.log
  166. else
  167. mv ${OUTPUTB}/${recipe}.log ${OUTPUTB}/ok/
  168. fi
  169. if [ "${TYPE}" != "2" ] ; then
  170. rm -rf $tmpdir/deploy $tmpdir/pkgdata $tmpdir/sstate-control $tmpdir/stamps $tmpdir/sysroots $tmpdir/work $tmpdir/work-shared 2>/dev/null
  171. fi
  172. i=`expr $i + 1`
  173. done
  174. echo "Copying buildhistory/packages to ${OUTPUTB}"
  175. cp -ra $buildhistory/packages ${OUTPUTB}
  176. # This will be usefull to see which library is pulling new dependency
  177. echo "Copying do_package logs to ${OUTPUTB}/do_package/"
  178. mkdir ${OUTPUTB}/do_package
  179. find $tmpdir/work/ -name log.do_package 2>/dev/null| while read f; do
  180. # pn is 3 levels back, but we don't know if there is just one log per pn (only one arch and version)
  181. # dest=`echo $f | sed 's#^.*/\([^/]*\)/\([^/]*\)/\([^/]*\)/log.do_package#\1#g'`
  182. dest=`echo $f | sed "s#$tmpdir/work/##g; s#/#_#g"`
  183. cp $f ${OUTPUTB}/do_package/$dest
  184. done
  185. }
  186. compare_deps() {
  187. # you can run just compare task with command like this
  188. # OUTPUT_BASE=test-dependencies/1373140172 \
  189. # OUTPUT_MAX=${OUTPUT_BASE}/2_max \
  190. # OUTPUT_MIN=${OUTPUT_BASE}/3_min \
  191. # openembedded-core/scripts/test-dependencies.sh --tmpdir=tmp-eglibc --targets=glib-2.0 --recipes=recipe_list --buildtype=c
  192. echo "===== Compare dependencies recorded in \"${OUTPUT_MAX}\" and \"${OUTPUT_MIN}\" ====="
  193. [ -n "${OUTPUTC}" ] || OUTPUTC=${OUTPUT_BASE}/comp
  194. mkdir -p ${OUTPUTC}
  195. OUTPUT_FILE=${OUTPUTC}/dependency-changes
  196. echo "Differences will be stored in ${OUTPUT_FILE}, dot is shown for every 100 of checked packages"
  197. echo > ${OUTPUT_FILE}
  198. [ -d ${OUTPUT_MAX} ] || echo_error "Directory with output from build 2 \"${OUTPUT_MAX}\" does not exist"
  199. [ -d ${OUTPUT_MIN} ] || echo_error "Directory with output from build 3 \"${OUTPUT_MIN}\" does not exist"
  200. [ -d ${OUTPUT_MAX}/packages/ ] || echo_error "Directory with packages from build 2 \"${OUTPUT_MAX}/packages/\" does not exist"
  201. [ -d ${OUTPUT_MIN}/packages/ ] || echo_error "Directory with packages from build 3 \"${OUTPUT_MIN}/packages/\" does not exist"
  202. i=0
  203. find ${OUTPUT_MAX}/packages/ -name latest | sed "s#${OUTPUT_MAX}/##g" | while read pkg; do
  204. max_pkg=${OUTPUT_MAX}/${pkg}
  205. min_pkg=${OUTPUT_MIN}/${pkg}
  206. # pkg=packages/armv5te-oe-linux-gnueabi/libungif/libungif/latest
  207. recipe=`echo "${pkg}" | sed 's#packages/[^/]*/\([^/]*\)/\([^/]*\)/latest#\1#g'`
  208. package=`echo "${pkg}" | sed 's#packages/[^/]*/\([^/]*\)/\([^/]*\)/latest#\2#g'`
  209. if [ ! -f "${min_pkg}" ] ; then
  210. echo "ERROR: ${recipe}: ${package} package isn't created when building with minimal dependencies?" | tee -a ${OUTPUT_FILE}
  211. echo ${recipe} >> ${OUTPUTC}/failed-recipes.log
  212. continue
  213. fi
  214. # strip version information in parenthesis
  215. max_deps=`grep "^RDEPENDS = " ${max_pkg} | sed 's/^RDEPENDS = / /g; s/$/ /g; s/([^(]*)//g'`
  216. min_deps=`grep "^RDEPENDS = " ${min_pkg} | sed 's/^RDEPENDS = / /g; s/$/ /g; s/([^(]*)//g'`
  217. if [ "$i" = 100 ] ; then
  218. echo -n "." # cheap progressbar
  219. i=0
  220. fi
  221. if [ "${max_deps}" = "${min_deps}" ] ; then
  222. # it's annoying long, but at least it's showing some progress, warnings are grepped at the end
  223. echo "NOTE: ${recipe}: ${package} rdepends weren't changed" >> ${OUTPUT_FILE}
  224. else
  225. missing_deps=
  226. for dep in ${max_deps}; do
  227. if ! echo "${min_deps}" | grep -q " ${dep} " ; then
  228. missing_deps="${missing_deps} ${dep}"
  229. echo # to get rid of dots on last line
  230. echo "WARN: ${recipe}: ${package} rdepends on ${dep}, but it isn't a build dependency?" | tee -a ${OUTPUT_FILE}
  231. fi
  232. done
  233. if [ -n "${missing_deps}" ] ; then
  234. echo ${recipe} >> ${OUTPUTC}/failed-recipes.log
  235. fi
  236. fi
  237. i=`expr $i + 1`
  238. done
  239. echo # to get rid of dots on last line
  240. echo "Found differences: "
  241. grep "^WARN: " ${OUTPUT_FILE} | tee ${OUTPUT_FILE}.warn.log
  242. echo "Found errors: "
  243. grep "^ERROR: " ${OUTPUT_FILE} | tee ${OUTPUT_FILE}.error.log
  244. RESULT+=`cat ${OUTPUT_FILE}.warn.log | wc -l`
  245. RESULT+=`cat ${OUTPUT_FILE}.error.log | wc -l`
  246. }
  247. for TYPE in $buildtype; do
  248. case ${TYPE} in
  249. 1) build_all;;
  250. 2) build_every_recipe;;
  251. 3) build_every_recipe;;
  252. 4) build_every_recipe;;
  253. c) compare_deps;;
  254. *) echo_error "Invalid buildtype \"$TYPE\""
  255. esac
  256. done
  257. cat ${OUTPUT_BASE}/*/failed-recipes.log | sort -u >> ${OUTPUT_BASE}/failed-recipes.log
  258. if [ "${RESULT}" != "0" ] ; then
  259. echo "ERROR: ${RESULT} issues were found in these recipes: `cat ${OUTPUT_BASE}/failed-recipes.log | xargs`"
  260. fi
  261. echo "INFO: Output written in: ${OUTPUT_BASE}"
  262. exit ${RESULT}