sstate-diff-machines.sh 6.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178
  1. #!/bin/bash
  2. #
  3. # Copyright OpenEmbedded Contributors
  4. #
  5. # SPDX-License-Identifier: GPL-2.0-only
  6. #
  7. # Used to compare sstate checksums between MACHINES.
  8. # Execute script and compare generated list.M files.
  9. # Using bash to have PIPESTATUS variable.
  10. # It's also usefull to keep older sstate checksums
  11. # to be able to find out why something is rebuilding
  12. # after updating metadata
  13. # $ diff \
  14. # sstate-diff/1349348392/fake-cortexa8/list.M \
  15. # sstate-diff/1349348392/fake-cortexa9/list.M \
  16. # | wc -l
  17. # 538
  18. # Then to compare sigdata use something like:
  19. # $ ls sstate-diff/1349348392/*/armv7a-vfp-neon*/linux-libc-headers/*do_configure*sigdata*
  20. # sstate-diff/1349348392/fake-cortexa8/armv7a-vfp-neon-oe-linux-gnueabi/linux-libc-headers/3.4.3-r0.do_configure.sigdata.cb73b3630a7b8191e72fc469c5137025
  21. # sstate-diff/1349348392/fake-cortexa9/armv7a-vfp-neon-oe-linux-gnueabi/linux-libc-headers/3.4.3-r0.do_configure.sigdata.f37ada177bf99ce8af85914df22b5a0b
  22. # $ bitbake-diffsigs stamps.1349348392/*/armv7a-vfp-neon*/linux-libc-headers/*do_configure*sigdata*
  23. # basehash changed from 8d0bd67bb1da6f68717760fc3ef43171 to e869fa61426e88e9c30726ba88a1216a
  24. # Variable TUNE_CCARGS value changed from -march=armv7-a -mthumb-interwork -mfloat-abi=softfp -mfpu=neon -mtune=cortex-a8 to -march=armv7-a -mthumb-interwork -mfloat-abi=softfp -mfpu=neon -mtune=cortex-a9
  25. # Global vars
  26. tmpdir=
  27. machines=
  28. targets=
  29. default_machines="qemuarm qemux86 qemux86-64"
  30. default_targets="core-image-base"
  31. analyze="N"
  32. usage () {
  33. cat << EOF
  34. Welcome to utility to compare sstate checksums between different MACHINEs.
  35. $0 <OPTION>
  36. Options:
  37. -h, --help
  38. Display this help and exit.
  39. --tmpdir=<tmpdir>
  40. Specify tmpdir, will use the environment variable TMPDIR if it is not specified.
  41. Something like /OE/oe-core/tmp-eglibc (no / at the end).
  42. --machines=<machines>
  43. List of MACHINEs separated by space, will use the environment variable MACHINES if it is not specified.
  44. Default value is "qemuarm qemux86 qemux86-64".
  45. --targets=<targets>
  46. List of targets separated by space, will use the environment variable TARGETS if it is not specified.
  47. Default value is "core-image-base".
  48. --analyze
  49. Show the differences between MACHINEs. It assumes:
  50. * First 2 MACHINEs in --machines parameter have the same TUNE_PKGARCH
  51. * Third optional MACHINE has different TUNE_PKGARCH - only native and allarch recipes are compared).
  52. * Next MACHINEs are ignored
  53. EOF
  54. }
  55. # Print error information and exit.
  56. echo_error () {
  57. echo "ERROR: $1" >&2
  58. exit 1
  59. }
  60. while [ -n "$1" ]; do
  61. case $1 in
  62. --tmpdir=*)
  63. tmpdir=`echo $1 | sed -e 's#^--tmpdir=##' | xargs readlink -e`
  64. [ -d "$tmpdir" ] || echo_error "Invalid argument to --tmpdir"
  65. shift
  66. ;;
  67. --machines=*)
  68. machines=`echo $1 | sed -e 's#^--machines="*\([^"]*\)"*#\1#'`
  69. shift
  70. ;;
  71. --targets=*)
  72. targets=`echo $1 | sed -e 's#^--targets="*\([^"]*\)"*#\1#'`
  73. shift
  74. ;;
  75. --analyze)
  76. analyze="Y"
  77. shift
  78. ;;
  79. --help|-h)
  80. usage
  81. exit 0
  82. ;;
  83. *)
  84. echo "Invalid arguments $*"
  85. echo_error "Try '$0 -h' for more information."
  86. ;;
  87. esac
  88. done
  89. # tmpdir directory, use environment variable TMPDIR
  90. # if it was not specified, otherwise, error.
  91. [ -n "$tmpdir" ] || tmpdir=$TMPDIR
  92. [ -n "$tmpdir" ] || echo_error "No tmpdir found!"
  93. [ -d "$tmpdir" ] || echo_error "Invalid tmpdir \"$tmpdir\""
  94. [ -n "$machines" ] || machines=$MACHINES
  95. [ -n "$machines" ] || machines=$default_machines
  96. [ -n "$targets" ] || targets=$TARGETS
  97. [ -n "$targets" ] || targets=$default_targets
  98. OUTPUT=${tmpdir}/sstate-diff/`date "+%s"`
  99. declare -i RESULT=0
  100. for M in ${machines}; do
  101. [ -d ${tmpdir}/stamps/ ] && find ${tmpdir}/stamps/ -name \*sigdata\* | xargs rm -f
  102. mkdir -p ${OUTPUT}/${M}
  103. export MACHINE=${M}
  104. bitbake -S none ${targets} 2>&1 | tee -a ${OUTPUT}/${M}/log;
  105. RESULT+=${PIPESTATUS[0]}
  106. if ls ${tmpdir}/stamps/* >/dev/null 2>/dev/null ; then
  107. cp -ra ${tmpdir}/stamps/* ${OUTPUT}/${M}
  108. find ${OUTPUT}/${M} -name \*sigdata\* | sed "s#${OUTPUT}/${M}/##g" | sort > ${OUTPUT}/${M}/list
  109. M_UNDERSCORE=`echo ${M} | sed 's/-/_/g'`
  110. sed "s/^${M_UNDERSCORE}-/MACHINE/g" ${OUTPUT}/${M}/list | sort > ${OUTPUT}/${M}/list.M
  111. find ${tmpdir}/stamps/ -name \*sigdata\* | xargs rm -f
  112. else
  113. printf "ERROR: no sigdata files were generated for MACHINE $M in ${tmpdir}/stamps\n";
  114. fi
  115. done
  116. COMPARE_TASKS="do_configure.sigdata do_populate_sysroot.sigdata do_package_write_ipk.sigdata do_package_write_rpm.sigdata do_package_write_deb.sigdata do_package_write_tar.sigdata"
  117. function compareSignatures() {
  118. MACHINE1=$1
  119. MACHINE2=$2
  120. PATTERN="$3"
  121. PRE_PATTERN=""
  122. [ -n "${PATTERN}" ] || PRE_PATTERN="-v"
  123. [ -n "${PATTERN}" ] || PATTERN="MACHINE"
  124. for TASK in $COMPARE_TASKS; do
  125. printf "\n\n === Comparing signatures for task ${TASK} between ${MACHINE1} and ${MACHINE2} ===\n" | tee -a ${OUTPUT}/signatures.${MACHINE2}.${TASK}.log
  126. diff ${OUTPUT}/${MACHINE1}/list.M ${OUTPUT}/${MACHINE2}/list.M | grep ${PRE_PATTERN} "${PATTERN}" | grep ${TASK} > ${OUTPUT}/signatures.${MACHINE2}.${TASK}
  127. for i in `cat ${OUTPUT}/signatures.${MACHINE2}.${TASK} | sed 's#[^/]*/\([^/]*\)/.*#\1#g' | sort -u | xargs`; do
  128. [ -e ${OUTPUT}/${MACHINE1}/*/$i/*${TASK}* ] || echo "INFO: ${i} task ${TASK} doesn't exist in ${MACHINE1}" >&2
  129. [ -e ${OUTPUT}/${MACHINE1}/*/$i/*${TASK}* ] || continue
  130. [ -e ${OUTPUT}/${MACHINE2}/*/$i/*${TASK}* ] || echo "INFO: ${i} task ${TASK} doesn't exist in ${MACHINE2}" >&2
  131. [ -e ${OUTPUT}/${MACHINE2}/*/$i/*${TASK}* ] || continue
  132. printf "ERROR: $i different signature for task ${TASK} between ${MACHINE1} and ${MACHINE2}\n";
  133. bitbake-diffsigs ${OUTPUT}/${MACHINE1}/*/$i/*${TASK}* ${OUTPUT}/${MACHINE2}/*/$i/*${TASK}*;
  134. echo "$i" >> ${OUTPUT}/failed-recipes.log
  135. echo
  136. done | tee -a ${OUTPUT}/signatures.${MACHINE2}.${TASK}.log
  137. # don't create empty files
  138. ERRORS=`grep "^ERROR.*" ${OUTPUT}/signatures.${MACHINE2}.${TASK}.log | wc -l`
  139. if [ "${ERRORS}" != "0" ] ; then
  140. echo "ERROR: ${ERRORS} errors found in ${OUTPUT}/signatures.${MACHINE2}.${TASK}.log"
  141. RESULT+=${ERRORS}
  142. fi
  143. done
  144. }
  145. function compareMachines() {
  146. [ "$#" -ge 2 ] && compareSignatures $1 $2
  147. [ "$#" -ge 3 ] && compareSignatures $1 $3 "\(^< all\)\|\(^< x86_64-linux\)\|\(^< i586-linux\)"
  148. }
  149. if [ "${analyze}" = "Y" ] ; then
  150. compareMachines ${machines}
  151. fi
  152. if [ "${RESULT}" != "0" -a -f ${OUTPUT}/failed-recipes.log ] ; then
  153. cat ${OUTPUT}/failed-recipes.log | sort -u >${OUTPUT}/failed-recipes.log.u && mv ${OUTPUT}/failed-recipes.log.u ${OUTPUT}/failed-recipes.log
  154. echo "ERROR: ${RESULT} issues were found in these recipes: `cat ${OUTPUT}/failed-recipes.log | xargs`"
  155. fi
  156. echo "INFO: Output written in: ${OUTPUT}"
  157. exit ${RESULT}