sbom_test.sh 7.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226
  1. #!/bin/bash
  2. # Copyright (C) 2023 The Android Open Source Project
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License");
  5. # you may not use this file except in compliance with the License.
  6. # You may obtain a copy of the License at
  7. #
  8. # http://www.apache.org/licenses/LICENSE-2.0
  9. #
  10. # Unless required by applicable law or agreed to in writing, software
  11. # distributed under the License is distributed on an "AS IS" BASIS,
  12. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13. # See the License for the specific language governing permissions and
  14. # limitations under the License.
  15. set -uo pipefail
  16. # Integration test for verifying generated SBOM for cuttlefish device.
  17. if [ ! -e "build/make/core/Makefile" ]; then
  18. echo "$0 must be run from the top of the Android source tree."
  19. exit 1
  20. fi
  21. function setup {
  22. tmp_dir="$(mktemp -d tmp.XXXXXX)"
  23. trap 'cleanup "${tmp_dir}"' EXIT
  24. echo "${tmp_dir}"
  25. }
  26. function cleanup {
  27. tmp_dir="$1"; shift
  28. rm -rf "${tmp_dir}"
  29. }
  30. function run_soong {
  31. target_product="$1";shift
  32. out_dir="$1"; shift
  33. targets="$1"; shift
  34. if [ "$#" -ge 1 ]; then
  35. apps=$1; shift
  36. TARGET_PRODUCT="${target_product}" TARGET_BUILD_VARIANT=userdebug OUT_DIR="${out_dir}" TARGET_BUILD_UNBUNDLED=true TARGET_BUILD_APPS=$apps build/soong/soong_ui.bash --make-mode ${targets}
  37. else
  38. TARGET_PRODUCT="${target_product}" TARGET_BUILD_VARIANT=userdebug OUT_DIR="${out_dir}" build/soong/soong_ui.bash --make-mode ${targets}
  39. fi
  40. }
  41. function diff_files {
  42. file_list_file="$1"; shift
  43. files_in_spdx_file="$1"; shift
  44. partition_name="$1"; shift
  45. exclude=
  46. if [ -v 'diff_excludes[$partition_name]' ]; then
  47. exclude=${diff_excludes[$partition_name]}
  48. fi
  49. diff "$file_list_file" "$files_in_spdx_file" $exclude
  50. if [ $? != "0" ]; then
  51. echo Found diffs in $f and SBOM.
  52. exit 1
  53. else
  54. echo No diffs.
  55. fi
  56. }
  57. function test_sbom_aosp_cf_x86_64_phone {
  58. # Setup
  59. out_dir="$(setup)"
  60. # Test
  61. # m droid, build sbom later in case additional dependencies might be built and included in partition images.
  62. run_soong "aosp_cf_x86_64_phone" "${out_dir}" "droid dump.erofs lz4"
  63. product_out=$out_dir/target/product/vsoc_x86_64
  64. sbom_test=$product_out/sbom_test
  65. mkdir -p $sbom_test
  66. cp $product_out/*.img $sbom_test
  67. # m sbom
  68. run_soong "aosp_cf_x86_64_phone" "${out_dir}" sbom
  69. # Generate installed file list from .img files in PRODUCT_OUT
  70. dump_erofs=$out_dir/host/linux-x86/bin/dump.erofs
  71. lz4=$out_dir/host/linux-x86/bin/lz4
  72. declare -A diff_excludes
  73. diff_excludes[vendor]="-I /vendor/lib64/libkeystore2_crypto.so"
  74. diff_excludes[system]="\
  75. -I /bin \
  76. -I /bugreports \
  77. -I /cache \
  78. -I /d \
  79. -I /etc \
  80. -I /init \
  81. -I /odm/app \
  82. -I /odm/bin \
  83. -I /odm_dlkm/etc \
  84. -I /odm/etc \
  85. -I /odm/firmware \
  86. -I /odm/framework \
  87. -I /odm/lib \
  88. -I /odm/lib64 \
  89. -I /odm/overlay \
  90. -I /odm/priv-app \
  91. -I /odm/usr \
  92. -I /sdcard \
  93. -I /system/lib64/android.hardware.confirmationui@1.0.so \
  94. -I /system/lib64/android.hardware.confirmationui-V1-ndk.so \
  95. -I /system/lib64/android.hardware.keymaster@4.1.so \
  96. -I /system/lib64/android.hardware.security.rkp-V3-ndk.so \
  97. -I /system/lib64/android.hardware.security.sharedsecret-V1-ndk.so \
  98. -I /system/lib64/android.security.compat-ndk.so \
  99. -I /system/lib64/libkeymaster4_1support.so \
  100. -I /system/lib64/libkeymaster4support.so \
  101. -I /system/lib64/libkeymint.so \
  102. -I /system/lib64/libkeystore2_aaid.so \
  103. -I /system/lib64/libkeystore2_apc_compat.so \
  104. -I /system/lib64/libkeystore2_crypto.so \
  105. -I /system/lib64/libkeystore-attestation-application-id.so \
  106. -I /system/lib64/libkm_compat_service.so \
  107. -I /system/lib64/libkm_compat.so \
  108. -I /system/lib64/vndk-29 \
  109. -I /system/lib64/vndk-sp-29 \
  110. -I /system/lib/vndk-29 \
  111. -I /system/lib/vndk-sp-29 \
  112. -I /system/usr/icu \
  113. -I /vendor_dlkm/etc"
  114. # Example output of dump.erofs is as below, and the data used in the test start
  115. # at line 11. Column 1 is inode id, column 2 is inode type and column 3 is name.
  116. # Each line is captured in variable "entry", awk is used to get type and name.
  117. # Output of dump.erofs:
  118. # File : /
  119. # Size: 160 On-disk size: 160 directory
  120. # NID: 39 Links: 10 Layout: 2 Compression ratio: 100.00%
  121. # Inode size: 64 Extent size: 0 Xattr size: 16
  122. # Uid: 0 Gid: 0 Access: 0755/rwxr-xr-x
  123. # Timestamp: 2023-02-14 01:15:54.000000000
  124. #
  125. # NID TYPE FILENAME
  126. # 39 2 .
  127. # 39 2 ..
  128. # 47 2 app
  129. # 1286748 2 bin
  130. # 1286754 2 etc
  131. # 5304814 2 lib
  132. # 5309056 2 lib64
  133. # 5309130 2 media
  134. # 5388910 2 overlay
  135. # 5479537 2 priv-app
  136. EROFS_IMAGES="\
  137. $sbom_test/product.img \
  138. $sbom_test/system.img \
  139. $sbom_test/system_ext.img \
  140. $sbom_test/system_dlkm.img \
  141. $sbom_test/system_other.img \
  142. $sbom_test/odm.img \
  143. $sbom_test/odm_dlkm.img \
  144. $sbom_test/vendor.img \
  145. $sbom_test/vendor_dlkm.img"
  146. for f in $EROFS_IMAGES; do
  147. partition_name=$(basename $f | cut -d. -f1)
  148. file_list_file="${sbom_test}/sbom-${partition_name}-files.txt"
  149. files_in_spdx_file="${sbom_test}/sbom-${partition_name}-files-in-spdx.txt"
  150. rm "$file_list_file" > /dev/null 2>&1 || true
  151. all_dirs="/"
  152. while [ ! -z "$all_dirs" ]; do
  153. dir=$(echo "$all_dirs" | cut -d ' ' -f1)
  154. all_dirs=$(echo "$all_dirs" | cut -d ' ' -f1 --complement -s)
  155. entries=$($dump_erofs --ls --path "$dir" $f | tail -n +11)
  156. while read -r entry; do
  157. inode_type=$(echo $entry | awk -F ' ' '{print $2}')
  158. name=$(echo $entry | awk -F ' ' '{print $3}')
  159. case $inode_type in
  160. "2") # directory
  161. all_dirs=$(echo "$all_dirs $dir/$name" | sed 's/^\s*//')
  162. ;;
  163. "1"|"7") # 1: file, 7: symlink
  164. (
  165. if [ "$partition_name" != "system" ]; then
  166. # system partition is mounted to /, not to prepend partition name.
  167. printf %s "/$partition_name"
  168. fi
  169. echo "$dir/$name" | sed 's#^//#/#'
  170. ) >> "$file_list_file"
  171. ;;
  172. esac
  173. done <<< "$entries"
  174. done
  175. sort -n -o "$file_list_file" "$file_list_file"
  176. grep "FileName: /${partition_name}/" $product_out/sbom.spdx | sed 's/^FileName: //' > "$files_in_spdx_file"
  177. if [ "$partition_name" = "system" ]; then
  178. # system partition is mounted to /, so include FileName starts with /root/ too.
  179. grep "FileName: /root/" $product_out/sbom.spdx | sed 's/^FileName: \/root//' >> "$files_in_spdx_file"
  180. fi
  181. sort -n -o "$files_in_spdx_file" "$files_in_spdx_file"
  182. echo ============ Diffing files in $f and SBOM
  183. diff_files "$file_list_file" "$files_in_spdx_file" "$partition_name"
  184. done
  185. RAMDISK_IMAGES="$product_out/ramdisk.img"
  186. for f in $RAMDISK_IMAGES; do
  187. partition_name=$(basename $f | cut -d. -f1)
  188. file_list_file="${sbom_test}/sbom-${partition_name}-files.txt"
  189. files_in_spdx_file="${sbom_test}/sbom-${partition_name}-files-in-spdx.txt"
  190. # lz4 decompress $f to stdout
  191. # cpio list all entries like ls -l
  192. # grep filter normal files and symlinks
  193. # awk get entry names
  194. # sed remove partition name from entry names
  195. $lz4 -c -d $f | cpio -tv 2>/dev/null | grep '^[-l]' | awk -F ' ' '{print $9}' | sed "s:^:/$partition_name/:" | sort -n > "$file_list_file"
  196. grep "FileName: /${partition_name}/" $product_out/sbom.spdx | sed 's/^FileName: //' | sort -n > "$files_in_spdx_file"
  197. echo ============ Diffing files in $f and SBOM
  198. diff_files "$file_list_file" "$files_in_spdx_file" "$partition_name"
  199. done
  200. # Teardown
  201. cleanup "${out_dir}"
  202. }
  203. test_sbom_aosp_cf_x86_64_phone