build_kzip.bash 3.5 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879
  1. #! /bin/bash -uv
  2. #
  3. # Build kzip files (source files for the indexing pipeline) for the given configuration,
  4. # merge them and place the resulting all.kzip into $DIST_DIR.
  5. # It is assumed that the current directory is the top of the source tree.
  6. # The following environment variables affect the result:
  7. # BUILD_NUMBER build number, used to generate unique ID (will use UUID if not set)
  8. # SUPERPROJECT_SHA superproject sha, used to generate unique id (will use BUILD_NUMBER if not set)
  9. # SUPERPROJECT_REVISION superproject revision, used for unique id if defined as a sha
  10. # KZIP_NAME name of the output file (will use SUPERPROJECT_REVISION|SUPERPROJECT_SHA|BUILD_NUMBER|UUID if not set)
  11. # DIST_DIR where the resulting all.kzip will be placed
  12. # KYTHE_KZIP_ENCODING proto or json (proto is default)
  13. # KYTHE_JAVA_SOURCE_BATCH_SIZE maximum number of the Java source files in a compilation unit
  14. # OUT_DIR output directory (out if not specified})
  15. # TARGET_BUILD_VARIANT variant, e.g., `userdebug`
  16. # TARGET_PRODUCT target device name, e.g., 'aosp_blueline'
  17. # XREF_CORPUS source code repository URI, e.g., 'android.googlesource.com/platform/superproject'
  18. # If the SUPERPROJECT_REVISION is defined as a sha, use this as the default value if no
  19. # SUPERPROJECT_SHA is specified.
  20. if [[ ${SUPERPROJECT_REVISION:-} =~ [0-9a-f]{40} ]]; then
  21. : ${KZIP_NAME:=${SUPERPROJECT_REVISION:-}}
  22. fi
  23. : ${KZIP_NAME:=${SUPERPROJECT_SHA:-}}
  24. : ${KZIP_NAME:=${BUILD_NUMBER:-}}
  25. : ${KZIP_NAME:=$(uuidgen)}
  26. : ${KYTHE_JAVA_SOURCE_BATCH_SIZE:=500}
  27. : ${KYTHE_KZIP_ENCODING:=proto}
  28. : ${XREF_CORPUS:?should be set}
  29. export KYTHE_JAVA_SOURCE_BATCH_SIZE KYTHE_KZIP_ENCODING
  30. # The extraction might fail for some source files, so run with -k and then check that
  31. # sufficiently many files were generated.
  32. declare -r out="${OUT_DIR:-out}"
  33. # Build extraction files and `merge_zips` which we use later.
  34. kzip_targets=(
  35. merge_zips
  36. xref_cxx
  37. xref_java
  38. # TODO: b/286390153 - reenable rust
  39. # xref_rust
  40. )
  41. build/soong/soong_ui.bash --build-mode --all-modules --skip-soong-tests --dir=$PWD -k "${kzip_targets[@]}"
  42. # Build extraction file for Go the files in build/{blueprint,soong} directories.
  43. declare -r abspath_out=$(realpath "${out}")
  44. declare -r go_extractor=$(realpath prebuilts/build-tools/linux-x86/bin/go_extractor)
  45. declare -r go_root=$(realpath prebuilts/go/linux-x86)
  46. declare -r source_root=$PWD
  47. # For the Go code, we invoke the extractor directly. The two caveats are that
  48. # the extractor's rewrite rules are generated on the fly as they depend on the
  49. # value of XREF_CORPUS, and that the name of the kzip file is derived from the
  50. # directory name by replacing '/' with '_'.
  51. # Go extractor should succeed.
  52. declare -ar go_modules=(build/blueprint build/soong
  53. build/make/tools/canoninja build/make/tools/compliance build/make/tools/rbcrun)
  54. set -e
  55. for dir in "${go_modules[@]}"; do
  56. (cd "$dir";
  57. outfile=$(echo "$dir" | sed -r 's|/|_|g;s|(.*)|\1.go.kzip|');
  58. KYTHE_ROOT_DIRECTORY="${source_root}" "$go_extractor" --goroot="$go_root" \
  59. --rules=<(printf '[{"pattern": "(.*)","vname": {"path": "@1@", "corpus":"%s"}}]' "${XREF_CORPUS}") \
  60. --canonicalize_package_corpus --output "${abspath_out}/soong/$outfile" ./...
  61. )
  62. done
  63. set +e
  64. declare -r kzip_count=$(find "$out" -name '*.kzip' | wc -l)
  65. (($kzip_count>100000)) || { printf "Too few kzip files were generated: %d\n" $kzip_count; exit 1; }
  66. # Pack
  67. declare -r allkzip="$KZIP_NAME.kzip"
  68. "$out/host/linux-x86/bin/merge_zips" "$DIST_DIR/$allkzip" @<(find "$out" -name '*.kzip')