123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491 |
- #!/bin/bash
- set -eu
- # Copyright 2020 Google Inc. All rights reserved.
- #
- # Licensed under the Apache License, Version 2.0 (the "License");
- # you may not use this file except in compliance with the License.
- # You may obtain a copy of the License at
- #
- # http://www.apache.org/licenses/LICENSE-2.0
- #
- # Unless required by applicable law or agreed to in writing, software
- # distributed under the License is distributed on an "AS IS" BASIS,
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- # See the License for the specific language governing permissions and
- # limitations under the License.
- # Tool to evaluate the transitive closure of the ninja dependency graph of the
- # files and targets a given target depends on.
- #
- # i.e. the list of things that, if changed, could cause a change to a target.
- readonly me=$(basename "${0}")
- readonly usage="usage: ${me} {options} target [target...]
- Evaluate the transitive closure of files and ninja targets that one or more
- targets depend on.
- Dependency Options:
- -(no)order_deps Whether to include order-only dependencies. (Default false)
- -(no)implicit Whether to include implicit dependencies. (Default true)
- -(no)explicit Whether to include regular / explicit deps. (Default true)
- -nofollow Unanchored regular expression. Matching paths and targets
- always get reported. Their dependencies do not get reported
- unless first encountered in a 'container' file type.
- Multiple allowed and combined using '|'.
- e.g. -nofollow='*.so' not -nofollow='.so$'
- -nofollow='*.so|*.dex' or -nofollow='*.so' -nofollow='.dex'
- (Defaults to no matches)
- -container Unanchored regular expression. Matching file extensions get
- treated as 'container' files for -nofollow option.
- Multiple allowed and combines using '|'
- (Default 'apex|apk|zip|jar|tar|tgz')
- Output Options:
- -(no)quiet Suppresses progress output to stderr and interactive
- alias -(no)q prompts. By default, when stderr is a tty, progress gets
- reported to stderr; when both stderr and stdin are tty,
- the script asks user whether to delete intermediate files.
- When suppressed or not prompted, script always deletes the
- temporary / intermediate files.
- -sep=<delim> Use 'delim' as output field separator between notice
- checksum and notice filename in notice output.
- e.g. sep='\\t'
- (Default space)
- -csv Shorthand for -sep=','
- -directories=<f> Output directory names of dependencies to 'f'.
- alias -d User '/dev/stdout' to send directories to stdout. Defaults
- to no directory output.
- -notices=<file> Output license and notice file paths to 'file'.
- alias -n Use '/dev/stdout' to send notices to stdout. Defaults to no
- license/notice output.
- -projects=<file> Output git project names to 'file'. Use '/dev/stdout' to
- alias -p send projects to stdout. Defaults to no project output.
- -targets=<fils> Output target dependencies to 'file'. Use '/dev/stdout' to
- alias -t send targets to stdout.
- When no directory, notice, project or target output options
- given, defaults to stdout. Otherwise, defaults to no target
- output.
- At minimum, before running this script, you must first run:
- $ source build/envsetup.sh
- $ lunch
- $ m nothing
- to setup the build environment, choose a target platform, and build the ninja
- dependency graph.
- "
- function die() { echo -e "${*}" >&2; exit 2; }
- # Reads one input target per line from stdin; outputs (isnotice target) tuples.
- #
- # output target is a ninja target that the input target depends on
- # isnotice in {0,1} with 1 for output targets believed to be license or notice
- function getDeps() {
- (tr '\n' '\0' | xargs -0 -r "${ninja_bin}" -f "${ninja_file}" -t query) \
- | awk -v include_order="${include_order_deps}" \
- -v include_implicit="${include_implicit_deps}" \
- -v include_explicit="${include_deps}" \
- -v containers="${container_types}" \
- '
- BEGIN {
- ininput = 0
- isnotice = 0
- currFileName = ""
- currExt = ""
- }
- $1 == "outputs:" || $1 == "validations:" {
- ininput = 0
- }
- ininput == 0 && $0 ~ /^\S\S*:$/ {
- isnotice = ($0 ~ /.*NOTICE.*[.]txt:$/)
- currFileName = gensub(/^.*[/]([^/]*)[:]$/, "\\1", "g")
- currExt = gensub(/^.*[.]([^./]*)[:]$/, "\\1", "g")
- }
- ininput != 0 && $1 !~ /^[|][|]?/ {
- if (include_explicit == "true") {
- fileName = gensub(/^.*[/]([^/]*)$/, "\\1", "g")
- print ( \
- (isnotice && $0 !~ /^\s*build[/]soong[/]scripts[/]/) \
- || $0 ~ /NOTICE|LICEN[CS]E/ \
- || $0 ~ /(notice|licen[cs]e)[.]txt/ \
- )" "(fileName == currFileName||currExt ~ "^(" containers ")$")" "gensub(/^\s*/, "", "g")
- }
- }
- ininput != 0 && $1 == "|" {
- if (include_implicit == "true") {
- fileName = gensub(/^.*[/]([^/]*)$/, "\\1", "g")
- $1 = ""
- print ( \
- (isnotice && $0 !~ /^\s*build[/]soong[/]scripts[/]/) \
- || $0 ~ /NOTICE|LICEN[CS]E/ \
- || $0 ~ /(notice|licen[cs]e)[.]txt/ \
- )" "(fileName == currFileName||currExt ~ "^(" containers ")$")" "gensub(/^\s*/, "", "g")
- }
- }
- ininput != 0 && $1 == "||" {
- if (include_order == "true") {
- fileName = gensub(/^.*[/]([^/]*)$/, "\\1", "g")
- $1 = ""
- print ( \
- (isnotice && $0 !~ /^\s*build[/]soong[/]scripts[/]/) \
- || $0 ~ /NOTICE|LICEN[CS]E/ \
- || $0 ~ /(notice|licen[cs]e)[.]txt/ \
- )" "(fileName == currFileName||currExt ~ "^(" containers ")$")" "gensub(/^\s*/, "", "g")
- }
- }
- $1 == "input:" {
- ininput = 1
- }
- '
- }
- # Reads one input directory per line from stdin; outputs unique git projects.
- function getProjects() {
- while read d; do
- while [ "${d}" != '.' ] && [ "${d}" != '/' ]; do
- if [ -d "${d}/.git/" ]; then
- echo "${d}"
- break
- fi
- d=$(dirname "${d}")
- done
- done | sort -u
- }
- if [ -z "${ANDROID_BUILD_TOP}" ]; then
- die "${me}: Run 'lunch' to configure the build environment"
- fi
- if [ -z "${TARGET_PRODUCT}" ]; then
- die "${me}: Run 'lunch' to configure the build environment"
- fi
- readonly ninja_file="${ANDROID_BUILD_TOP}/out/combined-${TARGET_PRODUCT}.ninja"
- if [ ! -f "${ninja_file}" ]; then
- die "${me}: Run 'm nothing' to build the dependency graph"
- fi
- readonly ninja_bin="${ANDROID_BUILD_TOP}/prebuilts/build-tools/linux-x86/bin/ninja"
- if [ ! -x "${ninja_bin}" ]; then
- die "${me}: Cannot find ninja executable expected at ${ninja_bin}"
- fi
- # parse the command-line
- declare -a targets # one or more targets to evaluate
- include_order_deps=false # whether to trace through || "order dependencies"
- include_implicit_deps=true # whether to trace through | "implicit deps"
- include_deps=true # whether to trace through regular explicit deps
- quiet=false # whether to suppress progress
- projects_out='' # where to output the list of projects
- directories_out='' # where to output the list of directories
- targets_out='' # where to output the list of targets/source files
- notices_out='' # where to output the list of license/notice files
- sep=" " # separator between md5sum and notice filename
- nofollow='' # regularexp must fully match targets to skip
- container_types='' # regularexp must full match file extension
- # defaults to 'apex|apk|zip|jar|tar|tgz' below.
- use_stdin=false # whether to read targets from stdin i.e. target -
- while [ $# -gt 0 ]; do
- case "${1:-}" in
- -)
- use_stdin=true
- ;;
- -*)
- flag=$(expr "${1}" : '^-*\(.*\)$')
- case "${flag:-}" in
- order_deps)
- include_order_deps=true;;
- noorder_deps)
- include_order_deps=false;;
- implicit)
- include_implicit_deps=true;;
- noimplicit)
- include_implicit_deps=false;;
- explicit)
- include_deps=true;;
- noexplicit)
- include_deps=false;;
- csv)
- sep=",";;
- sep)
- sep="${2?"${usage}"}"; shift;;
- sep=)
- sep=$(expr "${flag}" : '^sep=\(.*\)$');;
- q) ;&
- quiet)
- quiet=true;;
- noq) ;&
- noquiet)
- quiet=false;;
- nofollow)
- case "${nofollow}" in
- '')
- nofollow="${2?"${usage}"}";;
- *)
- nofollow="${nofollow}|${2?"${usage}"}";;
- esac
- shift
- ;;
- nofollow=*)
- case "${nofollow}" in
- '')
- nofollow=$(expr "${flag}" : '^nofollow=\(.*\)$');;
- *)
- nofollow="${nofollow}|"$(expr "${flag}" : '^nofollow=\(.*\)$');;
- esac
- ;;
- container)
- container_types="${container_types}|${2?"${usage}"}";;
- container=*)
- container_types="${container_types}|"$(expr "${flag}" : '^container=\(.*\)$');;
- p) ;&
- projects)
- projects_out="${2?"${usage}"}"; shift;;
- p=*) ;&
- projects=*)
- projects_out=$(expr "${flag}" : '^.*=\(.*\)$');;
- d) ;&
- directores)
- directories_out="${2?"${usage}"}"; shift;;
- d=*) ;&
- directories=*)
- directories_out=$(expr "${flag}" : '^.*=\(.*\)$');;
- t) ;&
- targets)
- targets_out="${2?"${usage}"}"; shift;;
- t=*) ;&
- targets=)
- targets_out=$(expr "${flag}" : '^.*=\(.*\)$');;
- n) ;&
- notices)
- notices_out="${2?"${usage}"}"; shift;;
- n=*) ;&
- notices=)
- notices_out=$(expr "${flag}" : '^.*=\(.*\)$');;
- *)
- die "${usage}\n\nUnknown flag ${1}";;
- esac
- ;;
- *)
- targets+=("${1:-}")
- ;;
- esac
- shift
- done
- # fail fast if command-line arguments are invalid
- if [ ! -v targets[0] ] && ! ${use_stdin}; then
- die "${usage}\n\nNo target specified."
- fi
- if [ -z "${projects_out}" ] \
- && [ -z "${directories_out}" ] \
- && [ -z "${targets_out}" ] \
- && [ -z "${notices_out}" ]
- then
- targets_out='/dev/stdout'
- fi
- if [ -z "${container_types}" ]; then
- container_types='apex|apk|zip|jar|tar|tgz'
- fi
- # showProgress when stderr is a tty
- if [ -t 2 ] && ! ${quiet}; then
- showProgress=true
- else
- showProgress=false
- fi
- # interactive when both stderr and stdin are tty
- if ${showProgress} && [ -t 0 ]; then
- interactive=true
- else
- interactive=false
- fi
- readonly tmpFiles=$(mktemp -d "${TMPDIR}.tdeps.XXXXXXXXX")
- if [ -z "${tmpFiles}" ]; then
- die "${me}: unable to create temporary directory"
- fi
- # The deps files contain unique (isnotice target) tuples where
- # isnotice in {0,1} with 1 when ninja target 'target' is a license or notice.
- readonly oldDeps="${tmpFiles}/old"
- readonly newDeps="${tmpFiles}/new"
- readonly allDeps="${tmpFiles}/all"
- if ${use_stdin}; then # start deps by reading 1 target per line from stdin
- awk '
- NF > 0 {
- print ( \
- $0 ~ /NOTICE|LICEN[CS]E/ \
- || $0 ~ /(notice|licen[cs]e)[.]txt/ \
- )" "gensub(/\s*$/, "", "g", gensub(/^\s*/, "", "g"))
- }
- ' > "${newDeps}"
- else # start with no deps by clearing file
- : > "${newDeps}"
- fi
- # extend deps by appending targets from command-line
- for idx in "${!targets[*]}"; do
- isnotice='0'
- case "${targets[${idx}]}" in
- *NOTICE*) ;&
- *LICEN[CS]E*) ;&
- *notice.txt) ;&
- *licen[cs]e.txt)
- isnotice='1';;
- esac
- echo "${isnotice} 1 ${targets[${idx}]}" >> "${newDeps}"
- done
- # remove duplicates and start with new, old and all the same
- sort -u < "${newDeps}" > "${allDeps}"
- cp "${allDeps}" "${newDeps}"
- cp "${allDeps}" "${oldDeps}"
- # report depth of dependenciens when showProgress
- depth=0
- # 1st iteration always unfiltered
- filter='cat'
- while [ $(wc -l < "${newDeps}") -gt 0 ]; do
- if ${showProgress}; then
- echo "depth ${depth} has "$(wc -l < "${newDeps}")" targets" >&2
- depth=$(expr ${depth} + 1)
- fi
- ( # recalculate dependencies by combining unique inputs of new deps w. old
- set +e
- sh -c "${filter}" < "${newDeps}" | cut -d\ -f3- | getDeps
- set -e
- cat "${oldDeps}"
- ) | sort -u > "${allDeps}"
- # recalculate new dependencies as net additions to old dependencies
- set +e
- diff "${oldDeps}" "${allDeps}" --old-line-format='' --new-line-format='%L' \
- --unchanged-line-format='' > "${newDeps}"
- set -e
- # apply filters on subsequent iterations
- case "${nofollow}" in
- '')
- filter='cat';;
- *)
- filter="egrep -v '^[01] 0 (${nofollow})$'"
- ;;
- esac
- # recalculate old dependencies for next iteration
- cp "${allDeps}" "${oldDeps}"
- done
- # found all deps -- clean up last iteration of old and new
- rm -f "${oldDeps}"
- rm -f "${newDeps}"
- if ${showProgress}; then
- echo $(wc -l < "${allDeps}")" targets" >&2
- fi
- if [ -n "${targets_out}" ]; then
- cut -d\ -f3- "${allDeps}" | sort -u > "${targets_out}"
- fi
- if [ -n "${directories_out}" ] \
- || [ -n "${projects_out}" ] \
- || [ -n "${notices_out}" ]
- then
- readonly allDirs="${tmpFiles}/dirs"
- (
- cut -d\ -f3- "${allDeps}" | tr '\n' '\0' | xargs -0 dirname
- ) | sort -u > "${allDirs}"
- if ${showProgress}; then
- echo $(wc -l < "${allDirs}")" directories" >&2
- fi
- case "${directories_out}" in
- '') : do nothing;;
- *)
- cat "${allDirs}" > "${directories_out}"
- ;;
- esac
- fi
- if [ -n "${projects_out}" ] \
- || [ -n "${notices_out}" ]
- then
- readonly allProj="${tmpFiles}/projects"
- set +e
- egrep -v '^out[/]' "${allDirs}" | getProjects > "${allProj}"
- set -e
- if ${showProgress}; then
- echo $(wc -l < "${allProj}")" projects" >&2
- fi
- case "${projects_out}" in
- '') : do nothing;;
- *)
- cat "${allProj}" > "${projects_out}"
- ;;
- esac
- fi
- case "${notices_out}" in
- '') : do nothing;;
- *)
- readonly allNotice="${tmpFiles}/notices"
- set +e
- egrep '^1' "${allDeps}" | cut -d\ -f3- | egrep -v '^out/' > "${allNotice}"
- set -e
- cat "${allProj}" | while read proj; do
- for f in LICENSE LICENCE NOTICE license.txt notice.txt; do
- if [ -f "${proj}/${f}" ]; then
- echo "${proj}/${f}"
- fi
- done
- done >> "${allNotice}"
- if ${showProgress}; then
- echo $(cat "${allNotice}" | sort -u | wc -l)" notice targets" >&2
- fi
- readonly hashedNotice="${tmpFiles}/hashednotices"
- ( # md5sum outputs checksum space indicator(space or *) filename newline
- set +e
- sort -u "${allNotice}" | tr '\n' '\0' | xargs -0 -r md5sum 2>/dev/null
- set -e
- # use sed to replace space and indicator with separator
- ) > "${hashedNotice}"
- if ${showProgress}; then
- echo $(cut -d\ -f2- "${hashedNotice}" | sort -u | wc -l)" notice files" >&2
- echo $(cut -d\ -f1 "${hashedNotice}" | sort -u | wc -l)" distinct notices" >&2
- fi
- sed 's/^\([^ ]*\) [* ]/\1'"${sep}"'/g' "${hashedNotice}" | sort > "${notices_out}"
- ;;
- esac
- if ${interactive}; then
- echo -n "$(date '+%F %-k:%M:%S') Delete ${tmpFiles} ? [n] " >&2
- read answer
- case "${answer}" in [yY]*) rm -fr "${tmpFiles}";; esac
- else
- rm -fr "${tmpFiles}"
- fi
|