aquery.go 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710
  1. // Copyright 2020 Google Inc. All rights reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package bazel
  15. import (
  16. "crypto/sha256"
  17. "encoding/base64"
  18. "fmt"
  19. "path/filepath"
  20. "reflect"
  21. "sort"
  22. "strings"
  23. "sync"
  24. analysis_v2_proto "prebuilts/bazel/common/proto/analysis_v2"
  25. "github.com/google/blueprint/metrics"
  26. "github.com/google/blueprint/proptools"
  27. "google.golang.org/protobuf/proto"
  28. )
  29. type artifactId int
  30. type depsetId int
  31. type pathFragmentId int
  32. // artifact contains relevant portions of Bazel's aquery proto, Artifact.
  33. // Represents a single artifact, whether it's a source file or a derived output file.
  34. type artifact struct {
  35. Id artifactId
  36. PathFragmentId pathFragmentId
  37. }
  38. type pathFragment struct {
  39. Id pathFragmentId
  40. Label string
  41. ParentId pathFragmentId
  42. }
  43. // KeyValuePair represents Bazel's aquery proto, KeyValuePair.
  44. type KeyValuePair struct {
  45. Key string
  46. Value string
  47. }
  48. // AqueryDepset is a depset definition from Bazel's aquery response. This is
  49. // akin to the `depSetOfFiles` in the response proto, except:
  50. // - direct artifacts are enumerated by full path instead of by ID
  51. // - it has a hash of the depset contents, instead of an int ID (for determinism)
  52. //
  53. // A depset is a data structure for efficient transitive handling of artifact
  54. // paths. A single depset consists of one or more artifact paths and one or
  55. // more "child" depsets.
  56. type AqueryDepset struct {
  57. ContentHash string
  58. DirectArtifacts []string
  59. TransitiveDepSetHashes []string
  60. }
  61. // depSetOfFiles contains relevant portions of Bazel's aquery proto, DepSetOfFiles.
  62. // Represents a data structure containing one or more files. Depsets in Bazel are an efficient
  63. // data structure for storing large numbers of file paths.
  64. type depSetOfFiles struct {
  65. Id depsetId
  66. DirectArtifactIds []artifactId
  67. TransitiveDepSetIds []depsetId
  68. }
  69. // action contains relevant portions of Bazel's aquery proto, Action.
  70. // Represents a single command line invocation in the Bazel build graph.
  71. type action struct {
  72. Arguments []string
  73. EnvironmentVariables []KeyValuePair
  74. InputDepSetIds []depsetId
  75. Mnemonic string
  76. OutputIds []artifactId
  77. TemplateContent string
  78. Substitutions []KeyValuePair
  79. FileContents string
  80. }
  81. // actionGraphContainer contains relevant portions of Bazel's aquery proto, ActionGraphContainer.
  82. // An aquery response from Bazel contains a single ActionGraphContainer proto.
  83. type actionGraphContainer struct {
  84. Artifacts []artifact
  85. Actions []action
  86. DepSetOfFiles []depSetOfFiles
  87. PathFragments []pathFragment
  88. }
  89. // BuildStatement contains information to register a build statement corresponding (one to one)
  90. // with a Bazel action from Bazel's action graph.
  91. type BuildStatement struct {
  92. Command string
  93. Depfile *string
  94. OutputPaths []string
  95. SymlinkPaths []string
  96. Env []*analysis_v2_proto.KeyValuePair
  97. Mnemonic string
  98. // Inputs of this build statement, either as unexpanded depsets or expanded
  99. // input paths. There should be no overlap between these fields; an input
  100. // path should either be included as part of an unexpanded depset or a raw
  101. // input path string, but not both.
  102. InputDepsetHashes []string
  103. InputPaths []string
  104. FileContents string
  105. }
  106. // A helper type for aquery processing which facilitates retrieval of path IDs from their
  107. // less readable Bazel structures (depset and path fragment).
  108. type aqueryArtifactHandler struct {
  109. // Maps depset id to AqueryDepset, a representation of depset which is
  110. // post-processed for middleman artifact handling, unhandled artifact
  111. // dropping, content hashing, etc.
  112. depsetIdToAqueryDepset map[depsetId]AqueryDepset
  113. emptyDepsetIds map[depsetId]struct{}
  114. // Maps content hash to AqueryDepset.
  115. depsetHashToAqueryDepset map[string]AqueryDepset
  116. // depsetIdToArtifactIdsCache is a memoization of depset flattening, because flattening
  117. // may be an expensive operation.
  118. depsetHashToArtifactPathsCache sync.Map
  119. // Maps artifact ids to fully expanded paths.
  120. artifactIdToPath map[artifactId]string
  121. }
  122. // The tokens should be substituted with the value specified here, instead of the
  123. // one returned in 'substitutions' of TemplateExpand action.
  124. var templateActionOverriddenTokens = map[string]string{
  125. // Uses "python3" for %python_binary% instead of the value returned by aquery
  126. // which is "py3wrapper.sh". See removePy3wrapperScript.
  127. "%python_binary%": "python3",
  128. }
  129. const (
  130. middlemanMnemonic = "Middleman"
  131. // The file name of py3wrapper.sh, which is used by py_binary targets.
  132. py3wrapperFileName = "/py3wrapper.sh"
  133. )
  134. func indexBy[K comparable, V any](values []V, keyFn func(v V) K) map[K]V {
  135. m := map[K]V{}
  136. for _, v := range values {
  137. m[keyFn(v)] = v
  138. }
  139. return m
  140. }
  141. func newAqueryHandler(aqueryResult *analysis_v2_proto.ActionGraphContainer) (*aqueryArtifactHandler, error) {
  142. pathFragments := indexBy(aqueryResult.PathFragments, func(pf *analysis_v2_proto.PathFragment) pathFragmentId {
  143. return pathFragmentId(pf.Id)
  144. })
  145. artifactIdToPath := make(map[artifactId]string, len(aqueryResult.Artifacts))
  146. for _, artifact := range aqueryResult.Artifacts {
  147. artifactPath, err := expandPathFragment(pathFragmentId(artifact.PathFragmentId), pathFragments)
  148. if err != nil {
  149. return nil, err
  150. }
  151. artifactIdToPath[artifactId(artifact.Id)] = artifactPath
  152. }
  153. // Map middleman artifact ContentHash to input artifact depset ID.
  154. // Middleman artifacts are treated as "substitute" artifacts for mixed builds. For example,
  155. // if we find a middleman action which has inputs [foo, bar], and output [baz_middleman], then,
  156. // for each other action which has input [baz_middleman], we add [foo, bar] to the inputs for
  157. // that action instead.
  158. middlemanIdToDepsetIds := map[artifactId][]uint32{}
  159. for _, actionEntry := range aqueryResult.Actions {
  160. if actionEntry.Mnemonic == middlemanMnemonic {
  161. for _, outputId := range actionEntry.OutputIds {
  162. middlemanIdToDepsetIds[artifactId(outputId)] = actionEntry.InputDepSetIds
  163. }
  164. }
  165. }
  166. depsetIdToDepset := indexBy(aqueryResult.DepSetOfFiles, func(d *analysis_v2_proto.DepSetOfFiles) depsetId {
  167. return depsetId(d.Id)
  168. })
  169. aqueryHandler := aqueryArtifactHandler{
  170. depsetIdToAqueryDepset: map[depsetId]AqueryDepset{},
  171. depsetHashToAqueryDepset: map[string]AqueryDepset{},
  172. depsetHashToArtifactPathsCache: sync.Map{},
  173. emptyDepsetIds: make(map[depsetId]struct{}, 0),
  174. artifactIdToPath: artifactIdToPath,
  175. }
  176. // Validate and adjust aqueryResult.DepSetOfFiles values.
  177. for _, depset := range aqueryResult.DepSetOfFiles {
  178. _, err := aqueryHandler.populateDepsetMaps(depset, middlemanIdToDepsetIds, depsetIdToDepset)
  179. if err != nil {
  180. return nil, err
  181. }
  182. }
  183. return &aqueryHandler, nil
  184. }
  185. // Ensures that the handler's depsetIdToAqueryDepset map contains an entry for the given
  186. // depset.
  187. func (a *aqueryArtifactHandler) populateDepsetMaps(depset *analysis_v2_proto.DepSetOfFiles, middlemanIdToDepsetIds map[artifactId][]uint32, depsetIdToDepset map[depsetId]*analysis_v2_proto.DepSetOfFiles) (*AqueryDepset, error) {
  188. if aqueryDepset, containsDepset := a.depsetIdToAqueryDepset[depsetId(depset.Id)]; containsDepset {
  189. return &aqueryDepset, nil
  190. }
  191. transitiveDepsetIds := depset.TransitiveDepSetIds
  192. directArtifactPaths := make([]string, 0, len(depset.DirectArtifactIds))
  193. for _, id := range depset.DirectArtifactIds {
  194. aId := artifactId(id)
  195. path, pathExists := a.artifactIdToPath[aId]
  196. if !pathExists {
  197. return nil, fmt.Errorf("undefined input artifactId %d", aId)
  198. }
  199. // Filter out any inputs which are universally dropped, and swap middleman
  200. // artifacts with their corresponding depsets.
  201. if depsetsToUse, isMiddleman := middlemanIdToDepsetIds[aId]; isMiddleman {
  202. // Swap middleman artifacts with their corresponding depsets and drop the middleman artifacts.
  203. transitiveDepsetIds = append(transitiveDepsetIds, depsetsToUse...)
  204. } else if strings.HasSuffix(path, py3wrapperFileName) ||
  205. strings.HasPrefix(path, "../bazel_tools") {
  206. continue
  207. // Drop these artifacts.
  208. // See go/python-binary-host-mixed-build for more details.
  209. // 1) Drop py3wrapper.sh, just use python binary, the launcher script generated by the
  210. // TemplateExpandAction handles everything necessary to launch a Pythin application.
  211. // 2) ../bazel_tools: they have MODIFY timestamp 10years in the future and would cause the
  212. // containing depset to always be considered newer than their outputs.
  213. } else {
  214. directArtifactPaths = append(directArtifactPaths, path)
  215. }
  216. }
  217. childDepsetHashes := make([]string, 0, len(transitiveDepsetIds))
  218. for _, id := range transitiveDepsetIds {
  219. childDepsetId := depsetId(id)
  220. childDepset, exists := depsetIdToDepset[childDepsetId]
  221. if !exists {
  222. if _, empty := a.emptyDepsetIds[childDepsetId]; empty {
  223. continue
  224. } else {
  225. return nil, fmt.Errorf("undefined input depsetId %d (referenced by depsetId %d)", childDepsetId, depset.Id)
  226. }
  227. }
  228. if childAqueryDepset, err := a.populateDepsetMaps(childDepset, middlemanIdToDepsetIds, depsetIdToDepset); err != nil {
  229. return nil, err
  230. } else if childAqueryDepset == nil {
  231. continue
  232. } else {
  233. childDepsetHashes = append(childDepsetHashes, childAqueryDepset.ContentHash)
  234. }
  235. }
  236. if len(directArtifactPaths) == 0 && len(childDepsetHashes) == 0 {
  237. a.emptyDepsetIds[depsetId(depset.Id)] = struct{}{}
  238. return nil, nil
  239. }
  240. aqueryDepset := AqueryDepset{
  241. ContentHash: depsetContentHash(directArtifactPaths, childDepsetHashes),
  242. DirectArtifacts: directArtifactPaths,
  243. TransitiveDepSetHashes: childDepsetHashes,
  244. }
  245. a.depsetIdToAqueryDepset[depsetId(depset.Id)] = aqueryDepset
  246. a.depsetHashToAqueryDepset[aqueryDepset.ContentHash] = aqueryDepset
  247. return &aqueryDepset, nil
  248. }
  249. // getInputPaths flattens the depsets of the given IDs and returns all transitive
  250. // input paths contained in these depsets.
  251. // This is a potentially expensive operation, and should not be invoked except
  252. // for actions which need specialized input handling.
  253. func (a *aqueryArtifactHandler) getInputPaths(depsetIds []uint32) ([]string, error) {
  254. var inputPaths []string
  255. for _, id := range depsetIds {
  256. inputDepSetId := depsetId(id)
  257. depset := a.depsetIdToAqueryDepset[inputDepSetId]
  258. inputArtifacts, err := a.artifactPathsFromDepsetHash(depset.ContentHash)
  259. if err != nil {
  260. return nil, err
  261. }
  262. for _, inputPath := range inputArtifacts {
  263. inputPaths = append(inputPaths, inputPath)
  264. }
  265. }
  266. return inputPaths, nil
  267. }
  268. func (a *aqueryArtifactHandler) artifactPathsFromDepsetHash(depsetHash string) ([]string, error) {
  269. if result, exists := a.depsetHashToArtifactPathsCache.Load(depsetHash); exists {
  270. return result.([]string), nil
  271. }
  272. if depset, exists := a.depsetHashToAqueryDepset[depsetHash]; exists {
  273. result := depset.DirectArtifacts
  274. for _, childHash := range depset.TransitiveDepSetHashes {
  275. childArtifactIds, err := a.artifactPathsFromDepsetHash(childHash)
  276. if err != nil {
  277. return nil, err
  278. }
  279. result = append(result, childArtifactIds...)
  280. }
  281. a.depsetHashToArtifactPathsCache.Store(depsetHash, result)
  282. return result, nil
  283. } else {
  284. return nil, fmt.Errorf("undefined input depset hash %s", depsetHash)
  285. }
  286. }
  287. // AqueryBuildStatements returns a slice of BuildStatements and a slice of AqueryDepset
  288. // which should be registered (and output to a ninja file) to correspond with Bazel's
  289. // action graph, as described by the given action graph json proto.
  290. // BuildStatements are one-to-one with actions in the given action graph, and AqueryDepsets
  291. // are one-to-one with Bazel's depSetOfFiles objects.
  292. func AqueryBuildStatements(aqueryJsonProto []byte, eventHandler *metrics.EventHandler) ([]*BuildStatement, []AqueryDepset, error) {
  293. aqueryProto := &analysis_v2_proto.ActionGraphContainer{}
  294. err := proto.Unmarshal(aqueryJsonProto, aqueryProto)
  295. if err != nil {
  296. return nil, nil, err
  297. }
  298. var aqueryHandler *aqueryArtifactHandler
  299. {
  300. eventHandler.Begin("init_handler")
  301. defer eventHandler.End("init_handler")
  302. aqueryHandler, err = newAqueryHandler(aqueryProto)
  303. if err != nil {
  304. return nil, nil, err
  305. }
  306. }
  307. // allocate both length and capacity so each goroutine can write to an index independently without
  308. // any need for synchronization for slice access.
  309. buildStatements := make([]*BuildStatement, len(aqueryProto.Actions))
  310. {
  311. eventHandler.Begin("build_statements")
  312. defer eventHandler.End("build_statements")
  313. wg := sync.WaitGroup{}
  314. var errOnce sync.Once
  315. for i, actionEntry := range aqueryProto.Actions {
  316. wg.Add(1)
  317. go func(i int, actionEntry *analysis_v2_proto.Action) {
  318. buildStatement, aErr := aqueryHandler.actionToBuildStatement(actionEntry)
  319. if aErr != nil {
  320. errOnce.Do(func() {
  321. err = aErr
  322. })
  323. } else {
  324. // set build statement at an index rather than appending such that each goroutine does not
  325. // impact other goroutines
  326. buildStatements[i] = buildStatement
  327. }
  328. wg.Done()
  329. }(i, actionEntry)
  330. }
  331. wg.Wait()
  332. }
  333. if err != nil {
  334. return nil, nil, err
  335. }
  336. depsetsByHash := map[string]AqueryDepset{}
  337. depsets := make([]AqueryDepset, 0, len(aqueryHandler.depsetIdToAqueryDepset))
  338. {
  339. eventHandler.Begin("depsets")
  340. defer eventHandler.End("depsets")
  341. for _, aqueryDepset := range aqueryHandler.depsetIdToAqueryDepset {
  342. if prevEntry, hasKey := depsetsByHash[aqueryDepset.ContentHash]; hasKey {
  343. // Two depsets collide on hash. Ensure that their contents are identical.
  344. if !reflect.DeepEqual(aqueryDepset, prevEntry) {
  345. return nil, nil, fmt.Errorf("two different depsets have the same hash: %v, %v", prevEntry, aqueryDepset)
  346. }
  347. } else {
  348. depsetsByHash[aqueryDepset.ContentHash] = aqueryDepset
  349. depsets = append(depsets, aqueryDepset)
  350. }
  351. }
  352. }
  353. eventHandler.Do("build_statement_sort", func() {
  354. // Build Statements and depsets must be sorted by their content hash to
  355. // preserve determinism between builds (this will result in consistent ninja file
  356. // output). Note they are not sorted by their original IDs nor their Bazel ordering,
  357. // as Bazel gives nondeterministic ordering / identifiers in aquery responses.
  358. sort.Slice(buildStatements, func(i, j int) bool {
  359. // Sort all nil statements to the end of the slice
  360. if buildStatements[i] == nil {
  361. return false
  362. } else if buildStatements[j] == nil {
  363. return true
  364. }
  365. //For build statements, compare output lists. In Bazel, each output file
  366. // may only have one action which generates it, so this will provide
  367. // a deterministic ordering.
  368. outputs_i := buildStatements[i].OutputPaths
  369. outputs_j := buildStatements[j].OutputPaths
  370. if len(outputs_i) != len(outputs_j) {
  371. return len(outputs_i) < len(outputs_j)
  372. }
  373. if len(outputs_i) == 0 {
  374. // No outputs for these actions, so compare commands.
  375. return buildStatements[i].Command < buildStatements[j].Command
  376. }
  377. // There may be multiple outputs, but the output ordering is deterministic.
  378. return outputs_i[0] < outputs_j[0]
  379. })
  380. })
  381. eventHandler.Do("depset_sort", func() {
  382. sort.Slice(depsets, func(i, j int) bool {
  383. return depsets[i].ContentHash < depsets[j].ContentHash
  384. })
  385. })
  386. return buildStatements, depsets, nil
  387. }
  388. // depsetContentHash computes and returns a SHA256 checksum of the contents of
  389. // the given depset. This content hash may serve as the depset's identifier.
  390. // Using a content hash for an identifier is superior for determinism. (For example,
  391. // using an integer identifier which depends on the order in which the depsets are
  392. // created would result in nondeterministic depset IDs.)
  393. func depsetContentHash(directPaths []string, transitiveDepsetHashes []string) string {
  394. h := sha256.New()
  395. // Use newline as delimiter, as paths cannot contain newline.
  396. h.Write([]byte(strings.Join(directPaths, "\n")))
  397. h.Write([]byte(strings.Join(transitiveDepsetHashes, "")))
  398. fullHash := base64.RawURLEncoding.EncodeToString(h.Sum(nil))
  399. return fullHash
  400. }
  401. func (a *aqueryArtifactHandler) depsetContentHashes(inputDepsetIds []uint32) ([]string, error) {
  402. var hashes []string
  403. for _, id := range inputDepsetIds {
  404. dId := depsetId(id)
  405. if aqueryDepset, exists := a.depsetIdToAqueryDepset[dId]; !exists {
  406. if _, empty := a.emptyDepsetIds[dId]; !empty {
  407. return nil, fmt.Errorf("undefined (not even empty) input depsetId %d", dId)
  408. }
  409. } else {
  410. hashes = append(hashes, aqueryDepset.ContentHash)
  411. }
  412. }
  413. return hashes, nil
  414. }
  415. // escapes the args received from aquery and creates a command string
  416. func commandString(actionEntry *analysis_v2_proto.Action) string {
  417. switch actionEntry.Mnemonic {
  418. case "GoCompilePkg":
  419. argsEscaped := []string{}
  420. for _, arg := range actionEntry.Arguments {
  421. if arg == "" {
  422. // If this is an empty string, add ''
  423. // And not
  424. // 1. (literal empty)
  425. // 2. `''\'''\'''` (escaped version of '')
  426. //
  427. // If we had used (1), then this would appear as a whitespace when we strings.Join
  428. argsEscaped = append(argsEscaped, "''")
  429. } else {
  430. argsEscaped = append(argsEscaped, proptools.ShellEscapeIncludingSpaces(arg))
  431. }
  432. }
  433. return strings.Join(argsEscaped, " ")
  434. default:
  435. return strings.Join(proptools.ShellEscapeListIncludingSpaces(actionEntry.Arguments), " ")
  436. }
  437. }
  438. func (a *aqueryArtifactHandler) normalActionBuildStatement(actionEntry *analysis_v2_proto.Action) (*BuildStatement, error) {
  439. command := commandString(actionEntry)
  440. inputDepsetHashes, err := a.depsetContentHashes(actionEntry.InputDepSetIds)
  441. if err != nil {
  442. return nil, err
  443. }
  444. outputPaths, depfile, err := a.getOutputPaths(actionEntry)
  445. if err != nil {
  446. return nil, err
  447. }
  448. buildStatement := &BuildStatement{
  449. Command: command,
  450. Depfile: depfile,
  451. OutputPaths: outputPaths,
  452. InputDepsetHashes: inputDepsetHashes,
  453. Env: actionEntry.EnvironmentVariables,
  454. Mnemonic: actionEntry.Mnemonic,
  455. }
  456. return buildStatement, nil
  457. }
  458. func (a *aqueryArtifactHandler) templateExpandActionBuildStatement(actionEntry *analysis_v2_proto.Action) (*BuildStatement, error) {
  459. outputPaths, depfile, err := a.getOutputPaths(actionEntry)
  460. if err != nil {
  461. return nil, err
  462. }
  463. if len(outputPaths) != 1 {
  464. return nil, fmt.Errorf("Expect 1 output to template expand action, got: output %q", outputPaths)
  465. }
  466. expandedTemplateContent := expandTemplateContent(actionEntry)
  467. // The expandedTemplateContent is escaped for being used in double quotes and shell unescape,
  468. // and the new line characters (\n) are also changed to \\n which avoids some Ninja escape on \n, which might
  469. // change \n to space and mess up the format of Python programs.
  470. // sed is used to convert \\n back to \n before saving to output file.
  471. // See go/python-binary-host-mixed-build for more details.
  472. command := fmt.Sprintf(`/bin/bash -c 'echo "%[1]s" | sed "s/\\\\n/\\n/g" > %[2]s && chmod a+x %[2]s'`,
  473. escapeCommandlineArgument(expandedTemplateContent), outputPaths[0])
  474. inputDepsetHashes, err := a.depsetContentHashes(actionEntry.InputDepSetIds)
  475. if err != nil {
  476. return nil, err
  477. }
  478. buildStatement := &BuildStatement{
  479. Command: command,
  480. Depfile: depfile,
  481. OutputPaths: outputPaths,
  482. InputDepsetHashes: inputDepsetHashes,
  483. Env: actionEntry.EnvironmentVariables,
  484. Mnemonic: actionEntry.Mnemonic,
  485. }
  486. return buildStatement, nil
  487. }
  488. func (a *aqueryArtifactHandler) fileWriteActionBuildStatement(actionEntry *analysis_v2_proto.Action) (*BuildStatement, error) {
  489. outputPaths, _, err := a.getOutputPaths(actionEntry)
  490. var depsetHashes []string
  491. if err == nil {
  492. depsetHashes, err = a.depsetContentHashes(actionEntry.InputDepSetIds)
  493. }
  494. if err != nil {
  495. return nil, err
  496. }
  497. return &BuildStatement{
  498. Depfile: nil,
  499. OutputPaths: outputPaths,
  500. Env: actionEntry.EnvironmentVariables,
  501. Mnemonic: actionEntry.Mnemonic,
  502. InputDepsetHashes: depsetHashes,
  503. FileContents: actionEntry.FileContents,
  504. }, nil
  505. }
  506. func (a *aqueryArtifactHandler) symlinkTreeActionBuildStatement(actionEntry *analysis_v2_proto.Action) (*BuildStatement, error) {
  507. outputPaths, _, err := a.getOutputPaths(actionEntry)
  508. if err != nil {
  509. return nil, err
  510. }
  511. inputPaths, err := a.getInputPaths(actionEntry.InputDepSetIds)
  512. if err != nil {
  513. return nil, err
  514. }
  515. if len(inputPaths) != 1 || len(outputPaths) != 1 {
  516. return nil, fmt.Errorf("Expect 1 input and 1 output to symlink action, got: input %q, output %q", inputPaths, outputPaths)
  517. }
  518. // The actual command is generated in bazelSingleton.GenerateBuildActions
  519. return &BuildStatement{
  520. Depfile: nil,
  521. OutputPaths: outputPaths,
  522. Env: actionEntry.EnvironmentVariables,
  523. Mnemonic: actionEntry.Mnemonic,
  524. InputPaths: inputPaths,
  525. }, nil
  526. }
  527. func (a *aqueryArtifactHandler) symlinkActionBuildStatement(actionEntry *analysis_v2_proto.Action) (*BuildStatement, error) {
  528. outputPaths, depfile, err := a.getOutputPaths(actionEntry)
  529. if err != nil {
  530. return nil, err
  531. }
  532. inputPaths, err := a.getInputPaths(actionEntry.InputDepSetIds)
  533. if err != nil {
  534. return nil, err
  535. }
  536. if len(inputPaths) != 1 || len(outputPaths) != 1 {
  537. return nil, fmt.Errorf("Expect 1 input and 1 output to symlink action, got: input %q, output %q", inputPaths, outputPaths)
  538. }
  539. out := outputPaths[0]
  540. outDir := proptools.ShellEscapeIncludingSpaces(filepath.Dir(out))
  541. out = proptools.ShellEscapeIncludingSpaces(out)
  542. in := filepath.Join("$PWD", proptools.ShellEscapeIncludingSpaces(inputPaths[0]))
  543. // Use absolute paths, because some soong actions don't play well with relative paths (for example, `cp -d`).
  544. command := fmt.Sprintf("mkdir -p %[1]s && rm -f %[2]s && ln -sf %[3]s %[2]s", outDir, out, in)
  545. symlinkPaths := outputPaths[:]
  546. buildStatement := &BuildStatement{
  547. Command: command,
  548. Depfile: depfile,
  549. OutputPaths: outputPaths,
  550. InputPaths: inputPaths,
  551. Env: actionEntry.EnvironmentVariables,
  552. Mnemonic: actionEntry.Mnemonic,
  553. SymlinkPaths: symlinkPaths,
  554. }
  555. return buildStatement, nil
  556. }
  557. func (a *aqueryArtifactHandler) getOutputPaths(actionEntry *analysis_v2_proto.Action) (outputPaths []string, depfile *string, err error) {
  558. for _, outputId := range actionEntry.OutputIds {
  559. outputPath, exists := a.artifactIdToPath[artifactId(outputId)]
  560. if !exists {
  561. err = fmt.Errorf("undefined outputId %d", outputId)
  562. return
  563. }
  564. ext := filepath.Ext(outputPath)
  565. if ext == ".d" {
  566. if depfile != nil {
  567. err = fmt.Errorf("found multiple potential depfiles %q, %q", *depfile, outputPath)
  568. return
  569. } else {
  570. depfile = &outputPath
  571. }
  572. } else {
  573. outputPaths = append(outputPaths, outputPath)
  574. }
  575. }
  576. return
  577. }
  578. // expandTemplateContent substitutes the tokens in a template.
  579. func expandTemplateContent(actionEntry *analysis_v2_proto.Action) string {
  580. replacerString := make([]string, len(actionEntry.Substitutions)*2)
  581. for i, pair := range actionEntry.Substitutions {
  582. value := pair.Value
  583. if val, ok := templateActionOverriddenTokens[pair.Key]; ok {
  584. value = val
  585. }
  586. replacerString[i*2] = pair.Key
  587. replacerString[i*2+1] = value
  588. }
  589. replacer := strings.NewReplacer(replacerString...)
  590. return replacer.Replace(actionEntry.TemplateContent)
  591. }
  592. // \->\\, $->\$, `->\`, "->\", \n->\\n, '->'"'"'
  593. var commandLineArgumentReplacer = strings.NewReplacer(
  594. `\`, `\\`,
  595. `$`, `\$`,
  596. "`", "\\`",
  597. `"`, `\"`,
  598. "\n", "\\n",
  599. `'`, `'"'"'`,
  600. )
  601. func escapeCommandlineArgument(str string) string {
  602. return commandLineArgumentReplacer.Replace(str)
  603. }
  604. func (a *aqueryArtifactHandler) actionToBuildStatement(actionEntry *analysis_v2_proto.Action) (*BuildStatement, error) {
  605. switch actionEntry.Mnemonic {
  606. // Middleman actions are not handled like other actions; they are handled separately as a
  607. // preparatory step so that their inputs may be relayed to actions depending on middleman
  608. // artifacts.
  609. case middlemanMnemonic:
  610. return nil, nil
  611. // PythonZipper is bogus action returned by aquery, ignore it (b/236198693)
  612. case "PythonZipper":
  613. return nil, nil
  614. // Skip "Fail" actions, which are placeholder actions designed to always fail.
  615. case "Fail":
  616. return nil, nil
  617. case "BaselineCoverage":
  618. return nil, nil
  619. case "Symlink", "SolibSymlink", "ExecutableSymlink":
  620. return a.symlinkActionBuildStatement(actionEntry)
  621. case "TemplateExpand":
  622. if len(actionEntry.Arguments) < 1 {
  623. return a.templateExpandActionBuildStatement(actionEntry)
  624. }
  625. case "FileWrite", "SourceSymlinkManifest":
  626. return a.fileWriteActionBuildStatement(actionEntry)
  627. case "SymlinkTree":
  628. return a.symlinkTreeActionBuildStatement(actionEntry)
  629. }
  630. if len(actionEntry.Arguments) < 1 {
  631. return nil, fmt.Errorf("received action with no command: [%s]", actionEntry.Mnemonic)
  632. }
  633. return a.normalActionBuildStatement(actionEntry)
  634. }
  635. func expandPathFragment(id pathFragmentId, pathFragmentsMap map[pathFragmentId]*analysis_v2_proto.PathFragment) (string, error) {
  636. var labels []string
  637. currId := id
  638. // Only positive IDs are valid for path fragments. An ID of zero indicates a terminal node.
  639. for currId > 0 {
  640. currFragment, ok := pathFragmentsMap[currId]
  641. if !ok {
  642. return "", fmt.Errorf("undefined path fragment id %d", currId)
  643. }
  644. labels = append([]string{currFragment.Label}, labels...)
  645. parentId := pathFragmentId(currFragment.ParentId)
  646. if currId == parentId {
  647. return "", fmt.Errorf("fragment cannot refer to itself as parent %#v", currFragment)
  648. }
  649. currId = parentId
  650. }
  651. return filepath.Join(labels...), nil
  652. }