aquery.go 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656
  1. // Copyright 2020 Google Inc. All rights reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package bazel
  15. import (
  16. "crypto/sha256"
  17. "encoding/base64"
  18. "encoding/json"
  19. "fmt"
  20. "path/filepath"
  21. "reflect"
  22. "sort"
  23. "strings"
  24. "github.com/google/blueprint/proptools"
  25. )
  26. type artifactId int
  27. type depsetId int
  28. type pathFragmentId int
  29. // artifact contains relevant portions of Bazel's aquery proto, Artifact.
  30. // Represents a single artifact, whether it's a source file or a derived output file.
  31. type artifact struct {
  32. Id artifactId
  33. PathFragmentId pathFragmentId
  34. }
  35. type pathFragment struct {
  36. Id pathFragmentId
  37. Label string
  38. ParentId pathFragmentId
  39. }
  40. // KeyValuePair represents Bazel's aquery proto, KeyValuePair.
  41. type KeyValuePair struct {
  42. Key string
  43. Value string
  44. }
  45. // AqueryDepset is a depset definition from Bazel's aquery response. This is
  46. // akin to the `depSetOfFiles` in the response proto, except:
  47. // - direct artifacts are enumerated by full path instead of by ID
  48. // - it has a hash of the depset contents, instead of an int ID (for determinism)
  49. //
  50. // A depset is a data structure for efficient transitive handling of artifact
  51. // paths. A single depset consists of one or more artifact paths and one or
  52. // more "child" depsets.
  53. type AqueryDepset struct {
  54. ContentHash string
  55. DirectArtifacts []string
  56. TransitiveDepSetHashes []string
  57. }
  58. // depSetOfFiles contains relevant portions of Bazel's aquery proto, DepSetOfFiles.
  59. // Represents a data structure containing one or more files. Depsets in Bazel are an efficient
  60. // data structure for storing large numbers of file paths.
  61. type depSetOfFiles struct {
  62. Id depsetId
  63. DirectArtifactIds []artifactId
  64. TransitiveDepSetIds []depsetId
  65. }
  66. // action contains relevant portions of Bazel's aquery proto, Action.
  67. // Represents a single command line invocation in the Bazel build graph.
  68. type action struct {
  69. Arguments []string
  70. EnvironmentVariables []KeyValuePair
  71. InputDepSetIds []depsetId
  72. Mnemonic string
  73. OutputIds []artifactId
  74. TemplateContent string
  75. Substitutions []KeyValuePair
  76. FileContents string
  77. }
  78. // actionGraphContainer contains relevant portions of Bazel's aquery proto, ActionGraphContainer.
  79. // An aquery response from Bazel contains a single ActionGraphContainer proto.
  80. type actionGraphContainer struct {
  81. Artifacts []artifact
  82. Actions []action
  83. DepSetOfFiles []depSetOfFiles
  84. PathFragments []pathFragment
  85. }
  86. // BuildStatement contains information to register a build statement corresponding (one to one)
  87. // with a Bazel action from Bazel's action graph.
  88. type BuildStatement struct {
  89. Command string
  90. Depfile *string
  91. OutputPaths []string
  92. SymlinkPaths []string
  93. Env []KeyValuePair
  94. Mnemonic string
  95. // Inputs of this build statement, either as unexpanded depsets or expanded
  96. // input paths. There should be no overlap between these fields; an input
  97. // path should either be included as part of an unexpanded depset or a raw
  98. // input path string, but not both.
  99. InputDepsetHashes []string
  100. InputPaths []string
  101. FileContents string
  102. }
  103. // A helper type for aquery processing which facilitates retrieval of path IDs from their
  104. // less readable Bazel structures (depset and path fragment).
  105. type aqueryArtifactHandler struct {
  106. // Switches to true if any depset contains only `bazelToolsDependencySentinel`
  107. bazelToolsDependencySentinelNeeded bool
  108. // Maps depset id to AqueryDepset, a representation of depset which is
  109. // post-processed for middleman artifact handling, unhandled artifact
  110. // dropping, content hashing, etc.
  111. depsetIdToAqueryDepset map[depsetId]AqueryDepset
  112. // Maps content hash to AqueryDepset.
  113. depsetHashToAqueryDepset map[string]AqueryDepset
  114. // depsetIdToArtifactIdsCache is a memoization of depset flattening, because flattening
  115. // may be an expensive operation.
  116. depsetHashToArtifactPathsCache map[string][]string
  117. // Maps artifact ids to fully expanded paths.
  118. artifactIdToPath map[artifactId]string
  119. }
  120. // The tokens should be substituted with the value specified here, instead of the
  121. // one returned in 'substitutions' of TemplateExpand action.
  122. var templateActionOverriddenTokens = map[string]string{
  123. // Uses "python3" for %python_binary% instead of the value returned by aquery
  124. // which is "py3wrapper.sh". See removePy3wrapperScript.
  125. "%python_binary%": "python3",
  126. }
  127. // The file name of py3wrapper.sh, which is used by py_binary targets.
  128. const py3wrapperFileName = "/py3wrapper.sh"
  129. // A file to be put into depsets that are otherwise empty
  130. const bazelToolsDependencySentinel = "BAZEL_TOOLS_DEPENDENCY_SENTINEL"
  131. func indexBy[K comparable, V any](values []V, keyFn func(v V) K) map[K]V {
  132. m := map[K]V{}
  133. for _, v := range values {
  134. m[keyFn(v)] = v
  135. }
  136. return m
  137. }
  138. func newAqueryHandler(aqueryResult actionGraphContainer) (*aqueryArtifactHandler, error) {
  139. pathFragments := indexBy(aqueryResult.PathFragments, func(pf pathFragment) pathFragmentId {
  140. return pf.Id
  141. })
  142. artifactIdToPath := map[artifactId]string{}
  143. for _, artifact := range aqueryResult.Artifacts {
  144. artifactPath, err := expandPathFragment(artifact.PathFragmentId, pathFragments)
  145. if err != nil {
  146. return nil, err
  147. }
  148. artifactIdToPath[artifact.Id] = artifactPath
  149. }
  150. // Map middleman artifact ContentHash to input artifact depset ID.
  151. // Middleman artifacts are treated as "substitute" artifacts for mixed builds. For example,
  152. // if we find a middleman action which has inputs [foo, bar], and output [baz_middleman], then,
  153. // for each other action which has input [baz_middleman], we add [foo, bar] to the inputs for
  154. // that action instead.
  155. middlemanIdToDepsetIds := map[artifactId][]depsetId{}
  156. for _, actionEntry := range aqueryResult.Actions {
  157. if actionEntry.Mnemonic == "Middleman" {
  158. for _, outputId := range actionEntry.OutputIds {
  159. middlemanIdToDepsetIds[outputId] = actionEntry.InputDepSetIds
  160. }
  161. }
  162. }
  163. depsetIdToDepset := indexBy(aqueryResult.DepSetOfFiles, func(d depSetOfFiles) depsetId {
  164. return d.Id
  165. })
  166. aqueryHandler := aqueryArtifactHandler{
  167. depsetIdToAqueryDepset: map[depsetId]AqueryDepset{},
  168. depsetHashToAqueryDepset: map[string]AqueryDepset{},
  169. depsetHashToArtifactPathsCache: map[string][]string{},
  170. artifactIdToPath: artifactIdToPath,
  171. }
  172. // Validate and adjust aqueryResult.DepSetOfFiles values.
  173. for _, depset := range aqueryResult.DepSetOfFiles {
  174. _, err := aqueryHandler.populateDepsetMaps(depset, middlemanIdToDepsetIds, depsetIdToDepset)
  175. if err != nil {
  176. return nil, err
  177. }
  178. }
  179. return &aqueryHandler, nil
  180. }
  181. // Ensures that the handler's depsetIdToAqueryDepset map contains an entry for the given
  182. // depset.
  183. func (a *aqueryArtifactHandler) populateDepsetMaps(depset depSetOfFiles, middlemanIdToDepsetIds map[artifactId][]depsetId, depsetIdToDepset map[depsetId]depSetOfFiles) (AqueryDepset, error) {
  184. if aqueryDepset, containsDepset := a.depsetIdToAqueryDepset[depset.Id]; containsDepset {
  185. return aqueryDepset, nil
  186. }
  187. transitiveDepsetIds := depset.TransitiveDepSetIds
  188. var directArtifactPaths []string
  189. for _, artifactId := range depset.DirectArtifactIds {
  190. path, pathExists := a.artifactIdToPath[artifactId]
  191. if !pathExists {
  192. return AqueryDepset{}, fmt.Errorf("undefined input artifactId %d", artifactId)
  193. }
  194. // Filter out any inputs which are universally dropped, and swap middleman
  195. // artifacts with their corresponding depsets.
  196. if depsetsToUse, isMiddleman := middlemanIdToDepsetIds[artifactId]; isMiddleman {
  197. // Swap middleman artifacts with their corresponding depsets and drop the middleman artifacts.
  198. transitiveDepsetIds = append(transitiveDepsetIds, depsetsToUse...)
  199. } else if strings.HasSuffix(path, py3wrapperFileName) ||
  200. strings.HasPrefix(path, "../bazel_tools") {
  201. // Drop these artifacts.
  202. // See go/python-binary-host-mixed-build for more details.
  203. // 1) Drop py3wrapper.sh, just use python binary, the launcher script generated by the
  204. // TemplateExpandAction handles everything necessary to launch a Pythin application.
  205. // 2) ../bazel_tools: they have MODIFY timestamp 10years in the future and would cause the
  206. // containing depset to always be considered newer than their outputs.
  207. } else {
  208. directArtifactPaths = append(directArtifactPaths, path)
  209. }
  210. }
  211. var childDepsetHashes []string
  212. for _, childDepsetId := range transitiveDepsetIds {
  213. childDepset, exists := depsetIdToDepset[childDepsetId]
  214. if !exists {
  215. return AqueryDepset{}, fmt.Errorf("undefined input depsetId %d (referenced by depsetId %d)", childDepsetId, depset.Id)
  216. }
  217. childAqueryDepset, err := a.populateDepsetMaps(childDepset, middlemanIdToDepsetIds, depsetIdToDepset)
  218. if err != nil {
  219. return AqueryDepset{}, err
  220. }
  221. childDepsetHashes = append(childDepsetHashes, childAqueryDepset.ContentHash)
  222. }
  223. if len(directArtifactPaths) == 0 && len(childDepsetHashes) == 0 {
  224. // We could omit this depset altogether but that requires cleanup on
  225. // transitive dependents.
  226. // As a simpler alternative, we use this sentinel file as a dependency.
  227. directArtifactPaths = append(directArtifactPaths, bazelToolsDependencySentinel)
  228. a.bazelToolsDependencySentinelNeeded = true
  229. }
  230. aqueryDepset := AqueryDepset{
  231. ContentHash: depsetContentHash(directArtifactPaths, childDepsetHashes),
  232. DirectArtifacts: directArtifactPaths,
  233. TransitiveDepSetHashes: childDepsetHashes,
  234. }
  235. a.depsetIdToAqueryDepset[depset.Id] = aqueryDepset
  236. a.depsetHashToAqueryDepset[aqueryDepset.ContentHash] = aqueryDepset
  237. return aqueryDepset, nil
  238. }
  239. // getInputPaths flattens the depsets of the given IDs and returns all transitive
  240. // input paths contained in these depsets.
  241. // This is a potentially expensive operation, and should not be invoked except
  242. // for actions which need specialized input handling.
  243. func (a *aqueryArtifactHandler) getInputPaths(depsetIds []depsetId) ([]string, error) {
  244. var inputPaths []string
  245. for _, inputDepSetId := range depsetIds {
  246. depset := a.depsetIdToAqueryDepset[inputDepSetId]
  247. inputArtifacts, err := a.artifactPathsFromDepsetHash(depset.ContentHash)
  248. if err != nil {
  249. return nil, err
  250. }
  251. for _, inputPath := range inputArtifacts {
  252. inputPaths = append(inputPaths, inputPath)
  253. }
  254. }
  255. return inputPaths, nil
  256. }
  257. func (a *aqueryArtifactHandler) artifactPathsFromDepsetHash(depsetHash string) ([]string, error) {
  258. if result, exists := a.depsetHashToArtifactPathsCache[depsetHash]; exists {
  259. return result, nil
  260. }
  261. if depset, exists := a.depsetHashToAqueryDepset[depsetHash]; exists {
  262. result := depset.DirectArtifacts
  263. for _, childHash := range depset.TransitiveDepSetHashes {
  264. childArtifactIds, err := a.artifactPathsFromDepsetHash(childHash)
  265. if err != nil {
  266. return nil, err
  267. }
  268. result = append(result, childArtifactIds...)
  269. }
  270. a.depsetHashToArtifactPathsCache[depsetHash] = result
  271. return result, nil
  272. } else {
  273. return nil, fmt.Errorf("undefined input depset hash %s", depsetHash)
  274. }
  275. }
  276. // AqueryBuildStatements returns a slice of BuildStatements and a slice of AqueryDepset
  277. // which should be registered (and output to a ninja file) to correspond with Bazel's
  278. // action graph, as described by the given action graph json proto.
  279. // BuildStatements are one-to-one with actions in the given action graph, and AqueryDepsets
  280. // are one-to-one with Bazel's depSetOfFiles objects.
  281. func AqueryBuildStatements(aqueryJsonProto []byte) ([]BuildStatement, []AqueryDepset, error) {
  282. var aqueryResult actionGraphContainer
  283. err := json.Unmarshal(aqueryJsonProto, &aqueryResult)
  284. if err != nil {
  285. return nil, nil, err
  286. }
  287. aqueryHandler, err := newAqueryHandler(aqueryResult)
  288. if err != nil {
  289. return nil, nil, err
  290. }
  291. var buildStatements []BuildStatement
  292. if aqueryHandler.bazelToolsDependencySentinelNeeded {
  293. buildStatements = append(buildStatements, BuildStatement{
  294. Command: fmt.Sprintf("touch '%s'", bazelToolsDependencySentinel),
  295. OutputPaths: []string{bazelToolsDependencySentinel},
  296. Mnemonic: bazelToolsDependencySentinel,
  297. })
  298. }
  299. for _, actionEntry := range aqueryResult.Actions {
  300. if shouldSkipAction(actionEntry) {
  301. continue
  302. }
  303. var buildStatement BuildStatement
  304. if actionEntry.isSymlinkAction() {
  305. buildStatement, err = aqueryHandler.symlinkActionBuildStatement(actionEntry)
  306. } else if actionEntry.isTemplateExpandAction() && len(actionEntry.Arguments) < 1 {
  307. buildStatement, err = aqueryHandler.templateExpandActionBuildStatement(actionEntry)
  308. } else if actionEntry.isFileWriteAction() {
  309. buildStatement, err = aqueryHandler.fileWriteActionBuildStatement(actionEntry)
  310. } else if actionEntry.isSymlinkTreeAction() {
  311. buildStatement, err = aqueryHandler.symlinkTreeActionBuildStatement(actionEntry)
  312. } else if len(actionEntry.Arguments) < 1 {
  313. return nil, nil, fmt.Errorf("received action with no command: [%s]", actionEntry.Mnemonic)
  314. } else {
  315. buildStatement, err = aqueryHandler.normalActionBuildStatement(actionEntry)
  316. }
  317. if err != nil {
  318. return nil, nil, err
  319. }
  320. buildStatements = append(buildStatements, buildStatement)
  321. }
  322. depsetsByHash := map[string]AqueryDepset{}
  323. var depsets []AqueryDepset
  324. for _, aqueryDepset := range aqueryHandler.depsetIdToAqueryDepset {
  325. if prevEntry, hasKey := depsetsByHash[aqueryDepset.ContentHash]; hasKey {
  326. // Two depsets collide on hash. Ensure that their contents are identical.
  327. if !reflect.DeepEqual(aqueryDepset, prevEntry) {
  328. return nil, nil, fmt.Errorf("two different depsets have the same hash: %v, %v", prevEntry, aqueryDepset)
  329. }
  330. } else {
  331. depsetsByHash[aqueryDepset.ContentHash] = aqueryDepset
  332. depsets = append(depsets, aqueryDepset)
  333. }
  334. }
  335. // Build Statements and depsets must be sorted by their content hash to
  336. // preserve determinism between builds (this will result in consistent ninja file
  337. // output). Note they are not sorted by their original IDs nor their Bazel ordering,
  338. // as Bazel gives nondeterministic ordering / identifiers in aquery responses.
  339. sort.Slice(buildStatements, func(i, j int) bool {
  340. // For build statements, compare output lists. In Bazel, each output file
  341. // may only have one action which generates it, so this will provide
  342. // a deterministic ordering.
  343. outputs_i := buildStatements[i].OutputPaths
  344. outputs_j := buildStatements[j].OutputPaths
  345. if len(outputs_i) != len(outputs_j) {
  346. return len(outputs_i) < len(outputs_j)
  347. }
  348. if len(outputs_i) == 0 {
  349. // No outputs for these actions, so compare commands.
  350. return buildStatements[i].Command < buildStatements[j].Command
  351. }
  352. // There may be multiple outputs, but the output ordering is deterministic.
  353. return outputs_i[0] < outputs_j[0]
  354. })
  355. sort.Slice(depsets, func(i, j int) bool {
  356. return depsets[i].ContentHash < depsets[j].ContentHash
  357. })
  358. return buildStatements, depsets, nil
  359. }
  360. // depsetContentHash computes and returns a SHA256 checksum of the contents of
  361. // the given depset. This content hash may serve as the depset's identifier.
  362. // Using a content hash for an identifier is superior for determinism. (For example,
  363. // using an integer identifier which depends on the order in which the depsets are
  364. // created would result in nondeterministic depset IDs.)
  365. func depsetContentHash(directPaths []string, transitiveDepsetHashes []string) string {
  366. h := sha256.New()
  367. // Use newline as delimiter, as paths cannot contain newline.
  368. h.Write([]byte(strings.Join(directPaths, "\n")))
  369. h.Write([]byte(strings.Join(transitiveDepsetHashes, "")))
  370. fullHash := base64.RawURLEncoding.EncodeToString(h.Sum(nil))
  371. return fullHash
  372. }
  373. func (a *aqueryArtifactHandler) depsetContentHashes(inputDepsetIds []depsetId) ([]string, error) {
  374. var hashes []string
  375. for _, depsetId := range inputDepsetIds {
  376. if aqueryDepset, exists := a.depsetIdToAqueryDepset[depsetId]; !exists {
  377. return nil, fmt.Errorf("undefined input depsetId %d", depsetId)
  378. } else {
  379. hashes = append(hashes, aqueryDepset.ContentHash)
  380. }
  381. }
  382. return hashes, nil
  383. }
  384. func (a *aqueryArtifactHandler) normalActionBuildStatement(actionEntry action) (BuildStatement, error) {
  385. command := strings.Join(proptools.ShellEscapeListIncludingSpaces(actionEntry.Arguments), " ")
  386. inputDepsetHashes, err := a.depsetContentHashes(actionEntry.InputDepSetIds)
  387. if err != nil {
  388. return BuildStatement{}, err
  389. }
  390. outputPaths, depfile, err := a.getOutputPaths(actionEntry)
  391. if err != nil {
  392. return BuildStatement{}, err
  393. }
  394. buildStatement := BuildStatement{
  395. Command: command,
  396. Depfile: depfile,
  397. OutputPaths: outputPaths,
  398. InputDepsetHashes: inputDepsetHashes,
  399. Env: actionEntry.EnvironmentVariables,
  400. Mnemonic: actionEntry.Mnemonic,
  401. }
  402. return buildStatement, nil
  403. }
  404. func (a *aqueryArtifactHandler) templateExpandActionBuildStatement(actionEntry action) (BuildStatement, error) {
  405. outputPaths, depfile, err := a.getOutputPaths(actionEntry)
  406. if err != nil {
  407. return BuildStatement{}, err
  408. }
  409. if len(outputPaths) != 1 {
  410. return BuildStatement{}, fmt.Errorf("Expect 1 output to template expand action, got: output %q", outputPaths)
  411. }
  412. expandedTemplateContent := expandTemplateContent(actionEntry)
  413. // The expandedTemplateContent is escaped for being used in double quotes and shell unescape,
  414. // and the new line characters (\n) are also changed to \\n which avoids some Ninja escape on \n, which might
  415. // change \n to space and mess up the format of Python programs.
  416. // sed is used to convert \\n back to \n before saving to output file.
  417. // See go/python-binary-host-mixed-build for more details.
  418. command := fmt.Sprintf(`/bin/bash -c 'echo "%[1]s" | sed "s/\\\\n/\\n/g" > %[2]s && chmod a+x %[2]s'`,
  419. escapeCommandlineArgument(expandedTemplateContent), outputPaths[0])
  420. inputDepsetHashes, err := a.depsetContentHashes(actionEntry.InputDepSetIds)
  421. if err != nil {
  422. return BuildStatement{}, err
  423. }
  424. buildStatement := BuildStatement{
  425. Command: command,
  426. Depfile: depfile,
  427. OutputPaths: outputPaths,
  428. InputDepsetHashes: inputDepsetHashes,
  429. Env: actionEntry.EnvironmentVariables,
  430. Mnemonic: actionEntry.Mnemonic,
  431. }
  432. return buildStatement, nil
  433. }
  434. func (a *aqueryArtifactHandler) fileWriteActionBuildStatement(actionEntry action) (BuildStatement, error) {
  435. outputPaths, _, err := a.getOutputPaths(actionEntry)
  436. var depsetHashes []string
  437. if err == nil {
  438. depsetHashes, err = a.depsetContentHashes(actionEntry.InputDepSetIds)
  439. }
  440. if err != nil {
  441. return BuildStatement{}, err
  442. }
  443. return BuildStatement{
  444. Depfile: nil,
  445. OutputPaths: outputPaths,
  446. Env: actionEntry.EnvironmentVariables,
  447. Mnemonic: actionEntry.Mnemonic,
  448. InputDepsetHashes: depsetHashes,
  449. FileContents: actionEntry.FileContents,
  450. }, nil
  451. }
  452. func (a *aqueryArtifactHandler) symlinkTreeActionBuildStatement(actionEntry action) (BuildStatement, error) {
  453. outputPaths, _, err := a.getOutputPaths(actionEntry)
  454. if err != nil {
  455. return BuildStatement{}, err
  456. }
  457. inputPaths, err := a.getInputPaths(actionEntry.InputDepSetIds)
  458. if err != nil {
  459. return BuildStatement{}, err
  460. }
  461. if len(inputPaths) != 1 || len(outputPaths) != 1 {
  462. return BuildStatement{}, fmt.Errorf("Expect 1 input and 1 output to symlink action, got: input %q, output %q", inputPaths, outputPaths)
  463. }
  464. // The actual command is generated in bazelSingleton.GenerateBuildActions
  465. return BuildStatement{
  466. Depfile: nil,
  467. OutputPaths: outputPaths,
  468. Env: actionEntry.EnvironmentVariables,
  469. Mnemonic: actionEntry.Mnemonic,
  470. InputPaths: inputPaths,
  471. }, nil
  472. }
  473. func (a *aqueryArtifactHandler) symlinkActionBuildStatement(actionEntry action) (BuildStatement, error) {
  474. outputPaths, depfile, err := a.getOutputPaths(actionEntry)
  475. if err != nil {
  476. return BuildStatement{}, err
  477. }
  478. inputPaths, err := a.getInputPaths(actionEntry.InputDepSetIds)
  479. if err != nil {
  480. return BuildStatement{}, err
  481. }
  482. if len(inputPaths) != 1 || len(outputPaths) != 1 {
  483. return BuildStatement{}, fmt.Errorf("Expect 1 input and 1 output to symlink action, got: input %q, output %q", inputPaths, outputPaths)
  484. }
  485. out := outputPaths[0]
  486. outDir := proptools.ShellEscapeIncludingSpaces(filepath.Dir(out))
  487. out = proptools.ShellEscapeIncludingSpaces(out)
  488. in := filepath.Join("$PWD", proptools.ShellEscapeIncludingSpaces(inputPaths[0]))
  489. // Use absolute paths, because some soong actions don't play well with relative paths (for example, `cp -d`).
  490. command := fmt.Sprintf("mkdir -p %[1]s && rm -f %[2]s && ln -sf %[3]s %[2]s", outDir, out, in)
  491. symlinkPaths := outputPaths[:]
  492. buildStatement := BuildStatement{
  493. Command: command,
  494. Depfile: depfile,
  495. OutputPaths: outputPaths,
  496. InputPaths: inputPaths,
  497. Env: actionEntry.EnvironmentVariables,
  498. Mnemonic: actionEntry.Mnemonic,
  499. SymlinkPaths: symlinkPaths,
  500. }
  501. return buildStatement, nil
  502. }
  503. func (a *aqueryArtifactHandler) getOutputPaths(actionEntry action) (outputPaths []string, depfile *string, err error) {
  504. for _, outputId := range actionEntry.OutputIds {
  505. outputPath, exists := a.artifactIdToPath[outputId]
  506. if !exists {
  507. err = fmt.Errorf("undefined outputId %d", outputId)
  508. return
  509. }
  510. ext := filepath.Ext(outputPath)
  511. if ext == ".d" {
  512. if depfile != nil {
  513. err = fmt.Errorf("found multiple potential depfiles %q, %q", *depfile, outputPath)
  514. return
  515. } else {
  516. depfile = &outputPath
  517. }
  518. } else {
  519. outputPaths = append(outputPaths, outputPath)
  520. }
  521. }
  522. return
  523. }
  524. // expandTemplateContent substitutes the tokens in a template.
  525. func expandTemplateContent(actionEntry action) string {
  526. var replacerString []string
  527. for _, pair := range actionEntry.Substitutions {
  528. value := pair.Value
  529. if val, ok := templateActionOverriddenTokens[pair.Key]; ok {
  530. value = val
  531. }
  532. replacerString = append(replacerString, pair.Key, value)
  533. }
  534. replacer := strings.NewReplacer(replacerString...)
  535. return replacer.Replace(actionEntry.TemplateContent)
  536. }
  537. func escapeCommandlineArgument(str string) string {
  538. // \->\\, $->\$, `->\`, "->\", \n->\\n, '->'"'"'
  539. replacer := strings.NewReplacer(
  540. `\`, `\\`,
  541. `$`, `\$`,
  542. "`", "\\`",
  543. `"`, `\"`,
  544. "\n", "\\n",
  545. `'`, `'"'"'`,
  546. )
  547. return replacer.Replace(str)
  548. }
  549. func (a action) isSymlinkAction() bool {
  550. return a.Mnemonic == "Symlink" || a.Mnemonic == "SolibSymlink" || a.Mnemonic == "ExecutableSymlink"
  551. }
  552. func (a action) isTemplateExpandAction() bool {
  553. return a.Mnemonic == "TemplateExpand"
  554. }
  555. func (a action) isFileWriteAction() bool {
  556. return a.Mnemonic == "FileWrite" || a.Mnemonic == "SourceSymlinkManifest"
  557. }
  558. func (a action) isSymlinkTreeAction() bool {
  559. return a.Mnemonic == "SymlinkTree"
  560. }
  561. func shouldSkipAction(a action) bool {
  562. // Middleman actions are not handled like other actions; they are handled separately as a
  563. // preparatory step so that their inputs may be relayed to actions depending on middleman
  564. // artifacts.
  565. if a.Mnemonic == "Middleman" {
  566. return true
  567. }
  568. // PythonZipper is bogus action returned by aquery, ignore it (b/236198693)
  569. if a.Mnemonic == "PythonZipper" {
  570. return true
  571. }
  572. // Skip "Fail" actions, which are placeholder actions designed to always fail.
  573. if a.Mnemonic == "Fail" {
  574. return true
  575. }
  576. if a.Mnemonic == "BaselineCoverage" {
  577. return true
  578. }
  579. return false
  580. }
  581. func expandPathFragment(id pathFragmentId, pathFragmentsMap map[pathFragmentId]pathFragment) (string, error) {
  582. var labels []string
  583. currId := id
  584. // Only positive IDs are valid for path fragments. An ID of zero indicates a terminal node.
  585. for currId > 0 {
  586. currFragment, ok := pathFragmentsMap[currId]
  587. if !ok {
  588. return "", fmt.Errorf("undefined path fragment id %d", currId)
  589. }
  590. labels = append([]string{currFragment.Label}, labels...)
  591. if currId == currFragment.ParentId {
  592. return "", fmt.Errorf("fragment cannot refer to itself as parent %#v", currFragment)
  593. }
  594. currId = currFragment.ParentId
  595. }
  596. return filepath.Join(labels...), nil
  597. }