aquery_test.go 37 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226
  1. // Copyright 2020 Google Inc. All rights reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package bazel
  15. import (
  16. "encoding/json"
  17. "fmt"
  18. "reflect"
  19. "sort"
  20. "testing"
  21. analysis_v2_proto "prebuilts/bazel/common/proto/analysis_v2"
  22. "github.com/google/blueprint/metrics"
  23. "google.golang.org/protobuf/proto"
  24. )
  25. func TestAqueryMultiArchGenrule(t *testing.T) {
  26. // This input string is retrieved from a real build of bionic-related genrules.
  27. const inputString = `
  28. {
  29. "Artifacts": [
  30. { "Id": 1, "path_fragment_id": 1 },
  31. { "Id": 2, "path_fragment_id": 6 },
  32. { "Id": 3, "path_fragment_id": 8 },
  33. { "Id": 4, "path_fragment_id": 12 },
  34. { "Id": 5, "path_fragment_id": 19 },
  35. { "Id": 6, "path_fragment_id": 20 },
  36. { "Id": 7, "path_fragment_id": 21 }],
  37. "Actions": [{
  38. "target_id": 1,
  39. "action_key": "ab53f6ecbdc2ee8cb8812613b63205464f1f5083f6dca87081a0a398c0f1ecf7",
  40. "Mnemonic": "Genrule",
  41. "configuration_id": 1,
  42. "Arguments": ["/bin/bash", "-c", "source ../bazel_tools/tools/genrule/genrule-setup.sh; ../sourceroot/bionic/libc/tools/gensyscalls.py arm ../sourceroot/bionic/libc/SYSCALLS.TXT \u003e bazel-out/sourceroot/k8-fastbuild/bin/bionic/libc/syscalls-arm.S"],
  43. "environment_variables": [{
  44. "Key": "PATH",
  45. "Value": "/bin:/usr/bin:/usr/local/bin"
  46. }],
  47. "input_dep_set_ids": [1],
  48. "output_ids": [4],
  49. "primary_output_id": 4
  50. }, {
  51. "target_id": 2,
  52. "action_key": "9f4309ce165dac458498cb92811c18b0b7919782cc37b82a42d2141b8cc90826",
  53. "Mnemonic": "Genrule",
  54. "configuration_id": 1,
  55. "Arguments": ["/bin/bash", "-c", "source ../bazel_tools/tools/genrule/genrule-setup.sh; ../sourceroot/bionic/libc/tools/gensyscalls.py x86 ../sourceroot/bionic/libc/SYSCALLS.TXT \u003e bazel-out/sourceroot/k8-fastbuild/bin/bionic/libc/syscalls-x86.S"],
  56. "environment_variables": [{
  57. "Key": "PATH",
  58. "Value": "/bin:/usr/bin:/usr/local/bin"
  59. }],
  60. "input_dep_set_ids": [2],
  61. "output_ids": [5],
  62. "primary_output_id": 5
  63. }, {
  64. "target_id": 3,
  65. "action_key": "50d6c586103ebeed3a218195540bcc30d329464eae36377eb82f8ce7c36ac342",
  66. "Mnemonic": "Genrule",
  67. "configuration_id": 1,
  68. "Arguments": ["/bin/bash", "-c", "source ../bazel_tools/tools/genrule/genrule-setup.sh; ../sourceroot/bionic/libc/tools/gensyscalls.py x86_64 ../sourceroot/bionic/libc/SYSCALLS.TXT \u003e bazel-out/sourceroot/k8-fastbuild/bin/bionic/libc/syscalls-x86_64.S"],
  69. "environment_variables": [{
  70. "Key": "PATH",
  71. "Value": "/bin:/usr/bin:/usr/local/bin"
  72. }],
  73. "input_dep_set_ids": [3],
  74. "output_ids": [6],
  75. "primary_output_id": 6
  76. }, {
  77. "target_id": 4,
  78. "action_key": "f30cbe442f5216f4223cf16a39112cad4ec56f31f49290d85cff587e48647ffa",
  79. "Mnemonic": "Genrule",
  80. "configuration_id": 1,
  81. "Arguments": ["/bin/bash", "-c", "source ../bazel_tools/tools/genrule/genrule-setup.sh; ../sourceroot/bionic/libc/tools/gensyscalls.py arm64 ../sourceroot/bionic/libc/SYSCALLS.TXT \u003e bazel-out/sourceroot/k8-fastbuild/bin/bionic/libc/syscalls-arm64.S"],
  82. "environment_variables": [{
  83. "Key": "PATH",
  84. "Value": "/bin:/usr/bin:/usr/local/bin"
  85. }],
  86. "input_dep_set_ids": [4],
  87. "output_ids": [7],
  88. "primary_output_id": 7
  89. }],
  90. "Targets": [
  91. { "Id": 1, "Label": "@sourceroot//bionic/libc:syscalls-arm", "rule_class_id": 1 },
  92. { "Id": 2, "Label": "@sourceroot//bionic/libc:syscalls-x86", "rule_class_id": 1 },
  93. { "Id": 3, "Label": "@sourceroot//bionic/libc:syscalls-x86_64", "rule_class_id": 1 },
  94. { "Id": 4, "Label": "@sourceroot//bionic/libc:syscalls-arm64", "rule_class_id": 1 }],
  95. "dep_set_of_files": [
  96. { "Id": 1, "direct_artifact_ids": [1, 2, 3] },
  97. { "Id": 2, "direct_artifact_ids": [1, 2, 3] },
  98. { "Id": 3, "direct_artifact_ids": [1, 2, 3] },
  99. { "Id": 4, "direct_artifact_ids": [1, 2, 3] }],
  100. "Configuration": [{
  101. "Id": 1,
  102. "Mnemonic": "k8-fastbuild",
  103. "platform_name": "k8",
  104. "Checksum": "485c362832c178e367d972177f68e69e0981e51e67ef1c160944473db53fe046"
  105. }],
  106. "rule_classes": [{ "Id": 1, "Name": "genrule"}],
  107. "path_fragments": [
  108. { "Id": 5, "Label": ".." },
  109. { "Id": 4, "Label": "sourceroot", "parent_id": 5 },
  110. { "Id": 3, "Label": "bionic", "parent_id": 4 },
  111. { "Id": 2, "Label": "libc", "parent_id": 3 },
  112. { "Id": 1, "Label": "SYSCALLS.TXT", "parent_id": 2 },
  113. { "Id": 7, "Label": "tools", "parent_id": 2 },
  114. { "Id": 6, "Label": "gensyscalls.py", "parent_id": 7 },
  115. { "Id": 11, "Label": "bazel_tools", "parent_id": 5 },
  116. { "Id": 10, "Label": "tools", "parent_id": 11 },
  117. { "Id": 9, "Label": "genrule", "parent_id": 10 },
  118. { "Id": 8, "Label": "genrule-setup.sh", "parent_id": 9 },
  119. { "Id": 18, "Label": "bazel-out" },
  120. { "Id": 17, "Label": "sourceroot", "parent_id": 18 },
  121. { "Id": 16, "Label": "k8-fastbuild", "parent_id": 17 },
  122. { "Id": 15, "Label": "bin", "parent_id": 16 },
  123. { "Id": 14, "Label": "bionic", "parent_id": 15 },
  124. { "Id": 13, "Label": "libc", "parent_id": 14 },
  125. { "Id": 12, "Label": "syscalls-arm.S", "parent_id": 13 },
  126. { "Id": 19, "Label": "syscalls-x86.S", "parent_id": 13 },
  127. { "Id": 20, "Label": "syscalls-x86_64.S", "parent_id": 13 },
  128. { "Id": 21, "Label": "syscalls-arm64.S", "parent_id": 13 }]
  129. }
  130. `
  131. data, err := JsonToActionGraphContainer(inputString)
  132. if err != nil {
  133. t.Error(err)
  134. return
  135. }
  136. actualbuildStatements, actualDepsets, _ := AqueryBuildStatements(data, &metrics.EventHandler{})
  137. var expectedBuildStatements []*BuildStatement
  138. for _, arch := range []string{"arm", "arm64", "x86", "x86_64"} {
  139. expectedBuildStatements = append(expectedBuildStatements,
  140. &BuildStatement{
  141. Command: fmt.Sprintf(
  142. "/bin/bash -c 'source ../bazel_tools/tools/genrule/genrule-setup.sh; ../sourceroot/bionic/libc/tools/gensyscalls.py %s ../sourceroot/bionic/libc/SYSCALLS.TXT > bazel-out/sourceroot/k8-fastbuild/bin/bionic/libc/syscalls-%s.S'",
  143. arch, arch),
  144. OutputPaths: []string{
  145. fmt.Sprintf("bazel-out/sourceroot/k8-fastbuild/bin/bionic/libc/syscalls-%s.S", arch),
  146. },
  147. Env: []*analysis_v2_proto.KeyValuePair{
  148. {Key: "PATH", Value: "/bin:/usr/bin:/usr/local/bin"},
  149. },
  150. Mnemonic: "Genrule",
  151. })
  152. }
  153. assertBuildStatements(t, expectedBuildStatements, actualbuildStatements)
  154. expectedFlattenedInputs := []string{
  155. "../sourceroot/bionic/libc/SYSCALLS.TXT",
  156. "../sourceroot/bionic/libc/tools/gensyscalls.py",
  157. }
  158. // In this example, each depset should have the same expected inputs.
  159. for _, actualDepset := range actualDepsets {
  160. actualFlattenedInputs := flattenDepsets([]string{actualDepset.ContentHash}, actualDepsets)
  161. if !reflect.DeepEqual(actualFlattenedInputs, expectedFlattenedInputs) {
  162. t.Errorf("Expected flattened inputs %v, but got %v", expectedFlattenedInputs, actualFlattenedInputs)
  163. }
  164. }
  165. }
  166. func TestInvalidOutputId(t *testing.T) {
  167. const inputString = `
  168. {
  169. "artifacts": [
  170. { "id": 1, "path_fragment_id": 1 },
  171. { "id": 2, "path_fragment_id": 2 }],
  172. "actions": [{
  173. "target_id": 1,
  174. "action_key": "x",
  175. "mnemonic": "x",
  176. "arguments": ["touch", "foo"],
  177. "input_dep_set_ids": [1],
  178. "output_ids": [3],
  179. "primary_output_id": 3
  180. }],
  181. "dep_set_of_files": [
  182. { "id": 1, "direct_artifact_ids": [1, 2] }],
  183. "path_fragments": [
  184. { "id": 1, "label": "one" },
  185. { "id": 2, "label": "two" }]
  186. }`
  187. data, err := JsonToActionGraphContainer(inputString)
  188. if err != nil {
  189. t.Error(err)
  190. return
  191. }
  192. _, _, err = AqueryBuildStatements(data, &metrics.EventHandler{})
  193. assertError(t, err, "undefined outputId 3")
  194. }
  195. func TestInvalidInputDepsetIdFromAction(t *testing.T) {
  196. const inputString = `
  197. {
  198. "artifacts": [
  199. { "id": 1, "path_fragment_id": 1 },
  200. { "id": 2, "path_fragment_id": 2 }],
  201. "actions": [{
  202. "target_id": 1,
  203. "action_key": "x",
  204. "mnemonic": "x",
  205. "arguments": ["touch", "foo"],
  206. "input_dep_set_ids": [2],
  207. "output_ids": [1],
  208. "primary_output_id": 1
  209. }],
  210. "dep_set_of_files": [
  211. { "id": 1, "direct_artifact_ids": [1, 2] }],
  212. "path_fragments": [
  213. { "id": 1, "label": "one" },
  214. { "id": 2, "label": "two" }]
  215. }`
  216. data, err := JsonToActionGraphContainer(inputString)
  217. if err != nil {
  218. t.Error(err)
  219. return
  220. }
  221. _, _, err = AqueryBuildStatements(data, &metrics.EventHandler{})
  222. assertError(t, err, "undefined (not even empty) input depsetId 2")
  223. }
  224. func TestInvalidInputDepsetIdFromDepset(t *testing.T) {
  225. const inputString = `
  226. {
  227. "artifacts": [
  228. { "id": 1, "path_fragment_id": 1 },
  229. { "id": 2, "path_fragment_id": 2 }],
  230. "actions": [{
  231. "target_id": 1,
  232. "action_key": "x",
  233. "mnemonic": "x",
  234. "arguments": ["touch", "foo"],
  235. "input_dep_set_ids": [1],
  236. "output_ids": [1],
  237. "primary_output_id": 1
  238. }],
  239. "dep_set_of_files": [
  240. { "id": 1, "direct_artifact_ids": [1, 2], "transitive_dep_set_ids": [42] }],
  241. "path_fragments": [
  242. { "id": 1, "label": "one"},
  243. { "id": 2, "label": "two" }]
  244. }`
  245. data, err := JsonToActionGraphContainer(inputString)
  246. if err != nil {
  247. t.Error(err)
  248. return
  249. }
  250. _, _, err = AqueryBuildStatements(data, &metrics.EventHandler{})
  251. assertError(t, err, "undefined input depsetId 42 (referenced by depsetId 1)")
  252. }
  253. func TestInvalidInputArtifactId(t *testing.T) {
  254. const inputString = `
  255. {
  256. "artifacts": [
  257. { "id": 1, "path_fragment_id": 1 },
  258. { "id": 2, "path_fragment_id": 2 }],
  259. "actions": [{
  260. "target_id": 1,
  261. "action_key": "x",
  262. "mnemonic": "x",
  263. "arguments": ["touch", "foo"],
  264. "input_dep_set_ids": [1],
  265. "output_ids": [1],
  266. "primary_output_id": 1
  267. }],
  268. "dep_set_of_files": [
  269. { "id": 1, "direct_artifact_ids": [1, 3] }],
  270. "path_fragments": [
  271. { "id": 1, "label": "one" },
  272. { "id": 2, "label": "two" }]
  273. }`
  274. data, err := JsonToActionGraphContainer(inputString)
  275. if err != nil {
  276. t.Error(err)
  277. return
  278. }
  279. _, _, err = AqueryBuildStatements(data, &metrics.EventHandler{})
  280. assertError(t, err, "undefined input artifactId 3")
  281. }
  282. func TestInvalidPathFragmentId(t *testing.T) {
  283. const inputString = `
  284. {
  285. "artifacts": [
  286. { "id": 1, "path_fragment_id": 1 },
  287. { "id": 2, "path_fragment_id": 2 }],
  288. "actions": [{
  289. "target_id": 1,
  290. "action_key": "x",
  291. "mnemonic": "x",
  292. "arguments": ["touch", "foo"],
  293. "input_dep_set_ids": [1],
  294. "output_ids": [1],
  295. "primary_output_id": 1
  296. }],
  297. "dep_set_of_files": [
  298. { "id": 1, "direct_artifact_ids": [1, 2] }],
  299. "path_fragments": [
  300. { "id": 1, "label": "one" },
  301. { "id": 2, "label": "two", "parent_id": 3 }]
  302. }`
  303. data, err := JsonToActionGraphContainer(inputString)
  304. if err != nil {
  305. t.Error(err)
  306. return
  307. }
  308. _, _, err = AqueryBuildStatements(data, &metrics.EventHandler{})
  309. assertError(t, err, "undefined path fragment id 3")
  310. }
  311. func TestDepfiles(t *testing.T) {
  312. const inputString = `
  313. {
  314. "artifacts": [
  315. { "id": 1, "path_fragment_id": 1 },
  316. { "id": 2, "path_fragment_id": 2 },
  317. { "id": 3, "path_fragment_id": 3 }],
  318. "actions": [{
  319. "target_Id": 1,
  320. "action_Key": "x",
  321. "mnemonic": "x",
  322. "arguments": ["touch", "foo"],
  323. "input_dep_set_ids": [1],
  324. "output_ids": [2, 3],
  325. "primary_output_id": 2
  326. }],
  327. "dep_set_of_files": [
  328. { "id": 1, "direct_Artifact_Ids": [1, 2, 3] }],
  329. "path_fragments": [
  330. { "id": 1, "label": "one" },
  331. { "id": 2, "label": "two" },
  332. { "id": 3, "label": "two.d" }]
  333. }`
  334. data, err := JsonToActionGraphContainer(inputString)
  335. if err != nil {
  336. t.Error(err)
  337. return
  338. }
  339. actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{})
  340. if err != nil {
  341. t.Errorf("Unexpected error %q", err)
  342. }
  343. if expected := 1; len(actual) != expected {
  344. t.Fatalf("Expected %d build statements, got %d", expected, len(actual))
  345. }
  346. bs := actual[0]
  347. expectedDepfile := "two.d"
  348. if bs.Depfile == nil {
  349. t.Errorf("Expected depfile %q, but there was none found", expectedDepfile)
  350. } else if *bs.Depfile != expectedDepfile {
  351. t.Errorf("Expected depfile %q, but got %q", expectedDepfile, *bs.Depfile)
  352. }
  353. }
  354. func TestMultipleDepfiles(t *testing.T) {
  355. const inputString = `
  356. {
  357. "artifacts": [
  358. { "id": 1, "path_fragment_id": 1 },
  359. { "id": 2, "path_fragment_id": 2 },
  360. { "id": 3, "path_fragment_id": 3 },
  361. { "id": 4, "path_fragment_id": 4 }],
  362. "actions": [{
  363. "target_id": 1,
  364. "action_key": "x",
  365. "mnemonic": "x",
  366. "arguments": ["touch", "foo"],
  367. "input_dep_set_ids": [1],
  368. "output_ids": [2,3,4],
  369. "primary_output_id": 2
  370. }],
  371. "dep_set_of_files": [{
  372. "id": 1,
  373. "direct_artifact_ids": [1, 2, 3, 4]
  374. }],
  375. "path_fragments": [
  376. { "id": 1, "label": "one" },
  377. { "id": 2, "label": "two" },
  378. { "id": 3, "label": "two.d" },
  379. { "id": 4, "label": "other.d" }]
  380. }`
  381. data, err := JsonToActionGraphContainer(inputString)
  382. if err != nil {
  383. t.Error(err)
  384. return
  385. }
  386. _, _, err = AqueryBuildStatements(data, &metrics.EventHandler{})
  387. assertError(t, err, `found multiple potential depfiles "two.d", "other.d"`)
  388. }
  389. func TestTransitiveInputDepsets(t *testing.T) {
  390. // The input aquery for this test comes from a proof-of-concept starlark rule which registers
  391. // a single action with many inputs given via a deep depset.
  392. const inputString = `
  393. {
  394. "artifacts": [
  395. { "id": 1, "path_fragment_id": 1 },
  396. { "id": 2, "path_fragment_id": 7 },
  397. { "id": 3, "path_fragment_id": 8 },
  398. { "id": 4, "path_fragment_id": 9 },
  399. { "id": 5, "path_fragment_id": 10 },
  400. { "id": 6, "path_fragment_id": 11 },
  401. { "id": 7, "path_fragment_id": 12 },
  402. { "id": 8, "path_fragment_id": 13 },
  403. { "id": 9, "path_fragment_id": 14 },
  404. { "id": 10, "path_fragment_id": 15 },
  405. { "id": 11, "path_fragment_id": 16 },
  406. { "id": 12, "path_fragment_id": 17 },
  407. { "id": 13, "path_fragment_id": 18 },
  408. { "id": 14, "path_fragment_id": 19 },
  409. { "id": 15, "path_fragment_id": 20 },
  410. { "id": 16, "path_fragment_id": 21 },
  411. { "id": 17, "path_fragment_id": 22 },
  412. { "id": 18, "path_fragment_id": 23 },
  413. { "id": 19, "path_fragment_id": 24 },
  414. { "id": 20, "path_fragment_id": 25 },
  415. { "id": 21, "path_fragment_id": 26 }],
  416. "actions": [{
  417. "target_id": 1,
  418. "action_key": "3b826d17fadbbbcd8313e456b90ec47c078c438088891dd45b4adbcd8889dc50",
  419. "mnemonic": "Action",
  420. "configuration_id": 1,
  421. "arguments": ["/bin/bash", "-c", "touch bazel-out/sourceroot/k8-fastbuild/bin/testpkg/test_out"],
  422. "input_dep_set_ids": [1],
  423. "output_ids": [21],
  424. "primary_output_id": 21
  425. }],
  426. "dep_set_of_files": [
  427. { "id": 3, "direct_artifact_ids": [1, 2, 3, 4, 5] },
  428. { "id": 4, "direct_artifact_ids": [6, 7, 8, 9, 10] },
  429. { "id": 2, "transitive_dep_set_ids": [3, 4], "direct_artifact_ids": [11, 12, 13, 14, 15] },
  430. { "id": 5, "direct_artifact_ids": [16, 17, 18, 19] },
  431. { "id": 1, "transitive_dep_set_ids": [2, 5], "direct_artifact_ids": [20] }],
  432. "path_fragments": [
  433. { "id": 6, "label": "bazel-out" },
  434. { "id": 5, "label": "sourceroot", "parent_id": 6 },
  435. { "id": 4, "label": "k8-fastbuild", "parent_id": 5 },
  436. { "id": 3, "label": "bin", "parent_id": 4 },
  437. { "id": 2, "label": "testpkg", "parent_id": 3 },
  438. { "id": 1, "label": "test_1", "parent_id": 2 },
  439. { "id": 7, "label": "test_2", "parent_id": 2 },
  440. { "id": 8, "label": "test_3", "parent_id": 2 },
  441. { "id": 9, "label": "test_4", "parent_id": 2 },
  442. { "id": 10, "label": "test_5", "parent_id": 2 },
  443. { "id": 11, "label": "test_6", "parent_id": 2 },
  444. { "id": 12, "label": "test_7", "parent_id": 2 },
  445. { "id": 13, "label": "test_8", "parent_id": 2 },
  446. { "id": 14, "label": "test_9", "parent_id": 2 },
  447. { "id": 15, "label": "test_10", "parent_id": 2 },
  448. { "id": 16, "label": "test_11", "parent_id": 2 },
  449. { "id": 17, "label": "test_12", "parent_id": 2 },
  450. { "id": 18, "label": "test_13", "parent_id": 2 },
  451. { "id": 19, "label": "test_14", "parent_id": 2 },
  452. { "id": 20, "label": "test_15", "parent_id": 2 },
  453. { "id": 21, "label": "test_16", "parent_id": 2 },
  454. { "id": 22, "label": "test_17", "parent_id": 2 },
  455. { "id": 23, "label": "test_18", "parent_id": 2 },
  456. { "id": 24, "label": "test_19", "parent_id": 2 },
  457. { "id": 25, "label": "test_root", "parent_id": 2 },
  458. { "id": 26,"label": "test_out", "parent_id": 2 }]
  459. }`
  460. data, err := JsonToActionGraphContainer(inputString)
  461. if err != nil {
  462. t.Error(err)
  463. return
  464. }
  465. actualbuildStatements, actualDepsets, _ := AqueryBuildStatements(data, &metrics.EventHandler{})
  466. expectedBuildStatements := []*BuildStatement{
  467. &BuildStatement{
  468. Command: "/bin/bash -c 'touch bazel-out/sourceroot/k8-fastbuild/bin/testpkg/test_out'",
  469. OutputPaths: []string{"bazel-out/sourceroot/k8-fastbuild/bin/testpkg/test_out"},
  470. Mnemonic: "Action",
  471. SymlinkPaths: []string{},
  472. },
  473. }
  474. assertBuildStatements(t, expectedBuildStatements, actualbuildStatements)
  475. // Inputs for the action are test_{i} from 1 to 20, and test_root. These inputs
  476. // are given via a deep depset, but the depset is flattened when returned as a
  477. // BuildStatement slice.
  478. var expectedFlattenedInputs []string
  479. for i := 1; i < 20; i++ {
  480. expectedFlattenedInputs = append(expectedFlattenedInputs, fmt.Sprintf("bazel-out/sourceroot/k8-fastbuild/bin/testpkg/test_%d", i))
  481. }
  482. expectedFlattenedInputs = append(expectedFlattenedInputs, "bazel-out/sourceroot/k8-fastbuild/bin/testpkg/test_root")
  483. actualDepsetHashes := actualbuildStatements[0].InputDepsetHashes
  484. actualFlattenedInputs := flattenDepsets(actualDepsetHashes, actualDepsets)
  485. if !reflect.DeepEqual(actualFlattenedInputs, expectedFlattenedInputs) {
  486. t.Errorf("Expected flattened inputs %v, but got %v", expectedFlattenedInputs, actualFlattenedInputs)
  487. }
  488. }
  489. func TestSymlinkTree(t *testing.T) {
  490. const inputString = `
  491. {
  492. "artifacts": [
  493. { "id": 1, "path_fragment_id": 1 },
  494. { "id": 2, "path_fragment_id": 2 }],
  495. "actions": [{
  496. "target_id": 1,
  497. "action_key": "x",
  498. "mnemonic": "SymlinkTree",
  499. "configuration_id": 1,
  500. "input_dep_set_ids": [1],
  501. "output_ids": [2],
  502. "primary_output_id": 2,
  503. "execution_platform": "//build/bazel/platforms:linux_x86_64"
  504. }],
  505. "path_fragments": [
  506. { "id": 1, "label": "foo.manifest" },
  507. { "id": 2, "label": "foo.runfiles/MANIFEST" }],
  508. "dep_set_of_files": [
  509. { "id": 1, "direct_artifact_ids": [1] }]
  510. }
  511. `
  512. data, err := JsonToActionGraphContainer(inputString)
  513. if err != nil {
  514. t.Error(err)
  515. return
  516. }
  517. actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{})
  518. if err != nil {
  519. t.Errorf("Unexpected error %q", err)
  520. }
  521. assertBuildStatements(t, []*BuildStatement{
  522. &BuildStatement{
  523. Command: "",
  524. OutputPaths: []string{"foo.runfiles/MANIFEST"},
  525. Mnemonic: "SymlinkTree",
  526. InputPaths: []string{"foo.manifest"},
  527. SymlinkPaths: []string{},
  528. },
  529. }, actual)
  530. }
  531. func TestBazelOutRemovalFromInputDepsets(t *testing.T) {
  532. const inputString = `{
  533. "artifacts": [
  534. { "id": 1, "path_fragment_id": 10 },
  535. { "id": 2, "path_fragment_id": 20 },
  536. { "id": 3, "path_fragment_id": 30 },
  537. { "id": 4, "path_fragment_id": 40 }],
  538. "dep_set_of_files": [{
  539. "id": 1111,
  540. "direct_artifact_ids": [3 , 4]
  541. }, {
  542. "id": 2222,
  543. "direct_artifact_ids": [3]
  544. }],
  545. "actions": [{
  546. "target_id": 100,
  547. "action_key": "x",
  548. "input_dep_set_ids": [1111, 2222],
  549. "mnemonic": "x",
  550. "arguments": ["bogus", "command"],
  551. "output_ids": [2],
  552. "primary_output_id": 1
  553. }],
  554. "path_fragments": [
  555. { "id": 10, "label": "input" },
  556. { "id": 20, "label": "output" },
  557. { "id": 30, "label": "dep1", "parent_id": 50 },
  558. { "id": 40, "label": "dep2", "parent_id": 60 },
  559. { "id": 50, "label": "bazel_tools", "parent_id": 60 },
  560. { "id": 60, "label": ".."}
  561. ]
  562. }`
  563. /* depsets
  564. 1111 2222
  565. / \ |
  566. ../dep2 ../bazel_tools/dep1
  567. */
  568. data, err := JsonToActionGraphContainer(inputString)
  569. if err != nil {
  570. t.Error(err)
  571. return
  572. }
  573. actualBuildStatements, actualDepsets, _ := AqueryBuildStatements(data, &metrics.EventHandler{})
  574. if len(actualDepsets) != 1 {
  575. t.Errorf("expected 1 depset but found %#v", actualDepsets)
  576. return
  577. }
  578. dep2Found := false
  579. for _, dep := range flattenDepsets([]string{actualDepsets[0].ContentHash}, actualDepsets) {
  580. if dep == "../bazel_tools/dep1" {
  581. t.Errorf("dependency %s expected to be removed but still exists", dep)
  582. } else if dep == "../dep2" {
  583. dep2Found = true
  584. }
  585. }
  586. if !dep2Found {
  587. t.Errorf("dependency ../dep2 expected but not found")
  588. }
  589. expectedBuildStatement := &BuildStatement{
  590. Command: "bogus command",
  591. OutputPaths: []string{"output"},
  592. Mnemonic: "x",
  593. SymlinkPaths: []string{},
  594. }
  595. buildStatementFound := false
  596. for _, actualBuildStatement := range actualBuildStatements {
  597. if buildStatementEquals(actualBuildStatement, expectedBuildStatement) == "" {
  598. buildStatementFound = true
  599. break
  600. }
  601. }
  602. if !buildStatementFound {
  603. t.Errorf("expected but missing %#v in %#v", expectedBuildStatement, actualBuildStatements)
  604. return
  605. }
  606. }
  607. func TestBazelOutRemovalFromTransitiveInputDepsets(t *testing.T) {
  608. const inputString = `{
  609. "artifacts": [
  610. { "id": 1, "path_fragment_id": 10 },
  611. { "id": 2, "path_fragment_id": 20 },
  612. { "id": 3, "path_fragment_id": 30 }],
  613. "dep_set_of_files": [{
  614. "id": 1111,
  615. "transitive_dep_set_ids": [2222]
  616. }, {
  617. "id": 2222,
  618. "direct_artifact_ids": [3]
  619. }, {
  620. "id": 3333,
  621. "direct_artifact_ids": [3]
  622. }, {
  623. "id": 4444,
  624. "transitive_dep_set_ids": [3333]
  625. }],
  626. "actions": [{
  627. "target_id": 100,
  628. "action_key": "x",
  629. "input_dep_set_ids": [1111, 4444],
  630. "mnemonic": "x",
  631. "arguments": ["bogus", "command"],
  632. "output_ids": [2],
  633. "primary_output_id": 1
  634. }],
  635. "path_fragments": [
  636. { "id": 10, "label": "input" },
  637. { "id": 20, "label": "output" },
  638. { "id": 30, "label": "dep", "parent_id": 50 },
  639. { "id": 50, "label": "bazel_tools", "parent_id": 60 },
  640. { "id": 60, "label": ".."}
  641. ]
  642. }`
  643. /* depsets
  644. 1111 4444
  645. || ||
  646. 2222 3333
  647. | |
  648. ../bazel_tools/dep
  649. Note: in dep_set_of_files:
  650. 1111 appears BEFORE its dependency,2222 while
  651. 4444 appears AFTER its dependency 3333
  652. and this test shows that that order doesn't affect empty depset pruning
  653. */
  654. data, err := JsonToActionGraphContainer(inputString)
  655. if err != nil {
  656. t.Error(err)
  657. return
  658. }
  659. actualBuildStatements, actualDepsets, _ := AqueryBuildStatements(data, &metrics.EventHandler{})
  660. if len(actualDepsets) != 0 {
  661. t.Errorf("expected 0 depsets but found %#v", actualDepsets)
  662. return
  663. }
  664. expectedBuildStatement := &BuildStatement{
  665. Command: "bogus command",
  666. OutputPaths: []string{"output"},
  667. Mnemonic: "x",
  668. }
  669. buildStatementFound := false
  670. for _, actualBuildStatement := range actualBuildStatements {
  671. if buildStatementEquals(actualBuildStatement, expectedBuildStatement) == "" {
  672. buildStatementFound = true
  673. break
  674. }
  675. }
  676. if !buildStatementFound {
  677. t.Errorf("expected but missing %#v in %#v", expectedBuildStatement, actualBuildStatements)
  678. return
  679. }
  680. }
  681. func TestMiddlemenAction(t *testing.T) {
  682. const inputString = `
  683. {
  684. "artifacts": [
  685. { "id": 1, "path_fragment_id": 1 },
  686. { "id": 2, "path_fragment_id": 2 },
  687. { "id": 3, "path_fragment_id": 3 },
  688. { "id": 4, "path_fragment_id": 4 },
  689. { "id": 5, "path_fragment_id": 5 },
  690. { "id": 6, "path_fragment_id": 6 }],
  691. "path_fragments": [
  692. { "id": 1, "label": "middleinput_one" },
  693. { "id": 2, "label": "middleinput_two" },
  694. { "id": 3, "label": "middleman_artifact" },
  695. { "id": 4, "label": "maininput_one" },
  696. { "id": 5, "label": "maininput_two" },
  697. { "id": 6, "label": "output" }],
  698. "dep_set_of_files": [
  699. { "id": 1, "direct_artifact_ids": [1, 2] },
  700. { "id": 2, "direct_artifact_ids": [3, 4, 5] }],
  701. "actions": [{
  702. "target_id": 1,
  703. "action_key": "x",
  704. "mnemonic": "Middleman",
  705. "arguments": ["touch", "foo"],
  706. "input_dep_set_ids": [1],
  707. "output_ids": [3],
  708. "primary_output_id": 3
  709. }, {
  710. "target_id": 2,
  711. "action_key": "y",
  712. "mnemonic": "Main action",
  713. "arguments": ["touch", "foo"],
  714. "input_dep_set_ids": [2],
  715. "output_ids": [6],
  716. "primary_output_id": 6
  717. }]
  718. }`
  719. data, err := JsonToActionGraphContainer(inputString)
  720. if err != nil {
  721. t.Error(err)
  722. return
  723. }
  724. actualBuildStatements, actualDepsets, err := AqueryBuildStatements(data, &metrics.EventHandler{})
  725. if err != nil {
  726. t.Errorf("Unexpected error %q", err)
  727. }
  728. if expected := 2; len(actualBuildStatements) != expected {
  729. t.Fatalf("Expected %d build statements, got %d %#v", expected, len(actualBuildStatements), actualBuildStatements)
  730. }
  731. expectedDepsetFiles := [][]string{
  732. {"middleinput_one", "middleinput_two", "maininput_one", "maininput_two"},
  733. {"middleinput_one", "middleinput_two"},
  734. }
  735. assertFlattenedDepsets(t, actualDepsets, expectedDepsetFiles)
  736. bs := actualBuildStatements[0]
  737. if len(bs.InputPaths) > 0 {
  738. t.Errorf("Expected main action raw inputs to be empty, but got %q", bs.InputPaths)
  739. }
  740. expectedOutputs := []string{"output"}
  741. if !reflect.DeepEqual(bs.OutputPaths, expectedOutputs) {
  742. t.Errorf("Expected main action outputs %q, but got %q", expectedOutputs, bs.OutputPaths)
  743. }
  744. expectedFlattenedInputs := []string{"middleinput_one", "middleinput_two", "maininput_one", "maininput_two"}
  745. actualFlattenedInputs := flattenDepsets(bs.InputDepsetHashes, actualDepsets)
  746. if !reflect.DeepEqual(actualFlattenedInputs, expectedFlattenedInputs) {
  747. t.Errorf("Expected flattened inputs %v, but got %v", expectedFlattenedInputs, actualFlattenedInputs)
  748. }
  749. bs = actualBuildStatements[1]
  750. if bs != nil {
  751. t.Errorf("Expected nil action for skipped")
  752. }
  753. }
  754. // Returns the contents of given depsets in concatenated post order.
  755. func flattenDepsets(depsetHashesToFlatten []string, allDepsets []AqueryDepset) []string {
  756. depsetsByHash := map[string]AqueryDepset{}
  757. for _, depset := range allDepsets {
  758. depsetsByHash[depset.ContentHash] = depset
  759. }
  760. var result []string
  761. for _, depsetId := range depsetHashesToFlatten {
  762. result = append(result, flattenDepset(depsetId, depsetsByHash)...)
  763. }
  764. return result
  765. }
  766. // Returns the contents of a given depset in post order.
  767. func flattenDepset(depsetHashToFlatten string, allDepsets map[string]AqueryDepset) []string {
  768. depset := allDepsets[depsetHashToFlatten]
  769. var result []string
  770. for _, depsetId := range depset.TransitiveDepSetHashes {
  771. result = append(result, flattenDepset(depsetId, allDepsets)...)
  772. }
  773. result = append(result, depset.DirectArtifacts...)
  774. return result
  775. }
  776. func assertFlattenedDepsets(t *testing.T, actualDepsets []AqueryDepset, expectedDepsetFiles [][]string) {
  777. t.Helper()
  778. if len(actualDepsets) != len(expectedDepsetFiles) {
  779. t.Errorf("Expected %d depsets, but got %d depsets", len(expectedDepsetFiles), len(actualDepsets))
  780. }
  781. for i, actualDepset := range actualDepsets {
  782. actualFlattenedInputs := flattenDepsets([]string{actualDepset.ContentHash}, actualDepsets)
  783. if !reflect.DeepEqual(actualFlattenedInputs, expectedDepsetFiles[i]) {
  784. t.Errorf("Expected depset files: %v, but got %v", expectedDepsetFiles[i], actualFlattenedInputs)
  785. }
  786. }
  787. }
  788. func TestSimpleSymlink(t *testing.T) {
  789. const inputString = `
  790. {
  791. "artifacts": [
  792. { "id": 1, "path_fragment_id": 3 },
  793. { "id": 2, "path_fragment_id": 5 }],
  794. "actions": [{
  795. "target_id": 1,
  796. "action_key": "x",
  797. "mnemonic": "Symlink",
  798. "input_dep_set_ids": [1],
  799. "output_ids": [2],
  800. "primary_output_id": 2
  801. }],
  802. "dep_set_of_files": [
  803. { "id": 1, "direct_artifact_ids": [1] }],
  804. "path_fragments": [
  805. { "id": 1, "label": "one" },
  806. { "id": 2, "label": "file_subdir", "parent_id": 1 },
  807. { "id": 3, "label": "file", "parent_id": 2 },
  808. { "id": 4, "label": "symlink_subdir", "parent_id": 1 },
  809. { "id": 5, "label": "symlink", "parent_id": 4 }]
  810. }`
  811. data, err := JsonToActionGraphContainer(inputString)
  812. if err != nil {
  813. t.Error(err)
  814. return
  815. }
  816. actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{})
  817. if err != nil {
  818. t.Errorf("Unexpected error %q", err)
  819. }
  820. expectedBuildStatements := []*BuildStatement{
  821. &BuildStatement{
  822. Command: "mkdir -p one/symlink_subdir && " +
  823. "rm -f one/symlink_subdir/symlink && " +
  824. "ln -sf $PWD/one/file_subdir/file one/symlink_subdir/symlink",
  825. InputPaths: []string{"one/file_subdir/file"},
  826. OutputPaths: []string{"one/symlink_subdir/symlink"},
  827. SymlinkPaths: []string{"one/symlink_subdir/symlink"},
  828. Mnemonic: "Symlink",
  829. },
  830. }
  831. assertBuildStatements(t, actual, expectedBuildStatements)
  832. }
  833. func TestSymlinkQuotesPaths(t *testing.T) {
  834. const inputString = `
  835. {
  836. "artifacts": [
  837. { "id": 1, "path_fragment_id": 3 },
  838. { "id": 2, "path_fragment_id": 5 }],
  839. "actions": [{
  840. "target_id": 1,
  841. "action_key": "x",
  842. "mnemonic": "SolibSymlink",
  843. "input_dep_set_ids": [1],
  844. "output_ids": [2],
  845. "primary_output_id": 2
  846. }],
  847. "dep_set_of_files": [
  848. { "id": 1, "direct_artifact_ids": [1] }],
  849. "path_fragments": [
  850. { "id": 1, "label": "one" },
  851. { "id": 2, "label": "file subdir", "parent_id": 1 },
  852. { "id": 3, "label": "file", "parent_id": 2 },
  853. { "id": 4, "label": "symlink subdir", "parent_id": 1 },
  854. { "id": 5, "label": "symlink", "parent_id": 4 }]
  855. }`
  856. data, err := JsonToActionGraphContainer(inputString)
  857. if err != nil {
  858. t.Error(err)
  859. return
  860. }
  861. actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{})
  862. if err != nil {
  863. t.Errorf("Unexpected error %q", err)
  864. }
  865. expectedBuildStatements := []*BuildStatement{
  866. &BuildStatement{
  867. Command: "mkdir -p 'one/symlink subdir' && " +
  868. "rm -f 'one/symlink subdir/symlink' && " +
  869. "ln -sf $PWD/'one/file subdir/file' 'one/symlink subdir/symlink'",
  870. InputPaths: []string{"one/file subdir/file"},
  871. OutputPaths: []string{"one/symlink subdir/symlink"},
  872. SymlinkPaths: []string{"one/symlink subdir/symlink"},
  873. Mnemonic: "SolibSymlink",
  874. },
  875. }
  876. assertBuildStatements(t, expectedBuildStatements, actual)
  877. }
  878. func TestSymlinkMultipleInputs(t *testing.T) {
  879. const inputString = `
  880. {
  881. "artifacts": [
  882. { "id": 1, "path_fragment_id": 1 },
  883. { "id": 2, "path_fragment_id": 2 },
  884. { "id": 3, "path_fragment_id": 3 }],
  885. "actions": [{
  886. "target_id": 1,
  887. "action_key": "x",
  888. "mnemonic": "Symlink",
  889. "input_dep_set_ids": [1],
  890. "output_ids": [3],
  891. "primary_output_id": 3
  892. }],
  893. "dep_set_of_files": [{ "id": 1, "direct_artifact_ids": [1,2] }],
  894. "path_fragments": [
  895. { "id": 1, "label": "file" },
  896. { "id": 2, "label": "other_file" },
  897. { "id": 3, "label": "symlink" }]
  898. }`
  899. data, err := JsonToActionGraphContainer(inputString)
  900. if err != nil {
  901. t.Error(err)
  902. return
  903. }
  904. _, _, err = AqueryBuildStatements(data, &metrics.EventHandler{})
  905. assertError(t, err, `Expect 1 input and 1 output to symlink action, got: input ["file" "other_file"], output ["symlink"]`)
  906. }
  907. func TestSymlinkMultipleOutputs(t *testing.T) {
  908. const inputString = `
  909. {
  910. "artifacts": [
  911. { "id": 1, "path_fragment_id": 1 },
  912. { "id": 3, "path_fragment_id": 3 }],
  913. "actions": [{
  914. "target_id": 1,
  915. "action_key": "x",
  916. "mnemonic": "Symlink",
  917. "input_dep_set_ids": [1],
  918. "output_ids": [2,3],
  919. "primary_output_id": 2
  920. }],
  921. "dep_set_of_files": [
  922. { "id": 1, "direct_artifact_ids": [1] }],
  923. "path_fragments": [
  924. { "id": 1, "label": "file" },
  925. { "id": 2, "label": "symlink" },
  926. { "id": 3, "label": "other_symlink" }]
  927. }`
  928. data, err := JsonToActionGraphContainer(inputString)
  929. if err != nil {
  930. t.Error(err)
  931. return
  932. }
  933. _, _, err = AqueryBuildStatements(data, &metrics.EventHandler{})
  934. assertError(t, err, "undefined outputId 2")
  935. }
  936. func TestTemplateExpandActionSubstitutions(t *testing.T) {
  937. const inputString = `
  938. {
  939. "artifacts": [{
  940. "id": 1,
  941. "path_fragment_id": 1
  942. }],
  943. "actions": [{
  944. "target_id": 1,
  945. "action_key": "x",
  946. "mnemonic": "TemplateExpand",
  947. "configuration_id": 1,
  948. "output_ids": [1],
  949. "primary_output_id": 1,
  950. "execution_platform": "//build/bazel/platforms:linux_x86_64",
  951. "template_content": "Test template substitutions: %token1%, %python_binary%",
  952. "substitutions": [
  953. { "key": "%token1%", "value": "abcd" },
  954. { "key": "%python_binary%", "value": "python3" }]
  955. }],
  956. "path_fragments": [
  957. { "id": 1, "label": "template_file" }]
  958. }`
  959. data, err := JsonToActionGraphContainer(inputString)
  960. if err != nil {
  961. t.Error(err)
  962. return
  963. }
  964. actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{})
  965. if err != nil {
  966. t.Errorf("Unexpected error %q", err)
  967. }
  968. expectedBuildStatements := []*BuildStatement{
  969. &BuildStatement{
  970. Command: "/bin/bash -c 'echo \"Test template substitutions: abcd, python3\" | sed \"s/\\\\\\\\n/\\\\n/g\" > template_file && " +
  971. "chmod a+x template_file'",
  972. OutputPaths: []string{"template_file"},
  973. Mnemonic: "TemplateExpand",
  974. SymlinkPaths: []string{},
  975. },
  976. }
  977. assertBuildStatements(t, expectedBuildStatements, actual)
  978. }
  979. func TestTemplateExpandActionNoOutput(t *testing.T) {
  980. const inputString = `
  981. {
  982. "artifacts": [
  983. { "id": 1, "path_fragment_id": 1 }],
  984. "actions": [{
  985. "target_id": 1,
  986. "action_key": "x",
  987. "mnemonic": "TemplateExpand",
  988. "configuration_id": 1,
  989. "primary_output_id": 1,
  990. "execution_platform": "//build/bazel/platforms:linux_x86_64",
  991. "templateContent": "Test template substitutions: %token1%, %python_binary%",
  992. "substitutions": [
  993. { "key": "%token1%", "value": "abcd" },
  994. { "key": "%python_binary%", "value": "python3" }]
  995. }],
  996. "path_fragments": [
  997. { "id": 1, "label": "template_file" }]
  998. }`
  999. data, err := JsonToActionGraphContainer(inputString)
  1000. if err != nil {
  1001. t.Error(err)
  1002. return
  1003. }
  1004. _, _, err = AqueryBuildStatements(data, &metrics.EventHandler{})
  1005. assertError(t, err, `Expect 1 output to template expand action, got: output []`)
  1006. }
  1007. func TestFileWrite(t *testing.T) {
  1008. const inputString = `
  1009. {
  1010. "artifacts": [
  1011. { "id": 1, "path_fragment_id": 1 }],
  1012. "actions": [{
  1013. "target_id": 1,
  1014. "action_key": "x",
  1015. "mnemonic": "FileWrite",
  1016. "configuration_id": 1,
  1017. "output_ids": [1],
  1018. "primary_output_id": 1,
  1019. "execution_platform": "//build/bazel/platforms:linux_x86_64",
  1020. "file_contents": "file data\n"
  1021. }],
  1022. "path_fragments": [
  1023. { "id": 1, "label": "foo.manifest" }]
  1024. }
  1025. `
  1026. data, err := JsonToActionGraphContainer(inputString)
  1027. if err != nil {
  1028. t.Error(err)
  1029. return
  1030. }
  1031. actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{})
  1032. if err != nil {
  1033. t.Errorf("Unexpected error %q", err)
  1034. }
  1035. assertBuildStatements(t, []*BuildStatement{
  1036. &BuildStatement{
  1037. OutputPaths: []string{"foo.manifest"},
  1038. Mnemonic: "FileWrite",
  1039. FileContents: "file data\n",
  1040. SymlinkPaths: []string{},
  1041. },
  1042. }, actual)
  1043. }
  1044. func TestSourceSymlinkManifest(t *testing.T) {
  1045. const inputString = `
  1046. {
  1047. "artifacts": [
  1048. { "id": 1, "path_fragment_id": 1 }],
  1049. "actions": [{
  1050. "target_id": 1,
  1051. "action_key": "x",
  1052. "mnemonic": "SourceSymlinkManifest",
  1053. "configuration_id": 1,
  1054. "output_ids": [1],
  1055. "primary_output_id": 1,
  1056. "execution_platform": "//build/bazel/platforms:linux_x86_64",
  1057. "file_contents": "symlink target\n"
  1058. }],
  1059. "path_fragments": [
  1060. { "id": 1, "label": "foo.manifest" }]
  1061. }
  1062. `
  1063. data, err := JsonToActionGraphContainer(inputString)
  1064. if err != nil {
  1065. t.Error(err)
  1066. return
  1067. }
  1068. actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{})
  1069. if err != nil {
  1070. t.Errorf("Unexpected error %q", err)
  1071. }
  1072. assertBuildStatements(t, []*BuildStatement{
  1073. &BuildStatement{
  1074. OutputPaths: []string{"foo.manifest"},
  1075. Mnemonic: "SourceSymlinkManifest",
  1076. SymlinkPaths: []string{},
  1077. },
  1078. }, actual)
  1079. }
  1080. func assertError(t *testing.T, err error, expected string) {
  1081. t.Helper()
  1082. if err == nil {
  1083. t.Errorf("expected error '%s', but got no error", expected)
  1084. } else if err.Error() != expected {
  1085. t.Errorf("expected error:\n\t'%s', but got:\n\t'%s'", expected, err.Error())
  1086. }
  1087. }
  1088. // Asserts that the given actual build statements match the given expected build statements.
  1089. // Build statement equivalence is determined using buildStatementEquals.
  1090. func assertBuildStatements(t *testing.T, expected []*BuildStatement, actual []*BuildStatement) {
  1091. t.Helper()
  1092. if len(expected) != len(actual) {
  1093. t.Errorf("expected %d build statements, but got %d,\n expected: %#v,\n actual: %#v",
  1094. len(expected), len(actual), expected, actual)
  1095. return
  1096. }
  1097. type compareFn = func(i int, j int) bool
  1098. byCommand := func(slice []*BuildStatement) compareFn {
  1099. return func(i int, j int) bool {
  1100. if slice[i] == nil {
  1101. return false
  1102. } else if slice[j] == nil {
  1103. return false
  1104. }
  1105. return slice[i].Command < slice[j].Command
  1106. }
  1107. }
  1108. sort.SliceStable(expected, byCommand(expected))
  1109. sort.SliceStable(actual, byCommand(actual))
  1110. for i, actualStatement := range actual {
  1111. expectedStatement := expected[i]
  1112. if differingField := buildStatementEquals(actualStatement, expectedStatement); differingField != "" {
  1113. t.Errorf("%s differs\nunexpected build statement %#v.\nexpected: %#v",
  1114. differingField, actualStatement, expectedStatement)
  1115. return
  1116. }
  1117. }
  1118. }
  1119. func buildStatementEquals(first *BuildStatement, second *BuildStatement) string {
  1120. if (first == nil) != (second == nil) {
  1121. return "Nil"
  1122. }
  1123. if first.Mnemonic != second.Mnemonic {
  1124. return "Mnemonic"
  1125. }
  1126. if first.Command != second.Command {
  1127. return "Command"
  1128. }
  1129. // Ordering is significant for environment variables.
  1130. if !reflect.DeepEqual(first.Env, second.Env) {
  1131. return "Env"
  1132. }
  1133. // Ordering is irrelevant for input and output paths, so compare sets.
  1134. if !reflect.DeepEqual(sortedStrings(first.InputPaths), sortedStrings(second.InputPaths)) {
  1135. return "InputPaths"
  1136. }
  1137. if !reflect.DeepEqual(sortedStrings(first.OutputPaths), sortedStrings(second.OutputPaths)) {
  1138. return "OutputPaths"
  1139. }
  1140. if !reflect.DeepEqual(sortedStrings(first.SymlinkPaths), sortedStrings(second.SymlinkPaths)) {
  1141. return "SymlinkPaths"
  1142. }
  1143. if first.Depfile != second.Depfile {
  1144. return "Depfile"
  1145. }
  1146. return ""
  1147. }
  1148. func sortedStrings(stringSlice []string) []string {
  1149. sorted := make([]string, len(stringSlice))
  1150. copy(sorted, stringSlice)
  1151. sort.Strings(sorted)
  1152. return sorted
  1153. }
  1154. // Transform the json format to ActionGraphContainer
  1155. func JsonToActionGraphContainer(inputString string) ([]byte, error) {
  1156. var aqueryProtoResult analysis_v2_proto.ActionGraphContainer
  1157. err := json.Unmarshal([]byte(inputString), &aqueryProtoResult)
  1158. if err != nil {
  1159. return []byte(""), err
  1160. }
  1161. data, _ := proto.Marshal(&aqueryProtoResult)
  1162. return data, err
  1163. }