Browse Source

Rollforward "Use proto output instead of jsonproto output for aquery""

Test: use m --bazel-mode-dev and mixed_droid.sh
Bug: 233798334

This reverts commit d27e28c2a7e64545a138a7f4d597fdf73a8b39a3.

Change-Id: Ifd37ac449979330529dd25c92a5eca2f5a80827e
Jason Wu 1 year ago
parent
commit
118fd2b443
7 changed files with 627 additions and 462 deletions
  1. 1 2
      android/bazel_handler.go
  2. 52 34
      android/bazel_handler_test.go
  3. 1 0
      bazel/Android.bp
  4. 72 3
      bazel/aquery.go
  5. 493 421
      bazel/aquery_test.go
  6. 7 2
      go.mod
  7. 1 0
      tests/lib.sh

+ 1 - 2
android/bazel_handler.go

@@ -558,7 +558,6 @@ func (r *builtinBazelRunner) createBazelCommand(paths *bazelPaths, runName bazel
 		// The actual platform values here may be overridden by configuration
 		// transitions from the buildroot.
 		fmt.Sprintf("--extra_toolchains=%s", "//prebuilts/clang/host/linux-x86:all"),
-
 		// This should be parameterized on the host OS, but let's restrict to linux
 		// to keep things simple for now.
 		fmt.Sprintf("--host_platform=%s", "//build/bazel/platforms:linux_x86_64"),
@@ -927,7 +926,7 @@ func (context *bazelContext) InvokeBazel(config Config) error {
 	//
 	// Use jsonproto instead of proto; actual proto parsing would require a dependency on Bazel's
 	// proto sources, which would add a number of unnecessary dependencies.
-	extraFlags := []string{"--output=jsonproto", "--include_file_write_contents"}
+	extraFlags := []string{"--output=proto", "--include_file_write_contents"}
 	if Bool(config.productVariables.ClangCoverage) {
 		extraFlags = append(extraFlags, "--collect_code_coverage")
 		paths := make([]string, 0, 2)

+ 52 - 34
android/bazel_handler_test.go

@@ -1,6 +1,7 @@
 package android
 
 import (
+	"encoding/json"
 	"os"
 	"path/filepath"
 	"reflect"
@@ -8,6 +9,8 @@ import (
 	"testing"
 
 	"android/soong/bazel/cquery"
+	"google.golang.org/protobuf/proto"
+	analysis_v2_proto "prebuilts/bazel/common/proto/analysis_v2"
 )
 
 var testConfig = TestConfig("out", nil, "", nil)
@@ -65,52 +68,56 @@ func TestInvokeBazelPopulatesBuildStatements(t *testing.T) {
 	var testCases = []testCase{
 		{`
 {
-  "artifacts": [
-    { "id": 1, "pathFragmentId": 1 },
-    { "id": 2, "pathFragmentId": 2 }],
-  "actions": [{
-    "targetId": 1,
-    "actionKey": "x",
-    "mnemonic": "x",
-    "arguments": ["touch", "foo"],
-    "inputDepSetIds": [1],
-    "outputIds": [1],
-    "primaryOutputId": 1
-  }],
-  "depSetOfFiles": [
-    { "id": 1, "directArtifactIds": [1, 2] }],
-  "pathFragments": [
-    { "id": 1, "label": "one" },
-    { "id": 2, "label": "two" }]
+ "artifacts": [
+   { "id": 1, "path_fragment_id": 1 },
+   { "id": 2, "path_fragment_id": 2 }],
+ "actions": [{
+   "target_Id": 1,
+   "action_Key": "x",
+   "mnemonic": "x",
+   "arguments": ["touch", "foo"],
+   "input_dep_set_ids": [1],
+   "output_Ids": [1],
+   "primary_output_id": 1
+ }],
+ "dep_set_of_files": [
+   { "id": 1, "direct_artifact_ids": [1, 2] }],
+ "path_fragments": [
+   { "id": 1, "label": "one" },
+   { "id": 2, "label": "two" }]
 }`,
 			"cd 'test/exec_root' && rm -f 'one' && touch foo",
 		}, {`
 {
-  "artifacts": [
-    { "id": 1, "pathFragmentId": 10 },
-    { "id": 2, "pathFragmentId": 20 }],
-  "actions": [{
-    "targetId": 100,
-    "actionKey": "x",
-    "mnemonic": "x",
-    "arguments": ["bogus", "command"],
-    "outputIds": [1, 2],
-    "primaryOutputId": 1
-  }],
-  "pathFragments": [
-    { "id": 10, "label": "one", "parentId": 30 },
-    { "id": 20, "label": "one.d", "parentId": 30 },
-    { "id": 30, "label": "parent" }]
+ "artifacts": [
+   { "id": 1, "path_fragment_id": 10 },
+   { "id": 2, "path_fragment_id": 20 }],
+ "actions": [{
+   "target_Id": 100,
+   "action_Key": "x",
+   "mnemonic": "x",
+   "arguments": ["bogus", "command"],
+   "output_Ids": [1, 2],
+   "primary_output_id": 1
+ }],
+ "path_fragments": [
+   { "id": 10, "label": "one", "parent_id": 30 },
+   { "id": 20, "label": "one.d", "parent_id": 30 },
+   { "id": 30, "label": "parent" }]
 }`,
 			`cd 'test/exec_root' && rm -f 'parent/one' && bogus command && sed -i'' -E 's@(^|\s|")bazel-out/@\1test/bazel_out/@g' 'parent/one.d'`,
 		},
 	}
 
 	for i, testCase := range testCases {
+		data, err := JsonToActionGraphContainer(testCase.input)
+		if err != nil {
+			t.Error(err)
+		}
 		bazelContext, _ := testBazelContext(t, map[bazelCommand]string{
-			bazelCommand{command: "aquery", expression: "deps(@soong_injection//mixed_builds:buildroot)"}: testCase.input})
+			bazelCommand{command: "aquery", expression: "deps(@soong_injection//mixed_builds:buildroot)"}: string(data)})
 
-		err := bazelContext.InvokeBazel(testConfig)
+		err = bazelContext.InvokeBazel(testConfig)
 		if err != nil {
 			t.Fatalf("testCase #%d: did not expect error invoking Bazel, but got %s", i+1, err)
 		}
@@ -194,3 +201,14 @@ func testBazelContext(t *testing.T, bazelCommandResults map[bazelCommand]string)
 		requests:    map[cqueryKey]bool{},
 	}, p.soongOutDir
 }
+
+// Transform the json format to ActionGraphContainer
+func JsonToActionGraphContainer(inputString string) ([]byte, error) {
+	var aqueryProtoResult analysis_v2_proto.ActionGraphContainer
+	err := json.Unmarshal([]byte(inputString), &aqueryProtoResult)
+	if err != nil {
+		return []byte(""), err
+	}
+	data, _ := proto.Marshal(&aqueryProtoResult)
+	return data, err
+}

+ 1 - 0
bazel/Android.bp

@@ -20,6 +20,7 @@ bootstrap_go_package {
         "soong_build",
     ],
     deps: [
+        "bazel_analysis_v2_proto",
         "blueprint",
     ],
 }

+ 72 - 3
bazel/aquery.go

@@ -17,7 +17,6 @@ package bazel
 import (
 	"crypto/sha256"
 	"encoding/base64"
-	"encoding/json"
 	"fmt"
 	"path/filepath"
 	"reflect"
@@ -25,6 +24,8 @@ import (
 	"strings"
 
 	"github.com/google/blueprint/proptools"
+	"google.golang.org/protobuf/proto"
+	analysis_v2_proto "prebuilts/bazel/common/proto/analysis_v2"
 )
 
 type artifactId int
@@ -312,11 +313,79 @@ func (a *aqueryArtifactHandler) artifactPathsFromDepsetHash(depsetHash string) (
 // BuildStatements are one-to-one with actions in the given action graph, and AqueryDepsets
 // are one-to-one with Bazel's depSetOfFiles objects.
 func AqueryBuildStatements(aqueryJsonProto []byte) ([]BuildStatement, []AqueryDepset, error) {
-	var aqueryResult actionGraphContainer
-	err := json.Unmarshal(aqueryJsonProto, &aqueryResult)
+	aqueryProto := &analysis_v2_proto.ActionGraphContainer{}
+	err := proto.Unmarshal(aqueryJsonProto, aqueryProto)
 	if err != nil {
 		return nil, nil, err
 	}
+	aqueryResult := actionGraphContainer{}
+
+	for _, protoArtifact := range aqueryProto.Artifacts {
+		aqueryResult.Artifacts = append(aqueryResult.Artifacts, artifact{artifactId(protoArtifact.Id),
+			pathFragmentId(protoArtifact.PathFragmentId)})
+	}
+
+	for _, protoAction := range aqueryProto.Actions {
+		var environmentVariable []KeyValuePair
+		var inputDepSetIds []depsetId
+		var outputIds []artifactId
+		var substitutions []KeyValuePair
+
+		for _, protoEnvironmentVariable := range protoAction.EnvironmentVariables {
+			environmentVariable = append(environmentVariable, KeyValuePair{
+				protoEnvironmentVariable.Key, protoEnvironmentVariable.Value,
+			})
+		}
+		for _, protoInputDepSetIds := range protoAction.InputDepSetIds {
+			inputDepSetIds = append(inputDepSetIds, depsetId(protoInputDepSetIds))
+		}
+		for _, protoOutputIds := range protoAction.OutputIds {
+			outputIds = append(outputIds, artifactId(protoOutputIds))
+		}
+		for _, protoSubstitutions := range protoAction.Substitutions {
+			substitutions = append(substitutions, KeyValuePair{
+				protoSubstitutions.Key, protoSubstitutions.Value,
+			})
+		}
+
+		aqueryResult.Actions = append(aqueryResult.Actions,
+			action{
+				Arguments:            protoAction.Arguments,
+				EnvironmentVariables: environmentVariable,
+				InputDepSetIds:       inputDepSetIds,
+				Mnemonic:             protoAction.Mnemonic,
+				OutputIds:            outputIds,
+				TemplateContent:      protoAction.TemplateContent,
+				Substitutions:        substitutions,
+				FileContents:         protoAction.FileContents})
+	}
+
+	for _, protoDepSetOfFiles := range aqueryProto.DepSetOfFiles {
+		var directArtifactIds []artifactId
+		var transitiveDepSetIds []depsetId
+
+		for _, protoDirectArtifactIds := range protoDepSetOfFiles.DirectArtifactIds {
+			directArtifactIds = append(directArtifactIds, artifactId(protoDirectArtifactIds))
+		}
+		for _, protoTransitiveDepSetIds := range protoDepSetOfFiles.TransitiveDepSetIds {
+			transitiveDepSetIds = append(transitiveDepSetIds, depsetId(protoTransitiveDepSetIds))
+		}
+		aqueryResult.DepSetOfFiles = append(aqueryResult.DepSetOfFiles,
+			depSetOfFiles{
+				Id:                  depsetId(protoDepSetOfFiles.Id),
+				DirectArtifactIds:   directArtifactIds,
+				TransitiveDepSetIds: transitiveDepSetIds})
+
+	}
+
+	for _, protoPathFragments := range aqueryProto.PathFragments {
+		aqueryResult.PathFragments = append(aqueryResult.PathFragments,
+			pathFragment{
+				Id:       pathFragmentId(protoPathFragments.Id),
+				Label:    protoPathFragments.Label,
+				ParentId: pathFragmentId(protoPathFragments.ParentId)})
+
+	}
 	aqueryHandler, err := newAqueryHandler(aqueryResult)
 	if err != nil {
 		return nil, nil, err

File diff suppressed because it is too large
+ 493 - 421
bazel/aquery_test.go


+ 7 - 2
go.mod

@@ -13,7 +13,12 @@ exclude github.com/golang/protobuf v1.5.0
 
 replace github.com/google/go-cmp v0.5.5 => ../../external/go-cmp
 
-// Indirect dep from go-cmp
-exclude golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543
+require prebuilts/bazel/common/proto/analysis_v2 v0.0.0
+
+replace prebuilts/bazel/common/proto/analysis_v2 => ../../prebuilts/bazel/common/proto/analysis_v2
+
+require prebuilts/bazel/common/proto/build v0.0.0 // indirect
+
+replace prebuilts/bazel/common/proto/build => ../../prebuilts/bazel/common/proto/build
 
 go 1.18

+ 1 - 0
tests/lib.sh

@@ -85,6 +85,7 @@ function create_mock_soong {
   copy_directory build/blueprint
   copy_directory build/soong
   copy_directory build/make/tools/rbcrun
+  copy_directory prebuilts/bazel/common/proto
 
   symlink_directory prebuilts/sdk
   symlink_directory prebuilts/go

Some files were not shown because too many files changed in this diff