dex.go 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395
  1. // Copyright 2017 Google Inc. All rights reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package java
  15. import (
  16. "strconv"
  17. "strings"
  18. "github.com/google/blueprint"
  19. "github.com/google/blueprint/proptools"
  20. "android/soong/android"
  21. "android/soong/remoteexec"
  22. )
  23. type DexProperties struct {
  24. // If set to true, compile dex regardless of installable. Defaults to false.
  25. Compile_dex *bool
  26. // list of module-specific flags that will be used for dex compiles
  27. Dxflags []string `android:"arch_variant"`
  28. // A list of files containing rules that specify the classes to keep in the main dex file.
  29. Main_dex_rules []string `android:"path"`
  30. Optimize struct {
  31. // If false, disable all optimization. Defaults to true for android_app and
  32. // android_test_helper_app modules, false for android_test, java_library, and java_test modules.
  33. Enabled *bool
  34. // True if the module containing this has it set by default.
  35. EnabledByDefault bool `blueprint:"mutated"`
  36. // Whether to continue building even if warnings are emitted. Defaults to true.
  37. Ignore_warnings *bool
  38. // If true, runs R8 in Proguard compatibility mode (default).
  39. // Otherwise, runs R8 in full mode.
  40. Proguard_compatibility *bool
  41. // If true, optimize for size by removing unused code. Defaults to true for apps,
  42. // false for libraries and tests.
  43. Shrink *bool
  44. // If true, optimize bytecode. Defaults to false.
  45. Optimize *bool
  46. // If true, obfuscate bytecode. Defaults to false.
  47. Obfuscate *bool
  48. // If true, do not use the flag files generated by aapt that automatically keep
  49. // classes referenced by the app manifest. Defaults to false.
  50. No_aapt_flags *bool
  51. Shrink_resources *bool
  52. // Flags to pass to proguard.
  53. Proguard_flags []string
  54. // Specifies the locations of files containing proguard flags.
  55. Proguard_flags_files []string `android:"path"`
  56. }
  57. // Keep the data uncompressed. We always need uncompressed dex for execution,
  58. // so this might actually save space by avoiding storing the same data twice.
  59. // This defaults to reasonable value based on module and should not be set.
  60. // It exists only to support ART tests.
  61. Uncompress_dex *bool
  62. // Exclude kotlinc generate files: *.kotlin_module, *.kotlin_builtins. Defaults to false.
  63. Exclude_kotlinc_generated_files *bool
  64. }
  65. type dexer struct {
  66. dexProperties DexProperties
  67. // list of extra proguard flag files
  68. extraProguardFlagFiles android.Paths
  69. proguardDictionary android.OptionalPath
  70. proguardUsageZip android.OptionalPath
  71. }
  72. func (d *dexer) effectiveOptimizeEnabled() bool {
  73. return BoolDefault(d.dexProperties.Optimize.Enabled, d.dexProperties.Optimize.EnabledByDefault)
  74. }
  75. var d8, d8RE = pctx.MultiCommandRemoteStaticRules("d8",
  76. blueprint.RuleParams{
  77. Command: `rm -rf "$outDir" && mkdir -p "$outDir" && ` +
  78. `mkdir -p $$(dirname $tmpJar) && ` +
  79. `${config.Zip2ZipCmd} -i $in -o $tmpJar -x '**/*.dex' && ` +
  80. `$d8Template${config.D8Cmd} ${config.D8Flags} --output $outDir $d8Flags $tmpJar && ` +
  81. `$zipTemplate${config.SoongZipCmd} $zipFlags -o $outDir/classes.dex.jar -C $outDir -f "$outDir/classes*.dex" && ` +
  82. `${config.MergeZipsCmd} -D -stripFile "**/*.class" $mergeZipsFlags $out $outDir/classes.dex.jar $in`,
  83. CommandDeps: []string{
  84. "${config.D8Cmd}",
  85. "${config.Zip2ZipCmd}",
  86. "${config.SoongZipCmd}",
  87. "${config.MergeZipsCmd}",
  88. },
  89. }, map[string]*remoteexec.REParams{
  90. "$d8Template": &remoteexec.REParams{
  91. Labels: map[string]string{"type": "compile", "compiler": "d8"},
  92. Inputs: []string{"${config.D8Jar}"},
  93. ExecStrategy: "${config.RED8ExecStrategy}",
  94. ToolchainInputs: []string{"${config.JavaCmd}"},
  95. Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},
  96. },
  97. "$zipTemplate": &remoteexec.REParams{
  98. Labels: map[string]string{"type": "tool", "name": "soong_zip"},
  99. Inputs: []string{"${config.SoongZipCmd}", "$outDir"},
  100. OutputFiles: []string{"$outDir/classes.dex.jar"},
  101. ExecStrategy: "${config.RED8ExecStrategy}",
  102. Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},
  103. },
  104. }, []string{"outDir", "d8Flags", "zipFlags", "tmpJar", "mergeZipsFlags"}, nil)
  105. var r8, r8RE = pctx.MultiCommandRemoteStaticRules("r8",
  106. blueprint.RuleParams{
  107. Command: `rm -rf "$outDir" && mkdir -p "$outDir" && ` +
  108. `rm -f "$outDict" && rm -rf "${outUsageDir}" && ` +
  109. `mkdir -p $$(dirname ${outUsage}) && ` +
  110. `mkdir -p $$(dirname $tmpJar) && ` +
  111. `${config.Zip2ZipCmd} -i $in -o $tmpJar -x '**/*.dex' && ` +
  112. `$r8Template${config.R8Cmd} ${config.R8Flags} -injars $tmpJar --output $outDir ` +
  113. `--no-data-resources ` +
  114. `-printmapping ${outDict} ` +
  115. `-printusage ${outUsage} ` +
  116. `--deps-file ${out}.d ` +
  117. `$r8Flags && ` +
  118. `touch "${outDict}" "${outUsage}" && ` +
  119. `${config.SoongZipCmd} -o ${outUsageZip} -C ${outUsageDir} -f ${outUsage} && ` +
  120. `rm -rf ${outUsageDir} && ` +
  121. `$zipTemplate${config.SoongZipCmd} $zipFlags -o $outDir/classes.dex.jar -C $outDir -f "$outDir/classes*.dex" && ` +
  122. `${config.MergeZipsCmd} -D -stripFile "**/*.class" $mergeZipsFlags $out $outDir/classes.dex.jar $in`,
  123. Depfile: "${out}.d",
  124. Deps: blueprint.DepsGCC,
  125. CommandDeps: []string{
  126. "${config.R8Cmd}",
  127. "${config.Zip2ZipCmd}",
  128. "${config.SoongZipCmd}",
  129. "${config.MergeZipsCmd}",
  130. },
  131. }, map[string]*remoteexec.REParams{
  132. "$r8Template": &remoteexec.REParams{
  133. Labels: map[string]string{"type": "compile", "compiler": "r8"},
  134. Inputs: []string{"$implicits", "${config.R8Jar}"},
  135. OutputFiles: []string{"${outUsage}"},
  136. ExecStrategy: "${config.RER8ExecStrategy}",
  137. ToolchainInputs: []string{"${config.JavaCmd}"},
  138. Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},
  139. },
  140. "$zipTemplate": &remoteexec.REParams{
  141. Labels: map[string]string{"type": "tool", "name": "soong_zip"},
  142. Inputs: []string{"${config.SoongZipCmd}", "$outDir"},
  143. OutputFiles: []string{"$outDir/classes.dex.jar"},
  144. ExecStrategy: "${config.RER8ExecStrategy}",
  145. Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},
  146. },
  147. "$zipUsageTemplate": &remoteexec.REParams{
  148. Labels: map[string]string{"type": "tool", "name": "soong_zip"},
  149. Inputs: []string{"${config.SoongZipCmd}", "${outUsage}"},
  150. OutputFiles: []string{"${outUsageZip}"},
  151. ExecStrategy: "${config.RER8ExecStrategy}",
  152. Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},
  153. },
  154. }, []string{"outDir", "outDict", "outUsage", "outUsageZip", "outUsageDir",
  155. "r8Flags", "zipFlags", "tmpJar", "mergeZipsFlags"}, []string{"implicits"})
  156. func (d *dexer) dexCommonFlags(ctx android.ModuleContext,
  157. minSdkVersion android.SdkSpec) (flags []string, deps android.Paths) {
  158. flags = d.dexProperties.Dxflags
  159. // Translate all the DX flags to D8 ones until all the build files have been migrated
  160. // to D8 flags. See: b/69377755
  161. flags = android.RemoveListFromList(flags,
  162. []string{"--core-library", "--dex", "--multi-dex"})
  163. for _, f := range android.PathsForModuleSrc(ctx, d.dexProperties.Main_dex_rules) {
  164. flags = append(flags, "--main-dex-rules", f.String())
  165. deps = append(deps, f)
  166. }
  167. if ctx.Config().Getenv("NO_OPTIMIZE_DX") != "" {
  168. flags = append(flags, "--debug")
  169. }
  170. if ctx.Config().Getenv("GENERATE_DEX_DEBUG") != "" {
  171. flags = append(flags,
  172. "--debug",
  173. "--verbose")
  174. }
  175. effectiveVersion, err := minSdkVersion.EffectiveVersion(ctx)
  176. if err != nil {
  177. ctx.PropertyErrorf("min_sdk_version", "%s", err)
  178. }
  179. flags = append(flags, "--min-api "+strconv.Itoa(effectiveVersion.FinalOrFutureInt()))
  180. return flags, deps
  181. }
  182. func d8Flags(flags javaBuilderFlags) (d8Flags []string, d8Deps android.Paths) {
  183. d8Flags = append(d8Flags, flags.bootClasspath.FormRepeatedClassPath("--lib ")...)
  184. d8Flags = append(d8Flags, flags.dexClasspath.FormRepeatedClassPath("--lib ")...)
  185. d8Deps = append(d8Deps, flags.bootClasspath...)
  186. d8Deps = append(d8Deps, flags.dexClasspath...)
  187. return d8Flags, d8Deps
  188. }
  189. func (d *dexer) r8Flags(ctx android.ModuleContext, flags javaBuilderFlags) (r8Flags []string, r8Deps android.Paths) {
  190. opt := d.dexProperties.Optimize
  191. // When an app contains references to APIs that are not in the SDK specified by
  192. // its LOCAL_SDK_VERSION for example added by support library or by runtime
  193. // classes added by desugaring, we artifically raise the "SDK version" "linked" by
  194. // ProGuard, to
  195. // - suppress ProGuard warnings of referencing symbols unknown to the lower SDK version.
  196. // - prevent ProGuard stripping subclass in the support library that extends class added in the higher SDK version.
  197. // See b/20667396
  198. var proguardRaiseDeps classpath
  199. ctx.VisitDirectDepsWithTag(proguardRaiseTag, func(m android.Module) {
  200. dep := ctx.OtherModuleProvider(m, JavaInfoProvider).(JavaInfo)
  201. proguardRaiseDeps = append(proguardRaiseDeps, dep.HeaderJars...)
  202. })
  203. r8Flags = append(r8Flags, proguardRaiseDeps.FormJavaClassPath("-libraryjars"))
  204. r8Flags = append(r8Flags, flags.bootClasspath.FormJavaClassPath("-libraryjars"))
  205. r8Flags = append(r8Flags, flags.dexClasspath.FormJavaClassPath("-libraryjars"))
  206. r8Deps = append(r8Deps, proguardRaiseDeps...)
  207. r8Deps = append(r8Deps, flags.bootClasspath...)
  208. r8Deps = append(r8Deps, flags.dexClasspath...)
  209. flagFiles := android.Paths{
  210. android.PathForSource(ctx, "build/make/core/proguard.flags"),
  211. }
  212. flagFiles = append(flagFiles, d.extraProguardFlagFiles...)
  213. // TODO(ccross): static android library proguard files
  214. flagFiles = append(flagFiles, android.PathsForModuleSrc(ctx, opt.Proguard_flags_files)...)
  215. r8Flags = append(r8Flags, android.JoinWithPrefix(flagFiles.Strings(), "-include "))
  216. r8Deps = append(r8Deps, flagFiles...)
  217. // TODO(b/70942988): This is included from build/make/core/proguard.flags
  218. r8Deps = append(r8Deps, android.PathForSource(ctx,
  219. "build/make/core/proguard_basic_keeps.flags"))
  220. r8Flags = append(r8Flags, opt.Proguard_flags...)
  221. if BoolDefault(opt.Proguard_compatibility, true) {
  222. r8Flags = append(r8Flags, "--force-proguard-compatibility")
  223. } else {
  224. // TODO(b/213833843): Allow configuration of the prefix via a build variable.
  225. var sourceFilePrefix = "go/retraceme "
  226. var sourceFileTemplate = "\"" + sourceFilePrefix + "%MAP_ID\""
  227. // TODO(b/200967150): Also tag the source file in compat builds.
  228. if Bool(opt.Optimize) || Bool(opt.Obfuscate) {
  229. r8Flags = append(r8Flags, "--map-id-template", "%MAP_HASH")
  230. r8Flags = append(r8Flags, "--source-file-template", sourceFileTemplate)
  231. }
  232. }
  233. // TODO(ccross): Don't shrink app instrumentation tests by default.
  234. if !Bool(opt.Shrink) {
  235. r8Flags = append(r8Flags, "-dontshrink")
  236. }
  237. if !Bool(opt.Optimize) {
  238. r8Flags = append(r8Flags, "-dontoptimize")
  239. }
  240. // TODO(ccross): error if obufscation + app instrumentation test.
  241. if !Bool(opt.Obfuscate) {
  242. r8Flags = append(r8Flags, "-dontobfuscate")
  243. }
  244. // TODO(ccross): if this is an instrumentation test of an obfuscated app, use the
  245. // dictionary of the app and move the app from libraryjars to injars.
  246. // Don't strip out debug information for eng builds.
  247. if ctx.Config().Eng() {
  248. r8Flags = append(r8Flags, "--debug")
  249. }
  250. // TODO(b/180878971): missing classes should be added to the relevant builds.
  251. // TODO(b/229727645): do not use true as default for Android platform builds.
  252. if proptools.BoolDefault(opt.Ignore_warnings, true) {
  253. r8Flags = append(r8Flags, "-ignorewarnings")
  254. }
  255. return r8Flags, r8Deps
  256. }
  257. func (d *dexer) compileDex(ctx android.ModuleContext, flags javaBuilderFlags, minSdkVersion android.SdkSpec,
  258. classesJar android.Path, jarName string) android.OutputPath {
  259. // Compile classes.jar into classes.dex and then javalib.jar
  260. javalibJar := android.PathForModuleOut(ctx, "dex", jarName).OutputPath
  261. outDir := android.PathForModuleOut(ctx, "dex")
  262. tmpJar := android.PathForModuleOut(ctx, "withres-withoutdex", jarName)
  263. zipFlags := "--ignore_missing_files"
  264. if proptools.Bool(d.dexProperties.Uncompress_dex) {
  265. zipFlags += " -L 0"
  266. }
  267. commonFlags, commonDeps := d.dexCommonFlags(ctx, minSdkVersion)
  268. // Exclude kotlinc generated files when "exclude_kotlinc_generated_files" is set to true.
  269. mergeZipsFlags := ""
  270. if proptools.BoolDefault(d.dexProperties.Exclude_kotlinc_generated_files, false) {
  271. mergeZipsFlags = "-stripFile META-INF/*.kotlin_module -stripFile **/*.kotlin_builtins"
  272. }
  273. useR8 := d.effectiveOptimizeEnabled()
  274. if useR8 {
  275. proguardDictionary := android.PathForModuleOut(ctx, "proguard_dictionary")
  276. d.proguardDictionary = android.OptionalPathForPath(proguardDictionary)
  277. proguardUsageDir := android.PathForModuleOut(ctx, "proguard_usage")
  278. proguardUsage := proguardUsageDir.Join(ctx, ctx.Namespace().Path,
  279. android.ModuleNameWithPossibleOverride(ctx), "unused.txt")
  280. proguardUsageZip := android.PathForModuleOut(ctx, "proguard_usage.zip")
  281. d.proguardUsageZip = android.OptionalPathForPath(proguardUsageZip)
  282. r8Flags, r8Deps := d.r8Flags(ctx, flags)
  283. r8Deps = append(r8Deps, commonDeps...)
  284. rule := r8
  285. args := map[string]string{
  286. "r8Flags": strings.Join(append(commonFlags, r8Flags...), " "),
  287. "zipFlags": zipFlags,
  288. "outDict": proguardDictionary.String(),
  289. "outUsageDir": proguardUsageDir.String(),
  290. "outUsage": proguardUsage.String(),
  291. "outUsageZip": proguardUsageZip.String(),
  292. "outDir": outDir.String(),
  293. "tmpJar": tmpJar.String(),
  294. "mergeZipsFlags": mergeZipsFlags,
  295. }
  296. if ctx.Config().UseRBE() && ctx.Config().IsEnvTrue("RBE_R8") {
  297. rule = r8RE
  298. args["implicits"] = strings.Join(r8Deps.Strings(), ",")
  299. }
  300. ctx.Build(pctx, android.BuildParams{
  301. Rule: rule,
  302. Description: "r8",
  303. Output: javalibJar,
  304. ImplicitOutputs: android.WritablePaths{proguardDictionary, proguardUsageZip},
  305. Input: classesJar,
  306. Implicits: r8Deps,
  307. Args: args,
  308. })
  309. } else {
  310. d8Flags, d8Deps := d8Flags(flags)
  311. d8Deps = append(d8Deps, commonDeps...)
  312. rule := d8
  313. if ctx.Config().UseRBE() && ctx.Config().IsEnvTrue("RBE_D8") {
  314. rule = d8RE
  315. }
  316. ctx.Build(pctx, android.BuildParams{
  317. Rule: rule,
  318. Description: "d8",
  319. Output: javalibJar,
  320. Input: classesJar,
  321. Implicits: d8Deps,
  322. Args: map[string]string{
  323. "d8Flags": strings.Join(append(commonFlags, d8Flags...), " "),
  324. "zipFlags": zipFlags,
  325. "outDir": outDir.String(),
  326. "tmpJar": tmpJar.String(),
  327. "mergeZipsFlags": mergeZipsFlags,
  328. },
  329. })
  330. }
  331. if proptools.Bool(d.dexProperties.Uncompress_dex) {
  332. alignedJavalibJar := android.PathForModuleOut(ctx, "aligned", jarName).OutputPath
  333. TransformZipAlign(ctx, alignedJavalibJar, javalibJar)
  334. javalibJar = alignedJavalibJar
  335. }
  336. return javalibJar
  337. }