dex.go 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442
  1. // Copyright 2017 Google Inc. All rights reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package java
  15. import (
  16. "strconv"
  17. "strings"
  18. "github.com/google/blueprint"
  19. "github.com/google/blueprint/proptools"
  20. "android/soong/android"
  21. "android/soong/remoteexec"
  22. )
  23. type DexProperties struct {
  24. // If set to true, compile dex regardless of installable. Defaults to false.
  25. Compile_dex *bool
  26. // list of module-specific flags that will be used for dex compiles
  27. Dxflags []string `android:"arch_variant"`
  28. // A list of files containing rules that specify the classes to keep in the main dex file.
  29. Main_dex_rules []string `android:"path"`
  30. Optimize struct {
  31. // If false, disable all optimization. Defaults to true for android_app and
  32. // android_test_helper_app modules, false for android_test, java_library, and java_test modules.
  33. Enabled *bool
  34. // True if the module containing this has it set by default.
  35. EnabledByDefault bool `blueprint:"mutated"`
  36. // Whether to continue building even if warnings are emitted. Defaults to true.
  37. Ignore_warnings *bool
  38. // If true, runs R8 in Proguard compatibility mode (default).
  39. // Otherwise, runs R8 in full mode.
  40. Proguard_compatibility *bool
  41. // If true, optimize for size by removing unused code. Defaults to true for apps,
  42. // false for libraries and tests.
  43. Shrink *bool
  44. // If true, optimize bytecode. Defaults to false.
  45. Optimize *bool
  46. // If true, obfuscate bytecode. Defaults to false.
  47. Obfuscate *bool
  48. // If true, do not use the flag files generated by aapt that automatically keep
  49. // classes referenced by the app manifest. Defaults to false.
  50. No_aapt_flags *bool
  51. // If true, optimize for size by removing unused resources. Defaults to false.
  52. Shrink_resources *bool
  53. // Flags to pass to proguard.
  54. Proguard_flags []string
  55. // Specifies the locations of files containing proguard flags.
  56. Proguard_flags_files []string `android:"path"`
  57. }
  58. // Keep the data uncompressed. We always need uncompressed dex for execution,
  59. // so this might actually save space by avoiding storing the same data twice.
  60. // This defaults to reasonable value based on module and should not be set.
  61. // It exists only to support ART tests.
  62. Uncompress_dex *bool
  63. // Exclude kotlinc generate files: *.kotlin_module, *.kotlin_builtins. Defaults to false.
  64. Exclude_kotlinc_generated_files *bool
  65. }
  66. type dexer struct {
  67. dexProperties DexProperties
  68. // list of extra proguard flag files
  69. extraProguardFlagFiles android.Paths
  70. proguardDictionary android.OptionalPath
  71. proguardConfiguration android.OptionalPath
  72. proguardUsageZip android.OptionalPath
  73. providesTransitiveHeaderJars
  74. }
  75. func (d *dexer) effectiveOptimizeEnabled() bool {
  76. return BoolDefault(d.dexProperties.Optimize.Enabled, d.dexProperties.Optimize.EnabledByDefault)
  77. }
  78. var d8, d8RE = pctx.MultiCommandRemoteStaticRules("d8",
  79. blueprint.RuleParams{
  80. Command: `rm -rf "$outDir" && mkdir -p "$outDir" && ` +
  81. `mkdir -p $$(dirname $tmpJar) && ` +
  82. `${config.Zip2ZipCmd} -i $in -o $tmpJar -x '**/*.dex' && ` +
  83. `$d8Template${config.D8Cmd} ${config.D8Flags} --output $outDir $d8Flags $tmpJar && ` +
  84. `$zipTemplate${config.SoongZipCmd} $zipFlags -o $outDir/classes.dex.jar -C $outDir -f "$outDir/classes*.dex" && ` +
  85. `${config.MergeZipsCmd} -D -stripFile "**/*.class" $mergeZipsFlags $out $outDir/classes.dex.jar $in`,
  86. CommandDeps: []string{
  87. "${config.D8Cmd}",
  88. "${config.Zip2ZipCmd}",
  89. "${config.SoongZipCmd}",
  90. "${config.MergeZipsCmd}",
  91. },
  92. }, map[string]*remoteexec.REParams{
  93. "$d8Template": &remoteexec.REParams{
  94. Labels: map[string]string{"type": "compile", "compiler": "d8"},
  95. Inputs: []string{"${config.D8Jar}"},
  96. ExecStrategy: "${config.RED8ExecStrategy}",
  97. ToolchainInputs: []string{"${config.JavaCmd}"},
  98. Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},
  99. },
  100. "$zipTemplate": &remoteexec.REParams{
  101. Labels: map[string]string{"type": "tool", "name": "soong_zip"},
  102. Inputs: []string{"${config.SoongZipCmd}", "$outDir"},
  103. OutputFiles: []string{"$outDir/classes.dex.jar"},
  104. ExecStrategy: "${config.RED8ExecStrategy}",
  105. Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},
  106. },
  107. }, []string{"outDir", "d8Flags", "zipFlags", "tmpJar", "mergeZipsFlags"}, nil)
  108. var r8, r8RE = pctx.MultiCommandRemoteStaticRules("r8",
  109. blueprint.RuleParams{
  110. Command: `rm -rf "$outDir" && mkdir -p "$outDir" && ` +
  111. `rm -f "$outDict" && rm -f "$outConfig" && rm -rf "${outUsageDir}" && ` +
  112. `mkdir -p $$(dirname ${outUsage}) && ` +
  113. `mkdir -p $$(dirname $tmpJar) && ` +
  114. `${config.Zip2ZipCmd} -i $in -o $tmpJar -x '**/*.dex' && ` +
  115. `$r8Template${config.R8Cmd} ${config.R8Flags} -injars $tmpJar --output $outDir ` +
  116. `--no-data-resources ` +
  117. `-printmapping ${outDict} ` +
  118. `-printconfiguration ${outConfig} ` +
  119. `-printusage ${outUsage} ` +
  120. `--deps-file ${out}.d ` +
  121. `$r8Flags && ` +
  122. `touch "${outDict}" "${outConfig}" "${outUsage}" && ` +
  123. `${config.SoongZipCmd} -o ${outUsageZip} -C ${outUsageDir} -f ${outUsage} && ` +
  124. `rm -rf ${outUsageDir} && ` +
  125. `$zipTemplate${config.SoongZipCmd} $zipFlags -o $outDir/classes.dex.jar -C $outDir -f "$outDir/classes*.dex" && ` +
  126. `${config.MergeZipsCmd} -D -stripFile "**/*.class" $mergeZipsFlags $out $outDir/classes.dex.jar $in`,
  127. Depfile: "${out}.d",
  128. Deps: blueprint.DepsGCC,
  129. CommandDeps: []string{
  130. "${config.R8Cmd}",
  131. "${config.Zip2ZipCmd}",
  132. "${config.SoongZipCmd}",
  133. "${config.MergeZipsCmd}",
  134. },
  135. }, map[string]*remoteexec.REParams{
  136. "$r8Template": &remoteexec.REParams{
  137. Labels: map[string]string{"type": "compile", "compiler": "r8"},
  138. Inputs: []string{"$implicits", "${config.R8Jar}"},
  139. OutputFiles: []string{"${outUsage}", "${outConfig}", "${outDict}"},
  140. ExecStrategy: "${config.RER8ExecStrategy}",
  141. ToolchainInputs: []string{"${config.JavaCmd}"},
  142. Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},
  143. },
  144. "$zipTemplate": &remoteexec.REParams{
  145. Labels: map[string]string{"type": "tool", "name": "soong_zip"},
  146. Inputs: []string{"${config.SoongZipCmd}", "$outDir"},
  147. OutputFiles: []string{"$outDir/classes.dex.jar"},
  148. ExecStrategy: "${config.RER8ExecStrategy}",
  149. Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},
  150. },
  151. "$zipUsageTemplate": &remoteexec.REParams{
  152. Labels: map[string]string{"type": "tool", "name": "soong_zip"},
  153. Inputs: []string{"${config.SoongZipCmd}", "${outUsage}"},
  154. OutputFiles: []string{"${outUsageZip}"},
  155. ExecStrategy: "${config.RER8ExecStrategy}",
  156. Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},
  157. },
  158. }, []string{"outDir", "outDict", "outConfig", "outUsage", "outUsageZip", "outUsageDir",
  159. "r8Flags", "zipFlags", "tmpJar", "mergeZipsFlags"}, []string{"implicits"})
  160. func (d *dexer) dexCommonFlags(ctx android.ModuleContext,
  161. dexParams *compileDexParams) (flags []string, deps android.Paths) {
  162. flags = d.dexProperties.Dxflags
  163. // Translate all the DX flags to D8 ones until all the build files have been migrated
  164. // to D8 flags. See: b/69377755
  165. flags = android.RemoveListFromList(flags,
  166. []string{"--core-library", "--dex", "--multi-dex"})
  167. for _, f := range android.PathsForModuleSrc(ctx, d.dexProperties.Main_dex_rules) {
  168. flags = append(flags, "--main-dex-rules", f.String())
  169. deps = append(deps, f)
  170. }
  171. if ctx.Config().Getenv("NO_OPTIMIZE_DX") != "" {
  172. flags = append(flags, "--debug")
  173. }
  174. if ctx.Config().Getenv("GENERATE_DEX_DEBUG") != "" {
  175. flags = append(flags,
  176. "--debug",
  177. "--verbose")
  178. }
  179. // Supplying the platform build flag disables various features like API modeling and desugaring.
  180. // For targets with a stable min SDK version (i.e., when the min SDK is both explicitly specified
  181. // and managed+versioned), we suppress this flag to ensure portability.
  182. // Note: Targets with a min SDK kind of core_platform (e.g., framework.jar) or unspecified (e.g.,
  183. // services.jar), are not classified as stable, which is WAI.
  184. // TODO(b/232073181): Expand to additional min SDK cases after validation.
  185. if !dexParams.sdkVersion.Stable() {
  186. flags = append(flags, "--android-platform-build")
  187. }
  188. effectiveVersion, err := dexParams.minSdkVersion.EffectiveVersion(ctx)
  189. if err != nil {
  190. ctx.PropertyErrorf("min_sdk_version", "%s", err)
  191. }
  192. flags = append(flags, "--min-api "+strconv.Itoa(effectiveVersion.FinalOrFutureInt()))
  193. return flags, deps
  194. }
  195. func d8Flags(flags javaBuilderFlags) (d8Flags []string, d8Deps android.Paths) {
  196. d8Flags = append(d8Flags, flags.bootClasspath.FormRepeatedClassPath("--lib ")...)
  197. d8Flags = append(d8Flags, flags.dexClasspath.FormRepeatedClassPath("--lib ")...)
  198. d8Deps = append(d8Deps, flags.bootClasspath...)
  199. d8Deps = append(d8Deps, flags.dexClasspath...)
  200. return d8Flags, d8Deps
  201. }
  202. func (d *dexer) r8Flags(ctx android.ModuleContext, flags javaBuilderFlags) (r8Flags []string, r8Deps android.Paths) {
  203. opt := d.dexProperties.Optimize
  204. // When an app contains references to APIs that are not in the SDK specified by
  205. // its LOCAL_SDK_VERSION for example added by support library or by runtime
  206. // classes added by desugaring, we artifically raise the "SDK version" "linked" by
  207. // ProGuard, to
  208. // - suppress ProGuard warnings of referencing symbols unknown to the lower SDK version.
  209. // - prevent ProGuard stripping subclass in the support library that extends class added in the higher SDK version.
  210. // See b/20667396
  211. var proguardRaiseDeps classpath
  212. ctx.VisitDirectDepsWithTag(proguardRaiseTag, func(m android.Module) {
  213. dep := ctx.OtherModuleProvider(m, JavaInfoProvider).(JavaInfo)
  214. proguardRaiseDeps = append(proguardRaiseDeps, dep.HeaderJars...)
  215. })
  216. r8Flags = append(r8Flags, proguardRaiseDeps.FormJavaClassPath("-libraryjars"))
  217. r8Deps = append(r8Deps, proguardRaiseDeps...)
  218. r8Flags = append(r8Flags, flags.bootClasspath.FormJavaClassPath("-libraryjars"))
  219. r8Deps = append(r8Deps, flags.bootClasspath...)
  220. r8Flags = append(r8Flags, flags.dexClasspath.FormJavaClassPath("-libraryjars"))
  221. r8Deps = append(r8Deps, flags.dexClasspath...)
  222. transitiveStaticLibsLookupMap := map[android.Path]bool{}
  223. if d.transitiveStaticLibsHeaderJars != nil {
  224. for _, jar := range d.transitiveStaticLibsHeaderJars.ToList() {
  225. transitiveStaticLibsLookupMap[jar] = true
  226. }
  227. }
  228. transitiveHeaderJars := android.Paths{}
  229. if d.transitiveLibsHeaderJars != nil {
  230. for _, jar := range d.transitiveLibsHeaderJars.ToList() {
  231. if _, ok := transitiveStaticLibsLookupMap[jar]; ok {
  232. // don't include a lib if it is already packaged in the current JAR as a static lib
  233. continue
  234. }
  235. transitiveHeaderJars = append(transitiveHeaderJars, jar)
  236. }
  237. }
  238. transitiveClasspath := classpath(transitiveHeaderJars)
  239. r8Flags = append(r8Flags, transitiveClasspath.FormJavaClassPath("-libraryjars"))
  240. r8Deps = append(r8Deps, transitiveClasspath...)
  241. flagFiles := android.Paths{
  242. android.PathForSource(ctx, "build/make/core/proguard.flags"),
  243. }
  244. flagFiles = append(flagFiles, d.extraProguardFlagFiles...)
  245. // TODO(ccross): static android library proguard files
  246. flagFiles = append(flagFiles, android.PathsForModuleSrc(ctx, opt.Proguard_flags_files)...)
  247. r8Flags = append(r8Flags, android.JoinWithPrefix(flagFiles.Strings(), "-include "))
  248. r8Deps = append(r8Deps, flagFiles...)
  249. // TODO(b/70942988): This is included from build/make/core/proguard.flags
  250. r8Deps = append(r8Deps, android.PathForSource(ctx,
  251. "build/make/core/proguard_basic_keeps.flags"))
  252. r8Flags = append(r8Flags, opt.Proguard_flags...)
  253. if BoolDefault(opt.Proguard_compatibility, true) {
  254. r8Flags = append(r8Flags, "--force-proguard-compatibility")
  255. } else {
  256. // TODO(b/213833843): Allow configuration of the prefix via a build variable.
  257. var sourceFilePrefix = "go/retraceme "
  258. var sourceFileTemplate = "\"" + sourceFilePrefix + "%MAP_ID\""
  259. // TODO(b/200967150): Also tag the source file in compat builds.
  260. if Bool(opt.Optimize) || Bool(opt.Obfuscate) {
  261. r8Flags = append(r8Flags, "--map-id-template", "%MAP_HASH")
  262. r8Flags = append(r8Flags, "--source-file-template", sourceFileTemplate)
  263. }
  264. }
  265. // TODO(ccross): Don't shrink app instrumentation tests by default.
  266. if !Bool(opt.Shrink) {
  267. r8Flags = append(r8Flags, "-dontshrink")
  268. }
  269. if !Bool(opt.Optimize) {
  270. r8Flags = append(r8Flags, "-dontoptimize")
  271. }
  272. // TODO(ccross): error if obufscation + app instrumentation test.
  273. if !Bool(opt.Obfuscate) {
  274. r8Flags = append(r8Flags, "-dontobfuscate")
  275. }
  276. // TODO(ccross): if this is an instrumentation test of an obfuscated app, use the
  277. // dictionary of the app and move the app from libraryjars to injars.
  278. // Don't strip out debug information for eng builds.
  279. if ctx.Config().Eng() {
  280. r8Flags = append(r8Flags, "--debug")
  281. }
  282. // TODO(b/180878971): missing classes should be added to the relevant builds.
  283. // TODO(b/229727645): do not use true as default for Android platform builds.
  284. if proptools.BoolDefault(opt.Ignore_warnings, true) {
  285. r8Flags = append(r8Flags, "-ignorewarnings")
  286. }
  287. return r8Flags, r8Deps
  288. }
  289. type compileDexParams struct {
  290. flags javaBuilderFlags
  291. sdkVersion android.SdkSpec
  292. minSdkVersion android.ApiLevel
  293. classesJar android.Path
  294. jarName string
  295. }
  296. func (d *dexer) compileDex(ctx android.ModuleContext, dexParams *compileDexParams) android.OutputPath {
  297. // Compile classes.jar into classes.dex and then javalib.jar
  298. javalibJar := android.PathForModuleOut(ctx, "dex", dexParams.jarName).OutputPath
  299. outDir := android.PathForModuleOut(ctx, "dex")
  300. tmpJar := android.PathForModuleOut(ctx, "withres-withoutdex", dexParams.jarName)
  301. zipFlags := "--ignore_missing_files"
  302. if proptools.Bool(d.dexProperties.Uncompress_dex) {
  303. zipFlags += " -L 0"
  304. }
  305. commonFlags, commonDeps := d.dexCommonFlags(ctx, dexParams)
  306. // Exclude kotlinc generated files when "exclude_kotlinc_generated_files" is set to true.
  307. mergeZipsFlags := ""
  308. if proptools.BoolDefault(d.dexProperties.Exclude_kotlinc_generated_files, false) {
  309. mergeZipsFlags = "-stripFile META-INF/*.kotlin_module -stripFile **/*.kotlin_builtins"
  310. }
  311. useR8 := d.effectiveOptimizeEnabled()
  312. if useR8 {
  313. proguardDictionary := android.PathForModuleOut(ctx, "proguard_dictionary")
  314. d.proguardDictionary = android.OptionalPathForPath(proguardDictionary)
  315. proguardConfiguration := android.PathForModuleOut(ctx, "proguard_configuration")
  316. d.proguardConfiguration = android.OptionalPathForPath(proguardConfiguration)
  317. proguardUsageDir := android.PathForModuleOut(ctx, "proguard_usage")
  318. proguardUsage := proguardUsageDir.Join(ctx, ctx.Namespace().Path,
  319. android.ModuleNameWithPossibleOverride(ctx), "unused.txt")
  320. proguardUsageZip := android.PathForModuleOut(ctx, "proguard_usage.zip")
  321. d.proguardUsageZip = android.OptionalPathForPath(proguardUsageZip)
  322. r8Flags, r8Deps := d.r8Flags(ctx, dexParams.flags)
  323. r8Deps = append(r8Deps, commonDeps...)
  324. rule := r8
  325. args := map[string]string{
  326. "r8Flags": strings.Join(append(commonFlags, r8Flags...), " "),
  327. "zipFlags": zipFlags,
  328. "outDict": proguardDictionary.String(),
  329. "outConfig": proguardConfiguration.String(),
  330. "outUsageDir": proguardUsageDir.String(),
  331. "outUsage": proguardUsage.String(),
  332. "outUsageZip": proguardUsageZip.String(),
  333. "outDir": outDir.String(),
  334. "tmpJar": tmpJar.String(),
  335. "mergeZipsFlags": mergeZipsFlags,
  336. }
  337. if ctx.Config().UseRBE() && ctx.Config().IsEnvTrue("RBE_R8") {
  338. rule = r8RE
  339. args["implicits"] = strings.Join(r8Deps.Strings(), ",")
  340. }
  341. ctx.Build(pctx, android.BuildParams{
  342. Rule: rule,
  343. Description: "r8",
  344. Output: javalibJar,
  345. ImplicitOutputs: android.WritablePaths{
  346. proguardDictionary,
  347. proguardUsageZip,
  348. proguardConfiguration},
  349. Input: dexParams.classesJar,
  350. Implicits: r8Deps,
  351. Args: args,
  352. })
  353. } else {
  354. d8Flags, d8Deps := d8Flags(dexParams.flags)
  355. d8Deps = append(d8Deps, commonDeps...)
  356. rule := d8
  357. if ctx.Config().UseRBE() && ctx.Config().IsEnvTrue("RBE_D8") {
  358. rule = d8RE
  359. }
  360. ctx.Build(pctx, android.BuildParams{
  361. Rule: rule,
  362. Description: "d8",
  363. Output: javalibJar,
  364. Input: dexParams.classesJar,
  365. Implicits: d8Deps,
  366. Args: map[string]string{
  367. "d8Flags": strings.Join(append(commonFlags, d8Flags...), " "),
  368. "zipFlags": zipFlags,
  369. "outDir": outDir.String(),
  370. "tmpJar": tmpJar.String(),
  371. "mergeZipsFlags": mergeZipsFlags,
  372. },
  373. })
  374. }
  375. if proptools.Bool(d.dexProperties.Uncompress_dex) {
  376. alignedJavalibJar := android.PathForModuleOut(ctx, "aligned", dexParams.jarName).OutputPath
  377. TransformZipAlign(ctx, alignedJavalibJar, javalibJar)
  378. javalibJar = alignedJavalibJar
  379. }
  380. return javalibJar
  381. }