merge_zips_test.go 8.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392
  1. // Copyright 2018 Google Inc. All rights reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package main
  15. import (
  16. "bytes"
  17. "fmt"
  18. "os"
  19. "strconv"
  20. "strings"
  21. "testing"
  22. "android/soong/jar"
  23. "android/soong/third_party/zip"
  24. )
  25. type testZipEntry struct {
  26. name string
  27. mode os.FileMode
  28. data []byte
  29. }
  30. var (
  31. A = testZipEntry{"A", 0755, []byte("foo")}
  32. a = testZipEntry{"a", 0755, []byte("foo")}
  33. a2 = testZipEntry{"a", 0755, []byte("FOO2")}
  34. a3 = testZipEntry{"a", 0755, []byte("Foo3")}
  35. bDir = testZipEntry{"b/", os.ModeDir | 0755, nil}
  36. bbDir = testZipEntry{"b/b/", os.ModeDir | 0755, nil}
  37. bbb = testZipEntry{"b/b/b", 0755, nil}
  38. ba = testZipEntry{"b/a", 0755, []byte("foob")}
  39. bc = testZipEntry{"b/c", 0755, []byte("bar")}
  40. bd = testZipEntry{"b/d", 0700, []byte("baz")}
  41. be = testZipEntry{"b/e", 0700, []byte("")}
  42. metainfDir = testZipEntry{jar.MetaDir, os.ModeDir | 0755, nil}
  43. manifestFile = testZipEntry{jar.ManifestFile, 0755, []byte("manifest")}
  44. manifestFile2 = testZipEntry{jar.ManifestFile, 0755, []byte("manifest2")}
  45. moduleInfoFile = testZipEntry{jar.ModuleInfoClass, 0755, []byte("module-info")}
  46. )
  47. type testInputZip struct {
  48. name string
  49. entries []testZipEntry
  50. reader *zip.Reader
  51. }
  52. func (tiz *testInputZip) Name() string {
  53. return tiz.name
  54. }
  55. func (tiz *testInputZip) Open() error {
  56. if tiz.reader == nil {
  57. tiz.reader = testZipEntriesToZipReader(tiz.entries)
  58. }
  59. return nil
  60. }
  61. func (tiz *testInputZip) Close() error {
  62. tiz.reader = nil
  63. return nil
  64. }
  65. func (tiz *testInputZip) Entries() []*zip.File {
  66. if tiz.reader == nil {
  67. panic(fmt.Errorf("%s: should be open to get entries", tiz.Name()))
  68. }
  69. return tiz.reader.File
  70. }
  71. func (tiz *testInputZip) IsOpen() bool {
  72. return tiz.reader != nil
  73. }
  74. func TestMergeZips(t *testing.T) {
  75. testCases := []struct {
  76. name string
  77. in [][]testZipEntry
  78. stripFiles []string
  79. stripDirs []string
  80. jar bool
  81. sort bool
  82. ignoreDuplicates bool
  83. stripDirEntries bool
  84. zipsToNotStrip map[string]bool
  85. out []testZipEntry
  86. err string
  87. }{
  88. {
  89. name: "duplicates error",
  90. in: [][]testZipEntry{
  91. {a},
  92. {a2},
  93. {a3},
  94. },
  95. out: []testZipEntry{a},
  96. err: "duplicate",
  97. },
  98. {
  99. name: "duplicates take first",
  100. in: [][]testZipEntry{
  101. {a},
  102. {a2},
  103. {a3},
  104. },
  105. out: []testZipEntry{a},
  106. ignoreDuplicates: true,
  107. },
  108. {
  109. name: "duplicates identical",
  110. in: [][]testZipEntry{
  111. {a},
  112. {a},
  113. },
  114. out: []testZipEntry{a},
  115. },
  116. {
  117. name: "sort",
  118. in: [][]testZipEntry{
  119. {be, bc, bDir, bbDir, bbb, A, metainfDir, manifestFile},
  120. },
  121. out: []testZipEntry{A, metainfDir, manifestFile, bDir, bbDir, bbb, bc, be},
  122. sort: true,
  123. },
  124. {
  125. name: "jar sort",
  126. in: [][]testZipEntry{
  127. {be, bc, bDir, A, metainfDir, manifestFile},
  128. },
  129. out: []testZipEntry{metainfDir, manifestFile, A, bDir, bc, be},
  130. jar: true,
  131. },
  132. {
  133. name: "jar merge",
  134. in: [][]testZipEntry{
  135. {metainfDir, manifestFile, bDir, be},
  136. {metainfDir, manifestFile2, bDir, bc},
  137. {metainfDir, manifestFile2, A},
  138. },
  139. out: []testZipEntry{metainfDir, manifestFile, A, bDir, bc, be},
  140. jar: true,
  141. },
  142. {
  143. name: "merge",
  144. in: [][]testZipEntry{
  145. {bDir, be},
  146. {bDir, bc},
  147. {A},
  148. },
  149. out: []testZipEntry{bDir, be, bc, A},
  150. },
  151. {
  152. name: "strip dir entries",
  153. in: [][]testZipEntry{
  154. {a, bDir, bbDir, bbb, bc, bd, be},
  155. },
  156. out: []testZipEntry{a, bbb, bc, bd, be},
  157. stripDirEntries: true,
  158. },
  159. {
  160. name: "strip files",
  161. in: [][]testZipEntry{
  162. {a, bDir, bbDir, bbb, bc, bd, be},
  163. },
  164. out: []testZipEntry{a, bDir, bbDir, bbb, bc},
  165. stripFiles: []string{"b/d", "b/e"},
  166. },
  167. {
  168. // merge_zips used to treat -stripFile a as stripping any file named a, it now only strips a in the
  169. // root of the zip.
  170. name: "strip file name",
  171. in: [][]testZipEntry{
  172. {a, bDir, ba},
  173. },
  174. out: []testZipEntry{bDir, ba},
  175. stripFiles: []string{"a"},
  176. },
  177. {
  178. name: "strip files glob",
  179. in: [][]testZipEntry{
  180. {a, bDir, ba},
  181. },
  182. out: []testZipEntry{bDir},
  183. stripFiles: []string{"**/a"},
  184. },
  185. {
  186. name: "strip dirs",
  187. in: [][]testZipEntry{
  188. {a, bDir, bbDir, bbb, bc, bd, be},
  189. },
  190. out: []testZipEntry{a},
  191. stripDirs: []string{"b"},
  192. },
  193. {
  194. name: "strip dirs glob",
  195. in: [][]testZipEntry{
  196. {a, bDir, bbDir, bbb, bc, bd, be},
  197. },
  198. out: []testZipEntry{a, bDir, bc, bd, be},
  199. stripDirs: []string{"b/*"},
  200. },
  201. {
  202. name: "zips to not strip",
  203. in: [][]testZipEntry{
  204. {a, bDir, bc},
  205. {bDir, bd},
  206. {bDir, be},
  207. },
  208. out: []testZipEntry{a, bDir, bd},
  209. stripDirs: []string{"b"},
  210. zipsToNotStrip: map[string]bool{
  211. "in1": true,
  212. },
  213. },
  214. }
  215. for _, test := range testCases {
  216. t.Run(test.name, func(t *testing.T) {
  217. inputZips := make([]InputZip, len(test.in))
  218. for i, in := range test.in {
  219. inputZips[i] = &testInputZip{name: "in" + strconv.Itoa(i), entries: in}
  220. }
  221. want := testZipEntriesToBuf(test.out)
  222. out := &bytes.Buffer{}
  223. writer := zip.NewWriter(out)
  224. err := mergeZips(inputZips, writer, "", "",
  225. test.sort, test.jar, false, test.stripDirEntries, test.ignoreDuplicates,
  226. test.stripFiles, test.stripDirs, test.zipsToNotStrip)
  227. closeErr := writer.Close()
  228. if closeErr != nil {
  229. t.Fatal(err)
  230. }
  231. if test.err != "" {
  232. if err == nil {
  233. t.Fatal("missing err, expected: ", test.err)
  234. } else if !strings.Contains(strings.ToLower(err.Error()), strings.ToLower(test.err)) {
  235. t.Fatal("incorrect err, want:", test.err, "got:", err)
  236. }
  237. return
  238. }
  239. if !bytes.Equal(want, out.Bytes()) {
  240. t.Error("incorrect zip output")
  241. t.Errorf("want:\n%s", dumpZip(want))
  242. t.Errorf("got:\n%s", dumpZip(out.Bytes()))
  243. }
  244. })
  245. }
  246. }
  247. func testZipEntriesToBuf(entries []testZipEntry) []byte {
  248. b := &bytes.Buffer{}
  249. zw := zip.NewWriter(b)
  250. for _, e := range entries {
  251. fh := zip.FileHeader{
  252. Name: e.name,
  253. }
  254. fh.SetMode(e.mode)
  255. w, err := zw.CreateHeader(&fh)
  256. if err != nil {
  257. panic(err)
  258. }
  259. _, err = w.Write(e.data)
  260. if err != nil {
  261. panic(err)
  262. }
  263. }
  264. err := zw.Close()
  265. if err != nil {
  266. panic(err)
  267. }
  268. return b.Bytes()
  269. }
  270. func testZipEntriesToZipReader(entries []testZipEntry) *zip.Reader {
  271. b := testZipEntriesToBuf(entries)
  272. r := bytes.NewReader(b)
  273. zr, err := zip.NewReader(r, int64(len(b)))
  274. if err != nil {
  275. panic(err)
  276. }
  277. return zr
  278. }
  279. func dumpZip(buf []byte) string {
  280. r := bytes.NewReader(buf)
  281. zr, err := zip.NewReader(r, int64(len(buf)))
  282. if err != nil {
  283. panic(err)
  284. }
  285. var ret string
  286. for _, f := range zr.File {
  287. ret += fmt.Sprintf("%v: %v %v %08x\n", f.Name, f.Mode(), f.UncompressedSize64, f.CRC32)
  288. }
  289. return ret
  290. }
  291. type DummyInpuZip struct {
  292. isOpen bool
  293. }
  294. func (diz *DummyInpuZip) Name() string {
  295. return "dummy"
  296. }
  297. func (diz *DummyInpuZip) Open() error {
  298. diz.isOpen = true
  299. return nil
  300. }
  301. func (diz *DummyInpuZip) Close() error {
  302. diz.isOpen = false
  303. return nil
  304. }
  305. func (DummyInpuZip) Entries() []*zip.File {
  306. panic("implement me")
  307. }
  308. func (diz *DummyInpuZip) IsOpen() bool {
  309. return diz.isOpen
  310. }
  311. func TestInputZipsManager(t *testing.T) {
  312. const nInputZips = 20
  313. const nMaxOpenZips = 10
  314. izm := NewInputZipsManager(20, 10)
  315. managedZips := make([]InputZip, nInputZips)
  316. for i := 0; i < nInputZips; i++ {
  317. managedZips[i] = izm.Manage(&DummyInpuZip{})
  318. }
  319. t.Run("InputZipsManager", func(t *testing.T) {
  320. for i, iz := range managedZips {
  321. if err := iz.Open(); err != nil {
  322. t.Fatalf("Step %d: open failed: %s", i, err)
  323. return
  324. }
  325. if izm.nOpenZips > nMaxOpenZips {
  326. t.Errorf("Step %d: should be <=%d open zips", i, nMaxOpenZips)
  327. }
  328. }
  329. if !managedZips[nInputZips-1].IsOpen() {
  330. t.Error("The last input should stay open")
  331. }
  332. for _, iz := range managedZips {
  333. iz.Close()
  334. }
  335. if izm.nOpenZips > 0 {
  336. t.Error("Some input zips are still open")
  337. }
  338. })
  339. }