zip.go 22 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006
  1. // Copyright 2015 Google Inc. All rights reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package zip
  15. import (
  16. "bytes"
  17. "compress/flate"
  18. "errors"
  19. "fmt"
  20. "hash/crc32"
  21. "io"
  22. "io/ioutil"
  23. "os"
  24. "path/filepath"
  25. "sort"
  26. "strings"
  27. "sync"
  28. "syscall"
  29. "time"
  30. "android/soong/response"
  31. "github.com/google/blueprint/pathtools"
  32. "android/soong/jar"
  33. "android/soong/third_party/zip"
  34. )
  35. // Block size used during parallel compression of a single file.
  36. const parallelBlockSize = 1 * 1024 * 1024 // 1MB
  37. // Minimum file size to use parallel compression. It requires more
  38. // flate.Writer allocations, since we can't change the dictionary
  39. // during Reset
  40. const minParallelFileSize = parallelBlockSize * 6
  41. // Size of the ZIP compression window (32KB)
  42. const windowSize = 32 * 1024
  43. type nopCloser struct {
  44. io.Writer
  45. }
  46. func (nopCloser) Close() error {
  47. return nil
  48. }
  49. type byteReaderCloser struct {
  50. *bytes.Reader
  51. io.Closer
  52. }
  53. type pathMapping struct {
  54. dest, src string
  55. zipMethod uint16
  56. }
  57. type FileArg struct {
  58. PathPrefixInZip, SourcePrefixToStrip string
  59. SourceFiles []string
  60. JunkPaths bool
  61. GlobDir string
  62. }
  63. type FileArgsBuilder struct {
  64. state FileArg
  65. err error
  66. fs pathtools.FileSystem
  67. fileArgs []FileArg
  68. }
  69. func NewFileArgsBuilder() *FileArgsBuilder {
  70. return &FileArgsBuilder{
  71. fs: pathtools.OsFs,
  72. }
  73. }
  74. func (b *FileArgsBuilder) JunkPaths(v bool) *FileArgsBuilder {
  75. b.state.JunkPaths = v
  76. b.state.SourcePrefixToStrip = ""
  77. return b
  78. }
  79. func (b *FileArgsBuilder) SourcePrefixToStrip(prefixToStrip string) *FileArgsBuilder {
  80. b.state.JunkPaths = false
  81. b.state.SourcePrefixToStrip = prefixToStrip
  82. return b
  83. }
  84. func (b *FileArgsBuilder) PathPrefixInZip(rootPrefix string) *FileArgsBuilder {
  85. b.state.PathPrefixInZip = rootPrefix
  86. return b
  87. }
  88. func (b *FileArgsBuilder) File(name string) *FileArgsBuilder {
  89. if b.err != nil {
  90. return b
  91. }
  92. arg := b.state
  93. arg.SourceFiles = []string{name}
  94. b.fileArgs = append(b.fileArgs, arg)
  95. return b
  96. }
  97. func (b *FileArgsBuilder) Dir(name string) *FileArgsBuilder {
  98. if b.err != nil {
  99. return b
  100. }
  101. arg := b.state
  102. arg.GlobDir = name
  103. b.fileArgs = append(b.fileArgs, arg)
  104. return b
  105. }
  106. // List reads the file names from the given file and adds them to the source files list.
  107. func (b *FileArgsBuilder) List(name string) *FileArgsBuilder {
  108. if b.err != nil {
  109. return b
  110. }
  111. f, err := b.fs.Open(name)
  112. if err != nil {
  113. b.err = err
  114. return b
  115. }
  116. defer f.Close()
  117. list, err := ioutil.ReadAll(f)
  118. if err != nil {
  119. b.err = err
  120. return b
  121. }
  122. arg := b.state
  123. arg.SourceFiles = strings.Fields(string(list))
  124. b.fileArgs = append(b.fileArgs, arg)
  125. return b
  126. }
  127. // RspFile reads the file names from given .rsp file and adds them to the source files list.
  128. func (b *FileArgsBuilder) RspFile(name string) *FileArgsBuilder {
  129. if b.err != nil {
  130. return b
  131. }
  132. f, err := b.fs.Open(name)
  133. if err != nil {
  134. b.err = err
  135. return b
  136. }
  137. defer f.Close()
  138. arg := b.state
  139. arg.SourceFiles, err = response.ReadRspFile(f)
  140. if err != nil {
  141. b.err = err
  142. return b
  143. }
  144. for i := range arg.SourceFiles {
  145. arg.SourceFiles[i] = pathtools.MatchEscape(arg.SourceFiles[i])
  146. }
  147. b.fileArgs = append(b.fileArgs, arg)
  148. return b
  149. }
  150. func (b *FileArgsBuilder) Error() error {
  151. if b == nil {
  152. return nil
  153. }
  154. return b.err
  155. }
  156. func (b *FileArgsBuilder) FileArgs() []FileArg {
  157. if b == nil {
  158. return nil
  159. }
  160. return b.fileArgs
  161. }
  162. type IncorrectRelativeRootError struct {
  163. RelativeRoot string
  164. Path string
  165. }
  166. func (x IncorrectRelativeRootError) Error() string {
  167. return fmt.Sprintf("path %q is outside relative root %q", x.Path, x.RelativeRoot)
  168. }
  169. type ZipWriter struct {
  170. time time.Time
  171. createdFiles map[string]string
  172. createdDirs map[string]string
  173. directories bool
  174. errors chan error
  175. writeOps chan chan *zipEntry
  176. cpuRateLimiter *CPURateLimiter
  177. memoryRateLimiter *MemoryRateLimiter
  178. compressorPool sync.Pool
  179. compLevel int
  180. followSymlinks pathtools.ShouldFollowSymlinks
  181. ignoreMissingFiles bool
  182. stderr io.Writer
  183. fs pathtools.FileSystem
  184. }
  185. type zipEntry struct {
  186. fh *zip.FileHeader
  187. // List of delayed io.Reader
  188. futureReaders chan chan io.Reader
  189. // Only used for passing into the MemoryRateLimiter to ensure we
  190. // release as much memory as much as we request
  191. allocatedSize int64
  192. }
  193. type ZipArgs struct {
  194. FileArgs []FileArg
  195. OutputFilePath string
  196. EmulateJar bool
  197. SrcJar bool
  198. AddDirectoryEntriesToZip bool
  199. CompressionLevel int
  200. ManifestSourcePath string
  201. NumParallelJobs int
  202. NonDeflatedFiles map[string]bool
  203. WriteIfChanged bool
  204. StoreSymlinks bool
  205. IgnoreMissingFiles bool
  206. Stderr io.Writer
  207. Filesystem pathtools.FileSystem
  208. }
  209. func zipTo(args ZipArgs, w io.Writer) error {
  210. if args.EmulateJar {
  211. args.AddDirectoryEntriesToZip = true
  212. }
  213. // Have Glob follow symlinks if they are not being stored as symlinks in the zip file.
  214. followSymlinks := pathtools.ShouldFollowSymlinks(!args.StoreSymlinks)
  215. z := &ZipWriter{
  216. time: jar.DefaultTime,
  217. createdDirs: make(map[string]string),
  218. createdFiles: make(map[string]string),
  219. directories: args.AddDirectoryEntriesToZip,
  220. compLevel: args.CompressionLevel,
  221. followSymlinks: followSymlinks,
  222. ignoreMissingFiles: args.IgnoreMissingFiles,
  223. stderr: args.Stderr,
  224. fs: args.Filesystem,
  225. }
  226. if z.fs == nil {
  227. z.fs = pathtools.OsFs
  228. }
  229. if z.stderr == nil {
  230. z.stderr = os.Stderr
  231. }
  232. pathMappings := []pathMapping{}
  233. noCompression := args.CompressionLevel == 0
  234. for _, fa := range args.FileArgs {
  235. var srcs []string
  236. for _, s := range fa.SourceFiles {
  237. s = strings.TrimSpace(s)
  238. if s == "" {
  239. continue
  240. }
  241. result, err := z.fs.Glob(s, nil, followSymlinks)
  242. if err != nil {
  243. return err
  244. }
  245. if len(result.Matches) == 0 {
  246. err := &os.PathError{
  247. Op: "lstat",
  248. Path: s,
  249. Err: os.ErrNotExist,
  250. }
  251. if args.IgnoreMissingFiles {
  252. fmt.Fprintln(z.stderr, "warning:", err)
  253. } else {
  254. return err
  255. }
  256. }
  257. srcs = append(srcs, result.Matches...)
  258. }
  259. if fa.GlobDir != "" {
  260. if exists, isDir, err := z.fs.Exists(fa.GlobDir); err != nil {
  261. return err
  262. } else if !exists && !args.IgnoreMissingFiles {
  263. err := &os.PathError{
  264. Op: "lstat",
  265. Path: fa.GlobDir,
  266. Err: os.ErrNotExist,
  267. }
  268. if args.IgnoreMissingFiles {
  269. fmt.Fprintln(z.stderr, "warning:", err)
  270. } else {
  271. return err
  272. }
  273. } else if !isDir && !args.IgnoreMissingFiles {
  274. err := &os.PathError{
  275. Op: "lstat",
  276. Path: fa.GlobDir,
  277. Err: syscall.ENOTDIR,
  278. }
  279. if args.IgnoreMissingFiles {
  280. fmt.Fprintln(z.stderr, "warning:", err)
  281. } else {
  282. return err
  283. }
  284. }
  285. result, err := z.fs.Glob(filepath.Join(fa.GlobDir, "**/*"), nil, followSymlinks)
  286. if err != nil {
  287. return err
  288. }
  289. srcs = append(srcs, result.Matches...)
  290. }
  291. for _, src := range srcs {
  292. err := fillPathPairs(fa, src, &pathMappings, args.NonDeflatedFiles, noCompression)
  293. if err != nil {
  294. return err
  295. }
  296. }
  297. }
  298. return z.write(w, pathMappings, args.ManifestSourcePath, args.EmulateJar, args.SrcJar, args.NumParallelJobs)
  299. }
  300. // Zip creates an output zip archive from given sources.
  301. func Zip(args ZipArgs) error {
  302. if args.OutputFilePath == "" {
  303. return fmt.Errorf("output file path must be nonempty")
  304. }
  305. buf := &bytes.Buffer{}
  306. var out io.Writer = buf
  307. var zipErr error
  308. if !args.WriteIfChanged {
  309. f, err := os.Create(args.OutputFilePath)
  310. if err != nil {
  311. return err
  312. }
  313. defer f.Close()
  314. defer func() {
  315. if zipErr != nil {
  316. os.Remove(args.OutputFilePath)
  317. }
  318. }()
  319. out = f
  320. }
  321. zipErr = zipTo(args, out)
  322. if zipErr != nil {
  323. return zipErr
  324. }
  325. if args.WriteIfChanged {
  326. err := pathtools.WriteFileIfChanged(args.OutputFilePath, buf.Bytes(), 0666)
  327. if err != nil {
  328. return err
  329. }
  330. }
  331. return nil
  332. }
  333. func fillPathPairs(fa FileArg, src string, pathMappings *[]pathMapping,
  334. nonDeflatedFiles map[string]bool, noCompression bool) error {
  335. var dest string
  336. if fa.JunkPaths {
  337. dest = filepath.Base(src)
  338. } else {
  339. var err error
  340. dest, err = filepath.Rel(fa.SourcePrefixToStrip, src)
  341. if err != nil {
  342. return err
  343. }
  344. if strings.HasPrefix(dest, "../") {
  345. return IncorrectRelativeRootError{
  346. Path: src,
  347. RelativeRoot: fa.SourcePrefixToStrip,
  348. }
  349. }
  350. }
  351. dest = filepath.Join(fa.PathPrefixInZip, dest)
  352. zipMethod := zip.Deflate
  353. if _, found := nonDeflatedFiles[dest]; found || noCompression {
  354. zipMethod = zip.Store
  355. }
  356. *pathMappings = append(*pathMappings,
  357. pathMapping{dest: dest, src: src, zipMethod: zipMethod})
  358. return nil
  359. }
  360. func jarSort(mappings []pathMapping) {
  361. sort.SliceStable(mappings, func(i int, j int) bool {
  362. return jar.EntryNamesLess(mappings[i].dest, mappings[j].dest)
  363. })
  364. }
  365. func (z *ZipWriter) write(f io.Writer, pathMappings []pathMapping, manifest string, emulateJar, srcJar bool,
  366. parallelJobs int) error {
  367. z.errors = make(chan error)
  368. defer close(z.errors)
  369. // This channel size can be essentially unlimited -- it's used as a fifo
  370. // queue decouple the CPU and IO loads. Directories don't require any
  371. // compression time, but still cost some IO. Similar with small files that
  372. // can be very fast to compress. Some files that are more difficult to
  373. // compress won't take a corresponding longer time writing out.
  374. //
  375. // The optimum size here depends on your CPU and IO characteristics, and
  376. // the the layout of your zip file. 1000 was chosen mostly at random as
  377. // something that worked reasonably well for a test file.
  378. //
  379. // The RateLimit object will put the upper bounds on the number of
  380. // parallel compressions and outstanding buffers.
  381. z.writeOps = make(chan chan *zipEntry, 1000)
  382. z.cpuRateLimiter = NewCPURateLimiter(int64(parallelJobs))
  383. z.memoryRateLimiter = NewMemoryRateLimiter(0)
  384. defer func() {
  385. z.cpuRateLimiter.Stop()
  386. z.memoryRateLimiter.Stop()
  387. }()
  388. if manifest != "" && !emulateJar {
  389. return errors.New("must specify --jar when specifying a manifest via -m")
  390. }
  391. if emulateJar {
  392. // manifest may be empty, in which case addManifest will fill in a default
  393. pathMappings = append(pathMappings, pathMapping{jar.ManifestFile, manifest, zip.Deflate})
  394. jarSort(pathMappings)
  395. }
  396. go func() {
  397. var err error
  398. defer close(z.writeOps)
  399. for _, ele := range pathMappings {
  400. if emulateJar && ele.dest == jar.ManifestFile {
  401. err = z.addManifest(ele.dest, ele.src, ele.zipMethod)
  402. } else {
  403. err = z.addFile(ele.dest, ele.src, ele.zipMethod, emulateJar, srcJar)
  404. }
  405. if err != nil {
  406. z.errors <- err
  407. return
  408. }
  409. }
  410. }()
  411. zipw := zip.NewWriter(f)
  412. var currentWriteOpChan chan *zipEntry
  413. var currentWriter io.WriteCloser
  414. var currentReaders chan chan io.Reader
  415. var currentReader chan io.Reader
  416. var done bool
  417. for !done {
  418. var writeOpsChan chan chan *zipEntry
  419. var writeOpChan chan *zipEntry
  420. var readersChan chan chan io.Reader
  421. if currentReader != nil {
  422. // Only read and process errors
  423. } else if currentReaders != nil {
  424. readersChan = currentReaders
  425. } else if currentWriteOpChan != nil {
  426. writeOpChan = currentWriteOpChan
  427. } else {
  428. writeOpsChan = z.writeOps
  429. }
  430. select {
  431. case writeOp, ok := <-writeOpsChan:
  432. if !ok {
  433. done = true
  434. }
  435. currentWriteOpChan = writeOp
  436. case op := <-writeOpChan:
  437. currentWriteOpChan = nil
  438. var err error
  439. if op.fh.Method == zip.Deflate {
  440. currentWriter, err = zipw.CreateCompressedHeader(op.fh)
  441. } else {
  442. var zw io.Writer
  443. op.fh.CompressedSize64 = op.fh.UncompressedSize64
  444. zw, err = zipw.CreateHeaderAndroid(op.fh)
  445. currentWriter = nopCloser{zw}
  446. }
  447. if err != nil {
  448. return err
  449. }
  450. currentReaders = op.futureReaders
  451. if op.futureReaders == nil {
  452. currentWriter.Close()
  453. currentWriter = nil
  454. }
  455. z.memoryRateLimiter.Finish(op.allocatedSize)
  456. case futureReader, ok := <-readersChan:
  457. if !ok {
  458. // Done with reading
  459. currentWriter.Close()
  460. currentWriter = nil
  461. currentReaders = nil
  462. }
  463. currentReader = futureReader
  464. case reader := <-currentReader:
  465. _, err := io.Copy(currentWriter, reader)
  466. if err != nil {
  467. return err
  468. }
  469. currentReader = nil
  470. case err := <-z.errors:
  471. return err
  472. }
  473. }
  474. // One last chance to catch an error
  475. select {
  476. case err := <-z.errors:
  477. return err
  478. default:
  479. zipw.Close()
  480. return nil
  481. }
  482. }
  483. // imports (possibly with compression) <src> into the zip at sub-path <dest>
  484. func (z *ZipWriter) addFile(dest, src string, method uint16, emulateJar, srcJar bool) error {
  485. var fileSize int64
  486. var executable bool
  487. var s os.FileInfo
  488. var err error
  489. if z.followSymlinks {
  490. s, err = z.fs.Stat(src)
  491. } else {
  492. s, err = z.fs.Lstat(src)
  493. }
  494. if err != nil {
  495. if os.IsNotExist(err) && z.ignoreMissingFiles {
  496. fmt.Fprintln(z.stderr, "warning:", err)
  497. return nil
  498. }
  499. return err
  500. }
  501. createParentDirs := func(dest, src string) error {
  502. if err := z.writeDirectory(filepath.Dir(dest), src, emulateJar); err != nil {
  503. return err
  504. }
  505. if prev, exists := z.createdDirs[dest]; exists {
  506. return fmt.Errorf("destination %q is both a directory %q and a file %q", dest, prev, src)
  507. }
  508. if prev, exists := z.createdFiles[dest]; exists {
  509. return fmt.Errorf("destination %q has two files %q and %q", dest, prev, src)
  510. }
  511. z.createdFiles[dest] = src
  512. return nil
  513. }
  514. if s.IsDir() {
  515. if z.directories {
  516. return z.writeDirectory(dest, src, emulateJar)
  517. }
  518. return nil
  519. } else if s.Mode()&os.ModeSymlink != 0 {
  520. err = createParentDirs(dest, src)
  521. if err != nil {
  522. return err
  523. }
  524. return z.writeSymlink(dest, src)
  525. } else if s.Mode().IsRegular() {
  526. r, err := z.fs.Open(src)
  527. if err != nil {
  528. return err
  529. }
  530. if srcJar && filepath.Ext(src) == ".java" {
  531. // rewrite the destination using the package path if it can be determined
  532. pkg, err := jar.JavaPackage(r, src)
  533. if err != nil {
  534. // ignore errors for now, leaving the file at in its original location in the zip
  535. } else {
  536. dest = filepath.Join(filepath.Join(strings.Split(pkg, ".")...), filepath.Base(src))
  537. }
  538. _, err = r.Seek(0, io.SeekStart)
  539. if err != nil {
  540. return err
  541. }
  542. }
  543. fileSize = s.Size()
  544. executable = s.Mode()&0100 != 0
  545. header := &zip.FileHeader{
  546. Name: dest,
  547. Method: method,
  548. UncompressedSize64: uint64(fileSize),
  549. }
  550. mode := os.FileMode(0644)
  551. if executable {
  552. mode = 0755
  553. }
  554. header.SetMode(mode)
  555. err = createParentDirs(dest, src)
  556. if err != nil {
  557. return err
  558. }
  559. return z.writeFileContents(header, r)
  560. } else {
  561. return fmt.Errorf("%s is not a file, directory, or symlink", src)
  562. }
  563. }
  564. func (z *ZipWriter) addManifest(dest string, src string, _ uint16) error {
  565. if prev, exists := z.createdDirs[dest]; exists {
  566. return fmt.Errorf("destination %q is both a directory %q and a file %q", dest, prev, src)
  567. }
  568. if prev, exists := z.createdFiles[dest]; exists {
  569. return fmt.Errorf("destination %q has two files %q and %q", dest, prev, src)
  570. }
  571. if err := z.writeDirectory(filepath.Dir(dest), src, true); err != nil {
  572. return err
  573. }
  574. var contents []byte
  575. if src != "" {
  576. f, err := z.fs.Open(src)
  577. if err != nil {
  578. return err
  579. }
  580. contents, err = ioutil.ReadAll(f)
  581. f.Close()
  582. if err != nil {
  583. return err
  584. }
  585. }
  586. fh, buf, err := jar.ManifestFileContents(contents)
  587. if err != nil {
  588. return err
  589. }
  590. reader := &byteReaderCloser{bytes.NewReader(buf), ioutil.NopCloser(nil)}
  591. return z.writeFileContents(fh, reader)
  592. }
  593. func (z *ZipWriter) writeFileContents(header *zip.FileHeader, r pathtools.ReaderAtSeekerCloser) (err error) {
  594. header.SetModTime(z.time)
  595. compressChan := make(chan *zipEntry, 1)
  596. z.writeOps <- compressChan
  597. // Pre-fill a zipEntry, it will be sent in the compressChan once
  598. // we're sure about the Method and CRC.
  599. ze := &zipEntry{
  600. fh: header,
  601. }
  602. ze.allocatedSize = int64(header.UncompressedSize64)
  603. z.cpuRateLimiter.Request()
  604. z.memoryRateLimiter.Request(ze.allocatedSize)
  605. fileSize := int64(header.UncompressedSize64)
  606. if fileSize == 0 {
  607. fileSize = int64(header.UncompressedSize)
  608. }
  609. if header.Method == zip.Deflate && fileSize >= minParallelFileSize {
  610. wg := new(sync.WaitGroup)
  611. // Allocate enough buffer to hold all readers. We'll limit
  612. // this based on actual buffer sizes in RateLimit.
  613. ze.futureReaders = make(chan chan io.Reader, (fileSize/parallelBlockSize)+1)
  614. // Calculate the CRC in the background, since reading the entire
  615. // file could take a while.
  616. //
  617. // We could split this up into chunks as well, but it's faster
  618. // than the compression. Due to the Go Zip API, we also need to
  619. // know the result before we can begin writing the compressed
  620. // data out to the zipfile.
  621. wg.Add(1)
  622. go z.crcFile(r, ze, compressChan, wg)
  623. for start := int64(0); start < fileSize; start += parallelBlockSize {
  624. sr := io.NewSectionReader(r, start, parallelBlockSize)
  625. resultChan := make(chan io.Reader, 1)
  626. ze.futureReaders <- resultChan
  627. z.cpuRateLimiter.Request()
  628. last := !(start+parallelBlockSize < fileSize)
  629. var dict []byte
  630. if start >= windowSize {
  631. dict, err = ioutil.ReadAll(io.NewSectionReader(r, start-windowSize, windowSize))
  632. if err != nil {
  633. return err
  634. }
  635. }
  636. wg.Add(1)
  637. go z.compressPartialFile(sr, dict, last, resultChan, wg)
  638. }
  639. close(ze.futureReaders)
  640. // Close the file handle after all readers are done
  641. go func(wg *sync.WaitGroup, closer io.Closer) {
  642. wg.Wait()
  643. closer.Close()
  644. }(wg, r)
  645. } else {
  646. go func() {
  647. z.compressWholeFile(ze, r, compressChan)
  648. r.Close()
  649. }()
  650. }
  651. return nil
  652. }
  653. func (z *ZipWriter) crcFile(r io.Reader, ze *zipEntry, resultChan chan *zipEntry, wg *sync.WaitGroup) {
  654. defer wg.Done()
  655. defer z.cpuRateLimiter.Finish()
  656. crc := crc32.NewIEEE()
  657. _, err := io.Copy(crc, r)
  658. if err != nil {
  659. z.errors <- err
  660. return
  661. }
  662. ze.fh.CRC32 = crc.Sum32()
  663. resultChan <- ze
  664. close(resultChan)
  665. }
  666. func (z *ZipWriter) compressPartialFile(r io.Reader, dict []byte, last bool, resultChan chan io.Reader, wg *sync.WaitGroup) {
  667. defer wg.Done()
  668. result, err := z.compressBlock(r, dict, last)
  669. if err != nil {
  670. z.errors <- err
  671. return
  672. }
  673. z.cpuRateLimiter.Finish()
  674. resultChan <- result
  675. }
  676. func (z *ZipWriter) compressBlock(r io.Reader, dict []byte, last bool) (*bytes.Buffer, error) {
  677. buf := new(bytes.Buffer)
  678. var fw *flate.Writer
  679. var err error
  680. if len(dict) > 0 {
  681. // There's no way to Reset a Writer with a new dictionary, so
  682. // don't use the Pool
  683. fw, err = flate.NewWriterDict(buf, z.compLevel, dict)
  684. } else {
  685. var ok bool
  686. if fw, ok = z.compressorPool.Get().(*flate.Writer); ok {
  687. fw.Reset(buf)
  688. } else {
  689. fw, err = flate.NewWriter(buf, z.compLevel)
  690. }
  691. defer z.compressorPool.Put(fw)
  692. }
  693. if err != nil {
  694. return nil, err
  695. }
  696. _, err = io.Copy(fw, r)
  697. if err != nil {
  698. return nil, err
  699. }
  700. if last {
  701. fw.Close()
  702. } else {
  703. fw.Flush()
  704. }
  705. return buf, nil
  706. }
  707. func (z *ZipWriter) compressWholeFile(ze *zipEntry, r io.ReadSeeker, compressChan chan *zipEntry) {
  708. crc := crc32.NewIEEE()
  709. _, err := io.Copy(crc, r)
  710. if err != nil {
  711. z.errors <- err
  712. return
  713. }
  714. ze.fh.CRC32 = crc.Sum32()
  715. _, err = r.Seek(0, 0)
  716. if err != nil {
  717. z.errors <- err
  718. return
  719. }
  720. readFile := func(reader io.ReadSeeker) ([]byte, error) {
  721. _, err := reader.Seek(0, 0)
  722. if err != nil {
  723. return nil, err
  724. }
  725. buf, err := ioutil.ReadAll(reader)
  726. if err != nil {
  727. return nil, err
  728. }
  729. return buf, nil
  730. }
  731. ze.futureReaders = make(chan chan io.Reader, 1)
  732. futureReader := make(chan io.Reader, 1)
  733. ze.futureReaders <- futureReader
  734. close(ze.futureReaders)
  735. if ze.fh.Method == zip.Deflate {
  736. compressed, err := z.compressBlock(r, nil, true)
  737. if err != nil {
  738. z.errors <- err
  739. return
  740. }
  741. if uint64(compressed.Len()) < ze.fh.UncompressedSize64 {
  742. futureReader <- compressed
  743. } else {
  744. buf, err := readFile(r)
  745. if err != nil {
  746. z.errors <- err
  747. return
  748. }
  749. ze.fh.Method = zip.Store
  750. futureReader <- bytes.NewReader(buf)
  751. }
  752. } else {
  753. buf, err := readFile(r)
  754. if err != nil {
  755. z.errors <- err
  756. return
  757. }
  758. ze.fh.Method = zip.Store
  759. futureReader <- bytes.NewReader(buf)
  760. }
  761. z.cpuRateLimiter.Finish()
  762. close(futureReader)
  763. compressChan <- ze
  764. close(compressChan)
  765. }
  766. // writeDirectory annotates that dir is a directory created for the src file or directory, and adds
  767. // the directory entry to the zip file if directories are enabled.
  768. func (z *ZipWriter) writeDirectory(dir string, src string, emulateJar bool) error {
  769. // clean the input
  770. dir = filepath.Clean(dir)
  771. // discover any uncreated directories in the path
  772. var zipDirs []string
  773. for dir != "" && dir != "." {
  774. if _, exists := z.createdDirs[dir]; exists {
  775. break
  776. }
  777. if prev, exists := z.createdFiles[dir]; exists {
  778. return fmt.Errorf("destination %q is both a directory %q and a file %q", dir, src, prev)
  779. }
  780. z.createdDirs[dir] = src
  781. // parent directories precede their children
  782. zipDirs = append([]string{dir}, zipDirs...)
  783. dir = filepath.Dir(dir)
  784. }
  785. if z.directories {
  786. // make a directory entry for each uncreated directory
  787. for _, cleanDir := range zipDirs {
  788. var dirHeader *zip.FileHeader
  789. if emulateJar && cleanDir+"/" == jar.MetaDir {
  790. dirHeader = jar.MetaDirFileHeader()
  791. } else {
  792. dirHeader = &zip.FileHeader{
  793. Name: cleanDir + "/",
  794. }
  795. dirHeader.SetMode(0755 | os.ModeDir)
  796. }
  797. dirHeader.SetModTime(z.time)
  798. ze := make(chan *zipEntry, 1)
  799. ze <- &zipEntry{
  800. fh: dirHeader,
  801. }
  802. close(ze)
  803. z.writeOps <- ze
  804. }
  805. }
  806. return nil
  807. }
  808. func (z *ZipWriter) writeSymlink(rel, file string) error {
  809. fileHeader := &zip.FileHeader{
  810. Name: rel,
  811. }
  812. fileHeader.SetModTime(z.time)
  813. fileHeader.SetMode(0777 | os.ModeSymlink)
  814. dest, err := z.fs.Readlink(file)
  815. if err != nil {
  816. return err
  817. }
  818. fileHeader.UncompressedSize64 = uint64(len(dest))
  819. fileHeader.CRC32 = crc32.ChecksumIEEE([]byte(dest))
  820. ze := make(chan *zipEntry, 1)
  821. futureReaders := make(chan chan io.Reader, 1)
  822. futureReader := make(chan io.Reader, 1)
  823. futureReaders <- futureReader
  824. close(futureReaders)
  825. futureReader <- bytes.NewBufferString(dest)
  826. close(futureReader)
  827. ze <- &zipEntry{
  828. fh: fileHeader,
  829. futureReaders: futureReaders,
  830. }
  831. close(ze)
  832. z.writeOps <- ze
  833. return nil
  834. }