zip.go 25 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105
  1. // Copyright 2015 Google Inc. All rights reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package zip
  15. import (
  16. "bytes"
  17. "compress/flate"
  18. "crypto/sha256"
  19. "encoding/binary"
  20. "errors"
  21. "fmt"
  22. "hash"
  23. "hash/crc32"
  24. "io"
  25. "io/ioutil"
  26. "os"
  27. "path/filepath"
  28. "sort"
  29. "strings"
  30. "sync"
  31. "syscall"
  32. "time"
  33. "android/soong/response"
  34. "github.com/google/blueprint/pathtools"
  35. "android/soong/jar"
  36. "android/soong/third_party/zip"
  37. )
  38. // Sha256HeaderID is a custom Header ID for the `extra` field in
  39. // the file header to store the SHA checksum.
  40. const Sha256HeaderID = 0x4967
  41. // Sha256HeaderSignature is the signature to verify that the extra
  42. // data block is used to store the SHA checksum.
  43. const Sha256HeaderSignature = 0x9514
  44. // Block size used during parallel compression of a single file.
  45. const parallelBlockSize = 1 * 1024 * 1024 // 1MB
  46. // Minimum file size to use parallel compression. It requires more
  47. // flate.Writer allocations, since we can't change the dictionary
  48. // during Reset
  49. const minParallelFileSize = parallelBlockSize * 6
  50. // Size of the ZIP compression window (32KB)
  51. const windowSize = 32 * 1024
  52. type nopCloser struct {
  53. io.Writer
  54. }
  55. func (nopCloser) Close() error {
  56. return nil
  57. }
  58. type byteReaderCloser struct {
  59. *bytes.Reader
  60. io.Closer
  61. }
  62. type pathMapping struct {
  63. dest, src string
  64. zipMethod uint16
  65. }
  66. type FileArg struct {
  67. PathPrefixInZip, SourcePrefixToStrip string
  68. ExplicitPathInZip string
  69. SourceFiles []string
  70. JunkPaths bool
  71. GlobDir string
  72. }
  73. type FileArgsBuilder struct {
  74. state FileArg
  75. err error
  76. fs pathtools.FileSystem
  77. fileArgs []FileArg
  78. }
  79. func NewFileArgsBuilder() *FileArgsBuilder {
  80. return &FileArgsBuilder{
  81. fs: pathtools.OsFs,
  82. }
  83. }
  84. func (b *FileArgsBuilder) JunkPaths(v bool) *FileArgsBuilder {
  85. b.state.JunkPaths = v
  86. b.state.SourcePrefixToStrip = ""
  87. return b
  88. }
  89. func (b *FileArgsBuilder) SourcePrefixToStrip(prefixToStrip string) *FileArgsBuilder {
  90. b.state.JunkPaths = false
  91. b.state.SourcePrefixToStrip = prefixToStrip
  92. return b
  93. }
  94. func (b *FileArgsBuilder) PathPrefixInZip(rootPrefix string) *FileArgsBuilder {
  95. b.state.PathPrefixInZip = rootPrefix
  96. return b
  97. }
  98. func (b *FileArgsBuilder) File(name string) *FileArgsBuilder {
  99. if b.err != nil {
  100. return b
  101. }
  102. arg := b.state
  103. arg.SourceFiles = []string{name}
  104. b.fileArgs = append(b.fileArgs, arg)
  105. if b.state.ExplicitPathInZip != "" {
  106. b.state.ExplicitPathInZip = ""
  107. }
  108. return b
  109. }
  110. func (b *FileArgsBuilder) Dir(name string) *FileArgsBuilder {
  111. if b.err != nil {
  112. return b
  113. }
  114. arg := b.state
  115. arg.GlobDir = name
  116. b.fileArgs = append(b.fileArgs, arg)
  117. return b
  118. }
  119. // List reads the file names from the given file and adds them to the source files list.
  120. func (b *FileArgsBuilder) List(name string) *FileArgsBuilder {
  121. if b.err != nil {
  122. return b
  123. }
  124. f, err := b.fs.Open(name)
  125. if err != nil {
  126. b.err = err
  127. return b
  128. }
  129. defer f.Close()
  130. list, err := ioutil.ReadAll(f)
  131. if err != nil {
  132. b.err = err
  133. return b
  134. }
  135. arg := b.state
  136. arg.SourceFiles = strings.Fields(string(list))
  137. b.fileArgs = append(b.fileArgs, arg)
  138. return b
  139. }
  140. // RspFile reads the file names from given .rsp file and adds them to the source files list.
  141. func (b *FileArgsBuilder) RspFile(name string) *FileArgsBuilder {
  142. if b.err != nil {
  143. return b
  144. }
  145. f, err := b.fs.Open(name)
  146. if err != nil {
  147. b.err = err
  148. return b
  149. }
  150. defer f.Close()
  151. arg := b.state
  152. arg.SourceFiles, err = response.ReadRspFile(f)
  153. if err != nil {
  154. b.err = err
  155. return b
  156. }
  157. for i := range arg.SourceFiles {
  158. arg.SourceFiles[i] = pathtools.MatchEscape(arg.SourceFiles[i])
  159. }
  160. b.fileArgs = append(b.fileArgs, arg)
  161. return b
  162. }
  163. // ExplicitPathInZip sets the path in the zip file for the next File call.
  164. func (b *FileArgsBuilder) ExplicitPathInZip(s string) *FileArgsBuilder {
  165. b.state.ExplicitPathInZip = s
  166. return b
  167. }
  168. func (b *FileArgsBuilder) Error() error {
  169. if b == nil {
  170. return nil
  171. }
  172. return b.err
  173. }
  174. func (b *FileArgsBuilder) FileArgs() []FileArg {
  175. if b == nil {
  176. return nil
  177. }
  178. return b.fileArgs
  179. }
  180. type IncorrectRelativeRootError struct {
  181. RelativeRoot string
  182. Path string
  183. }
  184. func (x IncorrectRelativeRootError) Error() string {
  185. return fmt.Sprintf("path %q is outside relative root %q", x.Path, x.RelativeRoot)
  186. }
  187. type ConflictingFileError struct {
  188. Dest string
  189. Prev string
  190. Src string
  191. }
  192. func (x ConflictingFileError) Error() string {
  193. return fmt.Sprintf("destination %q has two files %q and %q", x.Dest, x.Prev, x.Src)
  194. }
  195. type ZipWriter struct {
  196. time time.Time
  197. createdFiles map[string]string
  198. createdDirs map[string]string
  199. directories bool
  200. errors chan error
  201. writeOps chan chan *zipEntry
  202. cpuRateLimiter *CPURateLimiter
  203. memoryRateLimiter *MemoryRateLimiter
  204. compressorPool sync.Pool
  205. compLevel int
  206. followSymlinks pathtools.ShouldFollowSymlinks
  207. ignoreMissingFiles bool
  208. stderr io.Writer
  209. fs pathtools.FileSystem
  210. sha256Checksum bool
  211. }
  212. type zipEntry struct {
  213. fh *zip.FileHeader
  214. // List of delayed io.Reader
  215. futureReaders chan chan io.Reader
  216. // Only used for passing into the MemoryRateLimiter to ensure we
  217. // release as much memory as much as we request
  218. allocatedSize int64
  219. }
  220. type ZipArgs struct {
  221. FileArgs []FileArg
  222. OutputFilePath string
  223. EmulateJar bool
  224. SrcJar bool
  225. AddDirectoryEntriesToZip bool
  226. CompressionLevel int
  227. ManifestSourcePath string
  228. NumParallelJobs int
  229. NonDeflatedFiles map[string]bool
  230. WriteIfChanged bool
  231. StoreSymlinks bool
  232. IgnoreMissingFiles bool
  233. Sha256Checksum bool
  234. Stderr io.Writer
  235. Filesystem pathtools.FileSystem
  236. }
  237. func zipTo(args ZipArgs, w io.Writer) error {
  238. if args.EmulateJar {
  239. args.AddDirectoryEntriesToZip = true
  240. }
  241. // Have Glob follow symlinks if they are not being stored as symlinks in the zip file.
  242. followSymlinks := pathtools.ShouldFollowSymlinks(!args.StoreSymlinks)
  243. z := &ZipWriter{
  244. time: jar.DefaultTime,
  245. createdDirs: make(map[string]string),
  246. createdFiles: make(map[string]string),
  247. directories: args.AddDirectoryEntriesToZip,
  248. compLevel: args.CompressionLevel,
  249. followSymlinks: followSymlinks,
  250. ignoreMissingFiles: args.IgnoreMissingFiles,
  251. stderr: args.Stderr,
  252. fs: args.Filesystem,
  253. sha256Checksum: args.Sha256Checksum,
  254. }
  255. if z.fs == nil {
  256. z.fs = pathtools.OsFs
  257. }
  258. if z.stderr == nil {
  259. z.stderr = os.Stderr
  260. }
  261. pathMappings := []pathMapping{}
  262. noCompression := args.CompressionLevel == 0
  263. for _, fa := range args.FileArgs {
  264. var srcs []string
  265. for _, s := range fa.SourceFiles {
  266. s = strings.TrimSpace(s)
  267. if s == "" {
  268. continue
  269. }
  270. result, err := z.fs.Glob(s, nil, followSymlinks)
  271. if err != nil {
  272. return err
  273. }
  274. if len(result.Matches) == 0 {
  275. err := &os.PathError{
  276. Op: "lstat",
  277. Path: s,
  278. Err: os.ErrNotExist,
  279. }
  280. if args.IgnoreMissingFiles {
  281. fmt.Fprintln(z.stderr, "warning:", err)
  282. } else {
  283. return err
  284. }
  285. }
  286. srcs = append(srcs, result.Matches...)
  287. }
  288. if fa.GlobDir != "" {
  289. if exists, isDir, err := z.fs.Exists(fa.GlobDir); err != nil {
  290. return err
  291. } else if !exists && !args.IgnoreMissingFiles {
  292. err := &os.PathError{
  293. Op: "lstat",
  294. Path: fa.GlobDir,
  295. Err: os.ErrNotExist,
  296. }
  297. if args.IgnoreMissingFiles {
  298. fmt.Fprintln(z.stderr, "warning:", err)
  299. } else {
  300. return err
  301. }
  302. } else if !isDir && !args.IgnoreMissingFiles {
  303. err := &os.PathError{
  304. Op: "lstat",
  305. Path: fa.GlobDir,
  306. Err: syscall.ENOTDIR,
  307. }
  308. if args.IgnoreMissingFiles {
  309. fmt.Fprintln(z.stderr, "warning:", err)
  310. } else {
  311. return err
  312. }
  313. }
  314. result, err := z.fs.Glob(filepath.Join(fa.GlobDir, "**/*"), nil, followSymlinks)
  315. if err != nil {
  316. return err
  317. }
  318. srcs = append(srcs, result.Matches...)
  319. }
  320. for _, src := range srcs {
  321. err := fillPathPairs(fa, src, &pathMappings, args.NonDeflatedFiles, noCompression)
  322. if err != nil {
  323. return err
  324. }
  325. }
  326. }
  327. return z.write(w, pathMappings, args.ManifestSourcePath, args.EmulateJar, args.SrcJar, args.NumParallelJobs)
  328. }
  329. // Zip creates an output zip archive from given sources.
  330. func Zip(args ZipArgs) error {
  331. if args.OutputFilePath == "" {
  332. return fmt.Errorf("output file path must be nonempty")
  333. }
  334. buf := &bytes.Buffer{}
  335. var out io.Writer = buf
  336. var zipErr error
  337. if !args.WriteIfChanged {
  338. f, err := os.Create(args.OutputFilePath)
  339. if err != nil {
  340. return err
  341. }
  342. defer f.Close()
  343. defer func() {
  344. if zipErr != nil {
  345. os.Remove(args.OutputFilePath)
  346. }
  347. }()
  348. out = f
  349. }
  350. zipErr = zipTo(args, out)
  351. if zipErr != nil {
  352. return zipErr
  353. }
  354. if args.WriteIfChanged {
  355. err := pathtools.WriteFileIfChanged(args.OutputFilePath, buf.Bytes(), 0666)
  356. if err != nil {
  357. return err
  358. }
  359. }
  360. return nil
  361. }
  362. func fillPathPairs(fa FileArg, src string, pathMappings *[]pathMapping,
  363. nonDeflatedFiles map[string]bool, noCompression bool) error {
  364. var dest string
  365. if fa.ExplicitPathInZip != "" {
  366. dest = fa.ExplicitPathInZip
  367. } else if fa.JunkPaths {
  368. dest = filepath.Base(src)
  369. } else {
  370. var err error
  371. dest, err = filepath.Rel(fa.SourcePrefixToStrip, src)
  372. if err != nil {
  373. return err
  374. }
  375. if strings.HasPrefix(dest, "../") {
  376. return IncorrectRelativeRootError{
  377. Path: src,
  378. RelativeRoot: fa.SourcePrefixToStrip,
  379. }
  380. }
  381. }
  382. dest = filepath.Join(fa.PathPrefixInZip, dest)
  383. zipMethod := zip.Deflate
  384. if _, found := nonDeflatedFiles[dest]; found || noCompression {
  385. zipMethod = zip.Store
  386. }
  387. *pathMappings = append(*pathMappings,
  388. pathMapping{dest: dest, src: src, zipMethod: zipMethod})
  389. return nil
  390. }
  391. func jarSort(mappings []pathMapping) {
  392. sort.SliceStable(mappings, func(i int, j int) bool {
  393. return jar.EntryNamesLess(mappings[i].dest, mappings[j].dest)
  394. })
  395. }
  396. func (z *ZipWriter) write(f io.Writer, pathMappings []pathMapping, manifest string, emulateJar, srcJar bool,
  397. parallelJobs int) error {
  398. z.errors = make(chan error)
  399. defer close(z.errors)
  400. // This channel size can be essentially unlimited -- it's used as a fifo
  401. // queue decouple the CPU and IO loads. Directories don't require any
  402. // compression time, but still cost some IO. Similar with small files that
  403. // can be very fast to compress. Some files that are more difficult to
  404. // compress won't take a corresponding longer time writing out.
  405. //
  406. // The optimum size here depends on your CPU and IO characteristics, and
  407. // the the layout of your zip file. 1000 was chosen mostly at random as
  408. // something that worked reasonably well for a test file.
  409. //
  410. // The RateLimit object will put the upper bounds on the number of
  411. // parallel compressions and outstanding buffers.
  412. z.writeOps = make(chan chan *zipEntry, 1000)
  413. z.cpuRateLimiter = NewCPURateLimiter(int64(parallelJobs))
  414. z.memoryRateLimiter = NewMemoryRateLimiter(0)
  415. defer func() {
  416. z.cpuRateLimiter.Stop()
  417. z.memoryRateLimiter.Stop()
  418. }()
  419. if manifest != "" && !emulateJar {
  420. return errors.New("must specify --jar when specifying a manifest via -m")
  421. }
  422. if emulateJar {
  423. // manifest may be empty, in which case addManifest will fill in a default
  424. pathMappings = append(pathMappings, pathMapping{jar.ManifestFile, manifest, zip.Deflate})
  425. jarSort(pathMappings)
  426. }
  427. go func() {
  428. var err error
  429. defer close(z.writeOps)
  430. for _, ele := range pathMappings {
  431. if emulateJar && ele.dest == jar.ManifestFile {
  432. err = z.addManifest(ele.dest, ele.src, ele.zipMethod)
  433. } else {
  434. err = z.addFile(ele.dest, ele.src, ele.zipMethod, emulateJar, srcJar)
  435. }
  436. if err != nil {
  437. z.errors <- err
  438. return
  439. }
  440. }
  441. }()
  442. zipw := zip.NewWriter(f)
  443. var currentWriteOpChan chan *zipEntry
  444. var currentWriter io.WriteCloser
  445. var currentReaders chan chan io.Reader
  446. var currentReader chan io.Reader
  447. var done bool
  448. for !done {
  449. var writeOpsChan chan chan *zipEntry
  450. var writeOpChan chan *zipEntry
  451. var readersChan chan chan io.Reader
  452. if currentReader != nil {
  453. // Only read and process errors
  454. } else if currentReaders != nil {
  455. readersChan = currentReaders
  456. } else if currentWriteOpChan != nil {
  457. writeOpChan = currentWriteOpChan
  458. } else {
  459. writeOpsChan = z.writeOps
  460. }
  461. select {
  462. case writeOp, ok := <-writeOpsChan:
  463. if !ok {
  464. done = true
  465. }
  466. currentWriteOpChan = writeOp
  467. case op := <-writeOpChan:
  468. currentWriteOpChan = nil
  469. var err error
  470. if op.fh.Method == zip.Deflate {
  471. currentWriter, err = zipw.CreateCompressedHeader(op.fh)
  472. } else {
  473. var zw io.Writer
  474. op.fh.CompressedSize64 = op.fh.UncompressedSize64
  475. zw, err = zipw.CreateHeaderAndroid(op.fh)
  476. currentWriter = nopCloser{zw}
  477. }
  478. if err != nil {
  479. return err
  480. }
  481. currentReaders = op.futureReaders
  482. if op.futureReaders == nil {
  483. currentWriter.Close()
  484. currentWriter = nil
  485. }
  486. z.memoryRateLimiter.Finish(op.allocatedSize)
  487. case futureReader, ok := <-readersChan:
  488. if !ok {
  489. // Done with reading
  490. currentWriter.Close()
  491. currentWriter = nil
  492. currentReaders = nil
  493. }
  494. currentReader = futureReader
  495. case reader := <-currentReader:
  496. _, err := io.Copy(currentWriter, reader)
  497. if err != nil {
  498. return err
  499. }
  500. currentReader = nil
  501. case err := <-z.errors:
  502. return err
  503. }
  504. }
  505. // One last chance to catch an error
  506. select {
  507. case err := <-z.errors:
  508. return err
  509. default:
  510. zipw.Close()
  511. return nil
  512. }
  513. }
  514. // imports (possibly with compression) <src> into the zip at sub-path <dest>
  515. func (z *ZipWriter) addFile(dest, src string, method uint16, emulateJar, srcJar bool) error {
  516. var fileSize int64
  517. var executable bool
  518. var s os.FileInfo
  519. var err error
  520. if z.followSymlinks {
  521. s, err = z.fs.Stat(src)
  522. } else {
  523. s, err = z.fs.Lstat(src)
  524. }
  525. if err != nil {
  526. if os.IsNotExist(err) && z.ignoreMissingFiles {
  527. fmt.Fprintln(z.stderr, "warning:", err)
  528. return nil
  529. }
  530. return err
  531. }
  532. createParentDirs := func(dest, src string) error {
  533. if err := z.writeDirectory(filepath.Dir(dest), src, emulateJar); err != nil {
  534. return err
  535. }
  536. if prev, exists := z.createdDirs[dest]; exists {
  537. return fmt.Errorf("destination %q is both a directory %q and a file %q", dest, prev, src)
  538. }
  539. return nil
  540. }
  541. checkDuplicateFiles := func(dest, src string) (bool, error) {
  542. if prev, exists := z.createdFiles[dest]; exists {
  543. if prev != src {
  544. return true, ConflictingFileError{
  545. Dest: dest,
  546. Prev: prev,
  547. Src: src,
  548. }
  549. }
  550. return true, nil
  551. }
  552. z.createdFiles[dest] = src
  553. return false, nil
  554. }
  555. if s.IsDir() {
  556. if z.directories {
  557. return z.writeDirectory(dest, src, emulateJar)
  558. }
  559. return nil
  560. } else if s.Mode()&os.ModeSymlink != 0 {
  561. err = createParentDirs(dest, src)
  562. if err != nil {
  563. return err
  564. }
  565. duplicate, err := checkDuplicateFiles(dest, src)
  566. if err != nil {
  567. return err
  568. }
  569. if duplicate {
  570. return nil
  571. }
  572. return z.writeSymlink(dest, src)
  573. } else if s.Mode().IsRegular() {
  574. r, err := z.fs.Open(src)
  575. if err != nil {
  576. return err
  577. }
  578. if srcJar && filepath.Ext(src) == ".java" {
  579. // rewrite the destination using the package path if it can be determined
  580. pkg, err := jar.JavaPackage(r, src)
  581. if err != nil {
  582. // ignore errors for now, leaving the file at in its original location in the zip
  583. } else {
  584. dest = filepath.Join(filepath.Join(strings.Split(pkg, ".")...), filepath.Base(src))
  585. }
  586. _, err = r.Seek(0, io.SeekStart)
  587. if err != nil {
  588. return err
  589. }
  590. }
  591. fileSize = s.Size()
  592. executable = s.Mode()&0100 != 0
  593. header := &zip.FileHeader{
  594. Name: dest,
  595. Method: method,
  596. UncompressedSize64: uint64(fileSize),
  597. }
  598. mode := os.FileMode(0644)
  599. if executable {
  600. mode = 0755
  601. }
  602. header.SetMode(mode)
  603. err = createParentDirs(dest, src)
  604. if err != nil {
  605. return err
  606. }
  607. duplicate, err := checkDuplicateFiles(dest, src)
  608. if err != nil {
  609. return err
  610. }
  611. if duplicate {
  612. return nil
  613. }
  614. return z.writeFileContents(header, r)
  615. } else {
  616. return fmt.Errorf("%s is not a file, directory, or symlink", src)
  617. }
  618. }
  619. func (z *ZipWriter) addManifest(dest string, src string, _ uint16) error {
  620. if prev, exists := z.createdDirs[dest]; exists {
  621. return fmt.Errorf("destination %q is both a directory %q and a file %q", dest, prev, src)
  622. }
  623. if prev, exists := z.createdFiles[dest]; exists {
  624. if prev != src {
  625. return ConflictingFileError{
  626. Dest: dest,
  627. Prev: prev,
  628. Src: src,
  629. }
  630. }
  631. return nil
  632. }
  633. if err := z.writeDirectory(filepath.Dir(dest), src, true); err != nil {
  634. return err
  635. }
  636. var contents []byte
  637. if src != "" {
  638. f, err := z.fs.Open(src)
  639. if err != nil {
  640. return err
  641. }
  642. contents, err = ioutil.ReadAll(f)
  643. f.Close()
  644. if err != nil {
  645. return err
  646. }
  647. }
  648. fh, buf, err := jar.ManifestFileContents(contents)
  649. if err != nil {
  650. return err
  651. }
  652. reader := &byteReaderCloser{bytes.NewReader(buf), ioutil.NopCloser(nil)}
  653. return z.writeFileContents(fh, reader)
  654. }
  655. func (z *ZipWriter) writeFileContents(header *zip.FileHeader, r pathtools.ReaderAtSeekerCloser) (err error) {
  656. header.SetModTime(z.time)
  657. compressChan := make(chan *zipEntry, 1)
  658. z.writeOps <- compressChan
  659. // Pre-fill a zipEntry, it will be sent in the compressChan once
  660. // we're sure about the Method and CRC.
  661. ze := &zipEntry{
  662. fh: header,
  663. }
  664. ze.allocatedSize = int64(header.UncompressedSize64)
  665. z.cpuRateLimiter.Request()
  666. z.memoryRateLimiter.Request(ze.allocatedSize)
  667. fileSize := int64(header.UncompressedSize64)
  668. if fileSize == 0 {
  669. fileSize = int64(header.UncompressedSize)
  670. }
  671. if header.Method == zip.Deflate && fileSize >= minParallelFileSize {
  672. wg := new(sync.WaitGroup)
  673. // Allocate enough buffer to hold all readers. We'll limit
  674. // this based on actual buffer sizes in RateLimit.
  675. ze.futureReaders = make(chan chan io.Reader, (fileSize/parallelBlockSize)+1)
  676. // Calculate the CRC and SHA256 in the background, since reading
  677. // the entire file could take a while.
  678. //
  679. // We could split this up into chunks as well, but it's faster
  680. // than the compression. Due to the Go Zip API, we also need to
  681. // know the result before we can begin writing the compressed
  682. // data out to the zipfile.
  683. //
  684. // We calculate SHA256 only if `-sha256` is set.
  685. wg.Add(1)
  686. go z.checksumFileAsync(r, ze, compressChan, wg)
  687. for start := int64(0); start < fileSize; start += parallelBlockSize {
  688. sr := io.NewSectionReader(r, start, parallelBlockSize)
  689. resultChan := make(chan io.Reader, 1)
  690. ze.futureReaders <- resultChan
  691. z.cpuRateLimiter.Request()
  692. last := !(start+parallelBlockSize < fileSize)
  693. var dict []byte
  694. if start >= windowSize {
  695. dict, err = ioutil.ReadAll(io.NewSectionReader(r, start-windowSize, windowSize))
  696. if err != nil {
  697. return err
  698. }
  699. }
  700. wg.Add(1)
  701. go z.compressPartialFile(sr, dict, last, resultChan, wg)
  702. }
  703. close(ze.futureReaders)
  704. // Close the file handle after all readers are done
  705. go func(wg *sync.WaitGroup, closer io.Closer) {
  706. wg.Wait()
  707. closer.Close()
  708. }(wg, r)
  709. } else {
  710. go func() {
  711. z.compressWholeFile(ze, r, compressChan)
  712. r.Close()
  713. }()
  714. }
  715. return nil
  716. }
  717. func (z *ZipWriter) checksumFileAsync(r io.ReadSeeker, ze *zipEntry, resultChan chan *zipEntry, wg *sync.WaitGroup) {
  718. defer wg.Done()
  719. defer z.cpuRateLimiter.Finish()
  720. z.checksumFile(r, ze)
  721. resultChan <- ze
  722. close(resultChan)
  723. }
  724. func (z *ZipWriter) checksumFile(r io.ReadSeeker, ze *zipEntry) {
  725. crc := crc32.NewIEEE()
  726. writers := []io.Writer{crc}
  727. var shaHasher hash.Hash
  728. if z.sha256Checksum && !ze.fh.Mode().IsDir() {
  729. shaHasher = sha256.New()
  730. writers = append(writers, shaHasher)
  731. }
  732. w := io.MultiWriter(writers...)
  733. _, err := io.Copy(w, r)
  734. if err != nil {
  735. z.errors <- err
  736. return
  737. }
  738. ze.fh.CRC32 = crc.Sum32()
  739. if shaHasher != nil {
  740. z.appendSHAToExtra(ze, shaHasher.Sum(nil))
  741. }
  742. }
  743. func (z *ZipWriter) appendSHAToExtra(ze *zipEntry, checksum []byte) {
  744. // The block of SHA256 checksum consist of:
  745. // - Header ID, equals to Sha256HeaderID (2 bytes)
  746. // - Data size (2 bytes)
  747. // - Data block:
  748. // - Signature, equals to Sha256HeaderSignature (2 bytes)
  749. // - Data, SHA checksum value
  750. var buf []byte
  751. buf = binary.LittleEndian.AppendUint16(buf, Sha256HeaderID)
  752. buf = binary.LittleEndian.AppendUint16(buf, uint16(len(checksum)+2))
  753. buf = binary.LittleEndian.AppendUint16(buf, Sha256HeaderSignature)
  754. buf = append(buf, checksum...)
  755. ze.fh.Extra = append(ze.fh.Extra, buf...)
  756. }
  757. func (z *ZipWriter) compressPartialFile(r io.Reader, dict []byte, last bool, resultChan chan io.Reader, wg *sync.WaitGroup) {
  758. defer wg.Done()
  759. result, err := z.compressBlock(r, dict, last)
  760. if err != nil {
  761. z.errors <- err
  762. return
  763. }
  764. z.cpuRateLimiter.Finish()
  765. resultChan <- result
  766. }
  767. func (z *ZipWriter) compressBlock(r io.Reader, dict []byte, last bool) (*bytes.Buffer, error) {
  768. buf := new(bytes.Buffer)
  769. var fw *flate.Writer
  770. var err error
  771. if len(dict) > 0 {
  772. // There's no way to Reset a Writer with a new dictionary, so
  773. // don't use the Pool
  774. fw, err = flate.NewWriterDict(buf, z.compLevel, dict)
  775. } else {
  776. var ok bool
  777. if fw, ok = z.compressorPool.Get().(*flate.Writer); ok {
  778. fw.Reset(buf)
  779. } else {
  780. fw, err = flate.NewWriter(buf, z.compLevel)
  781. }
  782. defer z.compressorPool.Put(fw)
  783. }
  784. if err != nil {
  785. return nil, err
  786. }
  787. _, err = io.Copy(fw, r)
  788. if err != nil {
  789. return nil, err
  790. }
  791. if last {
  792. fw.Close()
  793. } else {
  794. fw.Flush()
  795. }
  796. return buf, nil
  797. }
  798. func (z *ZipWriter) compressWholeFile(ze *zipEntry, r io.ReadSeeker, compressChan chan *zipEntry) {
  799. z.checksumFile(r, ze)
  800. _, err := r.Seek(0, 0)
  801. if err != nil {
  802. z.errors <- err
  803. return
  804. }
  805. readFile := func(reader io.ReadSeeker) ([]byte, error) {
  806. _, err := reader.Seek(0, 0)
  807. if err != nil {
  808. return nil, err
  809. }
  810. buf, err := ioutil.ReadAll(reader)
  811. if err != nil {
  812. return nil, err
  813. }
  814. return buf, nil
  815. }
  816. ze.futureReaders = make(chan chan io.Reader, 1)
  817. futureReader := make(chan io.Reader, 1)
  818. ze.futureReaders <- futureReader
  819. close(ze.futureReaders)
  820. if ze.fh.Method == zip.Deflate {
  821. compressed, err := z.compressBlock(r, nil, true)
  822. if err != nil {
  823. z.errors <- err
  824. return
  825. }
  826. if uint64(compressed.Len()) < ze.fh.UncompressedSize64 {
  827. futureReader <- compressed
  828. } else {
  829. buf, err := readFile(r)
  830. if err != nil {
  831. z.errors <- err
  832. return
  833. }
  834. ze.fh.Method = zip.Store
  835. futureReader <- bytes.NewReader(buf)
  836. }
  837. } else {
  838. buf, err := readFile(r)
  839. if err != nil {
  840. z.errors <- err
  841. return
  842. }
  843. ze.fh.Method = zip.Store
  844. futureReader <- bytes.NewReader(buf)
  845. }
  846. z.cpuRateLimiter.Finish()
  847. close(futureReader)
  848. compressChan <- ze
  849. close(compressChan)
  850. }
  851. // writeDirectory annotates that dir is a directory created for the src file or directory, and adds
  852. // the directory entry to the zip file if directories are enabled.
  853. func (z *ZipWriter) writeDirectory(dir string, src string, emulateJar bool) error {
  854. // clean the input
  855. dir = filepath.Clean(dir)
  856. // discover any uncreated directories in the path
  857. var zipDirs []string
  858. for dir != "" && dir != "." {
  859. if _, exists := z.createdDirs[dir]; exists {
  860. break
  861. }
  862. if prev, exists := z.createdFiles[dir]; exists {
  863. return fmt.Errorf("destination %q is both a directory %q and a file %q", dir, src, prev)
  864. }
  865. z.createdDirs[dir] = src
  866. // parent directories precede their children
  867. zipDirs = append([]string{dir}, zipDirs...)
  868. dir = filepath.Dir(dir)
  869. }
  870. if z.directories {
  871. // make a directory entry for each uncreated directory
  872. for _, cleanDir := range zipDirs {
  873. var dirHeader *zip.FileHeader
  874. if emulateJar && cleanDir+"/" == jar.MetaDir {
  875. dirHeader = jar.MetaDirFileHeader()
  876. } else {
  877. dirHeader = &zip.FileHeader{
  878. Name: cleanDir + "/",
  879. }
  880. dirHeader.SetMode(0755 | os.ModeDir)
  881. }
  882. dirHeader.SetModTime(z.time)
  883. ze := make(chan *zipEntry, 1)
  884. ze <- &zipEntry{
  885. fh: dirHeader,
  886. }
  887. close(ze)
  888. z.writeOps <- ze
  889. }
  890. }
  891. return nil
  892. }
  893. func (z *ZipWriter) writeSymlink(rel, file string) error {
  894. fileHeader := &zip.FileHeader{
  895. Name: rel,
  896. }
  897. fileHeader.SetModTime(z.time)
  898. fileHeader.SetMode(0777 | os.ModeSymlink)
  899. dest, err := z.fs.Readlink(file)
  900. if err != nil {
  901. return err
  902. }
  903. fileHeader.UncompressedSize64 = uint64(len(dest))
  904. fileHeader.CRC32 = crc32.ChecksumIEEE([]byte(dest))
  905. ze := make(chan *zipEntry, 1)
  906. futureReaders := make(chan chan io.Reader, 1)
  907. futureReader := make(chan io.Reader, 1)
  908. futureReaders <- futureReader
  909. close(futureReaders)
  910. futureReader <- bytes.NewBufferString(dest)
  911. close(futureReader)
  912. ze <- &zipEntry{
  913. fh: fileHeader,
  914. futureReaders: futureReaders,
  915. }
  916. close(ze)
  917. z.writeOps <- ze
  918. return nil
  919. }