    001/*
    002 *  Licensed to the Apache Software Foundation (ASF) under one or more
    003 *  contributor license agreements.  See the NOTICE file distributed with
    004 *  this work for additional information regarding copyright ownership.
    005 *  The ASF licenses this file to You under the Apache License, Version 2.0
    006 *  (the "License"); you may not use this file except in compliance with
    007 *  the License.  You may obtain a copy of the License at
    008 *
    009 *      http://www.apache.org/licenses/LICENSE-2.0
    010 *
    011 *  Unless required by applicable law or agreed to in writing, software
    012 *  distributed under the License is distributed on an "AS IS" BASIS,
    013 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    014 *  See the License for the specific language governing permissions and
    015 *  limitations under the License.
    016 *
    017 */
    018package org.apache.commons.compress.archivers.zip;
    019
    020
    021import java.io.Closeable;
    022import java.io.File;
    023import java.io.FileNotFoundException;
    024import java.io.IOException;
    025import java.io.InputStream;
    026import java.nio.file.Path;
    027import java.util.Iterator;
    028import java.util.Queue;
    029import java.util.concurrent.ConcurrentLinkedQueue;
    030import java.util.concurrent.atomic.AtomicBoolean;
    031import java.util.zip.Deflater;
    032
    033import org.apache.commons.compress.parallel.FileBasedScatterGatherBackingStore;
    034import org.apache.commons.compress.parallel.ScatterGatherBackingStore;
    035import org.apache.commons.compress.utils.BoundedInputStream;
    036
    037/**
    038 * A ZIP output stream that is optimized for multi-threaded scatter/gather construction of ZIP files.
    039 * <p>
    040 * The internal data format of the entries used by this class are entirely private to this class and are not part of any public api whatsoever.
    041 * </p>
    042 * <p>
    043 * It is possible to extend this class to support different kinds of backing storage, the default implementation only supports file-based backing.
    044 * </p>
    045 * <p>
    046 * Thread safety: This class supports multiple threads. But the "writeTo" method must be called by the thread that originally created the
    047 * {@link ZipArchiveEntry}.
    048 * </p>
    049 *
    050 * @since 1.10
    051 */
    052public class ScatterZipOutputStream implements Closeable {
    053
    054    private static class CompressedEntry {
    055        final ZipArchiveEntryRequest zipArchiveEntryRequest;
    056        final long crc;
    057        final long compressedSize;
    058        final long size;
    059
    060        public CompressedEntry(final ZipArchiveEntryRequest zipArchiveEntryRequest, final long crc, final long compressedSize, final long size) {
    061            this.zipArchiveEntryRequest = zipArchiveEntryRequest;
    062            this.crc = crc;
    063            this.compressedSize = compressedSize;
    064            this.size = size;
    065        }
    066
    067        /**
    068         * Updates the original {@link ZipArchiveEntry} with sizes/crc
    069         * Do not use this methods from threads that did not create the instance itself !
    070         * @return the zipArchiveEntry that is basis for this request
    071         */
    072
    073        public ZipArchiveEntry transferToArchiveEntry() {
    074            final ZipArchiveEntry entry = zipArchiveEntryRequest.getZipArchiveEntry();
    075            entry.setCompressedSize(compressedSize);
    076            entry.setSize(size);
    077            entry.setCrc(crc);
    078            entry.setMethod(zipArchiveEntryRequest.getMethod());
    079            return entry;
    080        }
    081    }
    082
    083    public static class ZipEntryWriter implements Closeable {
    084        private final Iterator<CompressedEntry> itemsIterator;
    085        private final InputStream itemsIteratorData;
    086
    087        public ZipEntryWriter(final ScatterZipOutputStream scatter) throws IOException {
    088            scatter.backingStore.closeForWriting();
    089            itemsIterator = scatter.items.iterator();
    090            itemsIteratorData = scatter.backingStore.getInputStream();
    091        }
    092
    093        @Override
    094        public void close() throws IOException {
    095            if (itemsIteratorData != null) {
    096                itemsIteratorData.close();
    097            }
    098        }
    099
    100        public void writeNextZipEntry(final ZipArchiveOutputStream target) throws IOException {
    101            final CompressedEntry compressedEntry = itemsIterator.next();
    102            try (final BoundedInputStream rawStream = new BoundedInputStream(itemsIteratorData, compressedEntry.compressedSize)) {
    103                target.addRawArchiveEntry(compressedEntry.transferToArchiveEntry(), rawStream);
    104            }
    105        }
    106    }
    107
    108    /**
    109     * Creates a {@link ScatterZipOutputStream} with default compression level that is backed by a file
    110     *
    111     * @param file The file to offload compressed data into.
    112     * @return A ScatterZipOutputStream that is ready for use.
    113     * @throws FileNotFoundException if the file cannot be found
    114     */
    115    public static ScatterZipOutputStream fileBased(final File file) throws FileNotFoundException {
    116        return pathBased(file.toPath(), Deflater.DEFAULT_COMPRESSION);
    117    }
    118
    119    /**
    120     * Creates a {@link ScatterZipOutputStream} that is backed by a file
    121     *
    122     * @param file             The file to offload compressed data into.
    123     * @param compressionLevel The compression level to use, @see #Deflater
    124     * @return A  ScatterZipOutputStream that is ready for use.
    125     * @throws FileNotFoundException if the file cannot be found
    126     */
    127    public static ScatterZipOutputStream fileBased(final File file, final int compressionLevel) throws FileNotFoundException {
    128        return pathBased(file.toPath(), compressionLevel);
    129    }
    130
    131    /**
    132     * Creates a {@link ScatterZipOutputStream} with default compression level that is backed by a file
    133     * @param path The path to offload compressed data into.
    134     * @return A ScatterZipOutputStream that is ready for use.
    135     * @throws FileNotFoundException if the path cannot be found
    136     * @since 1.22
    137     */
    138    public static ScatterZipOutputStream pathBased(final Path path) throws FileNotFoundException {
    139        return pathBased(path, Deflater.DEFAULT_COMPRESSION);
    140    }
    141
    142    /**
    143     * Creates a {@link ScatterZipOutputStream} that is backed by a file
    144     * @param path The path to offload compressed data into.
    145     * @param compressionLevel The compression level to use, @see #Deflater
    146     * @return A ScatterZipOutputStream that is ready for use.
    147     * @throws FileNotFoundException if the path cannot be found
    148     * @since 1.22
    149     */
    150    public static ScatterZipOutputStream pathBased(final Path path, final int compressionLevel) throws FileNotFoundException {
    151        final ScatterGatherBackingStore bs = new FileBasedScatterGatherBackingStore(path);
    152        // lifecycle is bound to the ScatterZipOutputStream returned
    153        final StreamCompressor sc = StreamCompressor.create(compressionLevel, bs); //NOSONAR
    154        return new ScatterZipOutputStream(bs, sc);
    155    }
    156
    157    private final Queue<CompressedEntry> items = new ConcurrentLinkedQueue<>();
    158
    159    private final ScatterGatherBackingStore backingStore;
    160
    161    private final StreamCompressor streamCompressor;
    162
    163    private final AtomicBoolean isClosed = new AtomicBoolean();
    164
    165    private ZipEntryWriter zipEntryWriter;
    166
    167    public ScatterZipOutputStream(final ScatterGatherBackingStore backingStore,
    168                                  final StreamCompressor streamCompressor) {
    169        this.backingStore = backingStore;
    170        this.streamCompressor = streamCompressor;
    171    }
    172
    173    /**
    174     * Adds an archive entry to this scatter stream.
    175     *
    176     * @param zipArchiveEntryRequest The entry to write.
    177     * @throws IOException    If writing fails
    178     */
    179    public void addArchiveEntry(final ZipArchiveEntryRequest zipArchiveEntryRequest) throws IOException {
    180        try (final InputStream payloadStream = zipArchiveEntryRequest.getPayloadStream()) {
    181            streamCompressor.deflate(payloadStream, zipArchiveEntryRequest.getMethod());
    182        }
    183        items.add(new CompressedEntry(zipArchiveEntryRequest, streamCompressor.getCrc32(),
    184                                      streamCompressor.getBytesWrittenForLastEntry(), streamCompressor.getBytesRead()));
    185    }
    186
    187    /**
    188     * Closes this stream, freeing all resources involved in the creation of this stream.
    189     * @throws IOException If closing fails
    190     */
    191    @Override
    192    public void close() throws IOException {
    193        if (!isClosed.compareAndSet(false, true)) {
    194            return;
    195        }
    196        try {
    197            if (zipEntryWriter != null) {
    198                zipEntryWriter.close();
    199            }
    200            backingStore.close();
    201        } finally {
    202            streamCompressor.close();
    203        }
    204    }
    205
    206    /**
    207     * Writes the contents of this scatter stream to a target archive.
    208     *
    209     * @param target The archive to receive the contents of this {@link ScatterZipOutputStream}.
    210     * @throws IOException If writing fails
    211     * @see #zipEntryWriter()
    212     */
    213    public void writeTo(final ZipArchiveOutputStream target) throws IOException {
    214        backingStore.closeForWriting();
    215        try (final InputStream data = backingStore.getInputStream()) {
    216            for (final CompressedEntry compressedEntry : items) {
    217                try (final BoundedInputStream rawStream = new BoundedInputStream(data,
    218                        compressedEntry.compressedSize)) {
    219                    target.addRawArchiveEntry(compressedEntry.transferToArchiveEntry(), rawStream);
    220                }
    221            }
    222        }
    223    }
    224
    225    /**
    226     * Gets a ZIP entry writer for this scatter stream.
    227     * @throws IOException If getting scatter stream input stream
    228     * @return the ZipEntryWriter created on first call of the method
    229     */
    230    public ZipEntryWriter zipEntryWriter() throws IOException {
    231        if (zipEntryWriter == null) {
    232            zipEntryWriter = new ZipEntryWriter(this);
    233        }
    234        return zipEntryWriter;
    235    }
    236}