    001/*
    002 * Licensed to the Apache Software Foundation (ASF) under one
    003 * or more contributor license agreements.  See the NOTICE file
    004 * distributed with this work for additional information
    005 * regarding copyright ownership.  The ASF licenses this file
    006 * to you under the Apache License, Version 2.0 (the
    007 * "License"); you may not use this file except in compliance
    008 * with the License.  You may obtain a copy of the License at
    009 *
    010 * http://www.apache.org/licenses/LICENSE-2.0
    011 *
    012 * Unless required by applicable law or agreed to in writing,
    013 * software distributed under the License is distributed on an
    014 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
    015 * KIND, either express or implied.  See the License for the
    016 * specific language governing permissions and limitations
    017 * under the License.
    018 */
    019package org.apache.commons.compress.archivers.dump;
    020
    021import org.apache.commons.compress.archivers.ArchiveException;
    022import org.apache.commons.compress.archivers.ArchiveInputStream;
    023import org.apache.commons.compress.archivers.zip.ZipEncoding;
    024import org.apache.commons.compress.archivers.zip.ZipEncodingHelper;
    025import org.apache.commons.compress.utils.IOUtils;
    026
    027import java.io.EOFException;
    028import java.io.IOException;
    029import java.io.InputStream;
    030
    031import java.util.Arrays;
    032import java.util.HashMap;
    033import java.util.Map;
    034import java.util.PriorityQueue;
    035import java.util.Queue;
    036import java.util.Stack;
    037
    038/**
    039 * The DumpArchiveInputStream reads a UNIX dump archive as an InputStream.
    040 * Methods are provided to position at each successive entry in
    041 * the archive, and the read each entry as a normal input stream
    042 * using read().
    043 *
    044 * There doesn't seem to exist a hint on the encoding of string values
    045 * in any piece documentation.  Given the main purpose of dump/restore
    046 * is backing up a system it seems very likely the format uses the
    047 * current default encoding of the system.
    048 *
    049 * @NotThreadSafe
    050 */
    051public class DumpArchiveInputStream extends ArchiveInputStream {
    052    private final DumpArchiveSummary summary;
    053    private DumpArchiveEntry active;
    054    private boolean isClosed;
    055    private boolean hasHitEOF;
    056    private long entrySize;
    057    private long entryOffset;
    058    private int readIdx;
    059    private final byte[] readBuf = new byte[DumpArchiveConstants.TP_SIZE];
    060    private byte[] blockBuffer;
    061    private int recordOffset;
    062    private long filepos;
    063    protected TapeInputStream raw;
    064
    065    // map of ino -> dirent entry. We can use this to reconstruct full paths.
    066    private final Map<Integer, Dirent> names = new HashMap<>();
    067
    068    // map of ino -> (directory) entry when we're missing one or more elements in the path.
    069    private final Map<Integer, DumpArchiveEntry> pending = new HashMap<>();
    070
    071    // queue of (directory) entries where we now have the full path.
    072    private final Queue<DumpArchiveEntry> queue;
    073
    074    /**
    075     * The encoding to use for file names and labels.
    076     */
    077    private final ZipEncoding zipEncoding;
    078
    079    // the provided encoding (for unit tests)
    080    final String encoding;
    081
    082    /**
    083     * Constructor using the platform's default encoding for file
    084     * names.
    085     *
    086     * @param is stream to read from
    087     * @throws ArchiveException on error
    088     */
    089    public DumpArchiveInputStream(final InputStream is) throws ArchiveException {
    090        this(is, null);
    091    }
    092
    093    /**
    094     * Constructor.
    095     *
    096     * @param is stream to read from
    097     * @param encoding the encoding to use for file names, use null
    098     * for the platform's default encoding
    099     * @since 1.6
    100     * @throws ArchiveException on error
    101     */
    102    public DumpArchiveInputStream(final InputStream is, final String encoding)
    103        throws ArchiveException {
    104        this.raw = new TapeInputStream(is);
    105        this.hasHitEOF = false;
    106        this.encoding = encoding;
    107        this.zipEncoding = ZipEncodingHelper.getZipEncoding(encoding);
    108
    109        try {
    110            // read header, verify it's a dump archive.
    111            final byte[] headerBytes = raw.readRecord();
    112
    113            if (!DumpArchiveUtil.verify(headerBytes)) {
    114                throw new UnrecognizedFormatException();
    115            }
    116
    117            // get summary information
    118            summary = new DumpArchiveSummary(headerBytes, this.zipEncoding);
    119
    120            // reset buffer with actual block size.
    121            raw.resetBlockSize(summary.getNTRec(), summary.isCompressed());
    122
    123            // allocate our read buffer.
    124            blockBuffer = new byte[4 * DumpArchiveConstants.TP_SIZE];
    125
    126            // skip past CLRI and BITS segments since we don't handle them yet.
    127            readCLRI();
    128            readBITS();
    129        } catch (final IOException ex) {
    130            throw new ArchiveException(ex.getMessage(), ex);
    131        }
    132
    133        // put in a dummy record for the root node.
    134        final Dirent root = new Dirent(2, 2, 4, ".");
    135        names.put(2, root);
    136
    137        // use priority based on queue to ensure parent directories are
    138        // released first.
    139        queue = new PriorityQueue<>(10,
    140                (p, q) -> {
    141                    if (p.getOriginalName() == null || q.getOriginalName() == null) {
    142                        return Integer.MAX_VALUE;
    143                    }
    144
    145                    return p.getOriginalName().compareTo(q.getOriginalName());
    146                });
    147    }
    148
    149    @Deprecated
    150    @Override
    151    public int getCount() {
    152        return (int) getBytesRead();
    153    }
    154
    155    @Override
    156    public long getBytesRead() {
    157        return raw.getBytesRead();
    158    }
    159
    160    /**
    161     * Return the archive summary information.
    162     * @return the summary
    163     */
    164    public DumpArchiveSummary getSummary() {
    165        return summary;
    166    }
    167
    168    /**
    169     * Read CLRI (deleted inode) segment.
    170     */
    171    private void readCLRI() throws IOException {
    172        final byte[] buffer = raw.readRecord();
    173
    174        if (!DumpArchiveUtil.verify(buffer)) {
    175            throw new InvalidFormatException();
    176        }
    177
    178        active = DumpArchiveEntry.parse(buffer);
    179
    180        if (DumpArchiveConstants.SEGMENT_TYPE.CLRI != active.getHeaderType()) {
    181            throw new InvalidFormatException();
    182        }
    183
    184        // we don't do anything with this yet.
    185        if (raw.skip((long) DumpArchiveConstants.TP_SIZE * active.getHeaderCount())
    186            == -1) {
    187            throw new EOFException();
    188        }
    189        readIdx = active.getHeaderCount();
    190    }
    191
    192    /**
    193     * Read BITS segment.
    194     */
    195    private void readBITS() throws IOException {
    196        final byte[] buffer = raw.readRecord();
    197
    198        if (!DumpArchiveUtil.verify(buffer)) {
    199            throw new InvalidFormatException();
    200        }
    201
    202        active = DumpArchiveEntry.parse(buffer);
    203
    204        if (DumpArchiveConstants.SEGMENT_TYPE.BITS != active.getHeaderType()) {
    205            throw new InvalidFormatException();
    206        }
    207
    208        // we don't do anything with this yet.
    209        if (raw.skip((long) DumpArchiveConstants.TP_SIZE * active.getHeaderCount())
    210            == -1) {
    211            throw new EOFException();
    212        }
    213        readIdx = active.getHeaderCount();
    214    }
    215
    216    /**
    217     * Read the next entry.
    218     * @return the next entry
    219     * @throws IOException on error
    220     */
    221    public DumpArchiveEntry getNextDumpEntry() throws IOException {
    222        return getNextEntry();
    223    }
    224
    225    @Override
    226    public DumpArchiveEntry getNextEntry() throws IOException {
    227        DumpArchiveEntry entry = null;
    228        String path = null;
    229
    230        // is there anything in the queue?
    231        if (!queue.isEmpty()) {
    232            return queue.remove();
    233        }
    234
    235        while (entry == null) {
    236            if (hasHitEOF) {
    237                return null;
    238            }
    239
    240            // skip any remaining records in this segment for prior file.
    241            // we might still have holes... easiest to do it
    242            // block by block. We may want to revisit this if
    243            // the unnecessary decompression time adds up.
    244            while (readIdx < active.getHeaderCount()) {
    245                if (!active.isSparseRecord(readIdx++)
    246                    && raw.skip(DumpArchiveConstants.TP_SIZE) == -1) {
    247                    throw new EOFException();
    248                }
    249            }
    250
    251            readIdx = 0;
    252            filepos = raw.getBytesRead();
    253
    254            byte[] headerBytes = raw.readRecord();
    255
    256            if (!DumpArchiveUtil.verify(headerBytes)) {
    257                throw new InvalidFormatException();
    258            }
    259
    260            active = DumpArchiveEntry.parse(headerBytes);
    261
    262            // skip any remaining segments for prior file.
    263            while (DumpArchiveConstants.SEGMENT_TYPE.ADDR == active.getHeaderType()) {
    264                if (raw.skip((long) DumpArchiveConstants.TP_SIZE
    265                             * (active.getHeaderCount()
    266                                - active.getHeaderHoles())) == -1) {
    267                    throw new EOFException();
    268                }
    269
    270                filepos = raw.getBytesRead();
    271                headerBytes = raw.readRecord();
    272
    273                if (!DumpArchiveUtil.verify(headerBytes)) {
    274                    throw new InvalidFormatException();
    275                }
    276
    277                active = DumpArchiveEntry.parse(headerBytes);
    278            }
    279
    280            // check if this is an end-of-volume marker.
    281            if (DumpArchiveConstants.SEGMENT_TYPE.END == active.getHeaderType()) {
    282                hasHitEOF = true;
    283
    284                return null;
    285            }
    286
    287            entry = active;
    288
    289            if (entry.isDirectory()) {
    290                readDirectoryEntry(active);
    291
    292                // now we create an empty InputStream.
    293                entryOffset = 0;
    294                entrySize = 0;
    295                readIdx = active.getHeaderCount();
    296            } else {
    297                entryOffset = 0;
    298                entrySize = active.getEntrySize();
    299                readIdx = 0;
    300            }
    301
    302            recordOffset = readBuf.length;
    303
    304            path = getPath(entry);
    305
    306            if (path == null) {
    307                entry = null;
    308            }
    309        }
    310
    311        entry.setName(path);
    312        entry.setSimpleName(names.get(entry.getIno()).getName());
    313        entry.setOffset(filepos);
    314
    315        return entry;
    316    }
    317
    318    /**
    319     * Read directory entry.
    320     */
    321    private void readDirectoryEntry(DumpArchiveEntry entry)
    322        throws IOException {
    323        long size = entry.getEntrySize();
    324        boolean first = true;
    325
    326        while (first ||
    327                DumpArchiveConstants.SEGMENT_TYPE.ADDR == entry.getHeaderType()) {
    328            // read the header that we just peeked at.
    329            if (!first) {
    330                raw.readRecord();
    331            }
    332
    333            if (!names.containsKey(entry.getIno()) &&
    334                    DumpArchiveConstants.SEGMENT_TYPE.INODE == entry.getHeaderType()) {
    335                pending.put(entry.getIno(), entry);
    336            }
    337
    338            final int datalen = DumpArchiveConstants.TP_SIZE * entry.getHeaderCount();
    339
    340            if (blockBuffer.length < datalen) {
    341                blockBuffer = IOUtils.readRange(raw, datalen);
    342                if (blockBuffer.length != datalen) {
    343                    throw new EOFException();
    344                }
    345            } else if (raw.read(blockBuffer, 0, datalen) != datalen) {
    346                throw new EOFException();
    347            }
    348
    349            int reclen = 0;
    350
    351            for (int i = 0; i < datalen - 8 && i < size - 8;
    352                    i += reclen) {
    353                final int ino = DumpArchiveUtil.convert32(blockBuffer, i);
    354                reclen = DumpArchiveUtil.convert16(blockBuffer, i + 4);
    355
    356                final byte type = blockBuffer[i + 6];
    357
    358                final String name = DumpArchiveUtil.decode(zipEncoding, blockBuffer, i + 8, blockBuffer[i + 7]);
    359
    360                if (".".equals(name) || "..".equals(name)) {
    361                    // do nothing...
    362                    continue;
    363                }
    364
    365                final Dirent d = new Dirent(ino, entry.getIno(), type, name);
    366
    367                /*
    368                if ((type == 4) && names.containsKey(ino)) {
    369                    System.out.println("we already have ino: " +
    370                                       names.get(ino));
    371                }
    372                */
    373
    374                names.put(ino, d);
    375
    376                // check whether this allows us to fill anything in the pending list.
    377                for (final Map.Entry<Integer, DumpArchiveEntry> e : pending.entrySet()) {
    378                    final String path = getPath(e.getValue());
    379
    380                    if (path != null) {
    381                        e.getValue().setName(path);
    382                        e.getValue()
    383                         .setSimpleName(names.get(e.getKey()).getName());
    384                        queue.add(e.getValue());
    385                    }
    386                }
    387
    388                // remove anything that we found. (We can't do it earlier
    389                // because of concurrent modification exceptions.)
    390                for (final DumpArchiveEntry e : queue) {
    391                    pending.remove(e.getIno());
    392                }
    393            }
    394
    395            final byte[] peekBytes = raw.peek();
    396
    397            if (!DumpArchiveUtil.verify(peekBytes)) {
    398                throw new InvalidFormatException();
    399            }
    400
    401            entry = DumpArchiveEntry.parse(peekBytes);
    402            first = false;
    403            size -= DumpArchiveConstants.TP_SIZE;
    404        }
    405    }
    406
    407    /**
    408     * Get full path for specified archive entry, or null if there's a gap.
    409     *
    410     * @param entry
    411     * @return  full path for specified archive entry, or null if there's a gap.
    412     */
    413    private String getPath(final DumpArchiveEntry entry) {
    414        // build the stack of elements. It's possible that we're
    415        // still missing an intermediate value and if so we
    416        final Stack<String> elements = new Stack<>();
    417        Dirent dirent = null;
    418
    419        for (int i = entry.getIno();; i = dirent.getParentIno()) {
    420            if (!names.containsKey(i)) {
    421                elements.clear();
    422                break;
    423            }
    424
    425            dirent = names.get(i);
    426            elements.push(dirent.getName());
    427
    428            if (dirent.getIno() == dirent.getParentIno()) {
    429                break;
    430            }
    431        }
    432
    433        // if an element is missing defer the work and read next entry.
    434        if (elements.isEmpty()) {
    435            pending.put(entry.getIno(), entry);
    436
    437            return null;
    438        }
    439
    440        // generate full path from stack of elements.
    441        final StringBuilder sb = new StringBuilder(elements.pop());
    442
    443        while (!elements.isEmpty()) {
    444            sb.append('/');
    445            sb.append(elements.pop());
    446        }
    447
    448        return sb.toString();
    449    }
    450
    451    /**
    452     * Reads bytes from the current dump archive entry.
    453     *
    454     * This method is aware of the boundaries of the current
    455     * entry in the archive and will deal with them as if they
    456     * were this stream's start and EOF.
    457     *
    458     * @param buf The buffer into which to place bytes read.
    459     * @param off The offset at which to place bytes read.
    460     * @param len The number of bytes to read.
    461     * @return The number of bytes read, or -1 at EOF.
    462     * @throws IOException on error
    463     */
    464    @Override
    465    public int read(final byte[] buf, int off, int len) throws IOException {
    466        if (len == 0) {
    467            return 0;
    468        }
    469        int totalRead = 0;
    470
    471        if (hasHitEOF || isClosed || entryOffset >= entrySize) {
    472            return -1;
    473        }
    474
    475        if (active == null) {
    476            throw new IllegalStateException("No current dump entry");
    477        }
    478
    479        if (len + entryOffset > entrySize) {
    480            len = (int) (entrySize - entryOffset);
    481        }
    482
    483        while (len > 0) {
    484            final int sz = len > readBuf.length - recordOffset
    485                ? readBuf.length - recordOffset : len;
    486
    487            // copy any data we have
    488            if (recordOffset + sz <= readBuf.length) {
    489                System.arraycopy(readBuf, recordOffset, buf, off, sz);
    490                totalRead += sz;
    491                recordOffset += sz;
    492                len -= sz;
    493                off += sz;
    494            }
    495
    496            // load next block if necessary.
    497            if (len > 0) {
    498                if (readIdx >= 512) {
    499                    final byte[] headerBytes = raw.readRecord();
    500
    501                    if (!DumpArchiveUtil.verify(headerBytes)) {
    502                        throw new InvalidFormatException();
    503                    }
    504
    505                    active = DumpArchiveEntry.parse(headerBytes);
    506                    readIdx = 0;
    507                }
    508
    509                if (!active.isSparseRecord(readIdx++)) {
    510                    final int r = raw.read(readBuf, 0, readBuf.length);
    511                    if (r != readBuf.length) {
    512                        throw new EOFException();
    513                    }
    514                } else {
    515                    Arrays.fill(readBuf, (byte) 0);
    516                }
    517
    518                recordOffset = 0;
    519            }
    520        }
    521
    522        entryOffset += totalRead;
    523
    524        return totalRead;
    525    }
    526
    527    /**
    528     * Closes the stream for this entry.
    529     */
    530    @Override
    531    public void close() throws IOException {
    532        if (!isClosed) {
    533            isClosed = true;
    534            raw.close();
    535        }
    536    }
    537
    538    /**
    539     * Look at the first few bytes of the file to decide if it's a dump
    540     * archive. With 32 bytes we can look at the magic value, with a full
    541     * 1k we can verify the checksum.
    542     * @param buffer data to match
    543     * @param length length of data
    544     * @return whether the buffer seems to contain dump data
    545     */
    546    public static boolean matches(final byte[] buffer, final int length) {
    547        // do we have enough of the header?
    548        if (length < 32) {
    549            return false;
    550        }
    551
    552        // this is the best test
    553        if (length >= DumpArchiveConstants.TP_SIZE) {
    554            return DumpArchiveUtil.verify(buffer);
    555        }
    556
    557        // this will work in a pinch.
    558        return DumpArchiveConstants.NFS_MAGIC == DumpArchiveUtil.convert32(buffer,
    559            24);
    560    }
    561
    562}