001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing,
013 * software distributed under the License is distributed on an
014 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
015 * KIND, either express or implied.  See the License for the
016 * specific language governing permissions and limitations
017 * under the License.
018 */
019package org.apache.commons.compress.compressors.snappy;
020
021import java.io.IOException;
022import java.io.InputStream;
023import java.io.PushbackInputStream;
024import java.util.Arrays;
025
026import org.apache.commons.compress.compressors.CompressorInputStream;
027import org.apache.commons.compress.utils.BoundedInputStream;
028import org.apache.commons.compress.utils.IOUtils;
029
030/**
031 * CompressorInputStream for the framing Snappy format.
032 *
033 * <p>Based on the "spec" in the version "Last revised: 2013-10-25"</p>
034 *
035 * @see <a href="https://github.com/google/snappy/blob/master/framing_format.txt">Snappy framing format description</a>
036 * @since 1.7
037 */
038public class FramedSnappyCompressorInputStream extends CompressorInputStream {
039
040    /**
041     * package private for tests only.
042     */
043    static final long MASK_OFFSET = 0xa282ead8L;
044
045    private static final int STREAM_IDENTIFIER_TYPE = 0xff;
046    private static final int COMPRESSED_CHUNK_TYPE = 0;
047    private static final int UNCOMPRESSED_CHUNK_TYPE = 1;
048    private static final int PADDING_CHUNK_TYPE = 0xfe;
049    private static final int MIN_UNSKIPPABLE_TYPE = 2;
050    private static final int MAX_UNSKIPPABLE_TYPE = 0x7f;
051    private static final int MAX_SKIPPABLE_TYPE = 0xfd;
052
053    private static final byte[] SZ_SIGNATURE = new byte[] {
054        (byte) STREAM_IDENTIFIER_TYPE, // tag
055        6, 0, 0, // length
056        's', 'N', 'a', 'P', 'p', 'Y'
057    };
058
059    /** The underlying stream to read compressed data from */
060    private final PushbackInputStream in;
061    /** The dialect to expect */
062    private final FramedSnappyDialect dialect;
063
064    private SnappyCompressorInputStream currentCompressedChunk;
065
066    // used in no-arg read method
067    private final byte[] oneByte = new byte[1];
068
069    private boolean endReached, inUncompressedChunk;
070
071    private int uncompressedBytesRemaining;
072    private long expectedChecksum = -1;
073    private final PureJavaCrc32C checksum = new PureJavaCrc32C();
074
075    /**
076     * Constructs a new input stream that decompresses
077     * snappy-framed-compressed data from the specified input stream
078     * using the {@link FramedSnappyDialect#STANDARD} dialect.
079     * @param in  the InputStream from which to read the compressed data
080     * @throws IOException if reading fails
081     */
082    public FramedSnappyCompressorInputStream(final InputStream in) throws IOException {
083        this(in, FramedSnappyDialect.STANDARD);
084    }
085
086    /**
087     * Constructs a new input stream that decompresses snappy-framed-compressed data
088     * from the specified input stream.
089     * @param in  the InputStream from which to read the compressed data
090     * @param dialect the dialect used by the compressed stream
091     * @throws IOException if reading fails
092     */
093    public FramedSnappyCompressorInputStream(final InputStream in,
094                                             final FramedSnappyDialect dialect)
095        throws IOException {
096        this.in = new PushbackInputStream(in, 1);
097        this.dialect = dialect;
098        if (dialect.hasStreamIdentifier()) {
099            readStreamIdentifier();
100        }
101    }
102
103    /** {@inheritDoc} */
104    @Override
105    public int read() throws IOException {
106        return read(oneByte, 0, 1) == -1 ? -1 : oneByte[0] & 0xFF;
107    }
108
109    /** {@inheritDoc} */
110    @Override
111    public void close() throws IOException {
112        if (currentCompressedChunk != null) {
113            currentCompressedChunk.close();
114            currentCompressedChunk = null;
115        }
116        in.close();
117    }
118
119    /** {@inheritDoc} */
120    @Override
121    public int read(final byte[] b, final int off, final int len) throws IOException {
122        int read = readOnce(b, off, len);
123        if (read == -1) {
124            readNextBlock();
125            if (endReached) {
126                return -1;
127            }
128            read = readOnce(b, off, len);
129        }
130        return read;
131    }
132
133    /** {@inheritDoc} */
134    @Override
135    public int available() throws IOException {
136        if (inUncompressedChunk) {
137            return Math.min(uncompressedBytesRemaining,
138                            in.available());
139        } else if (currentCompressedChunk != null) {
140            return currentCompressedChunk.available();
141        }
142        return 0;
143    }
144
145    /**
146     * Read from the current chunk into the given array.
147     *
148     * @return -1 if there is no current chunk or the number of bytes
149     * read from the current chunk (which may be -1 if the end of the
150     * chunk is reached).
151     */
152    private int readOnce(final byte[] b, final int off, final int len) throws IOException {
153        int read = -1;
154        if (inUncompressedChunk) {
155            final int amount = Math.min(uncompressedBytesRemaining, len);
156            if (amount == 0) {
157                return -1;
158            }
159            read = in.read(b, off, amount);
160            if (read != -1) {
161                uncompressedBytesRemaining -= read;
162                count(read);
163            }
164        } else if (currentCompressedChunk != null) {
165            final long before = currentCompressedChunk.getBytesRead();
166            read = currentCompressedChunk.read(b, off, len);
167            if (read == -1) {
168                currentCompressedChunk.close();
169                currentCompressedChunk = null;
170            } else {
171                count(currentCompressedChunk.getBytesRead() - before);
172            }
173        }
174        if (read > 0) {
175            checksum.update(b, off, read);
176        }
177        return read;
178    }
179
180    private void readNextBlock() throws IOException {
181        verifyLastChecksumAndReset();
182        inUncompressedChunk = false;
183        final int type = readOneByte();
184        if (type == -1) {
185            endReached = true;
186        } else if (type == STREAM_IDENTIFIER_TYPE) {
187            in.unread(type);
188            pushedBackBytes(1);
189            readStreamIdentifier();
190            readNextBlock();
191        } else if (type == PADDING_CHUNK_TYPE
192                   || (type > MAX_UNSKIPPABLE_TYPE && type <= MAX_SKIPPABLE_TYPE)) {
193            skipBlock();
194            readNextBlock();
195        } else if (type >= MIN_UNSKIPPABLE_TYPE && type <= MAX_UNSKIPPABLE_TYPE) {
196            throw new IOException("unskippable chunk with type " + type
197                                  + " (hex " + Integer.toHexString(type) + ")"
198                                  + " detected.");
199        } else if (type == UNCOMPRESSED_CHUNK_TYPE) {
200            inUncompressedChunk = true;
201            uncompressedBytesRemaining = readSize() - 4 /* CRC */;
202            expectedChecksum = unmask(readCrc());
203        } else if (type == COMPRESSED_CHUNK_TYPE) {
204            boolean expectChecksum = dialect.usesChecksumWithCompressedChunks();
205            final long size = readSize() - (expectChecksum ? 4 : 0);
206            if (expectChecksum) {
207                expectedChecksum = unmask(readCrc());
208            } else {
209                expectedChecksum = -1;
210            }
211            currentCompressedChunk =
212                new SnappyCompressorInputStream(new BoundedInputStream(in, size));
213            // constructor reads uncompressed size
214            count(currentCompressedChunk.getBytesRead());
215        } else {
216            // impossible as all potential byte values have been covered
217            throw new IOException("unknown chunk type " + type
218                                  + " detected.");
219        }
220    }
221
222    private long readCrc() throws IOException {
223        final byte[] b = new byte[4];
224        final int read = IOUtils.readFully(in, b);
225        count(read);
226        if (read != 4) {
227            throw new IOException("premature end of stream");
228        }
229        long crc = 0;
230        for (int i = 0; i < 4; i++) {
231            crc |= (b[i] & 0xFFL) << (8 * i);
232        }
233        return crc;
234    }
235
236    static long unmask(long x) {
237        // ugly, maybe we should just have used ints and deal with the
238        // overflow
239        x -= MASK_OFFSET;
240        x &= 0xffffFFFFL;
241        return ((x >> 17) | (x << 15)) & 0xffffFFFFL;
242    }
243
244    private int readSize() throws IOException {
245        int b = 0;
246        int sz = 0;
247        for (int i = 0; i < 3; i++) {
248            b = readOneByte();
249            if (b == -1) {
250                throw new IOException("premature end of stream");
251            }
252            sz |= (b << (i * 8));
253        }
254        return sz;
255    }
256
257    private void skipBlock() throws IOException {
258        final int size = readSize();
259        final long read = IOUtils.skip(in, size);
260        count(read);
261        if (read != size) {
262            throw new IOException("premature end of stream");
263        }
264    }
265
266    private void readStreamIdentifier() throws IOException {
267        final byte[] b = new byte[10];
268        final int read = IOUtils.readFully(in, b);
269        count(read);
270        if (10 != read || !matches(b, 10)) {
271            throw new IOException("Not a framed Snappy stream");
272        }
273    }
274
275    private int readOneByte() throws IOException {
276        final int b = in.read();
277        if (b != -1) {
278            count(1);
279            return b & 0xFF;
280        }
281        return -1;
282    }
283
284    private void verifyLastChecksumAndReset() throws IOException {
285        if (expectedChecksum >= 0 && expectedChecksum != checksum.getValue()) {
286            throw new IOException("Checksum verification failed");
287        }
288        expectedChecksum = -1;
289        checksum.reset();
290    }
291
292    /**
293     * Checks if the signature matches what is expected for a .sz file.
294     *
295     * <p>.sz files start with a chunk with tag 0xff and content sNaPpY.</p>
296     * 
297     * @param signature the bytes to check
298     * @param length    the number of bytes to check
299     * @return          true if this is a .sz stream, false otherwise
300     */
301    public static boolean matches(final byte[] signature, final int length) {
302
303        if (length < SZ_SIGNATURE.length) {
304            return false;
305        }
306
307        byte[] shortenedSig = signature;
308        if (signature.length > SZ_SIGNATURE.length) {
309            shortenedSig = new byte[SZ_SIGNATURE.length];
310            System.arraycopy(signature, 0, shortenedSig, 0, SZ_SIGNATURE.length);
311        }
312
313        return Arrays.equals(shortenedSig, SZ_SIGNATURE);
314    }
315
316}