configure

@@ -367,8 +367,11 @@ else
   try()
   {
     show $*
-    ( $* ) >> configure.log 2>&1
+    got=`( $* ) 2>&1`
     ret=$?
+    if test "$got" != ""; then
+      printf "%s\n" "$got" >> configure.log
+    fi
     if test $ret -ne 0; then
       echo "(exit code "$ret")" >> configure.log
     fi
@@ -381,8 +384,11 @@ tryboth()
   show $*
   got=`( $* ) 2>&1`
   ret=$?
-  printf %s "$got" >> configure.log
+  if test "$got" != ""; then
+    printf "%s\n" "$got" >> configure.log
+  fi
   if test $ret -ne 0; then
+    echo "(exit code "$ret")" >> configure.log
     return $ret
   fi
   test "$got" = ""
@@ -457,17 +463,11 @@ size_t dummy = 0;
 EOF
 if try $CC -c $CFLAGS $test.c; then
   echo "Checking for size_t... Yes." | tee -a configure.log
-  need_sizet=0
 else
   echo "Checking for size_t... No." | tee -a configure.log
-  need_sizet=1
-fi
-
-echo >> configure.log
-
-# find the size_t integer type, if needed
-if test $need_sizet -eq 1; then
-  cat > $test.c <<EOF
+  # find a size_t integer type
+  # check for long long
+  cat > $test.c << EOF
 long long dummy = 0;
 EOF
   if try $CC -c $CFLAGS $test.c; then
@@ -495,17 +495,13 @@ EOF
   if try $CC $CFLAGS -o $test $test.c; then
     sizet=`./$test`
     echo "Checking for a pointer-size integer type..." $sizet"." | tee -a configure.log
+    CFLAGS="${CFLAGS} -DNO_SIZE_T=${sizet}"
+    SFLAGS="${SFLAGS} -DNO_SIZE_T=${sizet}"
   else
-    echo "Failed to find a pointer-size integer type." | tee -a configure.log
-    leave 1
+    echo "Checking for a pointer-size integer type... not found." | tee -a configure.log
   fi
 fi
-if test $need_sizet -eq 1; then
-  CFLAGS="${CFLAGS} -DNO_SIZE_T=${sizet}"
-  SFLAGS="${SFLAGS} -DNO_SIZE_T=${sizet}"
-fi
-
 echo >> configure.log
 # check for large file support, and if none, check for fseeko()
@@ -849,7 +845,6 @@ echo SHAREDLIBV = $SHAREDLIBV >> configure.log
 echo STATICLIB = $STATICLIB >> configure.log
 echo TEST = $TEST >> configure.log
 echo VER = $VER >> configure.log
-echo Z_U4 = $Z_U4 >> configure.log
 echo SRCDIR = $SRCDIR >> configure.log
 echo exec_prefix = $exec_prefix >> configure.log
 echo includedir = $includedir >> configure.log

deflate.c

@@ -255,11 +255,6 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy,
     int wrap = 1;
     static const char my_version[] = ZLIB_VERSION;
-    ushf *overlay;
-    /* We overlay pending_buf and d_buf+l_buf. This works since the average
-     * output size for (length,distance) codes is <= 24 bits.
-     */
-
     if (version == Z_NULL || version[0] != my_version[0] ||
         stream_size != sizeof(z_stream)) {
         return Z_VERSION_ERROR;
@@ -329,9 +324,47 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy,
     s->lit_bufsize = 1 << (memLevel + 6); /* 16K elements by default */
-    overlay = (ushf *) ZALLOC(strm, s->lit_bufsize, sizeof(ush)+2);
-    s->pending_buf = (uchf *) overlay;
-    s->pending_buf_size = (ulg)s->lit_bufsize * (sizeof(ush)+2L);
+    /* We overlay pending_buf and sym_buf. This works since the average size
+     * for length/distance pairs over any compressed block is assured to be 31
+     * bits or less.
+     *
+     * Analysis: The longest fixed codes are a length code of 8 bits plus 5
+     * extra bits, for lengths 131 to 257. The longest fixed distance codes are
+     * 5 bits plus 13 extra bits, for distances 16385 to 32768. The longest
+     * possible fixed-codes length/distance pair is then 31 bits total.
+     *
+     * sym_buf starts one-fourth of the way into pending_buf. So there are
+     * three bytes in sym_buf for every four bytes in pending_buf. Each symbol
+     * in sym_buf is three bytes -- two for the distance and one for the
+     * literal/length. As each symbol is consumed, the pointer to the next
+     * sym_buf value to read moves forward three bytes. From that symbol, up to
+     * 31 bits are written to pending_buf. The closest the written pending_buf
+     * bits gets to the next sym_buf symbol to read is just before the last
+     * code is written. At that time, 31*(n-2) bits have been written, just
+     * after 24*(n-2) bits have been consumed from sym_buf. sym_buf starts at
+     * 8*n bits into pending_buf. (Note that the symbol buffer fills when n-1
+     * symbols are written.) The closest the writing gets to what is unread is
+     * then n+14 bits. Here n is lit_bufsize, which is 16384 by default, and
+     * can range from 128 to 32768.
+     *
+     * Therefore, at a minimum, there are 142 bits of space between what is
+     * written and what is read in the overlain buffers, so the symbols cannot
+     * be overwritten by the compressed data. That space is actually 139 bits,
+     * due to the three-bit fixed-code block header.
+     *
+     * That covers the case where either Z_FIXED is specified, forcing fixed
+     * codes, or when the use of fixed codes is chosen, because that choice
+     * results in a smaller compressed block than dynamic codes. That latter
+     * condition then assures that the above analysis also covers all dynamic
+     * blocks. A dynamic-code block will only be chosen to be emitted if it has
+     * fewer bits than a fixed-code block would for the same set of symbols.
+     * Therefore its average symbol length is assured to be less than 31. So
+     * the compressed data for a dynamic block also cannot overwrite the
+     * symbols from which it is being constructed.
+     */
+
+    s->pending_buf = (uchf *) ZALLOC(strm, s->lit_bufsize, 4);
+    s->pending_buf_size = (ulg)s->lit_bufsize * 4;
     if (s->window == Z_NULL || s->prev == Z_NULL || s->head == Z_NULL ||
         s->pending_buf == Z_NULL) {
@@ -340,8 +373,12 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy,
         deflateEnd (strm);
         return Z_MEM_ERROR;
     }
-    s->d_buf = overlay + s->lit_bufsize/sizeof(ush);
-    s->l_buf = s->pending_buf + (1+sizeof(ush))*s->lit_bufsize;
+    s->sym_buf = s->pending_buf + s->lit_bufsize;
+    s->sym_end = (s->lit_bufsize - 1) * 3;
+    /* We avoid equality with lit_bufsize*3 because of wraparound at 64K
+     * on 16 bit machines and because stored blocks are restricted to
+     * 64K-1 bytes.
+     */
     s->level = level;
     s->strategy = strategy;
@@ -552,7 +589,8 @@ int ZEXPORT deflatePrime (strm, bits, value)
     if (deflateStateCheck(strm)) return Z_STREAM_ERROR;
     s = strm->state;
-    if ((Bytef *)(s->d_buf) < s->pending_out + ((Buf_size + 7) >> 3))
+    if (bits < 0 || bits > 16 ||
+        s->sym_buf < s->pending_out + ((Buf_size + 7) >> 3))
         return Z_BUF_ERROR;
     do {
         put = Buf_size - s->bi_valid;
@@ -1113,7 +1151,6 @@ int ZEXPORT deflateCopy (dest, source)
 #else
     deflate_state *ds;
     deflate_state *ss;
-    ushf *overlay;
     if (deflateStateCheck(source) || dest == Z_NULL) {
@@ -1133,8 +1170,7 @@ int ZEXPORT deflateCopy (dest, source)
     ds->window = (Bytef *) ZALLOC(dest, ds->w_size, 2*sizeof(Byte));
     ds->prev   = (Posf *)  ZALLOC(dest, ds->w_size, sizeof(Pos));
     ds->head   = (Posf *)  ZALLOC(dest, ds->hash_size, sizeof(Pos));
-    overlay = (ushf *) ZALLOC(dest, ds->lit_bufsize, sizeof(ush)+2);
-    ds->pending_buf = (uchf *) overlay;
+    ds->pending_buf = (uchf *) ZALLOC(dest, ds->lit_bufsize, 4);
     if (ds->window == Z_NULL || ds->prev == Z_NULL || ds->head == Z_NULL ||
         ds->pending_buf == Z_NULL) {
@@ -1148,8 +1184,7 @@ int ZEXPORT deflateCopy (dest, source)
     zmemcpy(ds->pending_buf, ss->pending_buf, (uInt)ds->pending_buf_size);
     ds->pending_out = ds->pending_buf + (ss->pending_out - ss->pending_buf);
-    ds->d_buf = overlay + ds->lit_bufsize/sizeof(ush);
-    ds->l_buf = ds->pending_buf + (1+sizeof(ush))*ds->lit_bufsize;
+    ds->sym_buf = ds->pending_buf + ds->lit_bufsize;
     ds->l_desc.dyn_tree = ds->dyn_ltree;
     ds->d_desc.dyn_tree = ds->dyn_dtree;
@@ -1925,7 +1960,7 @@ local block_state deflate_fast(s, flush)
         FLUSH_BLOCK(s, 1);
         return finish_done;
     }
-    if (s->last_lit)
+    if (s->sym_next)
         FLUSH_BLOCK(s, 0);
     return block_done;
 }
@@ -2056,7 +2091,7 @@ local block_state deflate_slow(s, flush)
         FLUSH_BLOCK(s, 1);
         return finish_done;
     }
-    if (s->last_lit)
+    if (s->sym_next)
         FLUSH_BLOCK(s, 0);
     return block_done;
 }
@@ -2131,7 +2166,7 @@ local block_state deflate_rle(s, flush)
         FLUSH_BLOCK(s, 1);
         return finish_done;
     }
-    if (s->last_lit)
+    if (s->sym_next)
         FLUSH_BLOCK(s, 0);
     return block_done;
 }
@@ -2170,7 +2205,7 @@ local block_state deflate_huff(s, flush)
         FLUSH_BLOCK(s, 1);
         return finish_done;
     }
-    if (s->last_lit)
+    if (s->sym_next)
         FLUSH_BLOCK(s, 0);
     return block_done;
 }

deflate.h

@@ -217,7 +217,7 @@ typedef struct internal_state {
     /* Depth of each subtree used as tie breaker for trees of equal frequency
      */
-    uchf *l_buf;          /* buffer for literals or lengths */
+    uchf *sym_buf;        /* buffer for distances and literals/lengths */
     uInt  lit_bufsize;
     /* Size of match buffer for literals/lengths.  There are 4 reasons for
@@ -239,13 +239,8 @@ typedef struct internal_state {
      *   - I can't count above 4
      */
-    uInt last_lit;      /* running index in l_buf */
-
-    ushf *d_buf;
-    /* Buffer for distances. To simplify the code, d_buf and l_buf have
-     * the same number of elements. To use different lengths, an extra flag
-     * array would be necessary.
-     */
+    uInt sym_next;      /* running index in sym_buf */
+    uInt sym_end;       /* symbol table full when sym_next reaches this */
     ulg opt_len;        /* bit length of current block with optimal trees */
     ulg static_len;     /* bit length of current block with static trees */
@@ -325,20 +320,22 @@ void ZLIB_INTERNAL _tr_stored_block OF((deflate_state *s, charf *buf,
 # define _tr_tally_lit(s, c, flush) \
   { uch cc = (c); \
-    s->d_buf[s->last_lit] = 0; \
-    s->l_buf[s->last_lit++] = cc; \
+    s->sym_buf[s->sym_next++] = 0; \
+    s->sym_buf[s->sym_next++] = 0; \
+    s->sym_buf[s->sym_next++] = cc; \
     s->dyn_ltree[cc].Freq++; \
-    flush = (s->last_lit == s->lit_bufsize-1); \
+    flush = (s->sym_next == s->sym_end); \
    }
 # define _tr_tally_dist(s, distance, length, flush) \
   { uch len = (uch)(length); \
     ush dist = (ush)(distance); \
-    s->d_buf[s->last_lit] = dist; \
-    s->l_buf[s->last_lit++] = len; \
+    s->sym_buf[s->sym_next++] = dist; \
+    s->sym_buf[s->sym_next++] = dist >> 8; \
+    s->sym_buf[s->sym_next++] = len; \
     dist--; \
     s->dyn_ltree[_length_code[len]+LITERALS+1].Freq++; \
     s->dyn_dtree[d_code(dist)].Freq++; \
-    flush = (s->last_lit == s->lit_bufsize-1); \
+    flush = (s->sym_next == s->sym_end); \
   }
 #else
 # define _tr_tally_lit(s, c, flush) flush = _tr_tally(s, 0, c)

examples/README.examples

@@ -34,6 +34,10 @@ gzlog.h
       and deflateSetDictionary()
     - illustrates use of a gzip header extra field
+gznorm.c
+    normalize a gzip file by combining members into a single member
+    - demonstrates how to concatenate deflate streams using Z_BLOCK
+
 zlib_how.html
     painfully comprehensive description of zpipe.c (see below)
     - describes in excruciating detail the use of deflate() and inflate()
@@ -44,6 +48,7 @@ zpipe.c
     - deeply commented in zlib_how.html (see above)
 zran.c
+zran.h
     index a zlib or gzip stream and randomly access it
     - illustrates the use of Z_BLOCK, inflatePrime(), and
       inflateSetDictionary() to provide random access

examples/zran.c

 /* zran.c -- example of zlib/gzip stream indexing and random access
- * Copyright (C) 2005, 2012 Mark Adler
+ * Copyright (C) 2005, 2012, 2018 Mark Adler
  * For conditions of distribution and use, see copyright notice in zlib.h
-   Version 1.1  29 Sep 2012  Mark Adler */
+ * Version 1.2  14 Oct 2018  Mark Adler */
 /* Version History:
  1.0  29 May 2005  First version
  1.1  29 Sep 2012  Fix memory reallocation error
+ 1.2  14 Oct 2018  Handle gzip streams with multiple members
+                   Add a header file to facilitate usage in applications
  */
 /* Illustrate the use of Z_BLOCK, inflatePrime(), and inflateSetDictionary()
@@ -20,11 +22,11 @@
    the starting file offset and bit of that block, and the 32K bytes of
    uncompressed data that precede that block.  Also the uncompressed offset of
    that block is saved to provide a referece for locating a desired starting
-   point in the uncompressed stream.  build_index() works by decompressing the
-   input zlib or gzip stream a block at a time, and at the end of each block
-   deciding if enough uncompressed data has gone by to justify the creation of
-   a new access point.  If so, that point is saved in a data structure that
-   grows as needed to accommodate the points.
+   point in the uncompressed stream.  deflate_index_build() works by
+   decompressing the input zlib or gzip stream a block at a time, and at the
+   end of each block deciding if enough uncompressed data has gone by to
+   justify the creation of a new access point.  If so, that point is saved in a
+   data structure that grows as needed to accommodate the points.
    To use the index, an offset in the uncompressed data is provided, for which
    the latest access point at or preceding that offset is located in the index.
@@ -43,7 +45,8 @@
    There is some fair bit of overhead to starting inflation for the random
    access, mainly copying the 32K byte dictionary.  So if small pieces of the
    file are being accessed, it would make sense to implement a cache to hold
-   some lookahead and avoid many calls to extract() for small lengths.
+   some lookahead and avoid many calls to deflate_index_extract() for small
+   lengths.
    Another way to build an index would be to use inflateCopy().  That would
    not be constrained to have access points at block boundaries, but requires
@@ -56,30 +59,21 @@
 #include <stdlib.h>
 #include <string.h>
 #include "zlib.h"
-#define local static
-
-#define SPAN 1048576L       /* desired distance between access points */
+#include "zran.h"
 #define WINSIZE 32768U      /* sliding window size */
 #define CHUNK 16384         /* file input buffer size */
-/* access point entry */
+/* Access point entry. */
 struct point {
     off_t out;          /* corresponding offset in uncompressed data */
     off_t in;           /* offset in input file of first full byte */
-    int bits;           /* number of bits (1-7) from byte at in - 1, or 0 */
+    int bits;           /* number of bits (1-7) from byte at in-1, or 0 */
     unsigned char window[WINSIZE];  /* preceding 32K of uncompressed data */
 };
-/* access point list */
-struct access {
-    int have;           /* number of list entries filled in */
-    int size;           /* number of list entries allocated */
-    struct point *list; /* allocated list */
-};
-
-/* Deallocate an index built by build_index() */
-local void free_index(struct access *index)
+/* See comments in zran.h. */
+void deflate_index_free(struct deflate_index *index)
 {
     if (index != NULL) {
         free(index->list);
@@ -87,39 +81,43 @@ local void free_index(struct access *index)
     }
 }
-/* Add an entry to the access point list.  If out of memory, deallocate the
-   existing list and return NULL. */
-local struct access *addpoint(struct access *index, int bits,
-    off_t in, off_t out, unsigned left, unsigned char *window)
+/* Add an entry to the access point list. If out of memory, deallocate the
+   existing list and return NULL. index->gzip is the allocated size of the
+   index in point entries, until it is time for deflate_index_build() to
+   return, at which point gzip is set to indicate a gzip file or not.
+ */
+static struct deflate_index *addpoint(struct deflate_index *index, int bits,
+                                      off_t in, off_t out, unsigned left,
+                                      unsigned char *window)
 {
     struct point *next;
     /* if list is empty, create it (start with eight points) */
     if (index == NULL) {
-        index = malloc(sizeof(struct access));
+        index = malloc(sizeof(struct deflate_index));
         if (index == NULL) return NULL;
         index->list = malloc(sizeof(struct point) << 3);
         if (index->list == NULL) {
             free(index);
             return NULL;
         }
-        index->size = 8;
+        index->gzip = 8;
         index->have = 0;
     }
     /* if list is full, make it bigger */
-    else if (index->have == index->size) {
-        index->size <<= 1;
-        next = realloc(index->list, sizeof(struct point) * index->size);
+    else if (index->have == index->gzip) {
+        index->gzip <<= 1;
+        next = realloc(index->list, sizeof(struct point) * index->gzip);
         if (next == NULL) {
-            free_index(index);
+            deflate_index_free(index);
             return NULL;
         }
         index->list = next;
     }
     /* fill in entry and increment how many we have */
-    next = index->list + index->have;
+    next = (struct point *)(index->list) + index->have;
     next->bits = bits;
     next->in = in;
     next->out = out;
@@ -133,20 +131,14 @@ local struct access *addpoint(struct access *index, int bits,
     return index;
 }
-/* Make one entire pass through the compressed stream and build an index, with
-   access points about every span bytes of uncompressed output -- span is
-   chosen to balance the speed of random access against the memory requirements
-   of the list, about 32K bytes per access point.  Note that data after the end
-   of the first zlib or gzip stream in the file is ignored.  build_index()
-   returns the number of access points on success (>= 1), Z_MEM_ERROR for out
-   of memory, Z_DATA_ERROR for an error in the input file, or Z_ERRNO for a
-   file read error.  On success, *built points to the resulting index. */
-local int build_index(FILE *in, off_t span, struct access **built)
+/* See comments in zran.h. */
+int deflate_index_build(FILE *in, off_t span, struct deflate_index **built)
 {
     int ret;
+    int gzip = 0;               /* true if reading a gzip file */
     off_t totin, totout;        /* our own total counters to avoid 4GB limit */
     off_t last;                 /* totout value of last access point */
-    struct access *index;       /* access points being generated */
+    struct deflate_index *index;    /* access points being generated */
     z_stream strm;
     unsigned char input[CHUNK];
     unsigned char window[WINSIZE];
@@ -163,7 +155,7 @@ local int build_index(FILE *in, off_t span, struct access **built)
     /* inflate the input, maintain a sliding window, and build an index -- this
        also validates the integrity of the compressed data using the check
-       information at the end of the gzip or zlib stream */
+       information in the gzip or zlib stream */
     totin = totout = last = 0;
     index = NULL;               /* will be allocated by first addpoint() */
     strm.avail_out = 0;
@@ -172,14 +164,19 @@ local int build_index(FILE *in, off_t span, struct access **built)
         strm.avail_in = fread(input, 1, CHUNK, in);
         if (ferror(in)) {
             ret = Z_ERRNO;
-            goto build_index_error;
+            goto deflate_index_build_error;
         }
         if (strm.avail_in == 0) {
             ret = Z_DATA_ERROR;
-            goto build_index_error;
+            goto deflate_index_build_error;
         }
         strm.next_in = input;
+        /* check for a gzip stream */
+        if (totin == 0 && strm.avail_in >= 3 &&
+            input[0] == 31 && input[1] == 139 && input[2] == 8)
+            gzip = 1;
+
         /* process all of that, or until end of stream */
         do {
             /* reset sliding window if necessary */
@@ -198,9 +195,17 @@ local int build_index(FILE *in, off_t span, struct access **built)
             if (ret == Z_NEED_DICT)
                 ret = Z_DATA_ERROR;
             if (ret == Z_MEM_ERROR || ret == Z_DATA_ERROR)
-                goto build_index_error;
-            if (ret == Z_STREAM_END)
+                goto deflate_index_build_error;
+            if (ret == Z_STREAM_END) {
+                if (gzip &&
+                    (strm.avail_in || ungetc(getc(in), in) != EOF)) {
+                    ret = inflateReset(&strm);
+                    if (ret != Z_OK)
+                        goto deflate_index_build_error;
+                    continue;
+                }
                 break;
+            }
             /* if at end of block, consider adding an index entry (note that if
                data_type indicates an end-of-block, then all of the
@@ -217,7 +222,7 @@ local int build_index(FILE *in, off_t span, struct access **built)
                                  totout, strm.avail_out, window);
                 if (index == NULL) {
                     ret = Z_MEM_ERROR;
-                    goto build_index_error;
+                    goto deflate_index_build_error;
                 }
                 last = totout;
             }
@@ -227,27 +232,21 @@ local int build_index(FILE *in, off_t span, struct access **built)
     /* clean up and return index (release unused entries in list) */
     (void)inflateEnd(&strm);
     index->list = realloc(index->list, sizeof(struct point) * index->have);
-    index->size = index->have;
+    index->gzip = gzip;
+    index->length = totout;
     *built = index;
-    return index->size;
+    return index->have;
     /* return error */
-  build_index_error:
+  deflate_index_build_error:
     (void)inflateEnd(&strm);
-    if (index != NULL)
-        free_index(index);
+    deflate_index_free(index);
     return ret;
 }
-/* Use the index to read len bytes from offset into buf, return bytes read or
-   negative for error (Z_DATA_ERROR or Z_MEM_ERROR).  If data is requested past
-   the end of the uncompressed data, then extract() will return a value less
-   than len, indicating how much as actually read into buf.  This function
-   should not return a data error unless the file was modified since the index
-   was generated.  extract() may also return Z_ERRNO if there is an error on
-   reading or seeking the input file. */
-local int extract(FILE *in, struct access *index, off_t offset,
-                  unsigned char *buf, int len)
+/* See comments in zran.h. */
+int deflate_index_extract(FILE *in, struct deflate_index *index, off_t offset,
+                          unsigned char *buf, int len)
 {
     int ret, skip;
     z_stream strm;
@@ -276,12 +275,12 @@ local int extract(FILE *in, struct access *index, off_t offset,
         return ret;
     ret = fseeko(in, here->in - (here->bits ? 1 : 0), SEEK_SET);
     if (ret == -1)
-        goto extract_ret;
+        goto deflate_index_extract_ret;
     if (here->bits) {
         ret = getc(in);
         if (ret == -1) {
             ret = ferror(in) ? Z_ERRNO : Z_DATA_ERROR;
-            goto extract_ret;
+            goto deflate_index_extract_ret;
         }
         (void)inflatePrime(&strm, here->bits, ret >> (8 - here->bits));
     }
@@ -293,21 +292,21 @@ local int extract(FILE *in, struct access *index, off_t offset,
     skip = 1;                               /* while skipping to offset */
     do {
         /* define where to put uncompressed data, and how much */
-        if (offset == 0 && skip) {          /* at offset now */
-            strm.avail_out = len;
-            strm.next_out = buf;
-            skip = 0;                       /* only do this once */
-        }
         if (offset > WINSIZE) {             /* skip WINSIZE bytes */
             strm.avail_out = WINSIZE;
             strm.next_out = discard;
             offset -= WINSIZE;
         }
-        else if (offset != 0) {             /* last skip */
+        else if (offset > 0) {              /* last skip */
             strm.avail_out = (unsigned)offset;
             strm.next_out = discard;
             offset = 0;
         }
+        else if (skip) {                    /* at offset now */
+            strm.avail_out = len;
+            strm.next_out = buf;
+            skip = 0;                       /* only do this once */
+        }
         /* uncompress until avail_out filled, or end of stream */
         do {
@@ -315,11 +314,11 @@ local int extract(FILE *in, struct access *index, off_t offset,
                 strm.avail_in = fread(input, 1, CHUNK, in);
                 if (ferror(in)) {
                     ret = Z_ERRNO;
-                    goto extract_ret;
+                    goto deflate_index_extract_ret;
                 }
                 if (strm.avail_in == 0) {
                     ret = Z_DATA_ERROR;
-                    goto extract_ret;
+                    goto deflate_index_extract_ret;
                 }
                 strm.next_in = input;
             }
@@ -327,41 +326,99 @@ local int extract(FILE *in, struct access *index, off_t offset,
             if (ret == Z_NEED_DICT)
                 ret = Z_DATA_ERROR;
             if (ret == Z_MEM_ERROR || ret == Z_DATA_ERROR)
-                goto extract_ret;
-            if (ret == Z_STREAM_END)
-                break;
+                goto deflate_index_extract_ret;
+            if (ret == Z_STREAM_END) {
+                /* the raw deflate stream has ended */
+                if (index->gzip == 0)
+                    /* this is a zlib stream that has ended -- done */
+                    break;
+
+                /* near the end of a gzip member, which might be followed by
+                   another gzip member -- skip the gzip trailer and see if
+                   there is more input after it */
+                if (strm.avail_in < 8) {
+                    fseeko(in, 8 - strm.avail_in, SEEK_CUR);
+                    strm.avail_in = 0;
+                }
+                else {
+                    strm.avail_in -= 8;
+                    strm.next_in += 8;
+                }
+                if (strm.avail_in == 0 && ungetc(getc(in), in) == EOF)
+                    /* the input ended after the gzip trailer -- done */
+                    break;
+
+                /* there is more input, so another gzip member should follow --
+                   validate and skip the gzip header */
+                ret = inflateReset2(&strm, 31);
+                if (ret != Z_OK)
+                    goto deflate_index_extract_ret;
+                do {
+                    if (strm.avail_in == 0) {
+                        strm.avail_in = fread(input, 1, CHUNK, in);
+                        if (ferror(in)) {
+                            ret = Z_ERRNO;
+                            goto deflate_index_extract_ret;
+                        }
+                        if (strm.avail_in == 0) {
+                            ret = Z_DATA_ERROR;
+                            goto deflate_index_extract_ret;
+                        }
+                        strm.next_in = input;
+                    }
+                    ret = inflate(&strm, Z_BLOCK);
+                    if (ret == Z_MEM_ERROR || ret == Z_DATA_ERROR)
+                        goto deflate_index_extract_ret;
+                } while ((strm.data_type & 128) == 0);
+
+                /* set up to continue decompression of the raw deflate stream
+                   that follows the gzip header */
+                ret = inflateReset2(&strm, -15);
+                if (ret != Z_OK)
+                    goto deflate_index_extract_ret;
+            }
+
+            /* continue to process the available input before reading more */
         } while (strm.avail_out != 0);
-        /* if reach end of stream, then don't keep trying to get more */
         if (ret == Z_STREAM_END)
+            /* reached the end of the compressed data -- return the data that
+               was available, possibly less than requested */
             break;
-        /* do until offset reached and requested data read, or stream ends */
+        /* do until offset reached and requested data read */
     } while (skip);
-    /* compute number of uncompressed bytes read after offset */
+    /* compute the number of uncompressed bytes read after the offset */
     ret = skip ? 0 : len - strm.avail_out;
-    /* clean up and return bytes read or error */
-  extract_ret:
+    /* clean up and return the bytes read, or the negative error */
+  deflate_index_extract_ret:
     (void)inflateEnd(&strm);
     return ret;
 }
-/* Demonstrate the use of build_index() and extract() by processing the file
-   provided on the command line, and the extracting 16K from about 2/3rds of
-   the way through the uncompressed output, and writing that to stdout. */
+#ifdef TEST
+
+#define SPAN 1048576L       /* desired distance between access points */
+#define LEN 16384           /* number of bytes to extract */
+
+/* Demonstrate the use of deflate_index_build() and deflate_index_extract() by
+   processing the file provided on the command line, and extracting LEN bytes
+   from 2/3rds of the way through the uncompressed output, writing that to
+   stdout. An offset can be provided as the second argument, in which case the
+   data is extracted from there instead. */
 int main(int argc, char **argv)
 {
     int len;
-    off_t offset;
+    off_t offset = -1;
     FILE *in;
-    struct access *index = NULL;
-    unsigned char buf[CHUNK];
+    struct deflate_index *index = NULL;
+    unsigned char buf[LEN];
     /* open input file */
-    if (argc != 2) {
-        fprintf(stderr, "usage: zran file.gz\n");
+    if (argc < 2 || argc > 3) {
+        fprintf(stderr, "usage: zran file.gz [offset]\n");
         return 1;
     }
     in = fopen(argv[1], "rb");
@@ -370,8 +427,18 @@ int main(int argc, char **argv)
         return 1;
     }
+    /* get optional offset */
+    if (argc == 3) {
+        char *end;
+        offset = strtoll(argv[2], &end, 10);
+        if (*end || offset < 0) {
+            fprintf(stderr, "zran: %s is not a valid offset\n", argv[2]);
+            return 1;
+        }
+    }
+
     /* build index */
-    len = build_index(in, SPAN, &index);
+    len = deflate_index_build(in, SPAN, &index);
     if (len < 0) {
         fclose(in);
         switch (len) {
@@ -392,8 +459,9 @@ int main(int argc, char **argv)
     fprintf(stderr, "zran: built index with %d access points\n", len);
     /* use index by reading some bytes from an arbitrary offset */
-    offset = (index->list[index->have - 1].out << 1) / 3;
-    len = extract(in, index, offset, buf, CHUNK);
+    if (offset == -1)
+        offset = (index->length << 1) / 3;
+    len = deflate_index_extract(in, index, offset, buf, LEN);
     if (len < 0)
         fprintf(stderr, "zran: extraction failed: %s error\n",
                 len == Z_MEM_ERROR ? "out of memory" : "input corrupted");
@@ -403,7 +471,9 @@ int main(int argc, char **argv)
     }
     /* clean up and exit */
-    free_index(index);
+    deflate_index_free(index);
     fclose(in);
     return 0;
 }
+
+#endif

examples/zran.h

+/* zran.h -- example of zlib/gzip stream indexing and random access
+ * Copyright (C) 2005, 2012, 2018 Mark Adler
+ * For conditions of distribution and use, see copyright notice in zlib.h
+ * Version 1.2  14 Oct 2018  Mark Adler */
+
+#include <stdio.h>
+#include "zlib.h"
+
+/* Access point list. */
+struct deflate_index {
+    int have;           /* number of list entries */
+    int gzip;           /* 1 if the index is of a gzip file, 0 if it is of a
+                           zlib stream */
+    off_t length;       /* total length of uncompressed data */
+    void *list;         /* allocated list of entries */
+};
+
+/* Make one entire pass through a zlib or gzip compressed stream and build an
+   index, with access points about every span bytes of uncompressed output.
+   gzip files with multiple members are indexed in their entirety. span should
+   be chosen to balance the speed of random access against the memory
+   requirements of the list, about 32K bytes per access point. The return value
+   is the number of access points on success (>= 1), Z_MEM_ERROR for out of
+   memory, Z_DATA_ERROR for an error in the input file, or Z_ERRNO for a file
+   read error. On success, *built points to the resulting index. */
+int deflate_index_build(FILE *in, off_t span, struct deflate_index **built);
+
+/* Deallocate an index built by deflate_index_build() */
+void deflate_index_free(struct deflate_index *index);
+
+/* Use the index to read len bytes from offset into buf. Return bytes read or
+   negative for error (Z_DATA_ERROR or Z_MEM_ERROR). If data is requested past
+   the end of the uncompressed data, then deflate_index_extract() will return a
+   value less than len, indicating how much was actually read into buf. This
+   function should not return a data error unless the file was modified since
+   the index was generated, since deflate_index_build() validated all of the
+   input. deflate_index_extract() will return Z_ERRNO if there is an error on
+   reading or seeking the input file. */
+int deflate_index_extract(FILE *in, struct deflate_index *index, off_t offset,
+                          unsigned char *buf, int len);

inflate.c

@@ -130,6 +130,7 @@ z_streamp strm;
     state->mode = HEAD;
     state->last = 0;
     state->havedict = 0;
+    state->flags = -1;
     state->dmax = 32768U;
     state->head = Z_NULL;
     state->hold = 0;
@@ -670,7 +671,6 @@ int flush;
                 state->mode = FLAGS;
                 break;
             }
-            state->flags = 0;           /* expect zlib header */
             if (state->head != Z_NULL)
                 state->head->done = -1;
             if (!(state->wrap & 1) ||   /* check if zlib header allowed */
@@ -697,6 +697,7 @@ int flush;
                 break;
             }
             state->dmax = 1U << len;
+            state->flags = 0;               /* indicate zlib header */
             Tracev((stderr, "inflate:   zlib header ok\n"));
             strm->adler = state->check = adler32(0L, Z_NULL, 0);
             state->mode = hold & 0x200 ? DICTID : TYPE;
@@ -1221,7 +1222,7 @@ int flush;
         case LENGTH:
             if (state->wrap && state->flags) {
                 NEEDBITS(32);
-                if (hold != (state->total & 0xffffffffUL)) {
+                if ((state->wrap & 4) && hold != (state->total & 0xffffffff)) {
                     strm->msg = (char *)"incorrect length check";
                     state->mode = BAD;
                     break;
@@ -1401,6 +1402,7 @@ int ZEXPORT inflateSync(strm)
 z_streamp strm;
 {
     unsigned len;               /* number of bytes to look at or looked at */
+    int flags;                  /* temporary to save header status */
     unsigned long in, out;      /* temporary to save total_in and total_out */
     unsigned char buf[4];       /* to restore bit buffer to byte string */
     struct inflate_state FAR *state;
@@ -1433,11 +1435,15 @@ z_streamp strm;
     /* return no joy or set up to restart inflate() on a new block */
     if (state->have != 4) return Z_DATA_ERROR;
-    if (state->mode == HEAD)
-        state->wrap = 0;    /* never processed header, so assume raw */
+    if (state->flags == -1)
+        state->wrap = 0;    /* if no header yet, treat as raw */
+    else
+        state->wrap &= ~4;  /* no point in computing a check value now */
+    flags = state->flags;
     in = strm->total_in;  out = strm->total_out;
     inflateReset(strm);
     strm->total_in = in;  strm->total_out = out;
+    state->flags = flags;
     state->mode = TYPE;
     return Z_OK;
 }

inflate.h

@@ -86,7 +86,8 @@ struct inflate_state {
     int wrap;                   /* bit 0 true for zlib, bit 1 true for gzip,
                                    bit 2 true to validate check value */
     int havedict;               /* true if dictionary provided */
-    int flags;                  /* gzip header method and flags (0 if zlib) */
+    int flags;                  /* gzip header method and flags, 0 if zlib, or
+                                   -1 if raw or no header yet */
     unsigned dmax;              /* zlib header max distance (INFLATE_STRICT) */
     unsigned long check;        /* protected copy of check value */
     unsigned long total;        /* protected copy of output count */

test/example.c

@@ -440,9 +440,8 @@ void test_sync(compr, comprLen, uncompr, uncomprLen)
     CHECK_ERR(err, "inflateSync");
     err = inflate(&d_stream, Z_FINISH);
-    if (err != Z_DATA_ERROR) {
-        fprintf(stderr, "inflate should report DATA_ERROR\n");
-        /* Because of incorrect adler32 */
+    if (err != Z_STREAM_END) {
+        fprintf(stderr, "inflate should report Z_STREAM_END\n");
         exit(1);
     }
     err = inflateEnd(&d_stream);

trees.c

@@ -416,7 +416,7 @@ local void init_block(s)
     s->dyn_ltree[END_BLOCK].Freq = 1;
     s->opt_len = s->static_len = 0L;
-    s->last_lit = s->matches = 0;
+    s->sym_next = s->matches = 0;
 }
 #define SMALLEST 1
@@ -948,7 +948,7 @@ void ZLIB_INTERNAL _tr_flush_block(s, buf, stored_len, last)
         Tracev((stderr, "\nopt %lu(%lu) stat %lu(%lu) stored %lu lit %u ",
                 opt_lenb, s->opt_len, static_lenb, s->static_len, stored_len,
-                s->last_lit));
+                s->sym_next / 3));
         if (static_lenb <= opt_lenb) opt_lenb = static_lenb;
@@ -1017,8 +1017,9 @@ int ZLIB_INTERNAL _tr_tally (s, dist, lc)
     unsigned dist;  /* distance of matched string */
     unsigned lc;    /* match length-MIN_MATCH or unmatched char (if dist==0) */
 {
-    s->d_buf[s->last_lit] = (ush)dist;
-    s->l_buf[s->last_lit++] = (uch)lc;
+    s->sym_buf[s->sym_next++] = dist;
+    s->sym_buf[s->sym_next++] = dist >> 8;
+    s->sym_buf[s->sym_next++] = lc;
     if (dist == 0) {
         /* lc is the unmatched char */
         s->dyn_ltree[lc].Freq++;
@@ -1033,30 +1034,7 @@ int ZLIB_INTERNAL _tr_tally (s, dist, lc)
         s->dyn_ltree[_length_code[lc]+LITERALS+1].Freq++;
         s->dyn_dtree[d_code(dist)].Freq++;
     }
-
-#ifdef TRUNCATE_BLOCK
-    /* Try to guess if it is profitable to stop the current block here */
-    if ((s->last_lit & 0x1fff) == 0 && s->level > 2) {
-        /* Compute an upper bound for the compressed length */
-        ulg out_length = (ulg)s->last_lit*8L;
-        ulg in_length = (ulg)((long)s->strstart - s->block_start);
-        int dcode;
-        for (dcode = 0; dcode < D_CODES; dcode++) {
-            out_length += (ulg)s->dyn_dtree[dcode].Freq *
-                (5L+extra_dbits[dcode]);
-        }
-        out_length >>= 3;
-        Tracev((stderr,"\nlast_lit %u, in %ld, out ~%ld(%ld%%) ",
-               s->last_lit, in_length, out_length,
-               100L - out_length*100L/in_length));
-        if (s->matches < s->last_lit/2 && out_length < in_length/2) return 1;
-    }
-#endif
-    return (s->last_lit == s->lit_bufsize-1);
-    /* We avoid equality with lit_bufsize because of wraparound at 64K
-     * on 16 bit machines and because stored blocks are restricted to
-     * 64K-1 bytes.
-     */
+    return (s->sym_next == s->sym_end);
 }
 /* ===========================================================================
@@ -1069,13 +1047,14 @@ local void compress_block(s, ltree, dtree)
 {
     unsigned dist;      /* distance of matched string */
     int lc;             /* match length or unmatched char (if dist == 0) */
-    unsigned lx = 0;    /* running index in l_buf */
+    unsigned sx = 0;    /* running index in sym_buf */
     unsigned code;      /* the code to send */
     int extra;          /* number of extra bits to send */
-    if (s->last_lit != 0) do {
-        dist = s->d_buf[lx];
-        lc = s->l_buf[lx++];
+    if (s->sym_next != 0) do {
+        dist = s->sym_buf[sx++] & 0xff;
+        dist += (unsigned)(s->sym_buf[sx++] & 0xff) << 8;
+        lc = s->sym_buf[sx++];
         if (dist == 0) {
             send_code(s, lc, ltree); /* send a literal byte */
             Tracecv(isgraph(lc), (stderr," '%c' ", lc));
@@ -1100,11 +1079,10 @@ local void compress_block(s, ltree, dtree)
             }
         } /* literal or match pair ? */
-        /* Check that the overlay between pending_buf and d_buf+l_buf is ok: */
-        Assert((uInt)(s->pending) < s->lit_bufsize + 2*lx,
-               "pendingBuf overflow");
-    } while (lx < s->last_lit);
+        /* Check that the overlay between pending_buf and sym_buf is ok: */
+        Assert(s->pending < s->lit_bufsize + sx, "pendingBuf overflow");
+    } while (sx < s->sym_next);
     send_code(s, END_BLOCK, ltree);
 }

zlib.h

@@ -543,8 +543,7 @@ ZEXTERN int ZEXPORT deflateInit2 OF((z_streamp strm,
                                      int  strategy));
      This is another version of deflateInit with more compression options.  The
-   fields next_in, zalloc, zfree and opaque must be initialized before by the
-   caller.
+   fields zalloc, zfree and opaque must be initialized before by the caller.
      The method parameter is the compression method.  It must be Z_DEFLATED in
    this version of the library.
@@ -866,9 +865,11 @@ ZEXTERN int ZEXPORT inflateInit2 OF((z_streamp strm,
    detection, or add 16 to decode only the gzip format (the zlib format will
    return a Z_DATA_ERROR).  If a gzip stream is being decoded, strm->adler is a
    CRC-32 instead of an Adler-32.  Unlike the gunzip utility and gzread() (see
-   below), inflate() will not automatically decode concatenated gzip streams.
-   inflate() will return Z_STREAM_END at the end of the gzip stream.  The state
-   would need to be reset to continue decoding a subsequent gzip stream.
+   below), inflate() will *not* automatically decode concatenated gzip members.
+   inflate() will return Z_STREAM_END at the end of the gzip member.  The state
+   would need to be reset to continue decoding a subsequent gzip member.  This
+   *must* be done if there is more data after a gzip member, in order for the
+   decompression to be compliant with the gzip standard (RFC 1952).
      inflateInit2 returns Z_OK if success, Z_MEM_ERROR if there was not enough
    memory, Z_VERSION_ERROR if the zlib library version is incompatible with the
@@ -1723,7 +1724,7 @@ ZEXTERN uLong ZEXPORT adler32_combine OF((uLong adler1, uLong adler2,
    negative, the result has no meaning or utility.
 */
-ZEXTERN uLong ZEXPORT crc32   OF((uLong crc, const Bytef *buf, uInt len));
+ZEXTERN uLong ZEXPORT crc32 OF((uLong crc, const Bytef *buf, uInt len));
 /*
      Update a running CRC-32 with the bytes buf[0..len-1] and return the
    updated CRC-32.  If buf is Z_NULL, this function returns the required
@@ -1756,6 +1757,20 @@ ZEXTERN uLong ZEXPORT crc32_combine OF((uLong crc1, uLong crc2, z_off_t len2));
    len2.
 */
+/*
+ZEXTERN uLong ZEXPORT crc32_combine_gen OF((z_off_t len2));
+
+     Return the operator corresponding to length len2, to be used with
+   crc32_combine_op().
+*/
+
+ZEXTERN uLong ZEXPORT crc32_combine_op OF((uLong crc1, uLong crc2, uLong op));
+/*
+     Give the same result as crc32_combine(), using op in place of len2. op is
+   is generated from len2 by crc32_combine_gen(). This will be faster than
+   crc32_combine() if the generated op is used more than once.
+*/
+
                         /* various hacks, don't look :) */
@@ -1843,6 +1858,7 @@ ZEXTERN int ZEXPORT gzgetc_ OF((gzFile file));  /* backward compatibility */
    ZEXTERN z_off64_t ZEXPORT gzoffset64 OF((gzFile));
    ZEXTERN uLong ZEXPORT adler32_combine64 OF((uLong, uLong, z_off64_t));
    ZEXTERN uLong ZEXPORT crc32_combine64 OF((uLong, uLong, z_off64_t));
+   ZEXTERN uLong ZEXPORT crc32_combine_gen64 OF((z_off64_t));
 #endif
 #if !defined(ZLIB_INTERNAL) && defined(Z_WANT64)
@@ -1853,6 +1869,7 @@ ZEXTERN int ZEXPORT gzgetc_ OF((gzFile file));  /* backward compatibility */
 #    define z_gzoffset z_gzoffset64
 #    define z_adler32_combine z_adler32_combine64
 #    define z_crc32_combine z_crc32_combine64
+#    define z_crc32_combine_gen z_crc32_combine64_gen
 #  else
 #    define gzopen gzopen64
 #    define gzseek gzseek64
@@ -1860,6 +1877,7 @@ ZEXTERN int ZEXPORT gzgetc_ OF((gzFile file));  /* backward compatibility */
 #    define gzoffset gzoffset64
 #    define adler32_combine adler32_combine64
 #    define crc32_combine crc32_combine64
+#    define crc32_combine_gen crc32_combine_gen64
 #  endif
 #  ifndef Z_LARGE64
      ZEXTERN gzFile ZEXPORT gzopen64 OF((const char *, const char *));
@@ -1868,6 +1886,7 @@ ZEXTERN int ZEXPORT gzgetc_ OF((gzFile file));  /* backward compatibility */
      ZEXTERN z_off_t ZEXPORT gzoffset64 OF((gzFile));
      ZEXTERN uLong ZEXPORT adler32_combine64 OF((uLong, uLong, z_off_t));
      ZEXTERN uLong ZEXPORT crc32_combine64 OF((uLong, uLong, z_off_t));
+     ZEXTERN uLong ZEXPORT crc32_combine_gen64 OF((z_off_t));
 #  endif
 #else
    ZEXTERN gzFile ZEXPORT gzopen OF((const char *, const char *));
@@ -1876,12 +1895,14 @@ ZEXTERN int ZEXPORT gzgetc_ OF((gzFile file));  /* backward compatibility */
    ZEXTERN z_off_t ZEXPORT gzoffset OF((gzFile));
    ZEXTERN uLong ZEXPORT adler32_combine OF((uLong, uLong, z_off_t));
    ZEXTERN uLong ZEXPORT crc32_combine OF((uLong, uLong, z_off_t));
+   ZEXTERN uLong ZEXPORT crc32_combine_gen OF((z_off_t));
 #endif
 #else /* Z_SOLO */
    ZEXTERN uLong ZEXPORT adler32_combine OF((uLong, uLong, z_off_t));
    ZEXTERN uLong ZEXPORT crc32_combine OF((uLong, uLong, z_off_t));
+   ZEXTERN uLong ZEXPORT crc32_combine_gen OF((z_off_t));
 #endif /* !Z_SOLO */

zutil.h

@@ -42,6 +42,17 @@ typedef unsigned short ush;
 typedef ush FAR ushf;
 typedef unsigned long  ulg;
+#if !defined(Z_U8) && !defined(Z_SOLO) && defined(STDC)
+#  include <limits.h>
+#  if (ULONG_MAX == 0xffffffffffffffffULL)
+#    define Z_U8 unsigned long
+#  elif (ULLONG_MAX == 0xffffffffffffffffULL)
+#    define Z_U8 unsigned long long
+#  elif (UINT_MAX == 0xffffffffffffffffULL)
+#    define Z_U8 unsigned
+#  endif
+#endif
+
 extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */
 /* (size given to avoid silly warnings with Visual C++) */