Support for statically compressed gzip content-encoding

Makefile

@@ -40,11 +40,11 @@ complexity:
 test:
 	go get golang.org/x/tools/cmd/cover
-	go test ./... -short -cover -v -timeout 1m
+	go test . -short -cover -v -timeout 1m
 acceptance: gitlab-pages
 	go get golang.org/x/tools/cmd/cover
-	go test ./... -cover -v -timeout 1m
+	go test . -cover -v -timeout 1m
 docker:
 	docker run --rm -it -v ${PWD}:/go/src/pages -w /go/src/pages golang:1.5 /bin/bash

domain.go

@@ -11,6 +11,8 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+
+	"gitlab.com/gitlab-org/gitlab-pages/internal/httputil"
 )
 type domain struct {
@@ -21,8 +23,33 @@ type domain struct {
 	certificateError error
 }
+func acceptsGZip(r *http.Request) bool {
+	if r.Header.Get("Range") != "" {
+		return false
+	}
+	offers := []string{"gzip", "identity"}
+	acceptedEncoding := httputil.NegotiateContentEncoding(r, offers)
+	return acceptedEncoding == "gzip"
+}
+
 func (d *domain) serveFile(w http.ResponseWriter, r *http.Request, fullPath string) error {
 	// Open and serve content of file
+	if acceptsGZip(r) {
+		_, err := os.Stat(fullPath + ".gz")
+		if err == nil {
+			// Set the content type based on the non-gzipped extension
+			_, haveType := w.Header()["Content-Type"]
+			if !haveType {
+				ctype := mime.TypeByExtension(filepath.Ext(fullPath))
+				w.Header().Set("Content-Type", ctype)
+			}
+
+			// Serve up the gzipped version
+			fullPath += ".gz"
+			w.Header().Set("Content-Encoding", "gzip")
+		}
+	}
+
 	file, err := os.Open(fullPath)
 	if err != nil {
 		return err
@@ -41,6 +68,15 @@ func (d *domain) serveFile(w http.ResponseWriter, r *http.Request, fullPath stri
 func (d *domain) serveCustomFile(w http.ResponseWriter, r *http.Request, code int, fullPath string) error {
 	// Open and serve content of file
+	ext := filepath.Ext(fullPath)
+	if acceptsGZip(r) {
+		_, err := os.Stat(fullPath + ".gz")
+		if err == nil {
+			// Serve up the gzipped version
+			fullPath += ".gz"
+			w.Header().Set("Content-Encoding", "gzip")
+		}
+	}
 	file, err := os.Open(fullPath)
 	if err != nil {
 		return err
@@ -57,7 +93,7 @@ func (d *domain) serveCustomFile(w http.ResponseWriter, r *http.Request, code in
 	// Serve the file
 	_, haveType := w.Header()["Content-Type"]
 	if !haveType {
-		ctype := mime.TypeByExtension(filepath.Ext(fullPath))
+		ctype := mime.TypeByExtension(ext)
 		w.Header().Set("Content-Type", ctype)
 	}
 	w.Header().Set("Content-Length", strconv.FormatInt(fi.Size(), 10))

domain_test.go

 package main
 import (
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
+	"compress/gzip"
+	"io/ioutil"
 	"mime"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 func TestGroupServeHTTP(t *testing.T) {
@@ -51,6 +54,80 @@ func TestDomainServeHTTP(t *testing.T) {
 	assert.HTTPError(t, testDomain.ServeHTTP, "GET", "/not-existing-file", nil)
 }
+func testHTTPGzip(t *testing.T, handler http.HandlerFunc, mode, url string, values url.Values, acceptEncoding string, str interface{}, ungzip bool) {
+	w := httptest.NewRecorder()
+	req, err := http.NewRequest(mode, url+"?"+values.Encode(), nil)
+	require.NoError(t, err)
+	if acceptEncoding != "" {
+		req.Header.Add("Accept-Encoding", acceptEncoding)
+	}
+	handler(w, req)
+
+	if ungzip {
+		reader, err := gzip.NewReader(w.Body)
+		require.NoError(t, err)
+		defer reader.Close()
+
+		contentEncoding := w.Header().Get("Content-Encoding")
+		assert.Equal(t, "gzip", contentEncoding, "Content-Encoding")
+
+		bytes, err := ioutil.ReadAll(reader)
+		require.NoError(t, err)
+		assert.Contains(t, string(bytes), str)
+	} else {
+		assert.Contains(t, w.Body.String(), str)
+	}
+}
+
+func TestGroupServeHTTPGzip(t *testing.T) {
+	setUpTests()
+
+	testGroup := &domain{
+		Group:   "group",
+		Project: "",
+	}
+
+	testSet := []struct {
+		mode           string      // HTTP mode
+		url            string      // Test URL
+		params         url.Values  // Test URL params
+		acceptEncoding string      // Accept encoding header
+		body           interface{} // Expected body at above URL
+		ungzip         bool        // Do we expect the request to require unzip?
+	}{
+		// No gzip encoding requested
+		{"GET", "http://group.test.io/", nil, "", "main-dir", false},
+		{"GET", "http://group.test.io/", nil, "identity", "main-dir", false},
+		{"GET", "http://group.test.io/", nil, "gzip; q=0", "main-dir", false},
+		// gzip encoding requeste},
+		{"GET", "http://group.test.io/", nil, "*", "main-dir", true},
+		{"GET", "http://group.test.io/", nil, "identity, gzip", "main-dir", true},
+		{"GET", "http://group.test.io/", nil, "gzip", "main-dir", true},
+		{"GET", "http://group.test.io/", nil, "gzip; q=1", "main-dir", true},
+		{"GET", "http://group.test.io/", nil, "gzip; q=0.9", "main-dir", true},
+		{"GET", "http://group.test.io/", nil, "gzip, deflate", "main-dir", true},
+		{"GET", "http://group.test.io/", nil, "gzip; q=1, deflate", "main-dir", true},
+		{"GET", "http://group.test.io/", nil, "gzip; q=0.9, deflate", "main-dir", true},
+		// gzip encoding requested, but url does not have compressed content on disk
+		{"GET", "http://group.test.io/project2/", nil, "*", "project2-main", false},
+		{"GET", "http://group.test.io/project2/", nil, "identity, gzip", "project2-main", false},
+		{"GET", "http://group.test.io/project2/", nil, "gzip", "project2-main", false},
+		{"GET", "http://group.test.io/project2/", nil, "gzip; q=1", "project2-main", false},
+		{"GET", "http://group.test.io/project2/", nil, "gzip; q=0.9", "project2-main", false},
+		{"GET", "http://group.test.io/project2/", nil, "gzip, deflate", "project2-main", false},
+		{"GET", "http://group.test.io/project2/", nil, "gzip; q=1, deflate", "project2-main", false},
+		{"GET", "http://group.test.io/project2/", nil, "gzip; q=0.9, deflate", "project2-main", false},
+		// malformed headers
+		{"GET", "http://group.test.io/", nil, ";; gzip", "main-dir", false},
+		{"GET", "http://group.test.io/", nil, "middle-out", "main-dir", false},
+		{"GET", "http://group.test.io/", nil, "gzip; quality=1", "main-dir", false},
+	}
+
+	for _, tt := range testSet {
+		testHTTPGzip(t, testGroup.ServeHTTP, tt.mode, tt.url, tt.params, tt.acceptEncoding, tt.body, tt.ungzip)
+	}
+}
+
 func testHTTP404(t *testing.T, handler http.HandlerFunc, mode, url string, values url.Values, str interface{}) {
 	w := httptest.NewRecorder()
 	req, err := http.NewRequest(mode, url+"?"+values.Encode(), nil)

internal/httputil/LICENSE

+Copyright (c) 2013 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

internal/httputil/README.md

+This folder is a partial import of the [GoDoc API package](https://github.com/golang/gddo),
+```
+github.com/golang/gddo/httputil
+```
+where the original license (see `LICENSE`) has been incorporated herein.

internal/httputil/header/header.go

+// Copyright 2013 The Go Authors. All rights reserved.
+//
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file or at
+// https://developers.google.com/open-source/licenses/bsd.
+
+// Package header provides functions for parsing HTTP headers.
+package header
+
+import (
+	"net/http"
+	"strings"
+	"time"
+)
+
+// Octet types from RFC 2616.
+var octetTypes [256]octetType
+
+type octetType byte
+
+const (
+	isToken octetType = 1 << iota
+	isSpace
+)
+
+func init() {
+	// OCTET      = <any 8-bit sequence of data>
+	// CHAR       = <any US-ASCII character (octets 0 - 127)>
+	// CTL        = <any US-ASCII control character (octets 0 - 31) and DEL (127)>
+	// CR         = <US-ASCII CR, carriage return (13)>
+	// LF         = <US-ASCII LF, linefeed (10)>
+	// SP         = <US-ASCII SP, space (32)>
+	// HT         = <US-ASCII HT, horizontal-tab (9)>
+	// <">        = <US-ASCII double-quote mark (34)>
+	// CRLF       = CR LF
+	// LWS        = [CRLF] 1*( SP | HT )
+	// TEXT       = <any OCTET except CTLs, but including LWS>
+	// separators = "(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\" | <">
+	//              | "/" | "[" | "]" | "?" | "=" | "{" | "}" | SP | HT
+	// token      = 1*<any CHAR except CTLs or separators>
+	// qdtext     = <any TEXT except <">>
+
+	for c := 0; c < 256; c++ {
+		var t octetType
+		isCtl := c <= 31 || c == 127
+		isChar := 0 <= c && c <= 127
+		isSeparator := strings.IndexRune(" \t\"(),/:;<=>?@[]\\{}", rune(c)) >= 0
+		if strings.IndexRune(" \t\r\n", rune(c)) >= 0 {
+			t |= isSpace
+		}
+		if isChar && !isCtl && !isSeparator {
+			t |= isToken
+		}
+		octetTypes[c] = t
+	}
+}
+
+// Copy returns a shallow copy of the header.
+func Copy(header http.Header) http.Header {
+	h := make(http.Header)
+	for k, vs := range header {
+		h[k] = vs
+	}
+	return h
+}
+
+var timeLayouts = []string{"Mon, 02 Jan 2006 15:04:05 GMT", time.RFC850, time.ANSIC}
+
+// ParseTime parses the header as time. The zero value is returned if the
+// header is not present or there is an error parsing the
+// header.
+func ParseTime(header http.Header, key string) time.Time {
+	if s := header.Get(key); s != "" {
+		for _, layout := range timeLayouts {
+			if t, err := time.Parse(layout, s); err == nil {
+				return t.UTC()
+			}
+		}
+	}
+	return time.Time{}
+}
+
+// ParseList parses a comma separated list of values. Commas are ignored in
+// quoted strings. Quoted values are not unescaped or unquoted. Whitespace is
+// trimmed.
+func ParseList(header http.Header, key string) []string {
+	var result []string
+	for _, s := range header[http.CanonicalHeaderKey(key)] {
+		begin := 0
+		end := 0
+		escape := false
+		quote := false
+		for i := 0; i < len(s); i++ {
+			b := s[i]
+			switch {
+			case escape:
+				escape = false
+				end = i + 1
+			case quote:
+				switch b {
+				case '\\':
+					escape = true
+				case '"':
+					quote = false
+				}
+				end = i + 1
+			case b == '"':
+				quote = true
+				end = i + 1
+			case octetTypes[b]&isSpace != 0:
+				if begin == end {
+					begin = i + 1
+					end = begin
+				}
+			case b == ',':
+				if begin < end {
+					result = append(result, s[begin:end])
+				}
+				begin = i + 1
+				end = begin
+			default:
+				end = i + 1
+			}
+		}
+		if begin < end {
+			result = append(result, s[begin:end])
+		}
+	}
+	return result
+}
+
+// ParseValueAndParams parses a comma separated list of values with optional
+// semicolon separated name-value pairs. Content-Type and Content-Disposition
+// headers are in this format.
+func ParseValueAndParams(header http.Header, key string) (value string, params map[string]string) {
+	params = make(map[string]string)
+	s := header.Get(key)
+	value, s = expectTokenSlash(s)
+	if value == "" {
+		return
+	}
+	value = strings.ToLower(value)
+	s = skipSpace(s)
+	for strings.HasPrefix(s, ";") {
+		var pkey string
+		pkey, s = expectToken(skipSpace(s[1:]))
+		if pkey == "" {
+			return
+		}
+		if !strings.HasPrefix(s, "=") {
+			return
+		}
+		var pvalue string
+		pvalue, s = expectTokenOrQuoted(s[1:])
+		if pvalue == "" {
+			return
+		}
+		pkey = strings.ToLower(pkey)
+		params[pkey] = pvalue
+		s = skipSpace(s)
+	}
+	return
+}
+
+// AcceptSpec describes an Accept* header.
+type AcceptSpec struct {
+	Value string
+	Q     float64
+}
+
+// ParseAccept parses Accept* headers.
+func ParseAccept(header http.Header, key string) (specs []AcceptSpec) {
+loop:
+	for _, s := range header[key] {
+		for {
+			var spec AcceptSpec
+			spec.Value, s = expectTokenSlash(s)
+			if spec.Value == "" {
+				continue loop
+			}
+			spec.Q = 1.0
+			s = skipSpace(s)
+			if strings.HasPrefix(s, ";") {
+				s = skipSpace(s[1:])
+				if !strings.HasPrefix(s, "q=") {
+					continue loop
+				}
+				spec.Q, s = expectQuality(s[2:])
+				if spec.Q < 0.0 {
+					continue loop
+				}
+			}
+			specs = append(specs, spec)
+			s = skipSpace(s)
+			if !strings.HasPrefix(s, ",") {
+				continue loop
+			}
+			s = skipSpace(s[1:])
+		}
+	}
+	return
+}
+
+func skipSpace(s string) (rest string) {
+	i := 0
+	for ; i < len(s); i++ {
+		if octetTypes[s[i]]&isSpace == 0 {
+			break
+		}
+	}
+	return s[i:]
+}
+
+func expectToken(s string) (token, rest string) {
+	i := 0
+	for ; i < len(s); i++ {
+		if octetTypes[s[i]]&isToken == 0 {
+			break
+		}
+	}
+	return s[:i], s[i:]
+}
+
+func expectTokenSlash(s string) (token, rest string) {
+	i := 0
+	for ; i < len(s); i++ {
+		b := s[i]
+		if (octetTypes[b]&isToken == 0) && b != '/' {
+			break
+		}
+	}
+	return s[:i], s[i:]
+}
+
+func expectQuality(s string) (q float64, rest string) {
+	switch {
+	case len(s) == 0:
+		return -1, ""
+	case s[0] == '0':
+		q = 0
+	case s[0] == '1':
+		q = 1
+	default:
+		return -1, ""
+	}
+	s = s[1:]
+	if !strings.HasPrefix(s, ".") {
+		return q, s
+	}
+	s = s[1:]
+	i := 0
+	n := 0
+	d := 1
+	for ; i < len(s); i++ {
+		b := s[i]
+		if b < '0' || b > '9' {
+			break
+		}
+		n = n*10 + int(b) - '0'
+		d *= 10
+	}
+	return q + float64(n)/float64(d), s[i:]
+}
+
+func expectTokenOrQuoted(s string) (value string, rest string) {
+	if !strings.HasPrefix(s, "\"") {
+		return expectToken(s)
+	}
+	s = s[1:]
+	for i := 0; i < len(s); i++ {
+		switch s[i] {
+		case '"':
+			return s[:i], s[i+1:]
+		case '\\':
+			p := make([]byte, len(s)-1)
+			j := copy(p, s[:i])
+			escape := true
+			for i = i + 1; i < len(s); i++ {
+				b := s[i]
+				switch {
+				case escape:
+					escape = false
+					p[j] = b
+					j++
+				case b == '\\':
+					escape = true
+				case b == '"':
+					return string(p[:j]), s[i+1:]
+				default:
+					p[j] = b
+					j++
+				}
+			}
+			return "", ""
+		}
+	}
+	return "", ""
+}

internal/httputil/negotiate.go

+// Copyright 2013 The Go Authors. All rights reserved.
+//
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file or at
+// https://developers.google.com/open-source/licenses/bsd.
+
+package httputil
+
+import (
+	"net/http"
+	"strings"
+
+	"gitlab.com/gitlab-org/gitlab-pages/internal/httputil/header"
+)
+
+// NegotiateContentEncoding returns the best offered content encoding for the
+// request's Accept-Encoding header. If two offers match with equal weight and
+// then the offer earlier in the list is preferred. If no offers are
+// acceptable, then "" is returned.
+func NegotiateContentEncoding(r *http.Request, offers []string) string {
+	bestOffer := "identity"
+	bestQ := -1.0
+	specs := header.ParseAccept(r.Header, "Accept-Encoding")
+	for _, offer := range offers {
+		for _, spec := range specs {
+			if spec.Q > bestQ &&
+				(spec.Value == "*" || spec.Value == offer) {
+				bestQ = spec.Q
+				bestOffer = offer
+			}
+		}
+	}
+	if bestQ == 0 {
+		bestOffer = ""
+	}
+	return bestOffer
+}
+
+// NegotiateContentType returns the best offered content type for the request's
+// Accept header. If two offers match with equal weight, then the more specific
+// offer is preferred.  For example, text/* trumps */*. If two offers match
+// with equal weight and specificity, then the offer earlier in the list is
+// preferred. If no offers match, then defaultOffer is returned.
+func NegotiateContentType(r *http.Request, offers []string, defaultOffer string) string {
+	bestOffer := defaultOffer
+	bestQ := -1.0
+	bestWild := 3
+	specs := header.ParseAccept(r.Header, "Accept")
+	for _, offer := range offers {
+		for _, spec := range specs {
+			switch {
+			case spec.Q == 0.0:
+				// ignore
+			case spec.Q < bestQ:
+				// better match found
+			case spec.Value == "*/*":
+				if spec.Q > bestQ || bestWild > 2 {
+					bestQ = spec.Q
+					bestWild = 2
+					bestOffer = offer
+				}
+			case strings.HasSuffix(spec.Value, "/*"):
+				if strings.HasPrefix(offer, spec.Value[:len(spec.Value)-1]) &&
+					(spec.Q > bestQ || bestWild > 1) {
+					bestQ = spec.Q
+					bestWild = 1
+					bestOffer = offer
+				}
+			default:
+				if spec.Value == offer &&
+					(spec.Q > bestQ || bestWild > 0) {
+					bestQ = spec.Q
+					bestWild = 0
+					bestOffer = offer
+				}
+			}
+		}
+	}
+	return bestOffer
+}