diff --git a/Makefile b/Makefile
index 197ca4b5cc820e7c1c96246a42d82bdc368ce247..6d65af45a833955dc07cd65a6e65e136b13d7b95 100644
--- a/Makefile
+++ b/Makefile
@@ -28,6 +28,9 @@ coverage: testdata/data/group/test.git
go tool cover -html=test.coverage -o coverage.html
rm -f test.coverage
+fmt:
+ go fmt ./...
+
testdata/data/group/test.git: testdata/data
git clone --bare https://gitlab.com/gitlab-org/gitlab-test.git $@
diff --git a/internal/api/api.go b/internal/api/api.go
index b23ea7ee309966cd6cf0a6a1c2323568ab9e0cae..02eb01c3f26c0151025c952f7c0ee848c472ee11 100644
--- a/internal/api/api.go
+++ b/internal/api/api.go
@@ -38,15 +38,6 @@ type Response struct {
// RepoPath is the full path on disk to the Git repository the request is
// about
RepoPath string
- // ArchivePath is the full path where we should find/create a cached copy
- // of a requested archive
- ArchivePath string
- // ArchivePrefix is used to put extracted archive contents in a
- // subdirectory
- ArchivePrefix string
- // CommitId is used do prevent race conditions between the 'time of check'
- // in the GitLab Rails app and the 'time of use' in gitlab-workhorse.
- CommitId string
// StoreLFSPath is provided by the GitLab Rails application
// to mark where the tmp file should be placed
StoreLFSPath string
diff --git a/internal/git/archive.go b/internal/git/archive.go
index 854c887c544e1c751400aac858c6bb9b41547fca..95016e9bfeb207e57135e09b15a527ea50bcbdf2 100644
--- a/internal/git/archive.go
+++ b/internal/git/archive.go
@@ -5,8 +5,8 @@ In this file we handle 'git archive' downloads
package git
import (
- "../api"
"../helper"
+ "../senddata"
"fmt"
"io"
"io/ioutil"
@@ -20,11 +20,23 @@ import (
"time"
)
-func GetArchive(a *api.API) http.Handler {
- return repoPreAuthorizeHandler(a, handleGetArchive)
+type archive struct{ senddata.Prefix }
+type archiveParams struct {
+ RepoPath string
+ ArchivePath string
+ ArchivePrefix string
+ CommitId string
}
-func handleGetArchive(w http.ResponseWriter, r *http.Request, a *api.Response) {
+var SendArchive = &archive{"git-archive:"}
+
+func (a *archive) Inject(w http.ResponseWriter, r *http.Request, sendData string) {
+ var params archiveParams
+ if err := a.Unpack(¶ms, sendData); err != nil {
+ helper.Fail500(w, fmt.Errorf("SendArchive: unpack sendData: %v", err))
+ return
+ }
+
var format string
urlPath := r.URL.Path
switch filepath.Base(urlPath) {
@@ -41,11 +53,11 @@ func handleGetArchive(w http.ResponseWriter, r *http.Request, a *api.Response) {
return
}
- archiveFilename := path.Base(a.ArchivePath)
+ archiveFilename := path.Base(params.ArchivePath)
- if cachedArchive, err := os.Open(a.ArchivePath); err == nil {
+ if cachedArchive, err := os.Open(params.ArchivePath); err == nil {
defer cachedArchive.Close()
- log.Printf("Serving cached file %q", a.ArchivePath)
+ log.Printf("Serving cached file %q", params.ArchivePath)
setArchiveHeaders(w, format, archiveFilename)
// Even if somebody deleted the cachedArchive from disk since we opened
// the file, Unix file semantics guarantee we can still read from the
@@ -58,7 +70,7 @@ func handleGetArchive(w http.ResponseWriter, r *http.Request, a *api.Response) {
// safe. We create the tempfile in the same directory as the final cached
// archive we want to create so that we can use an atomic link(2) operation
// to finalize the cached archive.
- tempFile, err := prepareArchiveTempfile(path.Dir(a.ArchivePath), archiveFilename)
+ tempFile, err := prepareArchiveTempfile(path.Dir(params.ArchivePath), archiveFilename)
if err != nil {
helper.Fail500(w, fmt.Errorf("handleGetArchive: create tempfile: %v", err))
return
@@ -68,7 +80,7 @@ func handleGetArchive(w http.ResponseWriter, r *http.Request, a *api.Response) {
compressCmd, archiveFormat := parseArchiveFormat(format)
- archiveCmd := gitCommand("", "git", "--git-dir="+a.RepoPath, "archive", "--format="+archiveFormat, "--prefix="+a.ArchivePrefix+"/", a.CommitId)
+ archiveCmd := gitCommand("", "git", "--git-dir="+params.RepoPath, "archive", "--format="+archiveFormat, "--prefix="+params.ArchivePrefix+"/", params.CommitId)
archiveStdout, err := archiveCmd.StdoutPipe()
if err != nil {
helper.Fail500(w, fmt.Errorf("handleGetArchive: archive stdout: %v", err))
@@ -125,13 +137,14 @@ func handleGetArchive(w http.ResponseWriter, r *http.Request, a *api.Response) {
}
}
- if err := finalizeCachedArchive(tempFile, a.ArchivePath); err != nil {
+ if err := finalizeCachedArchive(tempFile, params.ArchivePath); err != nil {
helper.LogError(fmt.Errorf("handleGetArchive: finalize cached archive: %v", err))
return
}
}
func setArchiveHeaders(w http.ResponseWriter, format string, archiveFilename string) {
+ w.Header().Del("Content-Length")
w.Header().Add("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, archiveFilename))
if format == "zip" {
w.Header().Add("Content-Type", "application/zip")
diff --git a/internal/git/blob.go b/internal/git/blob.go
index 6e99efb5ea2d1ba4ecf4291bd1b77c4f5e88a6d7..8243b73f13e6333637df08351e3f0ca471fa38b2 100644
--- a/internal/git/blob.go
+++ b/internal/git/blob.go
@@ -2,28 +2,25 @@ package git
import (
"../helper"
- "encoding/base64"
- "encoding/json"
+ "../senddata"
"fmt"
"io"
"log"
"net/http"
- "strings"
)
-type blobParams struct {
- RepoPath string
- BlobId string
-}
+type blob struct{ senddata.Prefix }
+type blobParams struct{ RepoPath, BlobId string }
-const SendBlobPrefix = "git-blob:"
+var SendBlob = &blob{"git-blob:"}
-func SendBlob(w http.ResponseWriter, r *http.Request, sendData string) {
- params, err := unpackSendData(sendData)
- if err != nil {
+func (b *blob) Inject(w http.ResponseWriter, r *http.Request, sendData string) {
+ var params blobParams
+ if err := b.Unpack(¶ms, sendData); err != nil {
helper.Fail500(w, fmt.Errorf("SendBlob: unpack sendData: %v", err))
return
}
+
log.Printf("SendBlob: sending %q for %q", params.BlobId, r.URL.Path)
gitShowCmd := gitCommand("", "git", "--git-dir="+params.RepoPath, "cat-file", "blob", params.BlobId)
@@ -49,15 +46,3 @@ func SendBlob(w http.ResponseWriter, r *http.Request, sendData string) {
return
}
}
-
-func unpackSendData(sendData string) (*blobParams, error) {
- jsonBytes, err := base64.URLEncoding.DecodeString(strings.TrimPrefix(sendData, SendBlobPrefix))
- if err != nil {
- return nil, err
- }
- result := &blobParams{}
- if err := json.Unmarshal([]byte(jsonBytes), result); err != nil {
- return nil, err
- }
- return result, nil
-}
diff --git a/internal/lfs/lfs.go b/internal/lfs/lfs.go
index 6d78cb6cd4122ec81096cba30ed8da50a6b2678b..2718d0a0f524ddb978c9b6cd67940885df58ebd7 100644
--- a/internal/lfs/lfs.go
+++ b/internal/lfs/lfs.go
@@ -7,7 +7,6 @@ package lfs
import (
"../api"
"../helper"
- "../proxy"
"bytes"
"crypto/sha256"
"encoding/hex"
@@ -20,8 +19,8 @@ import (
"path/filepath"
)
-func PutStore(a *api.API, p *proxy.Proxy) http.Handler {
- return lfsAuthorizeHandler(a, handleStoreLfsObject(p))
+func PutStore(a *api.API, h http.Handler) http.Handler {
+ return lfsAuthorizeHandler(a, handleStoreLfsObject(h))
}
func lfsAuthorizeHandler(myAPI *api.API, handleFunc api.HandleFunc) http.Handler {
diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go
index 39422f9cacf260cca6142a61c6008b1010c30dbe..ec7ea66babf5e3ceab5e1b39da25ffdfc7a0aafa 100644
--- a/internal/proxy/proxy.go
+++ b/internal/proxy/proxy.go
@@ -3,7 +3,6 @@ package proxy
import (
"../badgateway"
"../helper"
- "../senddata"
"net/http"
"net/http/httputil"
"net/url"
@@ -34,8 +33,6 @@ func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// Set Workhorse version
req.Header.Set("Gitlab-Workhorse", p.Version)
- rw := senddata.NewSendFileResponseWriter(w, &req)
- defer rw.Flush()
- p.reverseProxy.ServeHTTP(&rw, &req)
+ p.reverseProxy.ServeHTTP(w, &req)
}
diff --git a/internal/senddata/injecter.go b/internal/senddata/injecter.go
new file mode 100644
index 0000000000000000000000000000000000000000..72d8039aa9523fc163e7ea52b390f3f370da63fd
--- /dev/null
+++ b/internal/senddata/injecter.go
@@ -0,0 +1,32 @@
+package senddata
+
+import (
+ "encoding/base64"
+ "encoding/json"
+ "net/http"
+ "strings"
+)
+
+type Injecter interface {
+ Match(string) bool
+ Inject(http.ResponseWriter, *http.Request, string)
+}
+
+type Prefix string
+
+const HeaderKey = "Gitlab-Workhorse-Send-Data"
+
+func (p Prefix) Match(s string) bool {
+ return strings.HasPrefix(s, string(p))
+}
+
+func (p Prefix) Unpack(result interface{}, sendData string) error {
+ jsonBytes, err := base64.URLEncoding.DecodeString(strings.TrimPrefix(sendData, string(p)))
+ if err != nil {
+ return err
+ }
+ if err := json.Unmarshal([]byte(jsonBytes), result); err != nil {
+ return err
+ }
+ return nil
+}
diff --git a/internal/senddata/senddata.go b/internal/senddata/senddata.go
new file mode 100644
index 0000000000000000000000000000000000000000..d86585821940212bd71f11722e740bc51ccca46f
--- /dev/null
+++ b/internal/senddata/senddata.go
@@ -0,0 +1,74 @@
+package senddata
+
+import (
+ "net/http"
+)
+
+type sendDataResponseWriter struct {
+ rw http.ResponseWriter
+ status int
+ hijacked bool
+ req *http.Request
+ injecters []Injecter
+}
+
+func SendData(h http.Handler, injecters ...Injecter) http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ s := sendDataResponseWriter{
+ rw: w,
+ req: r,
+ injecters: injecters,
+ }
+ defer s.Flush()
+ h.ServeHTTP(&s, r)
+ })
+}
+
+func (s *sendDataResponseWriter) Header() http.Header {
+ return s.rw.Header()
+}
+
+func (s *sendDataResponseWriter) Write(data []byte) (n int, err error) {
+ if s.status == 0 {
+ s.WriteHeader(http.StatusOK)
+ }
+ if s.hijacked {
+ return
+ }
+ return s.rw.Write(data)
+}
+
+func (s *sendDataResponseWriter) WriteHeader(status int) {
+ if s.status != 0 {
+ return
+ }
+ s.status = status
+
+ if s.status == http.StatusOK && s.tryInject() {
+ return
+ }
+
+ s.rw.WriteHeader(s.status)
+}
+
+func (s *sendDataResponseWriter) tryInject() bool {
+ header := s.Header().Get(HeaderKey)
+ s.Header().Del(HeaderKey)
+ if header == "" {
+ return false
+ }
+
+ for _, injecter := range s.injecters {
+ if injecter.Match(header) {
+ s.hijacked = true
+ injecter.Inject(s.rw, s.req, header)
+ return true
+ }
+ }
+
+ return false
+}
+
+func (s *sendDataResponseWriter) Flush() {
+ s.WriteHeader(http.StatusOK)
+}
diff --git a/internal/senddata/sendfile.go b/internal/sendfile/sendfile.go
similarity index 69%
rename from internal/senddata/sendfile.go
rename to internal/sendfile/sendfile.go
index 7d851d633164bb96ca58260a8effa51f19fcffd1..bfa54e0d5d87ef4b1586681a2af880559acc0c63 100644
--- a/internal/senddata/sendfile.go
+++ b/internal/sendfile/sendfile.go
@@ -4,20 +4,15 @@ via the X-Sendfile mechanism. All that is needed in the Rails code is the
'send_file' method.
*/
-package senddata
+package sendfile
import (
- "../git"
"../helper"
"log"
"net/http"
- "strings"
)
-const (
- sendDataResponseHeader = "Gitlab-Workhorse-Send-Data"
- sendFileResponseHeader = "X-Sendfile"
-)
+const sendFileResponseHeader = "X-Sendfile"
type sendFileResponseWriter struct {
rw http.ResponseWriter
@@ -26,14 +21,17 @@ type sendFileResponseWriter struct {
req *http.Request
}
-func NewSendFileResponseWriter(rw http.ResponseWriter, req *http.Request) sendFileResponseWriter {
- s := sendFileResponseWriter{
- rw: rw,
- req: req,
- }
- // Advertise to upstream (Rails) that we support X-Sendfile
- req.Header.Set("X-Sendfile-Type", "X-Sendfile")
- return s
+func SendFile(h http.Handler) http.Handler {
+ return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+ s := &sendFileResponseWriter{
+ rw: rw,
+ req: req,
+ }
+ // Advertise to upstream (Rails) that we support X-Sendfile
+ req.Header.Set("X-Sendfile-Type", "X-Sendfile")
+ defer s.Flush()
+ h.ServeHTTP(s, req)
+ })
}
func (s *sendFileResponseWriter) Header() http.Header {
@@ -70,12 +68,6 @@ func (s *sendFileResponseWriter) WriteHeader(status int) {
sendFileFromDisk(s.rw, s.req, file)
return
}
- if sendData := s.Header().Get(sendDataResponseHeader); strings.HasPrefix(sendData, git.SendBlobPrefix) {
- s.Header().Del(sendDataResponseHeader)
- s.hijacked = true
- git.SendBlob(s.rw, s.req, sendData)
- return
- }
s.rw.WriteHeader(s.status)
return
diff --git a/internal/upstream/routes.go b/internal/upstream/routes.go
index 0fe09d44483d138b9362b414248f2d560c3606f4..a1cd83c3b43d092f2cabbb8cd9de5ca27a1f3201 100644
--- a/internal/upstream/routes.go
+++ b/internal/upstream/routes.go
@@ -6,6 +6,8 @@ import (
"../git"
"../lfs"
proxypkg "../proxy"
+ "../senddata"
+ "../sendfile"
"../staticpages"
"net/http"
"regexp"
@@ -37,10 +39,14 @@ func (u *Upstream) configureRoutes() {
u.RoundTripper,
)
static := &staticpages.Static{u.DocumentRoot}
- proxy := proxypkg.NewProxy(
- u.Backend,
- u.Version,
- u.RoundTripper,
+ proxy := senddata.SendData(
+ sendfile.SendFile(proxypkg.NewProxy(
+ u.Backend,
+ u.Version,
+ u.RoundTripper,
+ )),
+ git.SendArchive,
+ git.SendBlob,
)
u.Routes = []route{
@@ -50,20 +56,6 @@ func (u *Upstream) configureRoutes() {
route{"POST", regexp.MustCompile(gitProjectPattern + `git-receive-pack\z`), contentEncodingHandler(git.PostRPC(api))},
route{"PUT", regexp.MustCompile(gitProjectPattern + `gitlab-lfs/objects/([0-9a-f]{64})/([0-9]+)\z`), lfs.PutStore(api, proxy)},
- // Repository Archive
- route{"GET", regexp.MustCompile(projectPattern + `repository/archive\z`), git.GetArchive(api)},
- route{"GET", regexp.MustCompile(projectPattern + `repository/archive.zip\z`), git.GetArchive(api)},
- route{"GET", regexp.MustCompile(projectPattern + `repository/archive.tar\z`), git.GetArchive(api)},
- route{"GET", regexp.MustCompile(projectPattern + `repository/archive.tar.gz\z`), git.GetArchive(api)},
- route{"GET", regexp.MustCompile(projectPattern + `repository/archive.tar.bz2\z`), git.GetArchive(api)},
-
- // Repository Archive API
- route{"GET", regexp.MustCompile(projectsAPIPattern + `repository/archive\z`), git.GetArchive(api)},
- route{"GET", regexp.MustCompile(projectsAPIPattern + `repository/archive.zip\z`), git.GetArchive(api)},
- route{"GET", regexp.MustCompile(projectsAPIPattern + `repository/archive.tar\z`), git.GetArchive(api)},
- route{"GET", regexp.MustCompile(projectsAPIPattern + `repository/archive.tar.gz\z`), git.GetArchive(api)},
- route{"GET", regexp.MustCompile(projectsAPIPattern + `repository/archive.tar.bz2\z`), git.GetArchive(api)},
-
// CI Artifacts
route{"GET", regexp.MustCompile(projectPattern + `builds/[0-9]+/artifacts/file/`), contentEncodingHandler(artifacts.DownloadArtifact(api))},
route{"POST", regexp.MustCompile(ciAPIPattern + `v1/builds/[0-9]+/artifacts\z`), contentEncodingHandler(artifacts.UploadArtifacts(api, proxy))},
diff --git a/main_test.go b/main_test.go
index 0e7c2992c98936d2f2d77d6fba2cd1faa18437e5..d4e1eb0018183f14796bfffb59874c70bcede0db 100644
--- a/main_test.go
+++ b/main_test.go
@@ -115,7 +115,7 @@ func TestAllowedDownloadZip(t *testing.T) {
// Prepare test server and backend
archiveName := "foobar.zip"
- ts := testAuthServer(nil, 200, archiveOkBody(t, archiveName))
+ ts := archiveOKServer(t, archiveName)
defer ts.Close()
ws := startWorkhorseServer(ts.URL)
defer ws.Close()
@@ -134,7 +134,7 @@ func TestAllowedDownloadTar(t *testing.T) {
// Prepare test server and backend
archiveName := "foobar.tar"
- ts := testAuthServer(nil, 200, archiveOkBody(t, archiveName))
+ ts := archiveOKServer(t, archiveName)
defer ts.Close()
ws := startWorkhorseServer(ts.URL)
defer ws.Close()
@@ -153,7 +153,7 @@ func TestAllowedDownloadTarGz(t *testing.T) {
// Prepare test server and backend
archiveName := "foobar.tar.gz"
- ts := testAuthServer(nil, 200, archiveOkBody(t, archiveName))
+ ts := archiveOKServer(t, archiveName)
defer ts.Close()
ws := startWorkhorseServer(ts.URL)
defer ws.Close()
@@ -172,7 +172,7 @@ func TestAllowedDownloadTarBz2(t *testing.T) {
// Prepare test server and backend
archiveName := "foobar.tar.bz2"
- ts := testAuthServer(nil, 200, archiveOkBody(t, archiveName))
+ ts := archiveOKServer(t, archiveName)
defer ts.Close()
ws := startWorkhorseServer(ts.URL)
defer ws.Close()
@@ -191,7 +191,7 @@ func TestAllowedApiDownloadZip(t *testing.T) {
// Prepare test server and backend
archiveName := "foobar.zip"
- ts := testAuthServer(nil, 200, archiveOkBody(t, archiveName))
+ ts := archiveOKServer(t, archiveName)
defer ts.Close()
ws := startWorkhorseServer(ts.URL)
defer ws.Close()
@@ -210,7 +210,7 @@ func TestAllowedApiDownloadZipWithSlash(t *testing.T) {
// Prepare test server and backend
archiveName := "foobar.zip"
- ts := testAuthServer(nil, 200, archiveOkBody(t, archiveName))
+ ts := archiveOKServer(t, archiveName)
defer ts.Close()
ws := startWorkhorseServer(ts.URL)
defer ws.Close()
@@ -233,7 +233,7 @@ func TestDownloadCacheHit(t *testing.T) {
// Prepare test server and backend
archiveName := "foobar.zip"
- ts := testAuthServer(nil, 200, archiveOkBody(t, archiveName))
+ ts := archiveOKServer(t, archiveName)
defer ts.Close()
ws := startWorkhorseServer(ts.URL)
defer ws.Close()
@@ -264,7 +264,7 @@ func TestDownloadCacheCreate(t *testing.T) {
// Prepare test server and backend
archiveName := "foobar.zip"
- ts := testAuthServer(nil, 200, archiveOkBody(t, archiveName))
+ ts := archiveOKServer(t, archiveName)
defer ts.Close()
ws := startWorkhorseServer(ts.URL)
defer ws.Close()
@@ -657,6 +657,29 @@ func testAuthServer(url *regexp.Regexp, code int, body interface{}) *httptest.Se
})
}
+func archiveOKServer(t *testing.T, archiveName string) *httptest.Server {
+ return testhelper.TestServerWithHandler(regexp.MustCompile("."), func(w http.ResponseWriter, r *http.Request) {
+ cwd, err := os.Getwd()
+ if err != nil {
+ t.Fatal(err)
+ }
+ archivePath := path.Join(cwd, cacheDir, archiveName)
+
+ params := struct{ RepoPath, ArchivePath, CommitId, ArchivePrefix string }{
+ repoPath(t),
+ archivePath,
+ "c7fbe50c7c7419d9701eebe64b1fdacc3df5b9dd",
+ "foobar123",
+ }
+ jsonData, err := json.Marshal(params)
+ if err != nil {
+ t.Fatal(err)
+ }
+ encodedJSON := base64.StdEncoding.EncodeToString(jsonData)
+ w.Header().Set("Gitlab-Workhorse-Send-Data", "git-archive:"+encodedJSON)
+ })
+}
+
func startWorkhorseServer(authBackend string) *httptest.Server {
u := upstream.NewUpstream(
helper.URLMustParse(authBackend),
@@ -684,21 +707,6 @@ func gitOkBody(t *testing.T) interface{} {
}
}
-func archiveOkBody(t *testing.T, archiveName string) interface{} {
- cwd, err := os.Getwd()
- if err != nil {
- t.Fatal(err)
- }
- archivePath := path.Join(cwd, cacheDir, archiveName)
-
- return &api.Response{
- RepoPath: repoPath(t),
- ArchivePath: archivePath,
- CommitId: "c7fbe50c7c7419d9701eebe64b1fdacc3df5b9dd",
- ArchivePrefix: "foobar123",
- }
-}
-
func repoPath(t *testing.T) string {
cwd, err := os.Getwd()
if err != nil {