From 3ec457b4fedd36ed443326638d4d895edb66065a Mon Sep 17 00:00:00 2001
From: Jacob Vosmaer <contact@jacobvosmaer.nl>
Date: Tue, 2 Feb 2016 15:55:47 +0100
Subject: [PATCH 01/10] First version of "git archive" via headers
---
internal/api/api.go | 9 ------
internal/git/archive.go | 31 ++++++++++++-------
internal/git/blob.go | 21 +++++--------
internal/senddata/sendfile.go | 20 +++++++++----
internal/upstream/routes.go | 14 ---------
main_test.go | 56 +++++++++++++++++++++--------------
6 files changed, 76 insertions(+), 75 deletions(-)
diff --git a/internal/api/api.go b/internal/api/api.go
index b23ea7e..02eb01c 100644
--- a/internal/api/api.go
+++ b/internal/api/api.go
@@ -38,15 +38,6 @@ type Response struct {
// RepoPath is the full path on disk to the Git repository the request is
// about
RepoPath string
- // ArchivePath is the full path where we should find/create a cached copy
- // of a requested archive
- ArchivePath string
- // ArchivePrefix is used to put extracted archive contents in a
- // subdirectory
- ArchivePrefix string
- // CommitId is used do prevent race conditions between the 'time of check'
- // in the GitLab Rails app and the 'time of use' in gitlab-workhorse.
- CommitId string
// StoreLFSPath is provided by the GitLab Rails application
// to mark where the tmp file should be placed
StoreLFSPath string
diff --git a/internal/git/archive.go b/internal/git/archive.go
index 854c887..c8b232b 100644
--- a/internal/git/archive.go
+++ b/internal/git/archive.go
@@ -5,7 +5,6 @@ In this file we handle 'git archive' downloads
package git
import (
- "../api"
"../helper"
"fmt"
"io"
@@ -20,11 +19,20 @@ import (
"time"
)
-func GetArchive(a *api.API) http.Handler {
- return repoPreAuthorizeHandler(a, handleGetArchive)
-}
+const SendArchivePrefix = "git-archive:"
+
+func SendArchive(w http.ResponseWriter, r *http.Request, sendData string) {
+ var params struct {
+ RepoPath string
+ ArchivePath string
+ ArchivePrefix string
+ CommitId string
+ }
+ if err := unpackSendData(¶ms, sendData, SendArchivePrefix); err != nil {
+ helper.Fail500(w, fmt.Errorf("SendArchive: unpack sendData: %v", err))
+ return
+ }
-func handleGetArchive(w http.ResponseWriter, r *http.Request, a *api.Response) {
var format string
urlPath := r.URL.Path
switch filepath.Base(urlPath) {
@@ -41,11 +49,11 @@ func handleGetArchive(w http.ResponseWriter, r *http.Request, a *api.Response) {
return
}
- archiveFilename := path.Base(a.ArchivePath)
+ archiveFilename := path.Base(params.ArchivePath)
- if cachedArchive, err := os.Open(a.ArchivePath); err == nil {
+ if cachedArchive, err := os.Open(params.ArchivePath); err == nil {
defer cachedArchive.Close()
- log.Printf("Serving cached file %q", a.ArchivePath)
+ log.Printf("Serving cached file %q", params.ArchivePath)
setArchiveHeaders(w, format, archiveFilename)
// Even if somebody deleted the cachedArchive from disk since we opened
// the file, Unix file semantics guarantee we can still read from the
@@ -58,7 +66,7 @@ func handleGetArchive(w http.ResponseWriter, r *http.Request, a *api.Response) {
// safe. We create the tempfile in the same directory as the final cached
// archive we want to create so that we can use an atomic link(2) operation
// to finalize the cached archive.
- tempFile, err := prepareArchiveTempfile(path.Dir(a.ArchivePath), archiveFilename)
+ tempFile, err := prepareArchiveTempfile(path.Dir(params.ArchivePath), archiveFilename)
if err != nil {
helper.Fail500(w, fmt.Errorf("handleGetArchive: create tempfile: %v", err))
return
@@ -68,7 +76,7 @@ func handleGetArchive(w http.ResponseWriter, r *http.Request, a *api.Response) {
compressCmd, archiveFormat := parseArchiveFormat(format)
- archiveCmd := gitCommand("", "git", "--git-dir="+a.RepoPath, "archive", "--format="+archiveFormat, "--prefix="+a.ArchivePrefix+"/", a.CommitId)
+ archiveCmd := gitCommand("", "git", "--git-dir="+params.RepoPath, "archive", "--format="+archiveFormat, "--prefix="+params.ArchivePrefix+"/", params.CommitId)
archiveStdout, err := archiveCmd.StdoutPipe()
if err != nil {
helper.Fail500(w, fmt.Errorf("handleGetArchive: archive stdout: %v", err))
@@ -125,13 +133,14 @@ func handleGetArchive(w http.ResponseWriter, r *http.Request, a *api.Response) {
}
}
- if err := finalizeCachedArchive(tempFile, a.ArchivePath); err != nil {
+ if err := finalizeCachedArchive(tempFile, params.ArchivePath); err != nil {
helper.LogError(fmt.Errorf("handleGetArchive: finalize cached archive: %v", err))
return
}
}
func setArchiveHeaders(w http.ResponseWriter, format string, archiveFilename string) {
+ w.Header().Del("Content-Length")
w.Header().Add("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, archiveFilename))
if format == "zip" {
w.Header().Add("Content-Type", "application/zip")
diff --git a/internal/git/blob.go b/internal/git/blob.go
index aa1b7d6..b6e1b11 100644
--- a/internal/git/blob.go
+++ b/internal/git/blob.go
@@ -11,19 +11,15 @@ import (
"strings"
)
-type blobParams struct {
- RepoPath string
- BlobId string
-}
-
const SendBlobPrefix = "git-blob:"
func SendBlob(w http.ResponseWriter, r *http.Request, sendData string) {
- params, err := unpackSendData(sendData)
- if err != nil {
+ var params struct{ RepoPath, BlobId string }
+ if err := unpackSendData(¶ms, sendData, SendBlobPrefix); err != nil {
helper.Fail500(w, fmt.Errorf("SendBlob: unpack sendData: %v", err))
return
}
+
log.Printf("SendBlob: sending %q for %q", params.BlobId, r.URL.Path)
gitShowCmd := gitCommand("", "git", "--git-dir="+params.RepoPath, "cat-file", "blob", params.BlobId)
@@ -48,14 +44,13 @@ func SendBlob(w http.ResponseWriter, r *http.Request, sendData string) {
}
}
-func unpackSendData(sendData string) (*blobParams, error) {
- jsonBytes, err := base64.URLEncoding.DecodeString(strings.TrimPrefix(sendData, SendBlobPrefix))
+func unpackSendData(result interface{}, sendData string, prefix string) error {
+ jsonBytes, err := base64.URLEncoding.DecodeString(strings.TrimPrefix(sendData, prefix))
if err != nil {
- return nil, err
+ return err
}
- result := &blobParams{}
if err := json.Unmarshal([]byte(jsonBytes), result); err != nil {
- return nil, err
+ return err
}
- return result, nil
+ return nil
}
diff --git a/internal/senddata/sendfile.go b/internal/senddata/sendfile.go
index 7d851d6..8e087fd 100644
--- a/internal/senddata/sendfile.go
+++ b/internal/senddata/sendfile.go
@@ -70,11 +70,21 @@ func (s *sendFileResponseWriter) WriteHeader(status int) {
sendFileFromDisk(s.rw, s.req, file)
return
}
- if sendData := s.Header().Get(sendDataResponseHeader); strings.HasPrefix(sendData, git.SendBlobPrefix) {
- s.Header().Del(sendDataResponseHeader)
- s.hijacked = true
- git.SendBlob(s.rw, s.req, sendData)
- return
+
+ sendData := s.Header().Get(sendDataResponseHeader)
+ s.Header().Del(sendDataResponseHeader)
+ for _, handler := range []struct {
+ prefix string
+ f func(http.ResponseWriter, *http.Request, string)
+ }{
+ {git.SendBlobPrefix, git.SendBlob},
+ {git.SendArchivePrefix, git.SendArchive},
+ } {
+ if strings.HasPrefix(sendData, handler.prefix) {
+ s.hijacked = true
+ handler.f(s.rw, s.req, sendData)
+ return
+ }
}
s.rw.WriteHeader(s.status)
diff --git a/internal/upstream/routes.go b/internal/upstream/routes.go
index 0fe09d4..4462a5c 100644
--- a/internal/upstream/routes.go
+++ b/internal/upstream/routes.go
@@ -50,20 +50,6 @@ func (u *Upstream) configureRoutes() {
route{"POST", regexp.MustCompile(gitProjectPattern + `git-receive-pack\z`), contentEncodingHandler(git.PostRPC(api))},
route{"PUT", regexp.MustCompile(gitProjectPattern + `gitlab-lfs/objects/([0-9a-f]{64})/([0-9]+)\z`), lfs.PutStore(api, proxy)},
- // Repository Archive
- route{"GET", regexp.MustCompile(projectPattern + `repository/archive\z`), git.GetArchive(api)},
- route{"GET", regexp.MustCompile(projectPattern + `repository/archive.zip\z`), git.GetArchive(api)},
- route{"GET", regexp.MustCompile(projectPattern + `repository/archive.tar\z`), git.GetArchive(api)},
- route{"GET", regexp.MustCompile(projectPattern + `repository/archive.tar.gz\z`), git.GetArchive(api)},
- route{"GET", regexp.MustCompile(projectPattern + `repository/archive.tar.bz2\z`), git.GetArchive(api)},
-
- // Repository Archive API
- route{"GET", regexp.MustCompile(projectsAPIPattern + `repository/archive\z`), git.GetArchive(api)},
- route{"GET", regexp.MustCompile(projectsAPIPattern + `repository/archive.zip\z`), git.GetArchive(api)},
- route{"GET", regexp.MustCompile(projectsAPIPattern + `repository/archive.tar\z`), git.GetArchive(api)},
- route{"GET", regexp.MustCompile(projectsAPIPattern + `repository/archive.tar.gz\z`), git.GetArchive(api)},
- route{"GET", regexp.MustCompile(projectsAPIPattern + `repository/archive.tar.bz2\z`), git.GetArchive(api)},
-
// CI Artifacts
route{"GET", regexp.MustCompile(projectPattern + `builds/[0-9]+/artifacts/file/`), contentEncodingHandler(artifacts.DownloadArtifact(api))},
route{"POST", regexp.MustCompile(ciAPIPattern + `v1/builds/[0-9]+/artifacts\z`), contentEncodingHandler(artifacts.UploadArtifacts(api, proxy))},
diff --git a/main_test.go b/main_test.go
index a4fed1c..19ec0b3 100644
--- a/main_test.go
+++ b/main_test.go
@@ -115,7 +115,7 @@ func TestAllowedDownloadZip(t *testing.T) {
// Prepare test server and backend
archiveName := "foobar.zip"
- ts := testAuthServer(nil, 200, archiveOkBody(t, archiveName))
+ ts := archiveOKServer(t, archiveName)
defer ts.Close()
ws := startWorkhorseServer(ts.URL)
defer ws.Close()
@@ -134,7 +134,7 @@ func TestAllowedDownloadTar(t *testing.T) {
// Prepare test server and backend
archiveName := "foobar.tar"
- ts := testAuthServer(nil, 200, archiveOkBody(t, archiveName))
+ ts := archiveOKServer(t, archiveName)
defer ts.Close()
ws := startWorkhorseServer(ts.URL)
defer ws.Close()
@@ -153,7 +153,7 @@ func TestAllowedDownloadTarGz(t *testing.T) {
// Prepare test server and backend
archiveName := "foobar.tar.gz"
- ts := testAuthServer(nil, 200, archiveOkBody(t, archiveName))
+ ts := archiveOKServer(t, archiveName)
defer ts.Close()
ws := startWorkhorseServer(ts.URL)
defer ws.Close()
@@ -172,7 +172,7 @@ func TestAllowedDownloadTarBz2(t *testing.T) {
// Prepare test server and backend
archiveName := "foobar.tar.bz2"
- ts := testAuthServer(nil, 200, archiveOkBody(t, archiveName))
+ ts := archiveOKServer(t, archiveName)
defer ts.Close()
ws := startWorkhorseServer(ts.URL)
defer ws.Close()
@@ -191,7 +191,7 @@ func TestAllowedApiDownloadZip(t *testing.T) {
// Prepare test server and backend
archiveName := "foobar.zip"
- ts := testAuthServer(nil, 200, archiveOkBody(t, archiveName))
+ ts := archiveOKServer(t, archiveName)
defer ts.Close()
ws := startWorkhorseServer(ts.URL)
defer ws.Close()
@@ -210,7 +210,7 @@ func TestAllowedApiDownloadZipWithSlash(t *testing.T) {
// Prepare test server and backend
archiveName := "foobar.zip"
- ts := testAuthServer(nil, 200, archiveOkBody(t, archiveName))
+ ts := archiveOKServer(t, archiveName)
defer ts.Close()
ws := startWorkhorseServer(ts.URL)
defer ws.Close()
@@ -233,7 +233,7 @@ func TestDownloadCacheHit(t *testing.T) {
// Prepare test server and backend
archiveName := "foobar.zip"
- ts := testAuthServer(nil, 200, archiveOkBody(t, archiveName))
+ ts := archiveOKServer(t, archiveName)
defer ts.Close()
ws := startWorkhorseServer(ts.URL)
defer ws.Close()
@@ -264,7 +264,7 @@ func TestDownloadCacheCreate(t *testing.T) {
// Prepare test server and backend
archiveName := "foobar.zip"
- ts := testAuthServer(nil, 200, archiveOkBody(t, archiveName))
+ ts := archiveOKServer(t, archiveName)
defer ts.Close()
ws := startWorkhorseServer(ts.URL)
defer ws.Close()
@@ -659,6 +659,31 @@ func testAuthServer(url *regexp.Regexp, code int, body interface{}) *httptest.Se
})
}
+func archiveOKServer(t *testing.T, archiveName string) *httptest.Server {
+ return testhelper.TestServerWithHandler(regexp.MustCompile("."), func(w http.ResponseWriter, r *http.Request) {
+ cwd, err := os.Getwd()
+ if err != nil {
+ t.Fatal(err)
+ }
+ archivePath := path.Join(cwd, cacheDir, archiveName)
+
+ params := struct{ RepoPath, ArchivePath, CommitId, ArchivePrefix string }{
+ repoPath(t),
+ archivePath,
+ "c7fbe50c7c7419d9701eebe64b1fdacc3df5b9dd",
+ "foobar123",
+ }
+ jsonData, err := json.Marshal(params)
+ if err != nil {
+ t.Fatal(err)
+ }
+ encodedJSON := base64.StdEncoding.EncodeToString(jsonData)
+ w.Header().Set("Gitlab-Workhorse-Send-Data", "git-archive:"+encodedJSON)
+ // Prevent the Go HTTP server from setting the Content-Length to 0.
+ w.Header().Set("Transfer-Encoding", "chunked")
+ })
+}
+
func startWorkhorseServer(authBackend string) *httptest.Server {
u := upstream.NewUpstream(
helper.URLMustParse(authBackend),
@@ -686,21 +711,6 @@ func gitOkBody(t *testing.T) interface{} {
}
}
-func archiveOkBody(t *testing.T, archiveName string) interface{} {
- cwd, err := os.Getwd()
- if err != nil {
- t.Fatal(err)
- }
- archivePath := path.Join(cwd, cacheDir, archiveName)
-
- return &api.Response{
- RepoPath: repoPath(t),
- ArchivePath: archivePath,
- CommitId: "c7fbe50c7c7419d9701eebe64b1fdacc3df5b9dd",
- ArchivePrefix: "foobar123",
- }
-}
-
func repoPath(t *testing.T) string {
cwd, err := os.Getwd()
if err != nil {
--
GitLab
From ec168f4b14b638272f08334adf2ee94670daa0cf Mon Sep 17 00:00:00 2001
From: Jacob Vosmaer <contact@jacobvosmaer.nl>
Date: Tue, 2 Feb 2016 17:01:23 +0100
Subject: [PATCH 02/10] Use interfaces
---
internal/git/archive.go | 9 ++++---
internal/git/blob.go | 23 +++++-----------
internal/{senddata => inject}/sendfile.go | 26 +++++++-----------
internal/proxy/proxy.go | 4 +--
internal/senddata/handler.go | 32 +++++++++++++++++++++++
5 files changed, 56 insertions(+), 38 deletions(-)
rename internal/{senddata => inject}/sendfile.go (77%)
create mode 100644 internal/senddata/handler.go
diff --git a/internal/git/archive.go b/internal/git/archive.go
index c8b232b..14c7540 100644
--- a/internal/git/archive.go
+++ b/internal/git/archive.go
@@ -6,6 +6,7 @@ package git
import (
"../helper"
+ "../senddata"
"fmt"
"io"
"io/ioutil"
@@ -19,16 +20,18 @@ import (
"time"
)
-const SendArchivePrefix = "git-archive:"
+type archive struct{ senddata.Prefix }
-func SendArchive(w http.ResponseWriter, r *http.Request, sendData string) {
+var SendArchive = &archive{"git-archive:"}
+
+func (a *archive) Handle(w http.ResponseWriter, r *http.Request, sendData string) {
var params struct {
RepoPath string
ArchivePath string
ArchivePrefix string
CommitId string
}
- if err := unpackSendData(¶ms, sendData, SendArchivePrefix); err != nil {
+ if err := a.Unpack(¶ms, sendData); err != nil {
helper.Fail500(w, fmt.Errorf("SendArchive: unpack sendData: %v", err))
return
}
diff --git a/internal/git/blob.go b/internal/git/blob.go
index b6e1b11..1d63a82 100644
--- a/internal/git/blob.go
+++ b/internal/git/blob.go
@@ -2,20 +2,20 @@ package git
import (
"../helper"
- "encoding/base64"
- "encoding/json"
+ "../senddata"
"fmt"
"io"
"log"
"net/http"
- "strings"
)
-const SendBlobPrefix = "git-blob:"
+type blob struct{ senddata.Prefix }
-func SendBlob(w http.ResponseWriter, r *http.Request, sendData string) {
+var SendBlob = &blob{"git-blob:"}
+
+func (b *blob) Handle(w http.ResponseWriter, r *http.Request, sendData string) {
var params struct{ RepoPath, BlobId string }
- if err := unpackSendData(¶ms, sendData, SendBlobPrefix); err != nil {
+ if err := b.Unpack(¶ms, sendData); err != nil {
helper.Fail500(w, fmt.Errorf("SendBlob: unpack sendData: %v", err))
return
}
@@ -43,14 +43,3 @@ func SendBlob(w http.ResponseWriter, r *http.Request, sendData string) {
return
}
}
-
-func unpackSendData(result interface{}, sendData string, prefix string) error {
- jsonBytes, err := base64.URLEncoding.DecodeString(strings.TrimPrefix(sendData, prefix))
- if err != nil {
- return err
- }
- if err := json.Unmarshal([]byte(jsonBytes), result); err != nil {
- return err
- }
- return nil
-}
diff --git a/internal/senddata/sendfile.go b/internal/inject/sendfile.go
similarity index 77%
rename from internal/senddata/sendfile.go
rename to internal/inject/sendfile.go
index 8e087fd..df9cb0c 100644
--- a/internal/senddata/sendfile.go
+++ b/internal/inject/sendfile.go
@@ -4,20 +4,17 @@ via the X-Sendfile mechanism. All that is needed in the Rails code is the
'send_file' method.
*/
-package senddata
+package inject
import (
"../git"
"../helper"
+ "../senddata"
"log"
"net/http"
- "strings"
)
-const (
- sendDataResponseHeader = "Gitlab-Workhorse-Send-Data"
- sendFileResponseHeader = "X-Sendfile"
-)
+const sendFileResponseHeader = "X-Sendfile"
type sendFileResponseWriter struct {
rw http.ResponseWriter
@@ -71,18 +68,15 @@ func (s *sendFileResponseWriter) WriteHeader(status int) {
return
}
- sendData := s.Header().Get(sendDataResponseHeader)
- s.Header().Del(sendDataResponseHeader)
- for _, handler := range []struct {
- prefix string
- f func(http.ResponseWriter, *http.Request, string)
- }{
- {git.SendBlobPrefix, git.SendBlob},
- {git.SendArchivePrefix, git.SendArchive},
+ header := s.Header().Get(senddata.Header)
+ s.Header().Del(senddata.Header)
+ for _, handler := range []senddata.Handler{
+ git.SendBlob,
+ git.SendArchive,
} {
- if strings.HasPrefix(sendData, handler.prefix) {
+ if handler.Match(header) {
s.hijacked = true
- handler.f(s.rw, s.req, sendData)
+ handler.Handle(s.rw, s.req, header)
return
}
}
diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go
index 39422f9..e0511f5 100644
--- a/internal/proxy/proxy.go
+++ b/internal/proxy/proxy.go
@@ -3,7 +3,7 @@ package proxy
import (
"../badgateway"
"../helper"
- "../senddata"
+ "../inject"
"net/http"
"net/http/httputil"
"net/url"
@@ -34,7 +34,7 @@ func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// Set Workhorse version
req.Header.Set("Gitlab-Workhorse", p.Version)
- rw := senddata.NewSendFileResponseWriter(w, &req)
+ rw := inject.NewSendFileResponseWriter(w, &req)
defer rw.Flush()
p.reverseProxy.ServeHTTP(&rw, &req)
diff --git a/internal/senddata/handler.go b/internal/senddata/handler.go
new file mode 100644
index 0000000..d3c5080
--- /dev/null
+++ b/internal/senddata/handler.go
@@ -0,0 +1,32 @@
+package senddata
+
+import (
+ "encoding/base64"
+ "encoding/json"
+ "net/http"
+ "strings"
+)
+
+type Handler interface {
+ Match(string) bool
+ Handle(http.ResponseWriter, *http.Request, string)
+}
+
+type Prefix string
+
+const Header = "Gitlab-Workhorse-Send-Data"
+
+func (p Prefix) Match(s string) bool {
+ return strings.HasPrefix(s, string(p))
+}
+
+func (p Prefix) Unpack(result interface{}, sendData string) error {
+ jsonBytes, err := base64.URLEncoding.DecodeString(strings.TrimPrefix(sendData, string(p)))
+ if err != nil {
+ return err
+ }
+ if err := json.Unmarshal([]byte(jsonBytes), result); err != nil {
+ return err
+ }
+ return nil
+}
--
GitLab
From 2788c575786ac0dbfe44e0f154518d31919bcb5e Mon Sep 17 00:00:00 2001
From: Jacob Vosmaer <contact@jacobvosmaer.nl>
Date: Tue, 2 Feb 2016 17:37:10 +0100
Subject: [PATCH 03/10] Skip empty send-data header quicker
---
internal/inject/sendfile.go | 21 +++++++++++----------
1 file changed, 11 insertions(+), 10 deletions(-)
diff --git a/internal/inject/sendfile.go b/internal/inject/sendfile.go
index df9cb0c..65af336 100644
--- a/internal/inject/sendfile.go
+++ b/internal/inject/sendfile.go
@@ -68,16 +68,17 @@ func (s *sendFileResponseWriter) WriteHeader(status int) {
return
}
- header := s.Header().Get(senddata.Header)
- s.Header().Del(senddata.Header)
- for _, handler := range []senddata.Handler{
- git.SendBlob,
- git.SendArchive,
- } {
- if handler.Match(header) {
- s.hijacked = true
- handler.Handle(s.rw, s.req, header)
- return
+ if header := s.Header().Get(senddata.Header); header != "" {
+ s.Header().Del(senddata.Header)
+ for _, handler := range []senddata.Handler{
+ git.SendBlob,
+ git.SendArchive,
+ } {
+ if handler.Match(header) {
+ s.hijacked = true
+ handler.Handle(s.rw, s.req, header)
+ return
+ }
}
}
--
GitLab
From f819a3c5d71b2fef872c2418e2bc70945c7d3ebc Mon Sep 17 00:00:00 2001
From: Jacob Vosmaer <contact@jacobvosmaer.nl>
Date: Thu, 11 Feb 2016 16:56:06 +0100
Subject: [PATCH 04/10] Rename Handler to Injecter
---
internal/git/archive.go | 2 +-
internal/git/blob.go | 2 +-
internal/inject/sendfile.go | 4 ++--
internal/senddata/{handler.go => injecter.go} | 4 ++--
4 files changed, 6 insertions(+), 6 deletions(-)
rename internal/senddata/{handler.go => injecter.go} (87%)
diff --git a/internal/git/archive.go b/internal/git/archive.go
index 14c7540..e94f5cd 100644
--- a/internal/git/archive.go
+++ b/internal/git/archive.go
@@ -24,7 +24,7 @@ type archive struct{ senddata.Prefix }
var SendArchive = &archive{"git-archive:"}
-func (a *archive) Handle(w http.ResponseWriter, r *http.Request, sendData string) {
+func (a *archive) Inject(w http.ResponseWriter, r *http.Request, sendData string) {
var params struct {
RepoPath string
ArchivePath string
diff --git a/internal/git/blob.go b/internal/git/blob.go
index 1d63a82..fbaac2d 100644
--- a/internal/git/blob.go
+++ b/internal/git/blob.go
@@ -13,7 +13,7 @@ type blob struct{ senddata.Prefix }
var SendBlob = &blob{"git-blob:"}
-func (b *blob) Handle(w http.ResponseWriter, r *http.Request, sendData string) {
+func (b *blob) Inject(w http.ResponseWriter, r *http.Request, sendData string) {
var params struct{ RepoPath, BlobId string }
if err := b.Unpack(¶ms, sendData); err != nil {
helper.Fail500(w, fmt.Errorf("SendBlob: unpack sendData: %v", err))
diff --git a/internal/inject/sendfile.go b/internal/inject/sendfile.go
index 65af336..374b3f1 100644
--- a/internal/inject/sendfile.go
+++ b/internal/inject/sendfile.go
@@ -70,13 +70,13 @@ func (s *sendFileResponseWriter) WriteHeader(status int) {
if header := s.Header().Get(senddata.Header); header != "" {
s.Header().Del(senddata.Header)
- for _, handler := range []senddata.Handler{
+ for _, handler := range []senddata.Injecter{
git.SendBlob,
git.SendArchive,
} {
if handler.Match(header) {
s.hijacked = true
- handler.Handle(s.rw, s.req, header)
+ handler.Inject(s.rw, s.req, header)
return
}
}
diff --git a/internal/senddata/handler.go b/internal/senddata/injecter.go
similarity index 87%
rename from internal/senddata/handler.go
rename to internal/senddata/injecter.go
index d3c5080..0b6e635 100644
--- a/internal/senddata/handler.go
+++ b/internal/senddata/injecter.go
@@ -7,9 +7,9 @@ import (
"strings"
)
-type Handler interface {
+type Injecter interface {
Match(string) bool
- Handle(http.ResponseWriter, *http.Request, string)
+ Inject(http.ResponseWriter, *http.Request, string)
}
type Prefix string
--
GitLab
From 46680f427f1020156fed5291672526c492b06011 Mon Sep 17 00:00:00 2001
From: Jacob Vosmaer <contact@jacobvosmaer.nl>
Date: Thu, 11 Feb 2016 17:32:28 +0100
Subject: [PATCH 05/10] Configure sendfile and injecters in routes.go
---
internal/lfs/lfs.go | 5 +-
internal/proxy/proxy.go | 5 +-
internal/senddata/injecter.go | 2 +-
internal/senddata/senddata.go | 69 +++++++++++++++++++++++
internal/{inject => sendfile}/sendfile.go | 37 ++++--------
internal/upstream/routes.go | 14 +++--
6 files changed, 95 insertions(+), 37 deletions(-)
create mode 100644 internal/senddata/senddata.go
rename internal/{inject => sendfile}/sendfile.go (71%)
diff --git a/internal/lfs/lfs.go b/internal/lfs/lfs.go
index 6d78cb6..2718d0a 100644
--- a/internal/lfs/lfs.go
+++ b/internal/lfs/lfs.go
@@ -7,7 +7,6 @@ package lfs
import (
"../api"
"../helper"
- "../proxy"
"bytes"
"crypto/sha256"
"encoding/hex"
@@ -20,8 +19,8 @@ import (
"path/filepath"
)
-func PutStore(a *api.API, p *proxy.Proxy) http.Handler {
- return lfsAuthorizeHandler(a, handleStoreLfsObject(p))
+func PutStore(a *api.API, h http.Handler) http.Handler {
+ return lfsAuthorizeHandler(a, handleStoreLfsObject(h))
}
func lfsAuthorizeHandler(myAPI *api.API, handleFunc api.HandleFunc) http.Handler {
diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go
index e0511f5..ec7ea66 100644
--- a/internal/proxy/proxy.go
+++ b/internal/proxy/proxy.go
@@ -3,7 +3,6 @@ package proxy
import (
"../badgateway"
"../helper"
- "../inject"
"net/http"
"net/http/httputil"
"net/url"
@@ -34,8 +33,6 @@ func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// Set Workhorse version
req.Header.Set("Gitlab-Workhorse", p.Version)
- rw := inject.NewSendFileResponseWriter(w, &req)
- defer rw.Flush()
- p.reverseProxy.ServeHTTP(&rw, &req)
+ p.reverseProxy.ServeHTTP(w, &req)
}
diff --git a/internal/senddata/injecter.go b/internal/senddata/injecter.go
index 0b6e635..72d8039 100644
--- a/internal/senddata/injecter.go
+++ b/internal/senddata/injecter.go
@@ -14,7 +14,7 @@ type Injecter interface {
type Prefix string
-const Header = "Gitlab-Workhorse-Send-Data"
+const HeaderKey = "Gitlab-Workhorse-Send-Data"
func (p Prefix) Match(s string) bool {
return strings.HasPrefix(s, string(p))
diff --git a/internal/senddata/senddata.go b/internal/senddata/senddata.go
new file mode 100644
index 0000000..42f4091
--- /dev/null
+++ b/internal/senddata/senddata.go
@@ -0,0 +1,69 @@
+package senddata
+
+import (
+ "net/http"
+)
+
+type sendDataResponseWriter struct {
+ rw http.ResponseWriter
+ status int
+ hijacked bool
+ req *http.Request
+ injecters []Injecter
+}
+
+func SendData(h http.Handler, injecters ...Injecter) http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ s := sendDataResponseWriter{
+ rw: w,
+ req: r,
+ injecters: injecters,
+ }
+ defer s.Flush()
+ h.ServeHTTP(&s, r)
+ })
+}
+
+func (s *sendDataResponseWriter) Header() http.Header {
+ return s.rw.Header()
+}
+
+func (s *sendDataResponseWriter) Write(data []byte) (n int, err error) {
+ if s.status == 0 {
+ s.WriteHeader(http.StatusOK)
+ }
+ if s.hijacked {
+ return
+ }
+ return s.rw.Write(data)
+}
+
+func (s *sendDataResponseWriter) WriteHeader(status int) {
+ if s.status != 0 {
+ return
+ }
+
+ s.status = status
+ if s.status != http.StatusOK {
+ s.rw.WriteHeader(s.status)
+ return
+ }
+
+ if header := s.Header().Get(HeaderKey); header != "" {
+ s.Header().Del(HeaderKey)
+ for _, injecter := range s.injecters {
+ if injecter.Match(header) {
+ s.hijacked = true
+ injecter.Inject(s.rw, s.req, header)
+ return
+ }
+ }
+ }
+
+ s.rw.WriteHeader(s.status)
+ return
+}
+
+func (s *sendDataResponseWriter) Flush() {
+ s.WriteHeader(http.StatusOK)
+}
diff --git a/internal/inject/sendfile.go b/internal/sendfile/sendfile.go
similarity index 71%
rename from internal/inject/sendfile.go
rename to internal/sendfile/sendfile.go
index 374b3f1..bfa54e0 100644
--- a/internal/inject/sendfile.go
+++ b/internal/sendfile/sendfile.go
@@ -4,12 +4,10 @@ via the X-Sendfile mechanism. All that is needed in the Rails code is the
'send_file' method.
*/
-package inject
+package sendfile
import (
- "../git"
"../helper"
- "../senddata"
"log"
"net/http"
)
@@ -23,14 +21,17 @@ type sendFileResponseWriter struct {
req *http.Request
}
-func NewSendFileResponseWriter(rw http.ResponseWriter, req *http.Request) sendFileResponseWriter {
- s := sendFileResponseWriter{
- rw: rw,
- req: req,
- }
- // Advertise to upstream (Rails) that we support X-Sendfile
- req.Header.Set("X-Sendfile-Type", "X-Sendfile")
- return s
+func SendFile(h http.Handler) http.Handler {
+ return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+ s := &sendFileResponseWriter{
+ rw: rw,
+ req: req,
+ }
+ // Advertise to upstream (Rails) that we support X-Sendfile
+ req.Header.Set("X-Sendfile-Type", "X-Sendfile")
+ defer s.Flush()
+ h.ServeHTTP(s, req)
+ })
}
func (s *sendFileResponseWriter) Header() http.Header {
@@ -68,20 +69,6 @@ func (s *sendFileResponseWriter) WriteHeader(status int) {
return
}
- if header := s.Header().Get(senddata.Header); header != "" {
- s.Header().Del(senddata.Header)
- for _, handler := range []senddata.Injecter{
- git.SendBlob,
- git.SendArchive,
- } {
- if handler.Match(header) {
- s.hijacked = true
- handler.Inject(s.rw, s.req, header)
- return
- }
- }
- }
-
s.rw.WriteHeader(s.status)
return
}
diff --git a/internal/upstream/routes.go b/internal/upstream/routes.go
index 4462a5c..a1cd83c 100644
--- a/internal/upstream/routes.go
+++ b/internal/upstream/routes.go
@@ -6,6 +6,8 @@ import (
"../git"
"../lfs"
proxypkg "../proxy"
+ "../senddata"
+ "../sendfile"
"../staticpages"
"net/http"
"regexp"
@@ -37,10 +39,14 @@ func (u *Upstream) configureRoutes() {
u.RoundTripper,
)
static := &staticpages.Static{u.DocumentRoot}
- proxy := proxypkg.NewProxy(
- u.Backend,
- u.Version,
- u.RoundTripper,
+ proxy := senddata.SendData(
+ sendfile.SendFile(proxypkg.NewProxy(
+ u.Backend,
+ u.Version,
+ u.RoundTripper,
+ )),
+ git.SendArchive,
+ git.SendBlob,
)
u.Routes = []route{
--
GitLab
From 28c9928c81d4b93ecf619dc0f4aa0ef596fad528 Mon Sep 17 00:00:00 2001
From: Jacob Vosmaer <contact@jacobvosmaer.nl>
Date: Thu, 11 Feb 2016 17:36:45 +0100
Subject: [PATCH 06/10] Declare params structs outside Inject functions
---
internal/git/archive.go | 13 +++++++------
internal/git/blob.go | 3 ++-
2 files changed, 9 insertions(+), 7 deletions(-)
diff --git a/internal/git/archive.go b/internal/git/archive.go
index e94f5cd..95016e9 100644
--- a/internal/git/archive.go
+++ b/internal/git/archive.go
@@ -21,16 +21,17 @@ import (
)
type archive struct{ senddata.Prefix }
+type archiveParams struct {
+ RepoPath string
+ ArchivePath string
+ ArchivePrefix string
+ CommitId string
+}
var SendArchive = &archive{"git-archive:"}
func (a *archive) Inject(w http.ResponseWriter, r *http.Request, sendData string) {
- var params struct {
- RepoPath string
- ArchivePath string
- ArchivePrefix string
- CommitId string
- }
+ var params archiveParams
if err := a.Unpack(¶ms, sendData); err != nil {
helper.Fail500(w, fmt.Errorf("SendArchive: unpack sendData: %v", err))
return
diff --git a/internal/git/blob.go b/internal/git/blob.go
index fbaac2d..b9247f7 100644
--- a/internal/git/blob.go
+++ b/internal/git/blob.go
@@ -10,11 +10,12 @@ import (
)
type blob struct{ senddata.Prefix }
+type blobParams struct{ RepoPath, BlobId string }
var SendBlob = &blob{"git-blob:"}
func (b *blob) Inject(w http.ResponseWriter, r *http.Request, sendData string) {
- var params struct{ RepoPath, BlobId string }
+ var params blobParams
if err := b.Unpack(¶ms, sendData); err != nil {
helper.Fail500(w, fmt.Errorf("SendBlob: unpack sendData: %v", err))
return
--
GitLab
From 92e88955bf60b11833978feafeb62140b3573c19 Mon Sep 17 00:00:00 2001
From: Jacob Vosmaer <contact@jacobvosmaer.nl>
Date: Wed, 17 Feb 2016 12:37:04 +0100
Subject: [PATCH 07/10] Simplify WriteHeader in senddata
---
internal/senddata/senddata.go | 15 ++++++++-------
1 file changed, 8 insertions(+), 7 deletions(-)
diff --git a/internal/senddata/senddata.go b/internal/senddata/senddata.go
index 42f4091..982c243 100644
--- a/internal/senddata/senddata.go
+++ b/internal/senddata/senddata.go
@@ -42,26 +42,27 @@ func (s *sendDataResponseWriter) WriteHeader(status int) {
if s.status != 0 {
return
}
-
s.status = status
- if s.status != http.StatusOK {
- s.rw.WriteHeader(s.status)
+
+ if s.status == http.StatusOK && s.tryInject() {
return
}
+ s.rw.WriteHeader(s.status)
+}
+
+func (s *sendDataResponseWriter) tryInject() bool {
if header := s.Header().Get(HeaderKey); header != "" {
s.Header().Del(HeaderKey)
for _, injecter := range s.injecters {
if injecter.Match(header) {
s.hijacked = true
injecter.Inject(s.rw, s.req, header)
- return
+ return true
}
}
}
-
- s.rw.WriteHeader(s.status)
- return
+ return false
}
func (s *sendDataResponseWriter) Flush() {
--
GitLab
From 1a9126c9e6829e5a66f5e747f2673dcba04d1329 Mon Sep 17 00:00:00 2001
From: Jacob Vosmaer <contact@jacobvosmaer.nl>
Date: Wed, 17 Feb 2016 13:43:42 +0100
Subject: [PATCH 08/10] Add fmt task
---
Makefile | 3 +++
1 file changed, 3 insertions(+)
diff --git a/Makefile b/Makefile
index 197ca4b..6d65af4 100644
--- a/Makefile
+++ b/Makefile
@@ -28,6 +28,9 @@ coverage: testdata/data/group/test.git
go tool cover -html=test.coverage -o coverage.html
rm -f test.coverage
+fmt:
+ go fmt ./...
+
testdata/data/group/test.git: testdata/data
git clone --bare https://gitlab.com/gitlab-org/gitlab-test.git $@
--
GitLab
From f9e11b15d666d6160a75afc0ea8222897f12766f Mon Sep 17 00:00:00 2001
From: Jacob Vosmaer <contact@jacobvosmaer.nl>
Date: Wed, 17 Feb 2016 13:51:37 +0100
Subject: [PATCH 09/10] Remove unnecessary step from main_test.go
---
main_test.go | 2 --
1 file changed, 2 deletions(-)
diff --git a/main_test.go b/main_test.go
index 7003112..d4e1eb0 100644
--- a/main_test.go
+++ b/main_test.go
@@ -677,8 +677,6 @@ func archiveOKServer(t *testing.T, archiveName string) *httptest.Server {
}
encodedJSON := base64.StdEncoding.EncodeToString(jsonData)
w.Header().Set("Gitlab-Workhorse-Send-Data", "git-archive:"+encodedJSON)
- // Prevent the Go HTTP server from setting the Content-Length to 0.
- w.Header().Set("Transfer-Encoding", "chunked")
})
}
--
GitLab
From 85e6cbf861a09be89492433d40a16cf9a1106251 Mon Sep 17 00:00:00 2001
From: Jacob Vosmaer <contact@jacobvosmaer.nl>
Date: Wed, 17 Feb 2016 13:57:03 +0100
Subject: [PATCH 10/10] Use an early return in senddata.tryInject
---
internal/senddata/senddata.go | 20 ++++++++++++--------
1 file changed, 12 insertions(+), 8 deletions(-)
diff --git a/internal/senddata/senddata.go b/internal/senddata/senddata.go
index 982c243..d865858 100644
--- a/internal/senddata/senddata.go
+++ b/internal/senddata/senddata.go
@@ -52,16 +52,20 @@ func (s *sendDataResponseWriter) WriteHeader(status int) {
}
func (s *sendDataResponseWriter) tryInject() bool {
- if header := s.Header().Get(HeaderKey); header != "" {
- s.Header().Del(HeaderKey)
- for _, injecter := range s.injecters {
- if injecter.Match(header) {
- s.hijacked = true
- injecter.Inject(s.rw, s.req, header)
- return true
- }
+ header := s.Header().Get(HeaderKey)
+ s.Header().Del(HeaderKey)
+ if header == "" {
+ return false
+ }
+
+ for _, injecter := range s.injecters {
+ if injecter.Match(header) {
+ s.hijacked = true
+ injecter.Inject(s.rw, s.req, header)
+ return true
}
}
+
return false
}
--
GitLab