Skip to content
Snippets Groups Projects
Commit bc4aa822 authored by Kamil Trzciński's avatar Kamil Trzciński
Browse files

Merge branch 'listen-on-multiple-ips' into 'master'

Allow -listen-http, -listen-https and -listen-proxy to be given more than once

See merge request !6
parents 753b2973 b8ece518
No related branches found
No related tags found
1 merge request!6Allow -listen-http, -listen-https and -listen-proxy to be given more than once
Pipeline #
# Created by .ignore support plugin (hsz.mobi)
shared/pages/.update
/gitlab-pages
Loading
Loading
@@ -22,3 +22,4 @@ test:1.7:
image: golang:1.7
script:
- make verify
- make acceptance
v 0.2.5
- Allow listen-http, listen-https and listen-proxy to be specified multiple times
v 0.2.4
- Fix predefined 404 page content-type
 
Loading
Loading
Loading
Loading
@@ -42,5 +42,8 @@ test:
go get golang.org/x/tools/cmd/cover
go test ./... -cover
 
acceptance: gitlab-pages
go test ./... -run-acceptance-tests
docker:
docker run --rm -it -v ${PWD}:/go/src/pages -w /go/src/pages golang:1.5 /bin/bash
Loading
Loading
@@ -60,6 +60,18 @@ go build
sudo ./gitlab-pages -listen-http ":80" -pages-root path/to/gitlab/shared/pages -pages-domain example.com -daemon-uid 1000 -daemon-gid 1000
```
 
### Listen on multiple ports
Each of the `listen-http`, `listen-https` and `listen-proxy` arguments can be provided multiple times. Gitlab Pages will accept connections to them all.
Example:
```
go build
./gitlab-pages -listen-http "10.0.0.1:8080" -listen-https "[fd00::1]:8080" -pages-root path/to/gitlab/shared/pages -pages-domain example.com
```
This is most useful in dual-stack environments (IPv4+IPv6) where both Gitlab Pages and another HTTP server have to co-exist on the same server.
### License
 
MIT
0.2.4
0.2.5
package main
import (
"flag"
"github.com/stretchr/testify/assert"
"net/http"
"testing"
)
var shouldRun = flag.Bool("run-acceptance-tests", false, "Run the acceptance tests?")
var pagesBinary = flag.String("gitlab-pages-binary", "./gitlab-pages", "Path to the gitlab-pages binary")
// TODO: Use TCP port 0 everywhere to avoid conflicts. The binary could output
// the actual port (and type of listener) for us to read in place of the
// hardcoded values below.
var listeners = []ListenSpec{
{"http", "127.0.0.1", "37000"},
{"http", "::1", "37000"},
{"https", "127.0.0.1", "37001"},
{"https", "::1", "37001"},
{"proxy", "127.0.0.1", "37002"},
{"proxy", "::1", "37002"},
}
func skipUnlessEnabled(t *testing.T) {
if *shouldRun {
return
}
t.Log("Acceptance tests disabled")
t.SkipNow()
}
func TestUnknownHostReturnsNotFound(t *testing.T) {
skipUnlessEnabled(t)
teardown := RunPagesProcess(t, *pagesBinary, listeners)
defer teardown()
for _, spec := range listeners {
rsp, err := GetPageFromListener(t, spec, "invalid.invalid", "")
if assert.NoError(t, err) {
rsp.Body.Close()
assert.Equal(t, http.StatusNotFound, rsp.StatusCode)
}
}
}
func TestKnownHostReturns200(t *testing.T) {
skipUnlessEnabled(t)
teardown := RunPagesProcess(t, *pagesBinary, listeners)
defer teardown()
for _, spec := range listeners {
rsp, err := GetPageFromListener(t, spec, "group.gitlab-example.com", "project/")
if assert.NoError(t, err) {
rsp.Body.Close()
assert.Equal(t, http.StatusOK, rsp.StatusCode)
}
}
}
Loading
Loading
@@ -85,39 +85,40 @@ func (a *theApp) UpdateDomains(domains domains) {
func (a *theApp) Run() {
var wg sync.WaitGroup
 
if a.ListenHTTP != 0 {
// Listen for HTTP
for _, fd := range a.ListenHTTP {
wg.Add(1)
go func() {
go func(fd uintptr) {
defer wg.Done()
err := listenAndServe(a.ListenHTTP, a.ServeHTTP, a.HTTP2, nil)
err := listenAndServe(fd, a.ServeHTTP, a.HTTP2, nil)
if err != nil {
log.Fatal(err)
}
}()
}(fd)
}
 
// Listen for HTTPS
if a.ListenHTTPS != 0 {
for _, fd := range a.ListenHTTPS {
wg.Add(1)
go func() {
go func(fd uintptr) {
defer wg.Done()
err := listenAndServeTLS(a.ListenHTTPS, a.RootCertificate, a.RootKey, a.ServeHTTP, a.ServeTLS, a.HTTP2)
err := listenAndServeTLS(fd, a.RootCertificate, a.RootKey, a.ServeHTTP, a.ServeTLS, a.HTTP2)
if err != nil {
log.Fatal(err)
}
}()
}(fd)
}
 
// Listen for HTTP proxy requests
if a.ListenProxy != 0 {
for _, fd := range a.ListenProxy {
wg.Add(1)
go func() {
go func(fd uintptr) {
defer wg.Done()
err := listenAndServe(a.ListenProxy, a.ServeProxy, a.HTTP2, nil)
err := listenAndServe(fd, a.ServeProxy, a.HTTP2, nil)
if err != nil {
log.Fatal(err)
}
}()
}(fd)
}
 
go watchDomains(a.Domain, a.UpdateDomains, time.Second)
Loading
Loading
Loading
Loading
@@ -6,9 +6,9 @@ type appConfig struct {
RootCertificate []byte
RootKey []byte
 
ListenHTTP uintptr
ListenHTTPS uintptr
ListenProxy uintptr
ListenHTTP []uintptr
ListenHTTPS []uintptr
ListenProxy []uintptr
 
HTTP2 bool
RedirectHTTP bool
Loading
Loading
Loading
Loading
@@ -56,15 +56,20 @@ func daemonReexec(uid, gid uint, args ...string) (cmd *exec.Cmd, err error) {
return
}
 
func daemonUpdateFd(cmd *exec.Cmd, fd *uintptr) {
if *fd == 0 {
return
}
func daemonUpdateFd(cmd *exec.Cmd, fd uintptr) (childFd uintptr) {
file := os.NewFile(fd, "[socket]")
 
file := os.NewFile(*fd, "[socket]")
// we add 3 since, we have a 3 predefined FDs
*fd = uintptr(3 + len(cmd.ExtraFiles))
childFd = uintptr(3 + len(cmd.ExtraFiles))
cmd.ExtraFiles = append(cmd.ExtraFiles, file)
return
}
func daemonUpdateFds(cmd *exec.Cmd, fds []uintptr) {
for idx, fd := range fds {
fds[idx] = daemonUpdateFd(cmd, fd)
}
}
 
func killProcess(cmd *exec.Cmd) {
Loading
Loading
@@ -196,10 +201,10 @@ func daemonize(config appConfig, uid, gid uint) {
defer configWriter.Close()
cmd.ExtraFiles = append(cmd.ExtraFiles, configReader)
 
// Create a new file and store the FD
daemonUpdateFd(cmd, &config.ListenHTTP)
daemonUpdateFd(cmd, &config.ListenHTTPS)
daemonUpdateFd(cmd, &config.ListenProxy)
// Create a new file and store the FD for each listener
daemonUpdateFds(cmd, config.ListenHTTP)
daemonUpdateFds(cmd, config.ListenHTTPS)
daemonUpdateFds(cmd, config.ListenProxy)
 
// Start the process
if err = cmd.Start(); err != nil {
Loading
Loading
Loading
Loading
@@ -139,38 +139,9 @@ func TestDomainCertificate(t *testing.T) {
Group: "group",
Project: "project2",
Config: &domainConfig{
Domain: "test.domain.com",
Certificate: `-----BEGIN CERTIFICATE-----
MIICWDCCAcGgAwIBAgIJAMyzCfoGEwVNMA0GCSqGSIb3DQEBCwUAMEUxCzAJBgNV
BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX
aWRnaXRzIFB0eSBMdGQwHhcNMTYwMjExMTcxNzM2WhcNMjYwMjA4MTcxNzM2WjBF
MQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50
ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKB
gQC2ZSzGIlv2zRsELkmEA1JcvIdsFv80b0NbBftewDAQRuyPlhGNifFx6v7+3O1F
5+f+So43N0QbdrHu11K+ZuXNc6hUy0ofG/eRqXniGZEn8paUdQ98sWsbWelBDNeg
WX4FQomynjyxbG+3IuJR5UHoLWhrJ9+pbPrT915eObbaTQIDAQABo1AwTjAdBgNV
HQ4EFgQUGAhDu+gfckg4IkHRCQWBn4ltKV4wHwYDVR0jBBgwFoAUGAhDu+gfckg4
IkHRCQWBn4ltKV4wDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOBgQAaGx5U
JRW5HC9dXADLf9OnmJRqWi3VNXEXWFk5XgHKc1z7KIBWMsdj+1gzm5ltRO7hkHw9
bx6jQKZBRiUxyqTFw9Ywrk1fYFAxk8hxuqVYcGdonImJarHZTdVMBRWut9+EZBVm
77eYbz2zASNpy++QIg85YgQum9uqREClHRBsxQ==
-----END CERTIFICATE-----`,
Key: `-----BEGIN PRIVATE KEY-----
MIICdQIBADANBgkqhkiG9w0BAQEFAASCAl8wggJbAgEAAoGBALZlLMYiW/bNGwQu
SYQDUly8h2wW/zRvQ1sF+17AMBBG7I+WEY2J8XHq/v7c7UXn5/5Kjjc3RBt2se7X
Ur5m5c1zqFTLSh8b95GpeeIZkSfylpR1D3yxaxtZ6UEM16BZfgVCibKePLFsb7ci
4lHlQegtaGsn36ls+tP3Xl45ttpNAgMBAAECgYAAqZFmDs3isY/9jeV6c0CjUZP0
UokOubC27eihyXTjOj61rsfVicC0tzPB3S+HZ3YyODcYAD1hFCdFRMbqJhmDiewK
5GfATdNQeNARCfJdjYn57NKaXm7rc4C3so1YfxTL6k9QGJgTcybXiClQPDrhkZt3
YLIeeJbY3OppLqjzgQJBAN5AzwyUqX5eQIUncQKcFY0PIjfFTku62brT7hq+TlqY
1B6n3GUtIX+tyYg1qusy4KUUSzMslXJubHsxKanGqZ0CQQDSFwzK7KEYoZol5OMX
mRsavc3iXmmEkkNRdNb1R4UqrlasPeeIeO1CfoD2RPcQhZCwFtR8xS8u6X9ncfC4
qyxxAkAhpQvy6ppR7/Cyd4sLCxfUF8NlT/APVMTbHHQCBmcUHeiWj3C0vEVC78r/
XKh4HGaXdt//ajNhdEflykZ1VgadAkB6Zh934mEA3rXWOgHsb7EQ5WAb8HF9YVGD
FZVfFaoJ8cRhWTeZlQp14Qn1cLyYjZh8XvCxOJiCtlsZw5JBpMihAkBA6ltWb+aZ
EBjC8ZRwZE+cAzmxaYPSs2J7JhS7X7H7Ax7ShhvHI4br3nqf00H4LkvtcHkn5d9G
MwE1w2r4Deww
-----END PRIVATE KEY-----`,
Domain: "test.domain.com",
Certificate: CertificateFixture,
Key: KeyFixture,
},
}
 
Loading
Loading
package main
 
import (
"crypto/tls"
"fmt"
"github.com/stretchr/testify/assert"
"io/ioutil"
"log"
"net"
"net/http"
"os"
"os/exec"
"testing"
"time"
)
 
var chdirSet = false
Loading
Loading
@@ -19,3 +28,166 @@ func setUpTests() {
chdirSet = true
}
}
// The HTTPS certificate isn't signed by anyone. This http client is set up
// so it can talk to servers using it.
var InsecureHTTPSClient = &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
},
}
var CertificateFixture = `-----BEGIN CERTIFICATE-----
MIICWDCCAcGgAwIBAgIJAMyzCfoGEwVNMA0GCSqGSIb3DQEBCwUAMEUxCzAJBgNV
BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX
aWRnaXRzIFB0eSBMdGQwHhcNMTYwMjExMTcxNzM2WhcNMjYwMjA4MTcxNzM2WjBF
MQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50
ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKB
gQC2ZSzGIlv2zRsELkmEA1JcvIdsFv80b0NbBftewDAQRuyPlhGNifFx6v7+3O1F
5+f+So43N0QbdrHu11K+ZuXNc6hUy0ofG/eRqXniGZEn8paUdQ98sWsbWelBDNeg
WX4FQomynjyxbG+3IuJR5UHoLWhrJ9+pbPrT915eObbaTQIDAQABo1AwTjAdBgNV
HQ4EFgQUGAhDu+gfckg4IkHRCQWBn4ltKV4wHwYDVR0jBBgwFoAUGAhDu+gfckg4
IkHRCQWBn4ltKV4wDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOBgQAaGx5U
JRW5HC9dXADLf9OnmJRqWi3VNXEXWFk5XgHKc1z7KIBWMsdj+1gzm5ltRO7hkHw9
bx6jQKZBRiUxyqTFw9Ywrk1fYFAxk8hxuqVYcGdonImJarHZTdVMBRWut9+EZBVm
77eYbz2zASNpy++QIg85YgQum9uqREClHRBsxQ==
-----END CERTIFICATE-----`
var KeyFixture = `-----BEGIN PRIVATE KEY-----
MIICdQIBADANBgkqhkiG9w0BAQEFAASCAl8wggJbAgEAAoGBALZlLMYiW/bNGwQu
SYQDUly8h2wW/zRvQ1sF+17AMBBG7I+WEY2J8XHq/v7c7UXn5/5Kjjc3RBt2se7X
Ur5m5c1zqFTLSh8b95GpeeIZkSfylpR1D3yxaxtZ6UEM16BZfgVCibKePLFsb7ci
4lHlQegtaGsn36ls+tP3Xl45ttpNAgMBAAECgYAAqZFmDs3isY/9jeV6c0CjUZP0
UokOubC27eihyXTjOj61rsfVicC0tzPB3S+HZ3YyODcYAD1hFCdFRMbqJhmDiewK
5GfATdNQeNARCfJdjYn57NKaXm7rc4C3so1YfxTL6k9QGJgTcybXiClQPDrhkZt3
YLIeeJbY3OppLqjzgQJBAN5AzwyUqX5eQIUncQKcFY0PIjfFTku62brT7hq+TlqY
1B6n3GUtIX+tyYg1qusy4KUUSzMslXJubHsxKanGqZ0CQQDSFwzK7KEYoZol5OMX
mRsavc3iXmmEkkNRdNb1R4UqrlasPeeIeO1CfoD2RPcQhZCwFtR8xS8u6X9ncfC4
qyxxAkAhpQvy6ppR7/Cyd4sLCxfUF8NlT/APVMTbHHQCBmcUHeiWj3C0vEVC78r/
XKh4HGaXdt//ajNhdEflykZ1VgadAkB6Zh934mEA3rXWOgHsb7EQ5WAb8HF9YVGD
FZVfFaoJ8cRhWTeZlQp14Qn1cLyYjZh8XvCxOJiCtlsZw5JBpMihAkBA6ltWb+aZ
EBjC8ZRwZE+cAzmxaYPSs2J7JhS7X7H7Ax7ShhvHI4br3nqf00H4LkvtcHkn5d9G
MwE1w2r4Deww
-----END PRIVATE KEY-----`
func CreateHTTPSFixtureFiles(t *testing.T) (key string, cert string) {
keyfile, err := ioutil.TempFile("", "https-fixture")
assert.NoError(t, err)
key = keyfile.Name()
keyfile.Close()
certfile, err := ioutil.TempFile("", "https-fixture")
assert.NoError(t, err)
cert = certfile.Name()
certfile.Close()
assert.NoError(t, ioutil.WriteFile(key, []byte(KeyFixture), 0644))
assert.NoError(t, ioutil.WriteFile(cert, []byte(CertificateFixture), 0644))
return keyfile.Name(), certfile.Name()
}
// ListenSpec is used to point at a gitlab-pages http server, preserving the
// type of port it is (http, https, proxy)
type ListenSpec struct {
Type string
Host string
Port string
}
func (l ListenSpec) URL(suffix string) string {
scheme := "http"
if l.Type == "https" {
scheme = "https"
}
return fmt.Sprintf("%s://%s/%s", scheme, l.JoinHostPort(), suffix)
}
// Returns only once this spec points at a working TCP server
func (l ListenSpec) WaitUntilListening() {
for {
conn, _ := net.Dial("tcp", l.JoinHostPort())
if conn != nil {
conn.Close()
break
}
}
}
func (l ListenSpec) JoinHostPort() string {
return net.JoinHostPort(l.Host, l.Port)
}
// RunPagesProcess will start a gitlab-pages process with the specified listeners
// and return a function you can call to shut it down again. Use
// GetPageFromProcess to do a HTTP GET against a listener.
//
// If run as root via sudo, the gitlab-pages process will drop privileges
func RunPagesProcess(t *testing.T, pagesPath string, listeners []ListenSpec) (teardown func()) {
var tempfiles []string
var args []string
var hasHTTPS bool
_, err := os.Stat(pagesPath)
assert.NoError(t, err)
for _, spec := range listeners {
args = append(args, "-listen-"+spec.Type, spec.JoinHostPort())
if spec.Type == "https" {
hasHTTPS = true
}
}
if hasHTTPS {
key, cert := CreateHTTPSFixtureFiles(t)
tempfiles = []string{key, cert}
args = append(args, "-root-key", key, "-root-cert", cert)
}
if os.Geteuid() == 0 && os.Getenv("SUDO_UID") != "" && os.Getenv("SUDO_GID") != "" {
t.Log("Pages process will drop privileges")
args = append(args, "-daemon-uid", os.Getenv("SUDO_UID"), "-daemon-gid", os.Getenv("SUDO_GID"))
}
cmd := exec.Command(pagesPath, args...)
t.Logf("Running %s %v", pagesPath, args)
cmd.Start()
// Wait for all TCP servers to be open. Even with this, gitlab-pages
// will sometimes return 404 if a HTTP request comes in before it has
// updated its set of domains. This usually takes < 1ms, hence the sleep
// for now. Without it, intermittent failures occur.
//
// TODO: replace this with explicit status from the pages binary
// TODO: fix the first-request race
for _, spec := range listeners {
spec.WaitUntilListening()
}
time.Sleep(50 * time.Millisecond)
return func() {
cmd.Process.Kill()
cmd.Process.Wait()
for _, tempfile := range tempfiles {
os.Remove(tempfile)
}
}
}
// Does an insecure HTTP GET against the listener specified, setting a fake
// Host: and constructing the URL from the listener and the URL suffix.
func GetPageFromListener(t *testing.T, spec ListenSpec, host, urlsuffix string) (*http.Response, error) {
url := spec.URL(urlsuffix)
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
req.Host = host
t.Logf("curl -H'Host: %s' %s", host, url)
return InsecureHTTPSClient.Do(req)
}
Loading
Loading
@@ -3,7 +3,6 @@ package main
import (
"flag"
"log"
"net"
"os"
"strings"
)
Loading
Loading
@@ -16,9 +15,12 @@ var REVISION = "HEAD"
 
func appMain() {
var showVersion = flag.Bool("version", false, "Show version")
var listenHTTP = flag.String("listen-http", "", "The address to listen for HTTP requests")
var listenHTTPS = flag.String("listen-https", "", "The address to listen for HTTPS requests")
var listenProxy = flag.String("listen-proxy", "", "The address to listen for proxy requests")
var listenHTTP, listenHTTPS, listenProxy MultiStringFlag
flag.Var(&listenHTTP, "listen-http", "The address(es) to listen on for HTTP requests")
flag.Var(&listenHTTPS, "listen-https", "The address(es) to listen on for HTTPS requests")
flag.Var(&listenProxy, "listen-proxy", "The address(es) to listen on for proxy requests")
var pagesRootCert = flag.String("root-cert", "", "The default path to file certificate to serve static pages")
var pagesRootKey = flag.String("root-key", "", "The default path to file certificate to serve static pages")
var redirectHTTP = flag.Bool("redirect-http", true, "Serve the pages under HTTP")
Loading
Loading
@@ -53,22 +55,22 @@ func appMain() {
config.RootKey = readFile(*pagesRootKey)
}
 
if *listenHTTP != "" {
var l net.Listener
l, config.ListenHTTP = createSocket(*listenHTTP)
for _, addr := range listenHTTP {
l, fd := createSocket(addr)
defer l.Close()
config.ListenHTTP = append(config.ListenHTTP, fd)
}
 
if *listenHTTPS != "" {
var l net.Listener
l, config.ListenHTTPS = createSocket(*listenHTTPS)
for _, addr := range listenHTTPS {
l, fd := createSocket(addr)
defer l.Close()
config.ListenHTTPS = append(config.ListenHTTPS, fd)
}
 
if *listenProxy != "" {
var l net.Listener
l, config.ListenProxy = createSocket(*listenProxy)
for _, addr := range listenProxy {
l, fd := createSocket(addr)
defer l.Close()
config.ListenProxy = append(config.ListenProxy, fd)
}
 
if *daemonUID != 0 || *daemonGID != 0 {
Loading
Loading
package main
import (
"strings"
)
// MultiStringFlag implements the flag.Value interface and allows a string flag
// to be specified multiple times on the command line.
//
// e.g.: -listen-http 127.0.0.1:80 -listen-http [::1]:80
type MultiStringFlag []string
// String returns the list of parameters joined with a commas (",")
func (s *MultiStringFlag) String() string {
return strings.Join(*s, ",")
}
// Set appends the value to the list of parameters
func (s *MultiStringFlag) Set(value string) error {
*s = append(*s, value)
return nil
}
package main
import (
"flag"
"github.com/stretchr/testify/assert"
"testing"
)
func TestMultiStringFlagAppendsOnSet(t *testing.T) {
var concrete MultiStringFlag
var iface flag.Value
iface = &concrete
assert.NoError(t, iface.Set("foo"))
assert.NoError(t, iface.Set("bar"))
assert.Equal(t, MultiStringFlag{"foo", "bar"}, concrete)
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment