Support for TLS config on Workhorse

As part of a requirement to use end-to-end encryption, Workhorse needs
to be able to run HTTPS server.

It allows to specify [tls] config in config.toml.
The config must contain paths to certificate and private key.
After that, the server will be accessible via HTTPS.

Relates to #353010

Changelog: added

doc/development/workhorse/configuration.md

@@ -128,6 +128,25 @@ relative URL in the `authBackend` setting:
 gitlab-workhorse -authBackend http://localhost:8080/gitlab
 ```
+## TLS support
+
+A listener with TLS can be configured to be used for incoming requests.
+Paths to the files containing a certificate and matching private key for the server must be provided:
+
+```toml
+[[listeners]]
+network = "tcp"
+addr = "localhost:3443"
+[listeners.tls]
+  certificate = "/path/to/certificate"
+  key = "/path/to/private/key"
+  min_version = "tls1.2"
+  max_version = "tls1.3"
+```
+
+The `certificate` file should contain the concatenation
+of the server's certificate, any intermediates, and the CA's certificate.
+
 ## Interaction of authBackend and authSocket
 The interaction between `authBackend` and `authSocket` can be confusing.

workhorse/config.toml.example

@@ -20,3 +20,13 @@ URL = "unix:/home/git/gitlab/redis/redis.socket"
 [image_resizer]
   max_scaler_procs = 4 # Recommendation: CPUs / 2
   max_filesize = 250000
+
+[[listeners]]
+  network = "tcp"
+  addr = "127.0.0.1:3443"
+
+[listeners.tls]
+  certificate = "/path/to/certificate"
+  key = "/path/to/private/key"
+  min_version = "tls1.2"
+  max_version = "tls1.3"

workhorse/config_test.go

@@ -39,6 +39,14 @@ password = "redis password"
 provider = "test provider"
 [image_resizer]
 max_scaler_procs = 123
+[[listeners]]
+network = "tcp"
+addr = "localhost:3443"
+[listeners.tls]
+certificate = "/path/to/certificate"
+key = "/path/to/private/key"
+min_version = "tls1.1"
+max_version = "tls1.2"
 `
 	_, err = io.WriteString(f, data)
 	require.NoError(t, err)
@@ -57,6 +65,15 @@ max_scaler_procs = 123
 	require.Equal(t, []string{"127.0.0.1/8", "192.168.0.1/8"}, cfg.TrustedCIDRsForXForwardedFor)
 	require.Equal(t, []string{"10.0.0.1/8"}, cfg.TrustedCIDRsForPropagation)
 	require.Equal(t, 60*time.Second, cfg.ShutdownTimeout.Duration)
+
+	require.Len(t, cfg.Listeners, 1)
+	listener := cfg.Listeners[0]
+	require.Equal(t, "/path/to/certificate", listener.Tls.Certificate)
+	require.Equal(t, "/path/to/private/key", listener.Tls.Key)
+	require.Equal(t, "tls1.1", listener.Tls.MinVersion)
+	require.Equal(t, "tls1.2", listener.Tls.MaxVersion)
+	require.Equal(t, "tcp", listener.Network)
+	require.Equal(t, "localhost:3443", listener.Addr)
 }
 func TestConfigErrorHelp(t *testing.T) {

workhorse/internal/config/config.go

@@ -84,6 +84,19 @@ type ImageResizerConfig struct {
 	MaxFilesize    uint64 `toml:"max_filesize"`
 }
+type TlsConfig struct {
+	Certificate string `toml:"certificate"`
+	Key         string `toml:"key"`
+	MinVersion  string `toml:"min_version"`
+	MaxVersion  string `toml:"max_version"`
+}
+
+type ListenerConfig struct {
+	Network string     `toml:"network"`
+	Addr    string     `toml:"addr"`
+	Tls     *TlsConfig `toml:"tls"`
+}
+
 type Config struct {
 	Redis                        *RedisConfig             `toml:"redis"`
 	Backend                      *url.URL                 `toml:"-"`
@@ -106,6 +119,7 @@ type Config struct {
 	ShutdownTimeout              TomlDuration             `toml:"shutdown_timeout"`
 	TrustedCIDRsForXForwardedFor []string                 `toml:"trusted_cidrs_for_x_forwarded_for"`
 	TrustedCIDRsForPropagation   []string                 `toml:"trusted_cidrs_for_propagation"`
+	Listeners                    []ListenerConfig         `toml:"listeners"`
 }
 var DefaultImageResizerConfig = ImageResizerConfig{

workhorse/internal/server/server.go

+package server
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"net"
+	"net/http"
+	"syscall"
+
+	"gitlab.com/gitlab-org/labkit/log"
+
+	"gitlab.com/gitlab-org/gitlab/workhorse/internal/config"
+)
+
+var tlsVersions = map[string]uint16{
+	"":       0, // Default value in tls.Config
+	"tls1.0": tls.VersionTLS10,
+	"tls1.1": tls.VersionTLS11,
+	"tls1.2": tls.VersionTLS12,
+	"tls1.3": tls.VersionTLS13,
+}
+
+type Server struct {
+	Handler         http.Handler
+	Umask           int
+	ListenerConfigs []config.ListenerConfig
+	Errors          chan error
+
+	servers []*http.Server
+}
+
+func (s *Server) Run() error {
+	oldUmask := syscall.Umask(s.Umask)
+	defer syscall.Umask(oldUmask)
+
+	for _, cfg := range s.ListenerConfigs {
+		listener, err := s.newListener("upstream", cfg)
+		if err != nil {
+			return fmt.Errorf("server.Run: failed creating a listener: %v", err)
+		}
+
+		s.runUpstreamServer(listener)
+	}
+
+	return nil
+}
+
+func (s *Server) Close() error {
+	return s.allServers(func(srv *http.Server) error { return srv.Close() })
+}
+
+func (s *Server) Shutdown(ctx context.Context) error {
+	return s.allServers(func(srv *http.Server) error { return srv.Shutdown(ctx) })
+}
+
+func (s *Server) allServers(callback func(*http.Server) error) error {
+	var resultErr error
+	errC := make(chan error, len(s.servers))
+	for _, server := range s.servers {
+		server := server // Capture loop variable
+		go func() { errC <- callback(server) }()
+	}
+
+	for range s.servers {
+		if err := <-errC; err != nil {
+			resultErr = err
+		}
+	}
+
+	return resultErr
+}
+
+func (s *Server) runUpstreamServer(listener net.Listener) {
+	srv := &http.Server{
+		Addr:    listener.Addr().String(),
+		Handler: s.Handler,
+	}
+	go func() {
+		s.Errors <- srv.Serve(listener)
+	}()
+
+	s.servers = append(s.servers, srv)
+}
+
+func (s *Server) newListener(name string, cfg config.ListenerConfig) (net.Listener, error) {
+	if cfg.Tls == nil {
+		log.WithFields(log.Fields{"address": cfg.Addr, "network": cfg.Network}).Infof("Running %v server", name)
+
+		return net.Listen(cfg.Network, cfg.Addr)
+	}
+
+	cert, err := tls.LoadX509KeyPair(cfg.Tls.Certificate, cfg.Tls.Key)
+	if err != nil {
+		return nil, err
+	}
+
+	log.WithFields(log.Fields{"address": cfg.Addr, "network": cfg.Network}).Infof("Running %v server with tls", name)
+
+	tlsConfig := &tls.Config{
+		MinVersion:   tlsVersions[cfg.Tls.MinVersion],
+		MaxVersion:   tlsVersions[cfg.Tls.MaxVersion],
+		Certificates: []tls.Certificate{cert},
+	}
+
+	return tls.Listen(cfg.Network, cfg.Addr, tlsConfig)
+}

workhorse/internal/server/server_test.go

+package server
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"gitlab.com/gitlab-org/gitlab/workhorse/internal/config"
+)
+
+const (
+	certFile = "testdata/localhost.crt"
+	keyFile  = "testdata/localhost.key"
+)
+
+func TestRun(t *testing.T) {
+	srv := defaultServer()
+
+	require.NoError(t, srv.Run())
+	defer srv.Close()
+
+	require.Len(t, srv.servers, 2)
+
+	clients := buildClients(t, srv.servers)
+	for url, client := range clients {
+		resp, err := client.Get(url)
+		require.NoError(t, err)
+		require.Equal(t, 200, resp.StatusCode)
+	}
+}
+
+func TestShutdown(t *testing.T) {
+	ready := make(chan bool)
+	done := make(chan bool)
+	statusCodes := make(chan int)
+
+	srv := defaultServer()
+	srv.Handler = http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
+		ready <- true
+		<-done
+		rw.WriteHeader(200)
+	})
+
+	require.NoError(t, srv.Run())
+	defer srv.Close()
+
+	clients := buildClients(t, srv.servers)
+
+	for url, client := range clients {
+		go func(url string, client *http.Client) {
+			resp, err := client.Get(url)
+			require.NoError(t, err)
+			statusCodes <- resp.StatusCode
+		}(url, client)
+	}
+
+	for range clients {
+		<-ready
+	} // initiate requests
+
+	shutdownError := make(chan error)
+	go func() {
+		shutdownError <- srv.Shutdown(context.Background())
+	}()
+
+	for url, client := range clients {
+		require.Eventually(t, func() bool {
+			_, err := client.Get(url)
+			return err != nil
+		}, time.Second, 10*time.Millisecond, "server must stop accepting new requests")
+	}
+
+	for range clients {
+		done <- true
+	} // finish requests
+
+	require.NoError(t, <-shutdownError)
+	require.ElementsMatch(t, []int{200, 200}, []int{<-statusCodes, <-statusCodes})
+}
+
+func TestShutdown_withTimeout(t *testing.T) {
+	ready := make(chan bool)
+	done := make(chan bool)
+
+	srv := defaultServer()
+	srv.Handler = http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
+		ready <- true
+		<-done
+		rw.WriteHeader(200)
+	})
+
+	require.NoError(t, srv.Run())
+	defer srv.Close()
+
+	clients := buildClients(t, srv.servers)
+
+	for url, client := range clients {
+		go func(url string, client *http.Client) {
+			client.Get(url)
+		}(url, client)
+	}
+
+	for range clients {
+		<-ready
+	} // initiate requets
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond)
+	defer cancel()
+
+	err := srv.Shutdown(ctx)
+	require.Error(t, err)
+	require.EqualError(t, err, "context deadline exceeded")
+}
+
+func defaultServer() Server {
+	return Server{
+		ListenerConfigs: []config.ListenerConfig{
+			{
+				Addr:    "127.0.0.1:0",
+				Network: "tcp",
+			},
+			{
+				Addr:    "127.0.0.1:0",
+				Network: "tcp",
+				Tls: &config.TlsConfig{
+					Certificate: certFile,
+					Key:         keyFile,
+				},
+			},
+		},
+		Handler: http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
+			rw.WriteHeader(200)
+		}),
+		Errors: make(chan error),
+	}
+}
+
+func buildClients(t *testing.T, servers []*http.Server) map[string]*http.Client {
+	httpsClient := &http.Client{}
+	certpool := x509.NewCertPool()
+
+	tlsCertificate, err := tls.LoadX509KeyPair(certFile, keyFile)
+	require.NoError(t, err)
+
+	certificate, err := x509.ParseCertificate(tlsCertificate.Certificate[0])
+	require.NoError(t, err)
+
+	certpool.AddCert(certificate)
+	httpsClient.Transport = &http.Transport{
+		TLSClientConfig: &tls.Config{
+			RootCAs: certpool,
+		},
+	}
+
+	httpServer, httpsServer := servers[0], servers[1]
+	return map[string]*http.Client{
+		"http://" + httpServer.Addr:   http.DefaultClient,
+		"https://" + httpsServer.Addr: httpsClient,
+	}
+}

workhorse/internal/server/testdata/localhost.crt

+-----BEGIN CERTIFICATE-----
+MIIEjjCCAvagAwIBAgIQC2au+A/aGQ2Z21O0wVoEwjANBgkqhkiG9w0BAQsFADCB
+pTEeMBwGA1UEChMVbWtjZXJ0IGRldmVsb3BtZW50IENBMT0wOwYDVQQLDDRpZ29y
+ZHJvemRvdkBJZ29ycy1NYWNCb29rLVByby0yLmxvY2FsIChJZ29yIERyb3pkb3Yp
+MUQwQgYDVQQDDDtta2NlcnQgaWdvcmRyb3pkb3ZASWdvcnMtTWFjQm9vay1Qcm8t
+Mi5sb2NhbCAoSWdvciBEcm96ZG92KTAeFw0yMjAzMDcwNDMxMjRaFw0yNDA2MDcw
+NDMxMjRaMGgxJzAlBgNVBAoTHm1rY2VydCBkZXZlbG9wbWVudCBjZXJ0aWZpY2F0
+ZTE9MDsGA1UECww0aWdvcmRyb3pkb3ZASWdvcnMtTWFjQm9vay1Qcm8tMi5sb2Nh
+bCAoSWdvciBEcm96ZG92KTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB
+AMJ8ofGdcnenVRtNGViF4oxPv+CCFA6D2nfsjkJG8kmO6WW7VlbhJYxCMAuyFF1F
+b2UI2rrTFL8Aeq1KxeQzdrb3cpCquVH/UQ00G4ply28XVPRdbIyLQvOThMEeLL6v
+6gb4edL5oZmo/vWhdQxv0NGt282PAEt+bjnbdl28on8WVzmsw/m0nZ2BVWke+oUM
+krfsbyFaZj7aW8w0dNeK25ANy/Ldx55ENRDquphwYHDnpFOQpkHo5nPuoms5j2Sf
+GW3u3hgeFhRrFjqDstU3OKdA4AdHntDjl0gHm35w1m8PXiql/3EpkEMMx5ixQAqM
+cMZ7VVzy0HIjqsjdJZpzjx8CAwEAAaN2MHQwDgYDVR0PAQH/BAQDAgWgMBMGA1Ud
+JQQMMAoGCCsGAQUFBwMBMB8GA1UdIwQYMBaAFKTVZ2JsYLGJOP+UX0AwGO/81Kab
+MCwGA1UdEQQlMCOCCWxvY2FsaG9zdIcEfwAAAYcQAAAAAAAAAAAAAAAAAAAAATAN
+BgkqhkiG9w0BAQsFAAOCAYEAkGntoogSlhukGqTNbTXN9T/gXLtx9afWlgcBEafF
+MYQoJ1DOwXoYCQkMsxE0xWUyLDTpvjzfKkkyQwWzTwcYqRHOKafKYVSvENU5oaDY
+c2nk32SfkcF6bqJ50uBlFMEvKFExU1U+YSJhuEH/iqT9sSd52uwmnB0TJhSOc3J/
+1ZapKM2G71ezi8OyizwlwDJAwQ37CqrYS2slVO6Cy8zJ1l/ZsZ+kxRb+ME0LREI0
+J/rFTo9A6iyuXeBQ2jiRUrC6pmmbUQbVSjROx4RSmWoI/58/VnuZBY9P62OAOgUv
+pukfAbh3SUjN5++m4Py7WjP/y+L2ILPOFtxTY+CQPWQ5Hbff8iMB4NNfutdU1wSS
+CzXT1zWbU12kXod80wkMqWvNb3yU5spqXV6WYhOHiDIyqpPIqp5/i93Ck3Hd6/BQ
+DYlNOQsVHdSjWzNw9UubjpatiFqMK4hvJZE0haoLlmfDeZeqWk9oAuuCibLJGPg4
+TQri+lKgi0e76ynUr1zP1xUR
+-----END CERTIFICATE-----

workhorse/internal/server/testdata/localhost.key

+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDCfKHxnXJ3p1Ub
+TRlYheKMT7/gghQOg9p37I5CRvJJjullu1ZW4SWMQjALshRdRW9lCNq60xS/AHqt
+SsXkM3a293KQqrlR/1ENNBuKZctvF1T0XWyMi0Lzk4TBHiy+r+oG+HnS+aGZqP71
+oXUMb9DRrdvNjwBLfm4523ZdvKJ/Flc5rMP5tJ2dgVVpHvqFDJK37G8hWmY+2lvM
+NHTXituQDcvy3ceeRDUQ6rqYcGBw56RTkKZB6OZz7qJrOY9knxlt7t4YHhYUaxY6
+g7LVNzinQOAHR57Q45dIB5t+cNZvD14qpf9xKZBDDMeYsUAKjHDGe1Vc8tByI6rI
+3SWac48fAgMBAAECggEALuZXNyi8vdYAVAEXp51BsIxavQ0hQQ7S1DCbbagmLU7l
+Qb8XZwQMRfKAG5HqD0P7ROYJuRvF2PmIm9l4Nzuh2SV63yAMaJWlOgXizlEV6cg6
+mGMfFhVPI+XjEZ7xM1rAmMW6uwGv0ppKQXmZ/FHKjYXbh4qAi7QFaLZfqOMgXHzf
+C4nxf0xMzPP7rBnaxAGBRJWC+/UWxd1MVoHRjink4V/Tdy4zu+cEJ+2wuGawp4nz
+dEWYITzXMcBUKmZQHiOm+r58HpWK3mgXpJQBg3WqjR2iNa+ElyoPoGC6zu5Jd8Xg
+mMG2jHPFu+2F4UvymgxbKZqKHqcNjO7WMZRtIRiJgQKBgQDZGXUme0S5Bh8/y1us
+ltEfy4INFYJAejVxPwv7mRLtySqZLkWAPQTaSGgIk/XMTBYS3Ia9XD6Jl3zwo1qF
+R+y3ZkusGmk73o35kBxjc6purDei7CqMzwulbFTsUglDiF9T4X24bv1yK3lP2n8A
+Y6kLsscEC1wIEuwV5HFyQ2S9zwKBgQDlVepMrQ84FxQxN474LakwWLSkwo+6jS37
+61VPUqDUQpE4fGM6+F3fG+9YDMgvOVDneZ0MvzoiDRynbzF7K3k3fIBrYYbTRz7J
+p23BbTninzhrYTE/xd3LuFCZibCXA7nRa0QmYdXG4nUM2jjsjdR5AG7c/qJQDNun
+SXTbfM49sQKBgQCM9Jl6hbiGBTKO4gNAmJ9o7GIhCqEKKg6+23d1QNroZp9w23km
+nPeknjRltWN25MPENUiKc/Tqst/dAcLJHHzWSuXA9Vj0FTjLG0VDURsMRmbNMlci
+G1/tZNvyoAUBwu5Z8OMGt5F46j8WmL+yygI85TOQLavwVhDQ2gTKcnVbQwKBgQC0
+2VCf0KU8xS5eNYLgARn3jyw89VTkduq5S3aFzBIZ8LiWQ7j4yt0z0NKoq8O9QcSk
+FUocwDv2mEJtYwkxKTI46ExY4Zqxx/Aik47AxwKrzIVwYD+3G7DxMtMUkPkZzY1e
+MOmYHvS3FuPZE8lp+dqA5S+HxKF44Pria9HkOAJnsQKBgE853d9sR0DlJtEj64yu
+FX1rCle/UUODClktPgrwuM+xYutxOiEu6HUWHJI2yvWNk4oNL8Xd0IkR9NlwdatU
+E3+WDua+yYAsI9yWYn3+iqp+owNATkEDjWGivt0Onmgttt5kLHzPFCViIcgl32vv
+7V/plCsmgrS98xZHRrriTLvz
+-----END PRIVATE KEY-----

workhorse/main.go

@@ -22,6 +22,7 @@ import (
 	"gitlab.com/gitlab-org/gitlab/workhorse/internal/queueing"
 	"gitlab.com/gitlab-org/gitlab/workhorse/internal/redis"
 	"gitlab.com/gitlab-org/gitlab/workhorse/internal/secret"
+	"gitlab.com/gitlab-org/gitlab/workhorse/internal/server"
 	"gitlab.com/gitlab-org/gitlab/workhorse/internal/upstream"
 )
@@ -155,6 +156,7 @@ func buildConfig(arg0 string, args []string) (*bootConfig, *config.Config, error
 	cfg.ShutdownTimeout = cfgFromFile.ShutdownTimeout
 	cfg.TrustedCIDRsForXForwardedFor = cfgFromFile.TrustedCIDRsForXForwardedFor
 	cfg.TrustedCIDRsForPropagation = cfgFromFile.TrustedCIDRsForPropagation
+	cfg.Listeners = cfgFromFile.Listeners
 	return boot, cfg, nil
 }
@@ -177,14 +179,6 @@ func run(boot bootConfig, cfg config.Config) error {
 		}
 	}
-	// Change the umask only around net.Listen()
-	oldUmask := syscall.Umask(boot.listenUmask)
-	listener, err := net.Listen(boot.listenNetwork, boot.listenAddr)
-	syscall.Umask(oldUmask)
-	if err != nil {
-		return fmt.Errorf("main listener: %v", err)
-	}
-
 	finalErrors := make(chan error)
 	// The profiler will only be activated by HTTP requests. HTTP
@@ -241,8 +235,19 @@ func run(boot bootConfig, cfg config.Config) error {
 	done := make(chan os.Signal, 1)
 	signal.Notify(done, syscall.SIGINT, syscall.SIGTERM)
-	server := http.Server{Handler: up}
-	go func() { finalErrors <- server.Serve(listener) }()
+	listenerFromBootConfig := config.ListenerConfig{
+		Network: boot.listenNetwork,
+		Addr:    boot.listenAddr,
+	}
+	srv := &server.Server{
+		Handler:         up,
+		Umask:           boot.listenUmask,
+		ListenerConfigs: append(cfg.Listeners, listenerFromBootConfig),
+		Errors:          finalErrors,
+	}
+	if err := srv.Run(); err != nil {
+		return fmt.Errorf("running server: %v", err)
+	}
 	select {
 	case err := <-finalErrors:
@@ -254,6 +259,6 @@ func run(boot bootConfig, cfg config.Config) error {
 		defer cancel()
 		redis.Shutdown()
-		return server.Shutdown(ctx)
+		return srv.Shutdown(ctx)
 	}
 }