Merge branch 'geo-replication' into 'master'

Improve GitLab Geo replicated database support

See merge request !270

.gitignore

@@ -6,6 +6,7 @@
 /Procfile
 /postgresql/data
 /postgresql/.s.*
+/postgresql-primary/
 /postgresql-replica/data
 /postgresql-replica/.s.*
 /redis/*.conf

Makefile

@@ -164,18 +164,28 @@ postgresql/data:
 	${postgres_bin_dir}/initdb --locale=C -E utf-8 postgresql/data
 	support/bootstrap-rails
-postgresql-replication/cluster:
-	${postgres_bin_dir}/initdb --locale=C -E utf-8 postgresql-replica/data
+postgresql-replication-primary: postgresql-replication/access postgresql-replication/role postgresql-replication/config
+
+postgresql-replication-secondary: postgresql-replication/data postgresql-replication/access postgresql-replication/backup postgresql-replication/config
+
+postgresql-replication/data:
+	${postgres_bin_dir}/initdb --locale=C -E utf-8 postgresql/data
+
+postgresql-replication/access:
 	cat support/pg_hba.conf.add >> postgresql/data/pg_hba.conf
 postgresql-replication/role:
 	${postgres_bin_dir}/psql -h ${postgres_dir} -d postgres -c "CREATE ROLE ${postgres_replication_user} WITH REPLICATION LOGIN;"
 postgresql-replication/backup:
-	psql -h ${postgres_dir} -d postgres -c "select pg_start_backup('base backup for streaming rep')"
-	rsync -cva --inplace --exclude="*pg_xlog*" postgresql/data postgresql-replica
-	psql -h ${postgres_dir} -d postgres -c "select pg_stop_backup(), current_timestamp"
-	./support/recovery.conf ${postgres_dir} > postgresql-replica/data/recovery.conf
+	$(eval postgres_primary_dir := $(realpath postgresql-primary))
+	psql -h ${postgres_primary_dir} -d postgres -c "select pg_start_backup('base backup for streaming rep')"
+	rsync -cva --inplace --exclude="*pg_xlog*" --exclude="*.pid" ${postgres_primary_dir}/data postgresql
+	psql -h ${postgres_primary_dir} -d postgres -c "select pg_stop_backup(), current_timestamp"
+	./support/recovery.conf ${postgres_primary_dir} > postgresql/data/recovery.conf
+
+postgresql-replication/config:
+	./support/postgres-replication ${postgres_dir}
 .PHONY:	foreman
 foreman:

doc/howto/postgresql_replication.md

 # PostgreSQL replication
+## Prerequisites
+
+This guide assumes you've already set up one GDK instance with GitLab EE to act
+as the **primary** Geo node in a `gdk-ee` folder. If not, follow the [Set up GDK
+guide](../set-up-gdk.md#gitlab-enterprise-edition) before continuing!
+
+## Configuring replication
+
 For Gitlab Geo, you will need a master/slave database replication defined.
 There are a few extra steps to follow:
-You must start with a clean postgres setup, (jump to next if you are installing
-everything from scratch):
+In your primary instance (`gdk-ee`) you need to prepare the database for
+replication. This requires the PostgreSQL server to be running, so we'll start
+the server, perform the change (via a `make` task), and then kill and restart
+the server to pick up the change:
 ```
-rm -rf postgresql
-make postgresql
-```
-Initialize a slave database and setup replication:
-```
+cd gdk-ee
+# terminal window 1:
+foreman start postgresql
+
+# terminal window 2:
+make postgresql-replication-primary
 # terminal window 1:
-make postgresql-replication/cluster
+# stop foreman by hitting Ctrl-C, then restart it:
 foreman start postgresql
+```
+
+Because we'll be replicating the primary database to the secondary, we need to
+remove the secondary's PostgreSQL data folder:
+```
 # terminal window 2:
-make postgresql-replication/role
-make postgresql-replication/backup
-# go back to terminal window 1 and stop foreman by hitting "CTRL-C"
+cd gdk-geo
+rm -r postgresql
+```
+
+Now we need to add a symbolic link to the primary instance's data folder:
+
+```
+# From the gdk-geo folder:
+ln -s ../gdk-ee/postgresql postgresql-primary
+```
+
+Initialize a slave database and setup replication:
 ```
+# terminal window 2:
+make postgresql-replication-secondary
+```
+
+Now you can go back to **terminal window 1** and stop `foreman` by hitting
+<kbd>Ctrl-C</kbd>.
+
+## Next steps
+
+Continue to the [Post-installation
+section](../set-up-gdk.md#post-installation) of the setup guide.

doc/prepare.md

@@ -27,7 +27,7 @@ Please read [the prerequisites for all platforms](#prerequisites-for-all-platfor
 ```
 brew tap homebrew/dupes
 brew tap homebrew/versions
-brew install git redis postgresql libiconv icu4c pkg-config cmake nodejs go openssl node npm
+brew install git redis postgresql libiconv icu4c pkg-config cmake nodejs go openssl node npm coreutils
 bundle config build.eventmachine --with-cppflags=-I/usr/local/opt/openssl/include
 sudo npm install phantomjs-prebuilt@2.1.12 -g
 ```
@@ -37,7 +37,7 @@ sudo npm install phantomjs-prebuilt@2.1.12 -g
 We are using PostgreSQL-9.5 in the following example. If you want to use another version, please adjust paths accordingly.
 ```
-sudo port install git redis libiconv postgresql95-server icu pkgconfig cmake nodejs4 go openssl npm2
+sudo port install git redis libiconv postgresql95-server icu pkgconfig cmake nodejs4 go openssl npm2 coreutils
 bundle config build.eventmachine --with-cppflags=-I/opt/local/include/openssl
 sudo npm install phantomjs-prebuilt@2.1.12 -g
 echo 'export PATH=/opt/local/lib/postgresql95/bin/:$PATH' >> ~/.profile

doc/set-up-gdk.md

@@ -48,8 +48,6 @@ Alternatively, you can clone all components from their official source.
 gdk install
 ```
-If you are going to work on Gitlab Geo, you will need [PostgreSQL replication](./howto/postgresql_replication.md) setup before the "Post-installation" instructions.
-
 ## GitLab Enterprise Edition
 The recommended way to do development on GitLab Enterprise Edition is
@@ -74,6 +72,26 @@ might still be running.
 Instructions to generate a developer license can be found in the
 onboarding document: https://about.gitlab.com/handbook/developer-onboarding/#gitlab-enterprise-edition-ee
+### GitLab Geo
+
+Development on GitLab Geo requires two Enterprise Edition GDK instances running
+side-by-side. You can reuse the `gdk-ee` instance you set up in the previous
+section as your primary node, and now we'll create a secondary instance in a
+`gdk-geo` folder to act as a secondary node. We'll configure unique ports for
+the new instance so that it can run alongside the primary.
+
+```
+gdk init gdk-geo
+cd gdk-geo
+echo 3002 > port
+echo 3807 > webpack_port
+gdk install gitlab_repo=https://gitlab.com/gitlab-org/gitlab-ee.git
+```
+
+Now that you've installed a primary and a secondary GDK instance, follow the
+[PostgreSQL replication guide](./howto/postgresql_replication.md) before
+continuing to "Post-installation" below.
+
 ## Post-installation
 Start GitLab and all required services:

support/postgres-replication

+#!/bin/bash
+replication_include="include = 'replication.conf'"
+postgres_dir=$(realpath $1)
+if ! grep -Fxq "$replication_include" $postgres_dir/data/postgresql.conf; then
+  echo $replication_include >> $postgres_dir/data/postgresql.conf
+fi
+
+if [ ! -f "$postgres_dir/data/replication.conf" ]; then
+  cat <<EOF > $postgres_dir/data/replication.conf
+wal_level = hot_standby
+max_wal_senders = 10
+wal_keep_segments = 10
+hot_standby = on
+EOF
+fi