From c06a7b66450aa7eb7b7adea81043baa1f8b1c601 Mon Sep 17 00:00:00 2001
From: Gabriel Mazetto <brodock@gmail.com>
Date: Tue, 14 Mar 2017 17:33:00 +0100
Subject: [PATCH 1/3] Improve GitLab Geo replicated database support
---
.gitignore | 1 +
Makefile | 22 ++++++++++++-----
doc/howto/postgresql_replication.md | 37 +++++++++++++++++++++--------
doc/prepare.md | 4 ++--
doc/set-up-gdk.md | 20 ++++++++++++++--
support/postgres-replication | 15 ++++++++++++
6 files changed, 79 insertions(+), 20 deletions(-)
create mode 100755 support/postgres-replication
diff --git a/.gitignore b/.gitignore
index 0bbc648..c53335b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -53,3 +53,4 @@
/gitaly/
/gitlab-workhorse/config.toml
/gitlab-workhorse/.cache
+postgresql-primary
diff --git a/Makefile b/Makefile
index ec872df..734ded1 100644
--- a/Makefile
+++ b/Makefile
@@ -158,18 +158,28 @@ postgresql/data:
${postgres_bin_dir}/initdb --locale=C -E utf-8 postgresql/data
support/bootstrap-rails
-postgresql-replication/cluster:
- ${postgres_bin_dir}/initdb --locale=C -E utf-8 postgresql-replica/data
+postgresql-replication-primary: postgresql-replication/access postgresql-replication/role postgresql-replication/config
+
+postgresql-replication-secondary: postgresql-replication/data postgresql-replication/access postgresql-replication/backup postgresql-replication/config
+
+postgresql-replication/data:
+ ${postgres_bin_dir}/initdb --locale=C -E utf-8 postgresql/data
+
+postgresql-replication/access:
cat support/pg_hba.conf.add >> postgresql/data/pg_hba.conf
postgresql-replication/role:
${postgres_bin_dir}/psql -h ${postgres_dir} -d postgres -c "CREATE ROLE ${postgres_replication_user} WITH REPLICATION LOGIN;"
postgresql-replication/backup:
- psql -h ${postgres_dir} -d postgres -c "select pg_start_backup('base backup for streaming rep')"
- rsync -cva --inplace --exclude="*pg_xlog*" postgresql/data postgresql-replica
- psql -h ${postgres_dir} -d postgres -c "select pg_stop_backup(), current_timestamp"
- ./support/recovery.conf ${postgres_dir} > postgresql-replica/data/recovery.conf
+ $(eval postgres_primary_dir := $(realpath postgresql-primary))
+ psql -h ${postgres_primary_dir} -d postgres -c "select pg_start_backup('base backup for streaming rep')"
+ rsync -cva --inplace --exclude="*pg_xlog*" --exclude="*.pid" ${postgres_primary_dir}/data postgresql
+ psql -h ${postgres_primary_dir} -d postgres -c "select pg_stop_backup(), current_timestamp"
+ ./support/recovery.conf ${postgres_primary_dir} > postgresql/data/recovery.conf
+
+postgresql-replication/config:
+ ./support/postgres-replication ${postgres_dir}
.PHONY: foreman
foreman:
diff --git a/doc/howto/postgresql_replication.md b/doc/howto/postgresql_replication.md
index d07e9ff..330d9a7 100644
--- a/doc/howto/postgresql_replication.md
+++ b/doc/howto/postgresql_replication.md
@@ -3,24 +3,41 @@
For Gitlab Geo, you will need a master/slave database replication defined.
There are a few extra steps to follow:
-You must start with a clean postgres setup, (jump to next if you are installing
-everything from scratch):
+In your primary instance (`gdk-ee`) you need to prepare the database for
+replication:
```
+cd gdk-ee
+# terminal window 1:
+foreman start postgresql
+
+# terminal window 2:
+make postgresql-replication-primary
+
+# go back to terminal window 1 stop foreman by hitting "CTRL-C" and execute it again with:
+foreman start postgresql
+```
+
+You must start with a clean postgres setup on the secondary node:
+
+```
+# terminal window 2:
+cd ../gdk-geo
rm -rf postgresql
-make postgresql
```
-Initialize a slave database and setup replication:
+You need to setup a symbolic link to the `postgresql` folder from the
+primary instance (`gdk-ee`):
```
-# terminal window 1:
-make postgresql-replication/cluster
-foreman start postgresql
+# you must be in `gdk-geo` folder
+ln -s ../gdk-ee/postgresql postgresql-primary
+```
-# terminal window 2:
-make postgresql-replication/role
-make postgresql-replication/backup
+Initialize a slave database and setup replication:
+```
+# terminal window 2:
+make postgresql-replication-secondary
# go back to terminal window 1 and stop foreman by hitting "CTRL-C"
```
diff --git a/doc/prepare.md b/doc/prepare.md
index 3288833..a2a3e20 100644
--- a/doc/prepare.md
+++ b/doc/prepare.md
@@ -27,7 +27,7 @@ Please read [the prerequisites for all platforms](#prerequisites-for-all-platfor
```
brew tap homebrew/dupes
brew tap homebrew/versions
-brew install git redis postgresql libiconv icu4c pkg-config cmake nodejs go openssl node npm
+brew install git redis postgresql libiconv icu4c pkg-config cmake nodejs go openssl node npm coreutils
bundle config build.eventmachine --with-cppflags=-I/usr/local/opt/openssl/include
npm install phantomjs-prebuilt@2.1.12 -g
```
@@ -37,7 +37,7 @@ npm install phantomjs-prebuilt@2.1.12 -g
We are using PostgreSQL-9.5 in the following example. If you want to use another version, please adjust paths accordingly.
```
-sudo port install git redis libiconv postgresql95-server icu pkgconfig cmake nodejs4 go openssl npm2
+sudo port install git redis libiconv postgresql95-server icu pkgconfig cmake nodejs4 go openssl npm2 coreutils
bundle config build.eventmachine --with-cppflags=-I/opt/local/include/openssl
sudo npm install phantomjs-prebuilt@2.1.12 -g
echo 'export PATH=/opt/local/lib/postgresql95/bin/:$PATH' >> ~/.profile
diff --git a/doc/set-up-gdk.md b/doc/set-up-gdk.md
index c9efc13..405e9ca 100644
--- a/doc/set-up-gdk.md
+++ b/doc/set-up-gdk.md
@@ -48,8 +48,6 @@ Alternatively, you can clone all components from their official source.
gdk install
```
-If you are going to work on Gitlab Geo, you will need [PostgreSQL replication](./howto/postgresql_replication.md) setup before the "Post-installation" instructions.
-
## GitLab Enterprise Edition
The recommended way to do development on GitLab Enterprise Edition is
@@ -74,6 +72,24 @@ might still be running.
Instructions to generate a developer license can be found in the
onboarding document: https://about.gitlab.com/handbook/developer-onboarding/#gitlab-enterprise-edition-ee
+## GitLab Geo
+
+GitLab Geo requires two Enterprise Edition instances running most of the time.
+You can re-use your previously boostraped `gdk-ee` as your primary node and
+we suggest to bootstrap a `gdk-geo` that will act as your secondary node.
+We need to configure different ports for the new instances so they can talk to
+each other.
+
+```
+gdk init gdk-geo
+cd gdk-geo
+echo 3002 > port
+echo 3807 > webpack_port
+gdk install gitlab_repo=https://gitlab.com/gitlab-org/gitlab-ee.git
+```
+
+You will need [PostgreSQL replication](./howto/postgresql_replication.md) setup before the "Post-installation" instructions.
+
## Post-installation
Start GitLab and all required services:
diff --git a/support/postgres-replication b/support/postgres-replication
new file mode 100755
index 0000000..e643aca
--- /dev/null
+++ b/support/postgres-replication
@@ -0,0 +1,15 @@
+#!/bin/bash
+replication_include="include = 'replication.conf'"
+postgres_dir=$(realpath $1)
+if ! grep -Fxq "$replication_include" $postgres_dir/data/postgresql.conf; then
+ echo $replication_include >> $postgres_dir/data/postgresql.conf
+fi
+
+if [ ! -f "$postgres_dir/data/replication.conf" ]; then
+ cat <<EOF > $postgres_dir/data/replication.conf
+wal_level = hot_standby
+max_wal_senders = 10
+wal_keep_segments = 10
+hot_standby = on
+EOF
+fi
--
GitLab
From 8dadd157a27e325bc0d781499bd7e59f97305502 Mon Sep 17 00:00:00 2001
From: Robert Speicher <rspeicher@gmail.com>
Date: Mon, 10 Apr 2017 15:13:34 -0400
Subject: [PATCH 2/3] Update position of the postgresql-primary gitignore
---
.gitignore | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.gitignore b/.gitignore
index c53335b..6885722 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,6 +6,7 @@
/Procfile
/postgresql/data
/postgresql/.s.*
+/postgresql-primary/
/postgresql-replica/data
/postgresql-replica/.s.*
/redis/*.conf
@@ -53,4 +54,3 @@
/gitaly/
/gitlab-workhorse/config.toml
/gitlab-workhorse/.cache
-postgresql-primary
--
GitLab
From 4ab55ffefaa7ec81ba8bb1e391c53a6c07d546e1 Mon Sep 17 00:00:00 2001
From: Robert Speicher <rspeicher@gmail.com>
Date: Mon, 10 Apr 2017 15:14:18 -0400
Subject: [PATCH 3/3] Expand the Geo and Replication instructions
---
doc/howto/postgresql_replication.md | 37 ++++++++++++++++++++++-------
doc/set-up-gdk.md | 16 +++++++------
2 files changed, 37 insertions(+), 16 deletions(-)
diff --git a/doc/howto/postgresql_replication.md b/doc/howto/postgresql_replication.md
index 330d9a7..803af1c 100644
--- a/doc/howto/postgresql_replication.md
+++ b/doc/howto/postgresql_replication.md
@@ -1,36 +1,48 @@
# PostgreSQL replication
+## Prerequisites
+
+This guide assumes you've already set up one GDK instance with GitLab EE to act
+as the **primary** Geo node in a `gdk-ee` folder. If not, follow the [Set up GDK
+guide](../set-up-gdk.md#gitlab-enterprise-edition) before continuing!
+
+## Configuring replication
+
For Gitlab Geo, you will need a master/slave database replication defined.
There are a few extra steps to follow:
In your primary instance (`gdk-ee`) you need to prepare the database for
-replication:
+replication. This requires the PostgreSQL server to be running, so we'll start
+the server, perform the change (via a `make` task), and then kill and restart
+the server to pick up the change:
```
cd gdk-ee
+
# terminal window 1:
foreman start postgresql
# terminal window 2:
make postgresql-replication-primary
-# go back to terminal window 1 stop foreman by hitting "CTRL-C" and execute it again with:
+# terminal window 1:
+# stop foreman by hitting Ctrl-C, then restart it:
foreman start postgresql
```
-You must start with a clean postgres setup on the secondary node:
+Because we'll be replicating the primary database to the secondary, we need to
+remove the secondary's PostgreSQL data folder:
```
# terminal window 2:
-cd ../gdk-geo
-rm -rf postgresql
+cd gdk-geo
+rm -r postgresql
```
-You need to setup a symbolic link to the `postgresql` folder from the
-primary instance (`gdk-ee`):
+Now we need to add a symbolic link to the primary instance's data folder:
```
-# you must be in `gdk-geo` folder
+# From the gdk-geo folder:
ln -s ../gdk-ee/postgresql postgresql-primary
```
@@ -39,5 +51,12 @@ Initialize a slave database and setup replication:
```
# terminal window 2:
make postgresql-replication-secondary
-# go back to terminal window 1 and stop foreman by hitting "CTRL-C"
```
+
+Now you can go back to **terminal window 1** and stop `foreman` by hitting
+<kbd>Ctrl-C</kbd>.
+
+## Next steps
+
+Continue to the [Post-installation
+section](../set-up-gdk.md#post-installation) of the setup guide.
diff --git a/doc/set-up-gdk.md b/doc/set-up-gdk.md
index 405e9ca..4f191a8 100644
--- a/doc/set-up-gdk.md
+++ b/doc/set-up-gdk.md
@@ -72,13 +72,13 @@ might still be running.
Instructions to generate a developer license can be found in the
onboarding document: https://about.gitlab.com/handbook/developer-onboarding/#gitlab-enterprise-edition-ee
-## GitLab Geo
+### GitLab Geo
-GitLab Geo requires two Enterprise Edition instances running most of the time.
-You can re-use your previously boostraped `gdk-ee` as your primary node and
-we suggest to bootstrap a `gdk-geo` that will act as your secondary node.
-We need to configure different ports for the new instances so they can talk to
-each other.
+Development on GitLab Geo requires two Enterprise Edition GDK instances running
+side-by-side. You can reuse the `gdk-ee` instance you set up in the previous
+section as your primary node, and now we'll create a secondary instance in a
+`gdk-geo` folder to act as a secondary node. We'll configure unique ports for
+the new instance so that it can run alongside the primary.
```
gdk init gdk-geo
@@ -88,7 +88,9 @@ echo 3807 > webpack_port
gdk install gitlab_repo=https://gitlab.com/gitlab-org/gitlab-ee.git
```
-You will need [PostgreSQL replication](./howto/postgresql_replication.md) setup before the "Post-installation" instructions.
+Now that you've installed a primary and a secondary GDK instance, follow the
+[PostgreSQL replication guide](./howto/postgresql_replication.md) before
+continuing to "Post-installation" below.
## Post-installation
--
GitLab