Merge branch 'master' into gdk-ee-support

58b92fec · Zeger-Jan van de Weg · 127e7bd7 · c2e918d8 · 58b92fec · 58b92fec
Commit 58b92fec authored 9 years ago by Zeger-Jan van de Weg
--- a/.gitignore
+++ b/.gitignore
@@ -5,9 +5,12 @@ gitlab-shell/
 gitlab-satellites/
 repositories/
 gitlab-runner/
+gitlab-ci-multi-runner/
 Procfile
 postgresql/data
 postgresql/.s.*
+postgresql-replica/data
+postgresql-replica/.s.*
 redis/*.conf
 .bundle/
 gitlab-openldap/bin/
@@ -21,3 +24,4 @@ gitlab-openldap/sbin/
 gitlab-openldap/share/
 .vagrant/
 gitlab-workhorse/
+.vagrant_enabled
--- a/Makefile
+++ b/Makefile
 gitlab_ce_repo = https://gitlab.com/gitlab-org/gitlab-ce.git
 gitlab_ee_repo = https://gitlab.com/gitlab-org/gitlab-ee.git
 gitlab_shell_repo = https://gitlab.com/gitlab-org/gitlab-shell.git
-gitlab_runner_repo = https://gitlab.com/gitlab-org/gitlab-ci-runner.git
 gitlab_workhorse_repo = https://gitlab.com/gitlab-org/gitlab-workhorse.git
 gitlab_development_root = $(shell pwd)
 postgres_bin_dir = $(shell pg_config --bindir)
+postgres_replication_user = gitlab_replication
+postgres_dir = $(realpath ./postgresql)
+postgres_replica_dir = $(realpath ./postgresql-replica)
  
 all: gitlab-ce gitlab-ee gitlab-shell gitlab-workhorse-setup support-setup
  
@@ -82,6 +84,7 @@ gitlab-shell/.git:
  
 gitlab-shell/config.yml:
 	sed -e "s|/home/git|${gitlab_development_root}|"\
+	  -e "s|^gitlab_url:.*|gitlab_url: http+unix://${shell echo ${gitlab_development_root}/gitlab.socket | sed 's|/|%2F|g'}|"\
 	  -e "s|:8080/|:3000|"\
 	  -e "s|/usr/bin/redis-cli|$(shell which redis-cli)|"\
 	  -e "s|^  socket: .*|  socket: ${gitlab_development_root}/redis/redis.socket|"\
@@ -103,6 +106,11 @@ ee/update: ee/gitlab-update ee/gitlab-shell-update
 ce/gitlab-update: gitlab-ce/.git/pull
 	cd ${gitlab_development_root}/ce/gitlab && \
 	bundle install --without mysql production --jobs 4 && \
+	@echo ""
+	@echo "------------------------------------------------------------"
+	@echo "Make sure Postgres is running otherwise db:migrate will fail"
+	@echo "------------------------------------------------------------"
+	@echo ""
 	bundle exec rake db:migrate
  
 ce/gitlab-shell-update: ce/gitlab-shell/.git/pull
@@ -179,16 +187,35 @@ Procfile:
 			-e "s|gitlab/public|gitlab-ce/public|"\
 	  	-e "s|postgres |${postgres_bin_dir}/postgres |"\
 	  $@.example > $@
+	# Listen on external interface if inside a vagrant vm
+	if [ -f .vagrant_enabled ] ; \
+	then \
+		printf ',s/localhost:3000/0.0.0.0:3000/g\nwq\n' | ed $@ ; \
+	fi;
  
 redis: redis/redis.conf
  
 redis/redis.conf:
 	sed "s|/home/git|${gitlab_development_root}|" $@.example > $@
  
-postgresql: postgresql/data/PG_VERSION
+postgresql: postgresql/data
  
-postgresql/data/PG_VERSION:
+postgresql/data:
 	${postgres_bin_dir}/initdb --locale=C -E utf-8 postgresql/data
+	support/bootstrap-rails
+
+postgresql-replication/cluster:
+	${postgres_bin_dir}/initdb --locale=C -E utf-8 postgresql-replica/data
+	cat support/pg_hba.conf.add >> postgresql/data/pg_hba.conf
+
+postgresql-replication/role:
+	${postgres_bin_dir}/psql -h ${postgres_dir} -d postgres -c "CREATE ROLE ${postgres_replication_user} WITH REPLICATION LOGIN;"
+
+postgresql-replication/backup:
+	psql -h ${postgres_dir} -d postgres -c "select pg_start_backup('base backup for streaming rep')"
+	rsync -cva --inplace --exclude="*pg_xlog*" postgresql/data postgresql-replica
+	psql -h ${postgres_dir} -d postgres -c "select pg_stop_backup(), current_timestamp"
+	./support/recovery.conf ${postgres_dir} > postgresql-replica/data/recovery.conf
  
 .bundle:
 	bundle install --jobs 4

--- a/Procfile.example
+++ b/Procfile.example
 redis: redis-server /home/git/redis/redis.conf
 postgresql: postgres -D /home/git/postgresql/data -k /home/git/postgresql -h ''
+#postgresql-replica: postgres -D /home/git/postgresql-replica/data -k /home/git/postgresql-replica -h ''
 #openldap: cd gitlab-openldap && libexec/slapd -F slapd.d -d2 -h "ldap://127.0.0.1:3890"
-gitlab-workhorse: /home/git/gitlab-workhorse/gitlab-workhorse -authSocket /home/git/gitlab.socket -listenAddr localhost:3000 -documentRoot /home/git/gitlab/public -developmentMode
+gitlab-workhorse: PATH=/home/git/gitlab-workhorse:$PATH gitlab-workhorse -authSocket /home/git/gitlab.socket -listenAddr localhost:3000 -documentRoot /home/git/gitlab/public -developmentMode
+rails-web: cd gitlab && BUNDLE_GEMFILE=Gemfile RAILS_ENV=development bin/web start_foreground
+rails-background-jobs: cd gitlab && BUNDLE_GEMFILE=Gemfile RAILS_ENV=development bin/background_jobs start_foreground
--- a/README.md
+++ b/README.md
@@ -78,7 +78,7 @@ please see [the instuctions for our (experimental) Vagrant with Docker setup](#v
 If you do not have the dependencies below you will experience strange errors during installation.
  
 1. A non-root unix user, this can be your normal user but **DO NOT** run the installation as a root user
-1. Ruby 2.1.7 installed with a Ruby version manager (RVM, rbenv, chruby, etc.), **DO NOT** use the system Ruby
+1. Ruby 2.2.4 or newer installed with a Ruby version manager (RVM, rbenv, chruby, etc.), **DO NOT** use the system Ruby
 1. Bundler, which you can install with `gem install bundler`
  
 ##### OS X 10.9 (Mavericks), 10.10 (Yosemite), 10.11 (El Capitan)
@@ -88,9 +88,9 @@ Please read [the prerequisites for all platforms](#prerequisites-for-all-platfor
 ```
 brew tap homebrew/dupes
 brew tap homebrew/versions
-brew install git redis postgresql phantomjs198 libiconv icu4c pkg-config cmake nodejs go openssl
-brew link phantomjs198
-bundle config build.nokogiri --with-iconv-dir=/usr/local/opt/libiconv
+brew install git redis postgresql libiconv icu4c pkg-config cmake nodejs go openssl node npm
+bundle config build.eventmachine --with-cppflags=-I/usr/local/opt/openssl/include
+npm install phantomjs@1.9.8 -g
 ```
  
 ##### Ubuntu
@@ -98,7 +98,11 @@ bundle config build.nokogiri --with-iconv-dir=/usr/local/opt/libiconv
 Please read [the prerequisites for all platforms](#prerequisites-for-all-platforms).
  
 ```
-sudo apt-add-repository -y ppa:ubuntu-lxc/lxd-stable && sudo apt-get update
+# This PPA contains an up-to-date version of Go
+sudo apt-add-repository -y ppa:ubuntu-lxc/lxd-stable
+# This PPA gives us Redis 2.8 or newer
+sudo apt-add-repository -y ppa:chris-lea/redis-server
+sudo apt-get update
 sudo apt-get install git postgresql libpq-dev phantomjs redis-server libicu-dev cmake g++ nodejs libkrb5-dev golang ed pkg-config
 ```
  
@@ -118,6 +122,8 @@ Please read [the prerequisites for all platforms](#prerequisites-for-all-platfor
 sudo apt-get install postgresql libpq-dev redis-server libicu-dev cmake g++ nodejs libkrb5-dev ed pkg-config
 ```
  
+If you are running Debian Stretch or newer you will need to install Go compiler as well: `sudo apt-get install golang`
+
 You need to install phantomjs manually
  
 ```
@@ -130,6 +136,7 @@ sudo ln -s /usr/local/share/$PHANTOM_JS/bin/phantomjs /usr/local/bin
 phantomjs --version
 ```
  
+
 ##### Fedora
 ```
 sudo dnf install postgresql libpqxx-devel postgresql-libs redis linicu-devel nodejs git ed cmaker rpm-build lib-pq gcc-c++ krb5-devel
@@ -138,8 +145,6 @@ sudo dnf install postgresql libpqxx-devel postgresql-libs redis linicu-devel nod
 Install `phantomJS` manually, or download it and put in your $PATH. For instructions, follow the [Debian guide on phantomJS](#Debian).
  
 ##### RedHat
-You also need to install [Go](https://golang.org/dl) because the
-Go version included in most Ubuntu versions is too old for GitLab.
  
 ##### CentOS
  
@@ -155,8 +160,8 @@ sudo yum install postgresql93-server libicu-devel cmake gcc-c++ redis ed fontcon
 sudo gpg2 --keyserver hkp://keys.gnupg.net --recv-keys D39DC0E3
 sudo curl -sSL https://get.rvm.io | bash -s stable
 sudo source /etc/profile.d/rvm.sh
-sudo rvm install 2.1
-sudo rvm use 2.1
+sudo rvm install 2.2
+sudo rvm use 2.2
 #Ensure your user is in rvm group
 sudo usermod -a -G rvm <username>
 #add iptables exceptions, or sudo service stop iptables
@@ -169,6 +174,8 @@ fail due to a higher version requirement by GitLab.
 You can follow the instructions found [here](https://gitlab.com/gitlab-org/gitlab-recipes/tree/master/install/centos#add-puias-computational-repository)
 to install a newer binary version of git.
  
+You may need to install Redis 2.8 or newer manually.
+
 ##### Other platforms
  
 If you got GDK running an another platform please send a merge request to add it here.
@@ -197,6 +204,8 @@ To avoid usage of slow VirtualBox shared folders we use NFS here.
  a. Vagrant will download an OS image, bring it up, and install all the prerequisites.
 5. Run `vagrant ssh` to SSH into the box.
 6. Continue setup at *[Installation](#installation)* below.
+7.  After the installation is done, edit the 'gitlab-workhorse' line in
+    your Procfile and change `localhost:3000` to `0.0.0.0:3000`.
  
 ##### Development details
  
@@ -238,6 +247,8 @@ you will have to run the entire docker hypervisor in a VM
 2. Run `vagrant up --provider=docker` in this directory. Vagrant will build a docker image and start the container
 3. Run `vagrant ssh` to SSH into the container.
 5. Continue setup at *[Installation](#installation)* below.
+6.  After the installation is done, edit the 'gitlab-workhorse' line in
+    your Procfile and change `localhost:3000` to `0.0.0.0:3000`.
  
 See [development details](#development-details) and [exit](#exit) of Vagrant-Virtulabox setup, they apply here too.
  
@@ -260,6 +271,9 @@ make
 Alternatively, you can clone straight from your forked repositories or GitLab EE.
  
 ```
+# Install dependencies
+bundle install
+
 # Clone your own forked repositories
 make gitlab_ce_repo=git@gitlab.com:example/gitlab-ce.git gitlab_shell_repo=git@gitlab.com:example/gitlab-shell.git
 ```
@@ -275,6 +289,8 @@ make ce gitlab-workhorse support-setup
  
 Replace `ce` with `ee` to download and install GitLab EE.
  
+If you are going to work on Gitlab Geo, you will need [PostgreSQL replication](#postgresql-replication) setup before the "Post-installation" instructions.
+
 ## Post-installation
  
 First run the commands below to install the requirements for the development
@@ -293,6 +309,15 @@ and setup GitLab:
 ```bash
 cd ce/gitlab && bundle install && bundle exec rake db:create dev:setup
 ```
+## Post-installation
+
+Start GitLab and all required services:
+
+    bundle exec foreman start
+
+To access GitLab you may now go to http://localhost:3000 in your
+browser. The development login credentials are `root` and `5iveL!fe`.
+>>>>>>> master
  
 Finally, start the main GitLab Rails application while still in the `ce/gitlab/`
 subdirectory:
@@ -327,7 +352,6 @@ itself.
 First start Postgres and Redis.
  
 ```
-# terminal window 1
 # current directory: gitlab-development-kit
 bundle exec foreman start
 ```
@@ -438,6 +462,35 @@ remove an individual file (e.g. `rm Procfile`) and rebuild it by
 running `make`. If you want to rebuild _all_ configuration files
 created by the Makefile, run `make clean-config all`.
  
+## PostgreSQL replication
+
+For Gitlab Geo, you will need a master/slave database replication defined.
+There are a few extra steps to follow:
+
+You must start with a clean postgres setup, (jump to next if you are installing
+everything from scratch):
+
+```
+rm -rf postgresql
+make postgresql
+```
+
+Initialize a slave database and setup replication:
+
+```
+# terminal window 1:
+make postgresql-replication/cluster
+foreman start postgresql
+
+# terminal window 2:
+make postgresql-replication/role
+make postgresql-replication/backup
+
+# go back to terminal window 1 and stop foreman by hitting "CTRL-C"
+```
+
+Follow [Post-installation](#post-installation) instructions.
+
 ## OpenLDAP
  
 To run the OpenLDAP installation included in the GitLab development kit do the following:
@@ -479,6 +532,15 @@ ldap:
  
 The second database is optional, and will only work with Gitlab-EE.
  
+The following users are added to the LDAP server:
+
+| uid      | Password | DN                                          |
+| -------- | -------- | -------                                     |
+| john     | password | `uid=john,ou=people,dc=example,dc=com`      |
+| mary     | password | `uid=mary,ou=people,dc=example,dc=com`      |
+| bob      | password | `uid=bob,ou=people,dc=example-alt,dc=com`   |
+| alice    | password | `uid=alice,ou=people,dc=example-alt,dc=com` |
+
 ## NFS
  
 If you want to experiment with how GitLab behaves over NFS you can use a setup
@@ -529,7 +591,7 @@ is Homebrew on OS X, which encourages frequent updates (`brew update && brew
 upgrade`) which may break binary compatibility.
  
 ```
-bundle exec rake db:create gitlab:setup
+bundle exec rake db:create dev:setup
 rake aborted!
 LoadError: dlopen(/Users/janedoe/.rbenv/versions/2.1.2/lib/ruby/gems/2.1.0/extensions/x86_64-darwin-13/2.1.0-static/charlock_holmes-0.6.9.4/charlock_holmes/charlock_holmes.bundle, 9): Library not loaded: /usr/local/opt/icu4c/lib/libicui18n.52.1.dylib
  Referenced from: /Users/janedoe/.rbenv/versions/2.1.2/lib/ruby/gems/2.1.0/extensions/x86_64-darwin-13/2.1.0-static/charlock_holmes-0.6.9.4/charlock_holmes/charlock_holmes.bundle
@@ -590,6 +652,16 @@ bundle config build.eventmachine --with-cppflags=-I/usr/local/opt/openssl/includ
  
 and then do `bundle install` once again.
  
+### 'Invalid reference name' when creating a new tag
+
+Make sure that `git` is configured correctly on your development
+machine (where GDK runs).
+
+```
+git checkout -b can-I-commit
+git commit --allow-empty -m 'I can commit'
+```
+
 ### Other problems
  
 Please open an issue on the [GDK issue tracker](https://gitlab.com/gitlab-org/gitlab-development-kit/issues).

--- a/Vagrantfile
+++ b/Vagrantfile
@@ -5,12 +5,18 @@
 # Please see the Vagrant section in the readme for caveats and tips
 # https://gitlab.com/gitlab-org/gitlab-development-kit/tree/master#vagrant
  
+# Write a file so the rest of the code knows we're inside a vm
+require 'fileutils'
+FileUtils.touch('.vagrant_enabled')
+
 Vagrant.require_version ">= 1.6.0"
 VAGRANTFILE_API_VERSION = "2"
  
 def enable_shares(config, nfs)
 	# paths must be listed as shortest to longest per bug: https://github.com/GM-Alex/vagrant-winnfsd/issues/12#issuecomment-78195957
-	config.vm.synced_folder ".", "/vagrant", type: "rsync", rsync__exclude: ['gitlab/', 'gitlab-shell', 'gitlab-runner'], rsync__auto: false
+	config.vm.synced_folder ".", "/vagrant", type: "rsync",
+		rsync__exclude: ['gitlab', 'postgresql', 'gitlab-shell', 'gitlab-runner'],
+		rsync__auto: false
 	config.vm.synced_folder "gitlab/", "/vagrant/gitlab", :create => true, :nfs => nfs
 	config.vm.synced_folder "gitlab-shell/", "/vagrant/gitlab-shell", :create => true, :nfs => nfs
 	config.vm.synced_folder "gitlab-runner/", "/vagrant/gitlab-runner", :create => true, :nfs => nfs
@@ -18,8 +24,8 @@ end
  
 def running_in_admin_mode?
    return false unless Vagrant::Util::Platform.windows?
-    
-    (`reg query HKU\\S-1-5-19 2>&1` =~ /ERROR/).nil? 
+
+    (`reg query HKU\\S-1-5-19 2>&1` =~ /ERROR/).nil?
 end
  
 if Vagrant::Util::Platform.windows? && !running_in_admin_mode?
@@ -50,11 +56,20 @@ $apt_reqs = <<EOT
 apt-add-repository -y ppa:rael-gc/rvm
 apt-add-repository -y ppa:ubuntu-lxc/lxd-stable
 apt-get update
-apt-get -y install git postgresql libpq-dev phantomjs redis-server libicu-dev cmake g++ nodejs libkrb5-dev curl ruby ed golang nginx
+apt-get -y install git postgresql libpq-dev phantomjs redis-server libicu-dev cmake g++ nodejs libkrb5-dev curl ruby ed golang nginx libgmp-dev
 EOT
  
 # CentOS 6 kernel doesn't suppose UID mapping (affects vagrant-lxc mostly).
 $user_setup = <<EOT
+# create a swapfile
+sudo fallocate -l 4G /swapfile
+sudo chmod 600 /swapfile
+sudo mkswap /swapfile
+# enable swap now
+sudo swapon /swapfile
+# and on reboot
+echo '/swapfile   none    swap    sw    0   0' | sudo tee --append /etc/fstab
+
 if [ $(id -u vagrant) != $(stat -c %u /vagrant) ]; then
 	useradd -u $(stat -c %u /vagrant) -m build
 	echo "build ALL=(ALL) NOPASSWD:ALL" | tee /etc/sudoers.d/build
@@ -72,7 +87,15 @@ sudo ln -s /vagrant /home/vagrant/gitlab-development-kit
  
 # automatically move into the gitlab-development-kit folder, but only add the command
 # if it's not already there
-sudo -u $DEV_USER -i bash -c "grep -q 'cd /home/vagrant/gitlab-development-kit/' /home/vagrant/.bash_profile || echo 'cd /home/vagrant/gitlab-development-kit/' >> /home/vagrant/.bash_profile"
+if [ -f /home/vagrant/.bash_profile ]; then
+	sudo -u $DEV_USER -i bash -c "grep -q 'cd /home/vagrant/gitlab-development-kit/' /home/vagrant/.bash_profile || echo 'cd /home/vagrant/gitlab-development-kit/' >> /home/vagrant/.bash_profile"
+else
+	sudo -u $DEV_USER -i bash -c "touch /home/vagrant/.bash_profile && echo 'cd /home/vagrant/gitlab-development-kit/' >> /home/vagrant/.bash_profile"
+fi
+
+# set git defaults
+sudo -u $DEV_USER -i bash -c "git config --global user.name 'GitLab Development'"
+sudo -u $DEV_USER -i bash -c "git config --global user.email gitlab@local.local"
 EOT
  
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
@@ -121,9 +144,9 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 			enable_nfs = Vagrant::Util::Platform.platform =~ /darwin/ ? false : true
 			enable_shares(override, enable_nfs)
 		end
-		
-		# use 1/4 of memory or 2 GB, whichever is greatest
-		mem = [mem / 4, 2048].max
+
+		# use 1/4 of memory or 3 GB, whichever is greatest
+		mem = [mem / 4, 3072].max
  
 		# performance tweaks
 		# per https://www.virtualbox.org/manual/ch03.html#settings-processor set cpus to real cores, not hyperthreads

--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -2,7 +2,7 @@ FROM ubuntu:trusty
 MAINTAINER Gitlab.com
  
 RUN apt-get update ;\
-    apt-get -y install openssh-server supervisor ;\
+    apt-get -y install openssh-server supervisor software-properties-common rsync ;\
    rm -rf /var/lib/apt/lists/* ;\
    mkdir /var/run/sshd
  

--- a/gitlab-openldap/frontend.alt.example.com.ldif
+++ b/gitlab-openldap/frontend.alt.example.com.ldif
@@ -15,7 +15,7 @@ objectClass: simpleSecurityObject
 objectClass: organizationalRole
 cn: admin
 description: LDAP administrator
-# hased value for 'password'
+# hashed value for 'password'
 userPassword: {SSHA}ICMhr6Jxt5bk2awD7HL7GxRTM3BZ1pFI
  
 dn: ou=people,dc=example-alt,dc=com
@@ -37,7 +37,7 @@ cn: bob Doe
 displayName: bob Doe
 uidNumber: 1000
 gidNumber: 10000
-# hased value for 'password'
+# hashed value for 'password'
 userPassword: {SSHA}qqLFjamdd1cru4RV815+FiSxh/54rfbd
 gecos: bob Doe
 loginShell: /bin/bash
@@ -69,7 +69,7 @@ cn: alice Jane
 displayName: alice Jane
 uidNumber: 1001
 gidNumber: 10001
-# hased value for 'password'
+# hashed value for 'password'
 userPassword: {SSHA}qqLFjamdd1cru4RV815+FiSxh/54rfbd
 mail: alice.jane@example.com
 gecos: alice

--- a/gitlab-openldap/frontend.example.com.ldif
+++ b/gitlab-openldap/frontend.example.com.ldif
@@ -15,7 +15,7 @@ objectClass: simpleSecurityObject
 objectClass: organizationalRole
 cn: admin
 description: LDAP administrator
-# hased value for 'password'
+# hashed value for 'password'
 userPassword: {SSHA}ICMhr6Jxt5bk2awD7HL7GxRTM3BZ1pFI
  
 dn: ou=people,dc=example,dc=com
@@ -37,7 +37,7 @@ cn: John Doe
 displayName: John Doe
 uidNumber: 1000
 gidNumber: 10000
-# hased value for 'password'
+# hashed value for 'password'
 userPassword: {SSHA}qqLFjamdd1cru4RV815+FiSxh/54rfbd
 gecos: John Doe
 loginShell: /bin/bash
@@ -69,7 +69,7 @@ cn: Mary Jane
 displayName: Mary Jane
 uidNumber: 1001
 gidNumber: 10001
-# hased value for 'password'
+# hashed value for 'password'
 userPassword: {SSHA}qqLFjamdd1cru4RV815+FiSxh/54rfbd
 mail: mary.jane@example.com
 gecos: Mary

--- a/support/bootstrap-rails
+++ b/support/bootstrap-rails
+#!/usr/bin/env ruby
+
+begin
+  foreman_pid = spawn(*%W(bundle exec foreman start -c all=0,redis=1,postgresql=1))
+  system({'force' => 'yes', 'BOOTSTRAP' => '1'}, *%W(bundle exec rake db:create dev:setup), chdir: 'gitlab')
+ensure
+  Process.kill('TERM', foreman_pid)
+end
--- a/support/pg_hba.conf.add
+++ b/support/pg_hba.conf.add
+local    replication     gitlab_replication                     trust
--- a/support/recovery.conf
+++ b/support/recovery.conf
+#!/bin/bash
+# this script will output a recovery.conf file using first parameter as host/socket path
+socket_path=$1
+cat <<EOF
+standby_mode = 'on'
+primary_conninfo = 'host=${socket_path} port=5432 user=gitlab_replication'
+EOF