Merge branch 'master' of gitlab.com:gitlab-org/omnibus-gitlab into 6-6-stable

README.md

@@ -98,9 +98,20 @@ Omnibus-gitlab uses four different directories.
 - `/var/log/gitlab` contains all log data generated by components of
   omnibus-gitlab.
+### Storing Git data in an alternative directory
+
+By default, omnibus-gitlab stores Git repository data in `/var/opt/gitlab/git-data`.
+You can change this location by adding the following line to `/etc/gitlab/gitlab.rb`.
+
+```ruby
+git_data_dir "/mnt/nas/git-data"
+```
+
+Run `sudo gitlab-ctl reconfigure` for the change to take effect.
+
 ## Building your own package
 See [the separate build documentation](doc/build.md).
 [downloads]: https://www.gitlab.com/downloads
-[CE README]: https://gitlab.com/gitlab-org/gitlab-ce/blob/master/README.md
\ No newline at end of file
+[CE README]: https://gitlab.com/gitlab-org/gitlab-ce/blob/master/README.md

config/patches/gitlab-rails/backup_read_REVISION.patch

+diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb
+index efaefa4..8f0e38f 100644
+--- a/lib/backup/manager.rb
++++ b/lib/backup/manager.rb
+@@ -7,7 +7,7 @@ module Backup
+       s = {}
+       s[:db_version]         = "#{ActiveRecord::Migrator.current_version}"
+       s[:backup_created_at]  = Time.now
+-      s[:gitlab_version]     = %x{git rev-parse HEAD}.gsub(/\n/,"")
++      s[:gitlab_version]     = File.read(Rails.root.join('REVISION')).chomp
+       s[:tar_version]        = %x{tar --version | head -1}.gsub(/\n/,"")
+ 
+       Dir.chdir(Gitlab.config.backup.path)
+@@ -92,7 +92,7 @@ module Backup
+         Dir.chdir(Rails.root)
+ 
+         # restoring mismatching backups can lead to unexpected problems
+-        if settings[:gitlab_version] != %x{git rev-parse HEAD}.gsub(/\n/, "")
++        if settings[:gitlab_version] != File.read(Rails.root.join('REVISION')).chomp
+           puts "GitLab version mismatch:".red
+           puts "  Your current HEAD differs from the HEAD in the backup!".red
+           puts "  Please switch to the following revision and try again:".red

config/patches/gitlab-rails/backup_restore_hooks.patch

+diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb
+index 20fd5ba..552f7ea 100644
+--- a/lib/backup/repository.rb
++++ b/lib/backup/repository.rb
+@@ -72,8 +72,7 @@ module Backup
+       end
+ 
+       print 'Put GitLab hooks in repositories dirs'.yellow
+-      gitlab_shell_user_home = File.expand_path("~#{Gitlab.config.gitlab_shell.ssh_user}")
+-      if system("#{gitlab_shell_user_home}/gitlab-shell/support/rewrite-hooks.sh", Gitlab.config.gitlab_shell.repos_path)
++      if system("#{Gitlab.config.gitlab_shell.path}/support/rewrite-hooks.sh", Gitlab.config.gitlab_shell.repos_path)
+         puts " [DONE]".green
+       else
+         puts " [FAILED]".red

config/patches/gitlab-rails/backup_uploads_realpath.patch

+diff --git a/lib/backup/uploads.rb b/lib/backup/uploads.rb
+index e79da7e..e50e1ff 100644
+--- a/lib/backup/uploads.rb
++++ b/lib/backup/uploads.rb
+@@ -3,7 +3,7 @@ module Backup
+     attr_reader :app_uploads_dir, :backup_uploads_dir, :backup_dir
+ 
+     def initialize
+-      @app_uploads_dir = Rails.root.join('public', 'uploads')
++      @app_uploads_dir = File.realpath(Rails.root.join('public', 'uploads'))
+       @backup_dir = Gitlab.config.backup.path
+       @backup_uploads_dir = File.join(Gitlab.config.backup.path, 'uploads')
+     end
+@@ -21,8 +21,9 @@ module Backup
+     end
+ 
+     def backup_existing_uploads_dir
++      timestamped_uploads_path = File.join(app_uploads_dir, '..', "uploads.#{Time.now.to_i}")
+       if File.exists?(app_uploads_dir)
+-        FileUtils.mv(app_uploads_dir, Rails.root.join('public', "uploads.#{Time.now.to_i}"))
++        FileUtils.mv(app_uploads_dir, timestamped_uploads_path)
+       end
+     end
+   end

config/patches/gitlab-shell/create_hooks.patch

+diff --git a/bin/create-hooks b/bin/create-hooks
+new file mode 100755
+index 0000000..d6f07c7
+--- /dev/null
++++ b/bin/create-hooks
+@@ -0,0 +1,12 @@
++#!/usr/bin/env ruby
++
++# Recreate GitLab hooks in the Git repositories managed by GitLab.
++#
++# This script is used when restoring a GitLab backup.
++
++require_relative '../lib/gitlab_init'
++require File.join(ROOT_PATH, 'lib', 'gitlab_projects')
++
++Dir["#{GitlabConfig.new.repos_path}/*/*.git"].each do |repo|
++  GitlabProjects.create_hooks(repo)
++end
+diff --git a/lib/gitlab_projects.rb b/lib/gitlab_projects.rb
+index fec204c..b3ff372 100644
+--- a/lib/gitlab_projects.rb
++++ b/lib/gitlab_projects.rb
+@@ -42,6 +42,12 @@ class GitlabProjects
+     end
+   end
+ 
++  def self.create_hooks(path)
++    hook = File.join(path, 'hooks', 'update')
++    File.delete(hook) if File.exists?(hook)
++    File.symlink(File.join(ROOT_PATH, 'hooks', 'update'), hook)
++  end
++
+   protected
+ 
+   def create_branch
+@@ -74,13 +80,7 @@ class GitlabProjects
+     $logger.info "Adding project #{@project_name} at <#{full_path}>."
+     FileUtils.mkdir_p(full_path, mode: 0770)
+     cmd = %W(git --git-dir=#{full_path} init --bare)
+-    system(*cmd) && create_hooks(full_path)
+-  end
+-
+-  def create_hooks(path)
+-    hook = File.join(path, 'hooks', 'update')
+-    File.delete(hook) if File.exists?(hook)
+-    File.symlink(File.join(ROOT_PATH, 'hooks', 'update'), hook)
++    system(*cmd) && self.class.create_hooks(full_path)
+   end
+ 
+   def rm_project
+@@ -94,7 +94,7 @@ class GitlabProjects
+     @source = ARGV.shift
+     $logger.info "Importing project #{@project_name} from <#{@source}> to <#{full_path}>."
+     cmd = %W(git clone --bare -- #{@source} #{full_path})
+-    system(*cmd) && create_hooks(full_path)
++    system(*cmd) && self.class.create_hooks(full_path)
+   end
+ 
+   # Move repository from one directory to another
+@@ -156,7 +156,7 @@ class GitlabProjects
+ 
+     $logger.info "Forking project from <#{full_path}> to <#{full_destination_path}>."
+     cmd = %W(git clone --bare -- #{full_path} #{full_destination_path})
+-    system(*cmd) && create_hooks(full_destination_path)
++    system(*cmd) && self.class.create_hooks(full_destination_path)
+   end
+ 
+   def update_head
+diff --git a/spec/gitlab_projects_spec.rb b/spec/gitlab_projects_spec.rb
+index bbe27e2..4341ca5 100644
+--- a/spec/gitlab_projects_spec.rb
++++ b/spec/gitlab_projects_spec.rb
+@@ -95,7 +95,7 @@ describe GitlabProjects do
+ 
+     it "should create a directory" do
+       gl_projects.stub(system: true)
+-      gl_projects.stub(create_hooks: true)
++      GitlabProjects.stub(create_hooks: true)
+       gl_projects.exec
+       File.exists?(tmp_repo_path).should be_true
+     end
+@@ -103,7 +103,7 @@ describe GitlabProjects do
+     it "should receive valid cmd" do
+       valid_cmd = ['git', "--git-dir=#{tmp_repo_path}", 'init', '--bare']
+       gl_projects.should_receive(:system).with(*valid_cmd).and_return(true)
+-      gl_projects.should_receive(:create_hooks).with(tmp_repo_path)
++      GitlabProjects.should_receive(:create_hooks).with(tmp_repo_path)
+       gl_projects.exec
+     end
+ 
+diff --git a/support/rewrite-hooks.sh b/support/rewrite-hooks.sh
+index 3c96b6f..585eaeb 100755
+--- a/support/rewrite-hooks.sh
++++ b/support/rewrite-hooks.sh
+@@ -1,28 +1,5 @@
+ #!/bin/bash
++# This script is deprecated. Use bin/create-hooks instead.
+ 
+-# $1 is an optional argument specifying the location of the repositories directory.
+-# Defaults to /home/git/repositories if not provided
+-
+-home_dir="/home/git"
+-src=${1:-"$home_dir/repositories"}
+-
+-function create_link_in {
+-  ln -s -f "$home_dir/gitlab-shell/hooks/update" "$1/hooks/update"
+-}
+-
+-for dir in `ls "$src/"`
+-do
+-  if [ -d "$src/$dir" ]; then
+-    if [[ "$dir" =~ ^.*\.git$ ]]
+-    then
+-      create_link_in "$src/$dir"
+-    else
+-      for subdir in `ls "$src/$dir/"`
+-      do
+-        if [ -d "$src/$dir/$subdir" ] && [[ "$subdir" =~ ^.*\.git$ ]]; then
+-          create_link_in "$src/$dir/$subdir"
+-        fi
+-      done
+-    fi
+-  fi
+-done
++gitlab_shell_dir="$(cd $(dirname $0) && pwd)/.."
++exec ${gitlab_shell_dir}/bin/create-hooks

config/software/gitlab-rails.rb

@@ -43,6 +43,13 @@ build do
   # source code to include the Git revision of the code included in the omnibus
   # build.
   command "sed -i \"s/.*REVISION.*/REVISION = '$(git log --pretty=format:'%h' -n 1)'/\" config/initializers/2_app.rb"
+  patch :source => "backup_read_REVISION.patch"
+  command "git rev-parse HEAD > REVISION"
+
+  # The user uploads path is not (yet) configurable in gitlab-rails. As a
+  # workaround, omnibus-gitlab creates a symlink for public/uploads. This breaks
+  # the GitLab backup script.
+  patch :source => "backup_uploads_realpath.patch"
   bundle "install --without mysql development test --path=#{install_dir}/embedded/service/gem", :env => env
@@ -55,10 +62,10 @@ build do
   # database at this point so that is a problem. This bug is fixed in
   # acts-as-taggable-on 3.0.0 by
   # https://github.com/mbleigh/acts-as-taggable-on/commit/ad02dc9bb24ec8e1e79e7e35e2d4bb5910a66d8e
-  patch = "#{Omnibus.project_root}/config/patches/acts-as-taggable-on-ad02dc9bb24ec8e1e79e7e35e2d4bb5910a66d8e.diff"
+  aato_patch = "#{Omnibus.project_root}/config/patches/acts-as-taggable-on-ad02dc9bb24ec8e1e79e7e35e2d4bb5910a66d8e.diff"
   # To make this idempotent, we apply the patch (in case this is a first run) or
   # we revert and re-apply the patch (if this is a second or later run).
-  command "git apply #{patch} || (git apply -R #{patch} && git apply #{patch})",
+  command "git apply #{aato_patch} || (git apply -R #{aato_patch} && git apply #{aato_patch})",
     :cwd => "#{install_dir}/embedded/service/gem/ruby/1.9.1/gems/acts-as-taggable-on-2.4.1"
   rake "assets:precompile", :env => {"RAILS_ENV" => "production"}
   # Tear down now that the assets:precompile is done.

config/software/gitlab-shell.rb

@@ -25,6 +25,9 @@ dependency "rsync"
 source :git => "https://gitlab.com/gitlab-org/gitlab-shell.git"
 build do
+  # patch gitlab-shell 1.8.0 to correctly create hooks during backup restore
+  patch :source => "create_hooks.patch"
+
   command "mkdir -p #{install_dir}/embedded/service/gitlab-shell"
   command "#{install_dir}/embedded/bin/rsync -a --delete --exclude=.git/*** --exclude=.gitignore ./ #{install_dir}/embedded/service/gitlab-shell/"
   block do

files/gitlab-cookbooks/gitlab/attributes/default.rb

@@ -44,7 +44,7 @@ default['gitlab']['gitlab-rails']['log_directory'] = "/var/log/gitlab/gitlab-rai
 default['gitlab']['gitlab-rails']['environment'] = 'production'
 default['gitlab']['gitlab-rails']['internal_api_url'] = "http://localhost:8080"
-default['gitlab']['gitlab-rails']['uploads_directory'] = "/var/opt/gitlab/uploads"
+default['gitlab']['gitlab-rails']['uploads_directory'] = "/var/opt/gitlab/gitlab-rails/uploads"
 default['gitlab']['gitlab-rails']['rate_limit_requests_per_period'] = 10
 default['gitlab']['gitlab-rails']['rate_limit_period'] = 60
@@ -86,10 +86,10 @@ default['gitlab']['gitlab-rails']['ldap_method'] = "ssl"
 default['gitlab']['gitlab-rails']['ldap_bind_dn'] = "_the_full_dn_of_the_user_you_will_bind_with"
 default['gitlab']['gitlab-rails']['ldap_password'] = "_the_password_of_the_bind_user"
 default['gitlab']['gitlab-rails']['ldap_allow_username_or_email_login'] = true
-default['gitlab']['gitlab-rails']['satellites_path'] = "/var/opt/gitlab/gitlab-satellites"
+default['gitlab']['gitlab-rails']['satellites_path'] = "/var/opt/gitlab/git-data/gitlab-satellites"
 default['gitlab']['gitlab-rails']['backup_path'] = "tmp/backups"
 default['gitlab']['gitlab-rails']['gitlab_shell_path'] = "/opt/gitlab/embedded/service/gitlab-shell/"
-default['gitlab']['gitlab-rails']['gitlab_shell_repos_path'] = "/var/opt/gitlab/repositories"
+default['gitlab']['gitlab-rails']['gitlab_shell_repos_path'] = "/var/opt/gitlab/git-data/repositories"
 default['gitlab']['gitlab-rails']['gitlab_shell_hooks_path'] = "/opt/gitlab/embedded/service/gitlab-shell/hooks/"
 default['gitlab']['gitlab-rails']['gitlab_shell_upload_pack'] = true
 default['gitlab']['gitlab-rails']['gitlab_shell_receive_pack'] = true
@@ -125,6 +125,7 @@ default['gitlab']['sidekiq']['log_directory'] = "/var/log/gitlab/sidekiq"
 # gitlab-shell
 ###
 default['gitlab']['gitlab-shell']['log_directory'] = "/var/log/gitlab/gitlab-shell/"
+default['gitlab']['gitlab-shell']['git_data_directory'] = "/var/opt/gitlab/git-data"
 ###

files/gitlab-cookbooks/gitlab/libraries/gitlab.rb

@@ -31,11 +31,13 @@ module Gitlab
   postgresql Mash.new
   redis Mash.new
   gitlab_rails Mash.new
+  gitlab_shell Mash.new
   unicorn Mash.new
   sidekiq Mash.new
   nginx Mash.new
   node nil
   external_url nil
+  git_data_dir nil
   class << self
@@ -103,6 +105,14 @@ module Gitlab
       Gitlab['gitlab_rails']['gitlab_port'] = uri.port
     end
+    def parse_git_data_dir
+      return unless git_data_dir
+
+      Gitlab['gitlab_shell']['git_data_directory'] ||= git_data_dir
+      Gitlab['gitlab_rails']['gitlab_shell_repos_path'] ||= File.join(git_data_dir, "repositories")
+      Gitlab['gitlab_rails']['satellites_path'] ||= File.join(git_data_dir, "gitlab-satellites")
+    end
+
     def generate_hash
       results = { "gitlab" => {} }
       [
@@ -110,6 +120,7 @@ module Gitlab
         "user",
         "redis",
         "gitlab_rails",
+        "gitlab_shell",
         "unicorn",
         "sidekiq",
         "nginx",
@@ -125,6 +136,7 @@ module Gitlab
     def generate_config(node_name)
       generate_secrets(node_name)
       parse_external_url
+      parse_git_data_dir
       generate_hash
     end
   end

files/gitlab-cookbooks/gitlab/recipes/gitlab-shell.rb

@@ -44,16 +44,16 @@ execute "chcon -t ssh_home_t #{ssh_dir}" do
   only_if "id -Z"
 end
-directory log_directory do
-  owner git_user
-  mode "0700"
-  recursive true
-end
-
-directory gitlab_shell_var_dir do
-  owner git_user
-  mode '0700'
-  recursive true
+[
+  log_directory,
+  gitlab_shell_var_dir,
+  node['gitlab']['gitlab-shell']['git_data_directory']
+].each do |dir|
+  directory dir do
+    owner git_user
+    mode "0700"
+    recursive true
+  end
 end
 template_symlink File.join(gitlab_shell_var_dir, "config.yml") do