Merge branch 'master' into droplab-templating-xss-fix

features/steps/project/source/markdown_render.rb

@@ -7,7 +7,7 @@ class Spinach::Features::ProjectSourceMarkdownRender < Spinach::FeatureSteps
   include SharedMarkdown
   step 'I own project "Delta"' do
-    @project = Project.find_by(name: "Delta")
+    @project = ::Project.find_by(name: "Delta")
     @project ||= create(:project, :repository, name: "Delta", namespace: @user.namespace)
     @project.team << [@user, :master]
   end

features/steps/shared/authentication.rb

-require Rails.root.join('spec', 'support', 'login_helpers')
+require Rails.root.join('features', 'support', 'login_helpers')
 module SharedAuthentication
   include Spinach::DSL
   include LoginHelpers
   step 'I sign in as a user' do
-    login_as :user
+    sign_out(@user) if @user
+
+    @user = create(:user)
+    sign_in(@user)
+  end
+
+  step 'I sign in via the UI' do
+    gitlab_sign_in(create(:user))
   end
   step 'I sign in as an admin' do
-    login_as :admin
+    sign_out(@user) if @user
+
+    @user = create(:admin)
+    sign_in(@user)
   end
   step 'I sign in as "John Doe"' do
-    login_with(user_exists("John Doe"))
+    gitlab_sign_in(user_exists("John Doe"))
   end
   step 'I sign in as "Mary Jane"' do
-    login_with(user_exists("Mary Jane"))
+    gitlab_sign_in(user_exists("Mary Jane"))
   end
   step 'I should be redirected to sign in page' do
@@ -25,14 +35,41 @@ module SharedAuthentication
   end
   step "I logout" do
-    logout
+    gitlab_sign_out
   end
   step "I logout directly" do
-    logout_direct
+    gitlab_sign_out
   end
   def current_user
     @user || User.reorder(nil).first
   end
+
+  private
+
+  def gitlab_sign_in(user)
+    visit new_user_session_path
+
+    fill_in "user_login", with: user.email
+    fill_in "user_password", with: "12345678"
+    check 'user_remember_me'
+    click_button "Sign in"
+
+    @user = user
+  end
+
+  def gitlab_sign_out
+    return unless @user
+
+    if Capybara.current_driver == Capybara.javascript_driver
+      find('.header-user-dropdown-toggle').click
+      click_link 'Sign out'
+      expect(page).to have_button('Sign in')
+    else
+      sign_out(@user)
+    end
+
+    @user = nil
+  end
 end

features/support/login_helpers.rb

+module LoginHelpers
+  # After inclusion, IntegrationHelpers calls these two methods that aren't
+  # supported by Spinach, so we perform the end results ourselves
+  class << self
+    def setup(*args)
+      Spinach.hooks.before_scenario do
+        Warden.test_mode!
+      end
+    end
+
+    def teardown(*args)
+      Spinach.hooks.after_scenario do
+        Warden.test_reset!
+      end
+    end
+  end
+
+  include Devise::Test::IntegrationHelpers
+end

lib/api/merge_requests.rb

@@ -197,14 +197,15 @@ module API
       end
       put ':id/merge_requests/:merge_request_iid/merge' do
         merge_request = find_project_merge_request(params[:merge_request_iid])
+        merge_when_pipeline_succeeds = to_boolean(params[:merge_when_pipeline_succeeds])
         # Merge request can not be merged
         # because user dont have permissions to push into target branch
         unauthorized! unless merge_request.can_be_merged_by?(current_user)
-        not_allowed! unless merge_request.mergeable_state?
-        render_api_error!('Branch cannot be merged', 406) unless merge_request.mergeable?
+        not_allowed! unless merge_request.mergeable_state?(skip_ci_check: merge_when_pipeline_succeeds)
+        render_api_error!('Branch cannot be merged', 406) unless merge_request.mergeable?(skip_ci_check: merge_when_pipeline_succeeds)
         if params[:sha] && merge_request.diff_head_sha != params[:sha]
           render_api_error!("SHA does not match HEAD of source branch: #{merge_request.diff_head_sha}", 409)
@@ -215,7 +216,7 @@ module API
           should_remove_source_branch: params[:should_remove_source_branch]
         }
-        if params[:merge_when_pipeline_succeeds] && merge_request.head_pipeline && merge_request.head_pipeline.active?
+        if merge_when_pipeline_succeeds && merge_request.head_pipeline && merge_request.head_pipeline.active?
           ::MergeRequests::MergeWhenPipelineSucceedsService
             .new(merge_request.target_project, current_user, merge_params)
             .execute(merge_request)

lib/backup/database.rb

@@ -80,16 +80,32 @@ module Backup
         'port'      => '--port',
         'socket'    => '--socket',
         'username'  => '--user',
-        'encoding'  => '--default-character-set'
+        'encoding'  => '--default-character-set',
+        # SSL
+        'sslkey'    => '--ssl-key',
+        'sslcert'   => '--ssl-cert',
+        'sslca'     => '--ssl-ca',
+        'sslcapath' => '--ssl-capath',
+        'sslcipher' => '--ssl-cipher'
       }
       args.map { |opt, arg| "#{arg}=#{config[opt]}" if config[opt] }.compact
     end
     def pg_env
-      ENV['PGUSER']     = config["username"] if config["username"]
-      ENV['PGHOST']     = config["host"] if config["host"]
-      ENV['PGPORT']     = config["port"].to_s if config["port"]
-      ENV['PGPASSWORD'] = config["password"].to_s if config["password"]
+      args = {
+        'username'  => 'PGUSER',
+        'host'      => 'PGHOST',
+        'port'      => 'PGPORT',
+        'password'  => 'PGPASSWORD',
+        # SSL
+        'sslmode'         => 'PGSSLMODE',
+        'sslkey'          => 'PGSSLKEY',
+        'sslcert'         => 'PGSSLCERT',
+        'sslrootcert'     => 'PGSSLROOTCERT',
+        'sslcrl'          => 'PGSSLCRL',
+        'sslcompression'  => 'PGSSLCOMPRESSION'
+      }
+      args.each { |opt, arg| ENV[arg] = config[opt].to_s if config[opt] }
     end
     def report_success(success)

lib/gitlab/auth.rb

@@ -108,7 +108,7 @@ module Gitlab
           token = Doorkeeper::AccessToken.by_token(password)
           if valid_oauth_token?(token)
             user = User.find_by(id: token.resource_owner_id)
-            Gitlab::Auth::Result.new(user, nil, :oauth, read_authentication_abilities)
+            Gitlab::Auth::Result.new(user, nil, :oauth, full_authentication_abilities)
           end
         end
       end

lib/gitlab/diff/position_tracer.rb

@@ -82,7 +82,7 @@ module Gitlab
         file_diff, old_line, new_line = results
-        Position.new(
+        new_position = Position.new(
           old_path: file_diff.old_path,
           new_path: file_diff.new_path,
           head_sha: new_diff_refs.head_sha,
@@ -91,6 +91,13 @@ module Gitlab
           old_line: old_line,
           new_line: new_line
         )
+
+        # If a position is found, but is not actually contained in the diff, for example
+        # because it was an unchanged line in the context of a change that was undone,
+        # we cannot return this as a successful trace.
+        return unless new_position.diff_line(repository)
+
+        new_position
       end
       private

lib/gitlab/email/handler/base_handler.rb

@@ -16,6 +16,10 @@ module Gitlab
         def execute
           raise NotImplementedError
         end
+
+        def metrics_params
+          { handler: self.class.name }
+        end
       end
     end
   end

lib/gitlab/email/handler/create_issue_handler.rb

-
 require 'gitlab/email/handler/base_handler'
 module Gitlab
@@ -37,6 +36,10 @@ module Gitlab
           @project ||= Project.find_by_full_path(project_path)
         end
+        def metrics_params
+          super.merge(project: project)
+        end
+
         private
         def create_issue

lib/gitlab/email/handler/create_note_handler.rb

@@ -28,6 +28,10 @@ module Gitlab
             record_name: 'comment')
         end
+        def metrics_params
+          super.merge(project: project)
+        end
+
         private
         def author

lib/gitlab/email/handler/unsubscribe_handler.rb

@@ -19,6 +19,10 @@ module Gitlab
           noteable.unsubscribe(sent_notification.recipient)
         end
+        def metrics_params
+          super.merge(project: project)
+        end
+
         private
         def sent_notification

lib/gitlab/email/receiver.rb

-
 require_dependency 'gitlab/email/handler'
 # Inspired in great part by Discourse's Email::Receiver
@@ -32,9 +31,7 @@ module Gitlab
         raise UnknownIncomingEmail unless handler
-        Gitlab::Metrics.add_event(:receive_email,
-                                  project: handler.try(:project),
-                                  handler: handler.class.name)
+        Gitlab::Metrics.add_event(:receive_email, handler.metrics_params)
         handler.execute
       end

lib/gitlab/workhorse.rb

@@ -168,7 +168,7 @@ module Gitlab
       end
       def secret_path
-        Rails.root.join('.gitlab_workhorse_secret')
+        Gitlab.config.workhorse.secret_file
       end
       def set_key_and_notify(key, value, expire: nil, overwrite: true)

lib/tasks/gitlab/gitaly.rake

@@ -2,6 +2,8 @@ namespace :gitlab do
   namespace :gitaly do
     desc "GitLab | Install or upgrade gitaly"
     task :install, [:dir] => :environment do |t, args|
+      require 'toml'
+
       warn_user_is_not_gitlab
       unless args.dir.present?
         abort %(Please specify the directory where you want to install gitaly:\n  rake "gitlab:gitaly:install[/home/git/gitaly]")
@@ -16,6 +18,7 @@ namespace :gitlab do
       command = status.zero? ? 'gmake' : 'make'
       Dir.chdir(args.dir) do
+        create_gitaly_configuration
         run_command!([command])
       end
     end
@@ -33,5 +36,39 @@ namespace :gitlab do
       puts TOML.dump(storage: config)
     end
+
+    private
+
+    # We cannot create config.toml files for all possible Gitaly configuations.
+    # For instance, if Gitaly is running on another machine then it makes no
+    # sense to write a config.toml file on the current machine. This method will
+    # only write a config.toml file in the most common and simplest case: the
+    # case where we have exactly one Gitaly process and we are sure it is
+    # running locally because it uses a Unix socket.
+    def create_gitaly_configuration
+      storages = []
+      address = nil
+
+      Gitlab.config.repositories.storages.each do |key, val|
+        if address
+          if address != val['gitaly_address']
+            raise ArgumentError, "Your gitlab.yml contains more than one gitaly_address."
+          end
+        elsif URI(val['gitaly_address']).scheme != 'unix'
+          raise ArgumentError, "Automatic config.toml generation only supports 'unix:' addresses."
+        else
+          address = val['gitaly_address']
+        end
+
+        storages << { name: key, path: val['path'] }
+      end
+
+      File.open("config.toml", "w") do |f|
+        f.puts TOML.dump(socket_path: address.sub(%r{\Aunix:}, ''), storages: storages)
+      end
+    rescue ArgumentError => e
+      puts "Skipping config.toml generation:"
+      puts e.message
+    end
   end
 end

package.json

@@ -32,8 +32,10 @@
     "js-cookie": "^2.1.3",
     "jszip": "^3.1.3",
     "jszip-utils": "^0.0.2",
+    "marked": "^0.3.6",
     "mousetrap": "^1.4.6",
     "pikaday": "^1.5.1",
+    "prismjs": "^1.6.0",
     "raphael": "^2.2.7",
     "raw-loader": "^0.5.1",
     "react-dev-utils": "^0.5.2",

scripts/prepare_build.sh

 #!/bin/sh
-retry() {
-    if eval "$@"; then
-        return 0
-    fi
-    for i in 2 1; do
-        sleep 3s
-        echo "Retrying $i..."
-        if eval "$@"; then
-            return 0
-        fi
-    done
-    return 1
-}
-
-cp config/database.yml.mysql config/database.yml
-sed -i 's/username:.*/username: root/g' config/database.yml
-sed -i 's/password:.*/password:/g' config/database.yml
-sed -i 's/# socket:.*/host: mysql/g' config/database.yml
+. scripts/utils.sh
+
+export SETUP_DB=${SETUP_DB:-true}
+export USE_BUNDLE_INSTALL=${USE_BUNDLE_INSTALL:-true}
+export BUNDLE_INSTALL_FLAGS="--without production --jobs $(nproc) --path vendor --retry 3 --quiet"
+
+# Determine the database by looking at the job name.
+# For example, we'll get pg if the job is `rspec pg 19 20`
+export GITLAB_DATABASE=$(echo $CI_JOB_NAME | cut -f2 -d' ')
+
+# This would make the default database postgresql, and we could also use
+# pg to mean postgresql.
+if [ "$GITLAB_DATABASE" != 'mysql' ]; then
+    export GITLAB_DATABASE='postgresql'
+fi
+
+cp config/database.yml.$GITLAB_DATABASE config/database.yml
+if [ "$GITLAB_DATABASE" = 'postgresql' ]; then
+    sed -i 's/# host:.*/host: postgres/g' config/database.yml
+else # Assume it's mysql
+    sed -i 's/username:.*/username: root/g' config/database.yml
+    sed -i 's/password:.*/password:/g' config/database.yml
+    sed -i 's/# host:.*/host: mysql/g' config/database.yml
+fi
 cp config/resque.yml.example config/resque.yml
 sed -i 's/localhost/redis/g' config/resque.yml
-export FLAGS="--path vendor --retry 3 --quiet"
+cp config/gitlab.yml.example config/gitlab.yml
+
+if [ "$USE_BUNDLE_INSTALL" != "false" ]; then
+    retry bundle install --clean $BUNDLE_INSTALL_FLAGS
+fi
+
+# Only install knapsack after bundle install! Otherwise oddly some native
+# gems could not be found under some circumstance. No idea why, hours wasted.
+retry gem install knapsack fog-aws mime-types
+
+if [ "$SETUP_DB" != "false" ]; then
+    bundle exec rake db:drop db:create db:schema:load db:migrate
+
+    if [ "$GITLAB_DATABASE" = "mysql" ]; then
+        bundle exec rake add_limits_mysql
+    fi
+fi

scripts/utils.sh

+retry() {
+    if eval "$@"; then
+        return 0
+    fi
+
+    for i in 2 1; do
+        sleep 3s
+        echo "Retrying $i..."
+        if eval "$@"; then
+            return 0
+        fi
+    done
+    return 1
+}

spec/controllers/dashboard/todos_controller_spec.rb

 require 'spec_helper'
 describe Dashboard::TodosController do
-  include ApiHelpers
-
   let(:user) { create(:user) }
   let(:author)  { create(:user) }
   let(:project) { create(:empty_project) }

spec/controllers/projects/builds_controller_spec.rb

 require 'spec_helper'
 describe Projects::BuildsController do
-  include ApiHelpers
-
   let(:user) { create(:user) }
   let(:project) { create(:empty_project, :public) }
@@ -63,4 +61,44 @@ describe Projects::BuildsController do
       expect(json_response['favicon']).to eq "/assets/ci_favicons/#{status.favicon}.ico"
     end
   end
+
+  describe 'GET trace.json' do
+    let(:pipeline) { create(:ci_pipeline, project: project) }
+    let(:build) { create(:ci_build, pipeline: pipeline) }
+    let(:user) { create(:user) }
+
+    context 'when user is logged in as developer' do
+      before do
+        project.add_developer(user)
+        sign_in(user)
+        get_trace
+      end
+
+      it 'traces build log' do
+        expect(response).to have_http_status(:ok)
+        expect(json_response['id']).to eq build.id
+        expect(json_response['status']).to eq build.status
+      end
+    end
+
+    context 'when user is logged in as non member' do
+      before do
+        sign_in(user)
+        get_trace
+      end
+
+      it 'traces build log' do
+        expect(response).to have_http_status(:ok)
+        expect(json_response['id']).to eq build.id
+        expect(json_response['status']).to eq build.status
+      end
+    end
+
+    def get_trace
+      get :trace, namespace_id: project.namespace,
+                  project_id: project,
+                  id: build.id,
+                  format: :json
+    end
+  end
 end

spec/controllers/projects/builds_controller_specs.rb

-require 'spec_helper'
-
-describe Projects::BuildsController do
-  include ApiHelpers
-
-  let(:project) { create(:empty_project, :public) }
-
-  describe 'GET trace.json' do
-    let(:pipeline) { create(:ci_pipeline, project: project) }
-    let(:build) { create(:ci_build, pipeline: pipeline) }
-    let(:user) { create(:user) }
-
-    context 'when user is logged in as developer' do
-      before do
-        project.add_developer(user)
-        sign_in(user)
-        get_trace
-      end
-
-      it 'traces build log' do
-        expect(response).to have_http_status(:ok)
-        expect(json_response['id']).to eq build.id
-        expect(json_response['status']).to eq build.status
-      end
-    end
-
-    context 'when user is logged in as non member' do
-      before do
-        sign_in(user)
-        get_trace
-      end
-
-      it 'traces build log' do
-        expect(response).to have_http_status(:ok)
-        expect(json_response['id']).to eq build.id
-        expect(json_response['status']).to eq build.status
-      end
-    end
-
-    def get_trace
-      get :trace, namespace_id: project.namespace,
-                  project_id: project,
-                  id: build.id,
-                  format: :json
-    end
-  end
-end