Merge branch 'ci-and-ce-sitting-in-a-tree-k-i-s-s-i-n-g' into 'master'

Merge CI into CE

First step of #2164.

- [x] Merge latest CE master
- [x] Make application start
- [x] Re-use gitlab sessions (remove CI oauth part)
- [x] Get rid of gitlab_ci.yml config
- [x] Make tests start
- [x] Make most CI features works
- [x] Make tests green
- [x] Write migration documentation
- [x] Add CI builds to CE backup

See merge request !1204

app/controllers/ci/runners_controller.rb

+module Ci
+  class RunnersController < Ci::ApplicationController
+    before_action :authenticate_user!
+    before_action :project
+    before_action :set_runner, only: [:edit, :update, :destroy, :pause, :resume, :show]
+    before_action :authorize_access_project!
+    before_action :authorize_manage_project!
+
+    layout 'ci/project'
+
+    def index
+      @runners = @project.runners.order('id DESC')
+      @specific_runners =
+        Ci::Runner.specific.includes(:runner_projects).
+        where(Ci::RunnerProject.table_name => { project_id: current_user.authorized_projects } ).
+        where.not(id:  @runners).order("#{Ci::Runner.table_name}.id DESC").page(params[:page]).per(20)
+      @shared_runners = Ci::Runner.shared.active
+      @shared_runners_count = @shared_runners.count(:all)
+    end
+
+    def edit
+    end
+
+    def update
+      if @runner.update_attributes(runner_params)
+        redirect_to edit_ci_project_runner_path(@project, @runner), notice: 'Runner was successfully updated.'
+      else
+        redirect_to edit_ci_project_runner_path(@project, @runner), alert: 'Runner was not updated.'
+      end
+    end
+
+    def destroy
+      if @runner.only_for?(@project)
+        @runner.destroy
+      end
+
+      redirect_to ci_project_runners_path(@project)
+    end
+
+    def resume
+      if @runner.update_attributes(active: true)
+        redirect_to ci_project_runners_path(@project, @runner), notice: 'Runner was successfully updated.'
+      else
+        redirect_to ci_project_runners_path(@project, @runner), alert: 'Runner was not updated.'
+      end
+    end
+
+    def pause
+      if @runner.update_attributes(active: false)
+        redirect_to ci_project_runners_path(@project, @runner), notice: 'Runner was successfully updated.'
+      else
+        redirect_to ci_project_runners_path(@project, @runner), alert: 'Runner was not updated.'
+      end
+    end
+
+    def show
+    end
+
+    protected
+
+    def project
+      @project = Ci::Project.find(params[:project_id])
+    end
+
+    def set_runner
+      @runner ||= @project.runners.find(params[:id])
+    end
+
+    def runner_params
+      params.require(:runner).permit(:description, :tag_list, :contacted_at, :active)
+    end
+  end
+end

app/controllers/ci/services_controller.rb

+module Ci
+  class ServicesController < Ci::ApplicationController
+    before_action :authenticate_user!
+    before_action :project
+    before_action :authorize_access_project!
+    before_action :authorize_manage_project!
+    before_action :service, only: [:edit, :update, :test]
+
+    respond_to :html
+
+    layout 'ci/project'
+
+    def index
+      @project.build_missing_services
+      @services = @project.services.reload
+    end
+
+    def edit
+    end
+
+    def update
+      if @service.update_attributes(service_params)
+        redirect_to edit_ci_project_service_path(@project, @service.to_param)
+      else
+        render 'edit'
+      end
+    end
+
+    def test
+      last_build = @project.builds.last
+
+      if @service.execute(last_build)
+        message = { notice: 'We successfully tested the service' }
+      else
+        message = { alert: 'We tried to test the service but error occurred' }
+      end
+
+      redirect_to :back, message
+    end
+
+    private
+
+    def project
+      @project = Ci::Project.find(params[:project_id])
+    end
+
+    def service
+      @service ||= @project.services.find { |service| service.to_param == params[:id] }
+    end
+
+    def service_params
+      params.require(:service).permit(
+        :type, :active, :webhook, :notify_only_broken_builds,
+        :email_recipients, :email_only_broken_builds, :email_add_pusher,
+        :hipchat_token, :hipchat_room, :hipchat_server
+      )
+    end
+  end
+end

app/controllers/ci/triggers_controller.rb

+module Ci
+  class TriggersController < Ci::ApplicationController
+    before_action :authenticate_user!
+    before_action :project
+    before_action :authorize_access_project!
+    before_action :authorize_manage_project!
+
+    layout 'ci/project'
+
+    def index
+      @triggers = @project.triggers
+      @trigger = Ci::Trigger.new
+    end
+
+    def create
+      @trigger = @project.triggers.new
+      @trigger.save
+
+      if @trigger.valid?
+        redirect_to ci_project_triggers_path(@project)
+      else
+        @triggers = @project.triggers.select(&:persisted?)
+        render :index
+      end
+    end
+
+    def destroy
+      trigger.destroy
+
+      redirect_to ci_project_triggers_path(@project)
+    end
+
+    private
+
+    def trigger
+      @trigger ||= @project.triggers.find(params[:id])
+    end
+
+    def project
+      @project = Ci::Project.find(params[:project_id])
+    end
+  end
+end

app/controllers/ci/variables_controller.rb

+module Ci
+  class VariablesController < Ci::ApplicationController
+    before_action :authenticate_user!
+    before_action :project
+    before_action :authorize_access_project!
+    before_action :authorize_manage_project!
+
+    layout 'ci/project'
+
+    def show
+    end
+
+    def update
+      if project.update_attributes(project_params)
+        Ci::EventService.new.change_project_settings(current_user, project)
+
+        redirect_to ci_project_variables_path(project), notice: 'Variables were successfully updated.'
+      else
+        render action: 'show'
+      end
+    end
+
+    private
+
+    def project
+      @project ||= Ci::Project.find(params[:project_id])
+    end
+
+    def project_params
+      params.require(:project).permit({ variables_attributes: [:id, :key, :value, :_destroy] })
+    end
+  end
+end

app/controllers/ci/web_hooks_controller.rb

+module Ci
+  class WebHooksController < Ci::ApplicationController
+    before_action :authenticate_user!
+    before_action :project
+    before_action :authorize_access_project!
+    before_action :authorize_manage_project!
+
+    layout 'ci/project'
+
+    def index
+      @web_hooks = @project.web_hooks
+      @web_hook = Ci::WebHook.new
+    end
+
+    def create
+      @web_hook = @project.web_hooks.new(web_hook_params)
+      @web_hook.save
+
+      if @web_hook.valid?
+        redirect_to ci_project_web_hooks_path(@project)
+      else
+        @web_hooks = @project.web_hooks.select(&:persisted?)
+        render :index
+      end
+    end
+
+    def test
+      Ci::TestHookService.new.execute(hook, current_user)
+
+      redirect_to :back
+    end
+
+    def destroy
+      hook.destroy
+
+      redirect_to ci_project_web_hooks_path(@project)
+    end
+
+    private
+
+    def hook
+      @web_hook ||= @project.web_hooks.find(params[:id])
+    end
+
+    def project
+      @project = Ci::Project.find(params[:project_id])
+    end
+
+    def web_hook_params
+      params.require(:web_hook).permit(:url)
+    end
+  end
+end

app/controllers/oauth/applications_controller.rb

 class Oauth::ApplicationsController < Doorkeeper::ApplicationsController
   include Gitlab::CurrentSettings
   include PageLayoutHelper
-  
+
   before_action :verify_user_oauth_applications_enabled
   before_action :authenticate_user!

app/helpers/ci/application_helper.rb

+module Ci
+  module ApplicationHelper
+    def loader_html
+      image_tag 'ci/loader.gif', alt: 'Loading'
+    end
+
+    # Navigation link helper
+    #
+    # Returns an `li` element with an 'active' class if the supplied
+    # controller(s) and/or action(s) are currently active. The content of the
+    # element is the value passed to the block.
+    #
+    # options - The options hash used to determine if the element is "active" (default: {})
+    #           :controller   - One or more controller names to check (optional).
+    #           :action       - One or more action names to check (optional).
+    #           :path         - A shorthand path, such as 'dashboard#index', to check (optional).
+    #           :html_options - Extra options to be passed to the list element (optional).
+    # block   - An optional block that will become the contents of the returned
+    #           `li` element.
+    #
+    # When both :controller and :action are specified, BOTH must match in order
+    # to be marked as active. When only one is given, either can match.
+    #
+    # Examples
+    #
+    #   # Assuming we're on TreeController#show
+    #
+    #   # Controller matches, but action doesn't
+    #   nav_link(controller: [:tree, :refs], action: :edit) { "Hello" }
+    #   # => '<li>Hello</li>'
+    #
+    #   # Controller matches
+    #   nav_link(controller: [:tree, :refs]) { "Hello" }
+    #   # => '<li class="active">Hello</li>'
+    #
+    #   # Shorthand path
+    #   nav_link(path: 'tree#show') { "Hello" }
+    #   # => '<li class="active">Hello</li>'
+    #
+    #   # Supplying custom options for the list element
+    #   nav_link(controller: :tree, html_options: {class: 'home'}) { "Hello" }
+    #   # => '<li class="home active">Hello</li>'
+    #
+    # Returns a list item element String
+    def nav_link(options = {}, &block)
+      if path = options.delete(:path)
+        if path.respond_to?(:each)
+          c = path.map { |p| p.split('#').first }
+          a = path.map { |p| p.split('#').last }
+        else
+          c, a, _ = path.split('#')
+        end
+      else
+        c = options.delete(:controller)
+        a = options.delete(:action)
+      end
+
+      if c && a
+        # When given both options, make sure BOTH are active
+        klass = current_controller?(*c) && current_action?(*a) ? 'active' : ''
+      else
+        # Otherwise check EITHER option
+        klass = current_controller?(*c) || current_action?(*a) ? 'active' : ''
+      end
+
+      # Add our custom class into the html_options, which may or may not exist
+      # and which may or may not already have a :class key
+      o = options.delete(:html_options) || {}
+      o[:class] ||= ''
+      o[:class] += ' ' + klass
+      o[:class].strip!
+
+      if block_given?
+        content_tag(:li, capture(&block), o)
+      else
+        content_tag(:li, nil, o)
+      end
+    end
+
+    # Check if a particular controller is the current one
+    #
+    # args - One or more controller names to check
+    #
+    # Examples
+    #
+    #   # On TreeController
+    #   current_controller?(:tree)           # => true
+    #   current_controller?(:commits)        # => false
+    #   current_controller?(:commits, :tree) # => true
+    def current_controller?(*args)
+      args.any? { |v| v.to_s.downcase == controller.controller_name }
+    end
+
+    # Check if a particular action is the current one
+    #
+    # args - One or more action names to check
+    #
+    # Examples
+    #
+    #   # On Projects#new
+    #   current_action?(:new)           # => true
+    #   current_action?(:create)        # => false
+    #   current_action?(:new, :create)  # => true
+    def current_action?(*args)
+      args.any? { |v| v.to_s.downcase == action_name }
+    end
+
+    def date_from_to(from, to)
+      "#{from.to_s(:short)} - #{to.to_s(:short)}"
+    end
+
+    def body_data_page
+      path = controller.controller_path.split('/')
+      namespace = path.first if path.second
+
+      [namespace, controller.controller_name, controller.action_name].compact.join(":")
+    end
+
+    def duration_in_words(finished_at, started_at)
+      if finished_at && started_at
+        interval_in_seconds = finished_at.to_i - started_at.to_i
+      elsif started_at
+        interval_in_seconds = Time.now.to_i - started_at.to_i
+      end
+
+      time_interval_in_words(interval_in_seconds)
+    end
+
+    def time_interval_in_words(interval_in_seconds)
+      minutes = interval_in_seconds / 60
+      seconds = interval_in_seconds - minutes * 60
+
+      if minutes >= 1
+        "#{pluralize(minutes, "minute")} #{pluralize(seconds, "second")}"
+      else
+        "#{pluralize(seconds, "second")}"
+      end
+    end
+  end
+end

app/helpers/ci/builds_helper.rb

+module Ci
+  module BuildsHelper
+    def build_ref_link build
+      gitlab_ref_link build.project, build.ref
+    end
+
+    def build_compare_link build
+      gitlab_compare_link build.project, build.commit.short_before_sha, build.short_sha
+    end
+
+    def build_commit_link build
+      gitlab_commit_link build.project, build.short_sha
+    end
+
+    def build_url(build)
+      ci_project_build_url(build.project, build)
+    end
+
+    def build_status_alert_class(build)
+      if build.success?
+        'alert-success'
+      elsif build.failed?
+        'alert-danger'
+      elsif build.canceled?
+        'alert-disabled'
+      else
+        'alert-warning'
+      end
+    end
+
+    def build_icon_css_class(build)
+      if build.success?
+        'fa-circle cgreen'
+      elsif build.failed?
+        'fa-circle cred'
+      else
+        'fa-circle light'
+      end
+    end
+  end
+end

app/helpers/ci/commits_helper.rb

+module Ci
+  module CommitsHelper
+    def commit_status_alert_class(commit)
+      return 'alert-info' unless commit
+
+      case commit.status
+      when 'success'
+        'alert-success'
+      when 'failed', 'canceled'
+        'alert-danger'
+      when 'skipped'
+        'alert-disabled'
+      else
+        'alert-warning'
+      end
+    end
+
+    def ci_commit_path(commit)
+      ci_project_ref_commits_path(commit.project, commit.ref, commit.sha)
+    end
+
+    def commit_link(commit)
+      link_to(commit.short_sha, ci_commit_path(commit))
+    end
+
+    def truncate_first_line(message, length = 50)
+      truncate(message.each_line.first.chomp, length: length) if message
+    end
+
+    def ci_commit_title(commit)
+      content_tag :span do
+        link_to(
+          simple_sanitize(commit.project.name), ci_project_path(commit.project)
+        ) + ' @ ' +
+          gitlab_commit_link(@project, @commit.sha)
+      end
+    end
+  end
+end

app/helpers/ci/gitlab_helper.rb

+module Ci
+  module GitlabHelper
+    def no_turbolink
+      { :"data-no-turbolink" => "data-no-turbolink" }
+    end
+
+    def gitlab_ref_link project, ref
+      gitlab_url = project.gitlab_url.dup
+      gitlab_url << "/commits/#{ref}"
+      link_to ref, gitlab_url, no_turbolink
+    end
+
+    def gitlab_compare_link project, before, after
+      gitlab_url = project.gitlab_url.dup
+      gitlab_url << "/compare/#{before}...#{after}"
+
+      link_to "#{before}...#{after}", gitlab_url, no_turbolink
+    end
+
+    def gitlab_commit_link project, sha
+      gitlab_url = project.gitlab_url.dup
+      gitlab_url << "/commit/#{sha}"
+      link_to Ci::Commit.truncate_sha(sha), gitlab_url, no_turbolink
+    end
+
+    def yaml_web_editor_link(project)
+      commits = project.commits
+
+      if commits.any? && commits.last.push_data[:ci_yaml_file]
+        "#{@project.gitlab_url}/edit/master/.gitlab-ci.yml"
+      else
+        "#{@project.gitlab_url}/new/master"
+      end
+    end
+  end
+end

app/helpers/ci/icons_helper.rb

+module Ci
+  module IconsHelper
+    def boolean_to_icon(value)
+      if value.to_s == "true"
+        content_tag :i, nil, class: 'fa fa-circle cgreen'
+      else
+        content_tag :i, nil, class: 'fa fa-power-off clgray'
+      end
+    end
+  end
+end

app/helpers/ci/projects_helper.rb

+module Ci
+  module ProjectsHelper
+    def ref_tab_class ref = nil
+      'active' if ref == @ref
+    end
+
+    def success_ratio(success_builds, failed_builds)
+      failed_builds = failed_builds.count(:all)
+      success_builds = success_builds.count(:all)
+
+      return 100 if failed_builds.zero?
+
+      ratio = (success_builds.to_f / (success_builds + failed_builds)) * 100
+      ratio.to_i
+    end
+
+    def markdown_badge_code(project, ref)
+      url = status_ci_project_url(project, ref: ref, format: 'png')
+      "[![build status](#{url})](#{ci_project_url(project, ref: ref)})"
+    end
+
+    def html_badge_code(project, ref)
+      url = status_ci_project_url(project, ref: ref, format: 'png')
+      "<a href='#{ci_project_url(project, ref: ref)}'><img src='#{url}' /></a>"
+    end
+
+    def project_uses_specific_runner?(project)
+      project.runners.any?
+    end
+
+    def no_runners_for_project?(project)
+      project.runners.blank? &&
+        Ci::Runner.shared.blank?
+    end
+  end
+end

app/helpers/ci/routes_helper.rb

+module Ci
+  module RoutesHelper
+    class Base
+      include Gitlab::Application.routes.url_helpers
+
+      def default_url_options
+        {
+          host: Settings.gitlab['host'],
+          protocol: Settings.gitlab['https'] ? "https" : "http",
+          port: Settings.gitlab['port']
+        }
+      end
+    end
+
+    def url_helpers
+      @url_helpers ||= Base.new
+    end
+
+    def self.method_missing(method, *args, &block)
+      @url_helpers ||= Base.new
+
+      if @url_helpers.respond_to?(method)
+        @url_helpers.send(method, *args, &block)
+      else
+        super method, *args, &block
+      end
+    end
+  end
+end

app/helpers/ci/runners_helper.rb

+module Ci
+  module RunnersHelper
+    def runner_status_icon(runner)
+      unless runner.contacted_at
+        return content_tag :i, nil,
+          class: "fa fa-warning-sign",
+          title: "New runner. Has not connected yet"
+      end
+
+      status =
+        if runner.active?
+          runner.contacted_at > 3.hour.ago ? :online : :offline
+        else
+          :paused
+        end
+
+      content_tag :i, nil,
+        class: "fa fa-circle runner-status-#{status}",
+        title: "Runner is #{status}, last contact was #{time_ago_in_words(runner.contacted_at)} ago"
+    end
+  end
+end

app/helpers/ci/triggers_helper.rb

+module Ci
+  module TriggersHelper
+    def ci_build_trigger_url(project_id, ref_name)
+      "#{Settings.gitlab_ci.url}/ci/api/v1/projects/#{project_id}/refs/#{ref_name}/trigger"
+    end
+  end
+end

app/helpers/ci/user_helper.rb

+module Ci
+  module UserHelper
+    def user_avatar_url(user = nil, size = nil, default = 'identicon')
+      size = 40 if size.nil? || size <= 0
+
+      if user.blank? || user.avatar_url.blank?
+        'ci/no_avatar.png'
+      elsif /^(http(s?):\/\/(www|secure)\.gravatar\.com\/avatar\/(\w*))/ =~ user.avatar_url
+        Regexp.last_match[0] + "?s=#{size}&d=#{default}"
+      else
+        user.avatar_url
+      end
+    end
+  end
+end

app/mailers/ci/emails/builds.rb

+module Ci
+  module Emails
+    module Builds
+      def build_fail_email(build_id, to)
+        @build = Ci::Build.find(build_id)
+        @project = @build.project
+        mail(to: to, subject: subject("Build failed for #{@project.name}", @build.short_sha))
+      end
+
+      def build_success_email(build_id, to)
+        @build = Ci::Build.find(build_id)
+        @project = @build.project
+        mail(to: to, subject: subject("Build success for #{@project.name}", @build.short_sha))
+      end
+    end
+  end
+end

app/mailers/ci/notify.rb

+module Ci
+  class Notify < ActionMailer::Base
+    include Ci::Emails::Builds
+
+    add_template_helper Ci::ApplicationHelper
+    add_template_helper Ci::GitlabHelper
+
+    default_url_options[:host]     = Gitlab.config.gitlab.host
+    default_url_options[:protocol] = Gitlab.config.gitlab.protocol
+    default_url_options[:port]     = Gitlab.config.gitlab.port unless Gitlab.config.gitlab_on_standard_port?
+    default_url_options[:script_name] = Gitlab.config.gitlab.relative_url_root
+
+    default from: Gitlab.config.gitlab.email_from
+
+    # Just send email with 3 seconds delay
+    def self.delay
+      delay_for(2.seconds)
+    end
+
+    private
+
+    # Formats arguments into a String suitable for use as an email subject
+    #
+    # extra - Extra Strings to be inserted into the subject
+    #
+    # Examples
+    #
+    #   >> subject('Lorem ipsum')
+    #   => "GitLab-CI | Lorem ipsum"
+    #
+    #   # Automatically inserts Project name when @project is set
+    #   >> @project = Project.last
+    #   => #<Project id: 1, name: "Ruby on Rails", path: "ruby_on_rails", ...>
+    #   >> subject('Lorem ipsum')
+    #   => "GitLab-CI | Ruby on Rails | Lorem ipsum "
+    #
+    #   # Accepts multiple arguments
+    #   >> subject('Lorem ipsum', 'Dolor sit amet')
+    #   => "GitLab-CI | Lorem ipsum | Dolor sit amet"
+    def subject(*extra)
+      subject = "GitLab-CI"
+      subject << (@project ? " | #{@project.name}" : "")
+      subject << " | " + extra.join(' | ') if extra.present?
+      subject
+    end
+  end
+end

app/mailers/notify.rb

@@ -100,7 +100,7 @@ class Notify < BaseMailer
   def mail_thread(model, headers = {})
     if @project
-      headers['X-GitLab-Project'] = @project.name 
+      headers['X-GitLab-Project'] = @project.name
       headers['X-GitLab-Project-Id'] = @project.id
       headers['X-GitLab-Project-Path'] = @project.path_with_namespace
     end

app/models/ability.rb

@@ -149,6 +149,7 @@ class Ability
         :admin_merge_request,
         :create_merge_request,
         :create_wiki,
+        :manage_builds,
         :push_code
       ]
     end