Merge branch 'ci-and-ce-sitting-in-a-tree-k-i-s-s-i-n-g' into 'master'

Merge CI into CE

First step of #2164.

- [x] Merge latest CE master
- [x] Make application start
- [x] Re-use gitlab sessions (remove CI oauth part)
- [x] Get rid of gitlab_ci.yml config
- [x] Make tests start
- [x] Make most CI features works
- [x] Make tests green
- [x] Write migration documentation
- [x] Add CI builds to CE backup

See merge request !1204

app/models/project_services/jira_service.rb

@@ -19,7 +19,7 @@
 #
 class JiraService < IssueTrackerService
-  include Rails.application.routes.url_helpers
+  include Gitlab::Application.routes.url_helpers
   prop_accessor :title, :description, :project_url, :issues_url, :new_issue_url

app/models/user.rb

@@ -753,4 +753,13 @@ class User < ActiveRecord::Base
   def can_be_removed?
     !solo_owned_groups.present?
   end
+
+  def ci_authorized_projects
+    @ci_authorized_projects ||= Ci::Project.where(gitlab_id: authorized_projects)
+  end
+
+  def ci_authorized_runners
+    Ci::Runner.specific.includes(:runner_projects).
+      where(ci_runner_projects: { project_id: ci_authorized_projects } )
+  end
 end

app/services/ci/create_commit_service.rb

+module Ci
+  class CreateCommitService
+    def execute(project, params)
+      before_sha = params[:before]
+      sha = params[:checkout_sha] || params[:after]
+      origin_ref = params[:ref]
+      
+      unless origin_ref && sha.present?
+        return false
+      end
+
+      ref = origin_ref.gsub(/\Arefs\/(tags|heads)\//, '')
+
+      # Skip branch removal
+      if sha == Ci::Git::BLANK_SHA
+        return false
+      end
+
+      commit = project.commits.find_by_sha_and_ref(sha, ref)
+
+      # Create commit if not exists yet
+      unless commit
+        data = {
+          ref: ref,
+          sha: sha,
+          tag: origin_ref.start_with?('refs/tags/'),
+          before_sha: before_sha,
+          push_data: {
+            before: before_sha,
+            after: sha,
+            ref: ref,
+            user_name: params[:user_name],
+            user_email: params[:user_email],
+            repository: params[:repository],
+            commits: params[:commits],
+            total_commits_count: params[:total_commits_count],
+            ci_yaml_file: params[:ci_yaml_file]
+          }
+        }
+
+        commit = project.commits.create(data)
+      end
+
+      commit.update_committed!
+      commit.create_builds unless commit.builds.any?
+
+      commit
+    end
+  end
+end

app/services/ci/create_project_service.rb

+module Ci
+  class CreateProjectService
+    include Gitlab::Application.routes.url_helpers
+
+    def execute(current_user, params, project_route, forked_project = nil)
+      @project = Ci::Project.parse(params)
+
+      Ci::Project.transaction do
+        @project.save!
+
+        data = {
+          token: @project.token,
+          project_url: project_route.gsub(":project_id", @project.id.to_s),
+        }
+
+        gl_project = ::Project.find(@project.gitlab_id)
+        gl_project.build_missing_services
+        gl_project.gitlab_ci_service.update_attributes(data.merge(active: true))
+      end
+
+      if forked_project
+        # Copy settings
+        settings = forked_project.attributes.select do |attr_name, value|
+          ["public", "shared_runners_enabled", "allow_git_fetch"].include? attr_name
+        end
+
+        @project.update(settings)
+      end
+
+      Ci::EventService.new.create_project(current_user, @project)
+
+      @project
+    end
+  end
+end

app/services/ci/create_trigger_request_service.rb

+module Ci
+  class CreateTriggerRequestService
+    def execute(project, trigger, ref, variables = nil)
+      commit = project.commits.where(ref: ref).last
+      return unless commit
+
+      trigger_request = trigger.trigger_requests.create!(
+        commit: commit,
+        variables: variables
+      )
+
+      if commit.create_builds(trigger_request)
+        trigger_request
+      end
+    end
+  end
+end

app/services/ci/event_service.rb

+module Ci
+  class EventService
+    def remove_project(user, project)
+      create(
+        description: "Project \"#{project.name}\" has been removed by #{user.username}",
+        user_id: user.id,
+        is_admin: true
+      )
+    end
+
+    def create_project(user, project)
+      create(
+        description: "Project \"#{project.name}\" has been created by #{user.username}",
+        user_id: user.id,
+        is_admin: true
+      )
+    end
+
+    def change_project_settings(user, project)
+      create(
+        project_id: project.id,
+        user_id: user.id,
+        description: "User \"#{user.username}\" updated projects settings"
+      )
+    end
+
+    def create(*args)
+      Ci::Event.create!(*args)
+    end
+  end
+end

app/services/ci/image_for_build_service.rb

+module Ci
+  class ImageForBuildService
+    def execute(project, params)
+      image_name =
+        if params[:sha]
+          commit = project.commits.find_by(sha: params[:sha])
+          image_for_commit(commit)
+        elsif params[:ref]
+          commit = project.last_commit_for_ref(params[:ref])
+          image_for_commit(commit)
+        else
+          'build-unknown.svg'
+        end
+
+      image_path = Rails.root.join('public/ci', image_name)
+
+      OpenStruct.new(
+        path: image_path,
+        name: image_name
+      )
+    end
+
+    private
+
+    def image_for_commit(commit)
+      return 'build-unknown.svg' unless commit
+
+      'build-' + commit.status + ".svg"
+    end
+  end
+end

app/services/ci/register_build_service.rb

+module Ci
+  # This class responsible for assigning
+  # proper pending build to runner on runner API request
+  class RegisterBuildService
+    def execute(current_runner)
+      builds = Ci::Build.pending.unstarted
+
+      builds =
+        if current_runner.shared?
+          # don't run projects which have not enables shared runners
+          builds.includes(:project).where(ci_projects: { shared_runners_enabled: true })
+        else
+          # do run projects which are only assigned to this runner
+          builds.where(project_id: current_runner.projects)
+        end
+
+      builds = builds.order('created_at ASC')
+
+      build = builds.find do |build|
+        (build.tag_list - current_runner.tag_list).empty?
+      end
+        
+
+      if build
+        # In case when 2 runners try to assign the same build, second runner will be declined
+        # with StateMachine::InvalidTransition in run! method.
+        build.with_lock do
+          build.runner_id = current_runner.id
+          build.save!
+          build.run!
+        end
+      end
+
+      build
+
+    rescue StateMachine::InvalidTransition
+      nil
+    end
+  end
+end

app/services/ci/test_hook_service.rb

+module Ci
+  class TestHookService
+    def execute(hook, current_user)
+      Ci::WebHookService.new.build_end(hook.project.commits.last.last_build)
+    end
+  end
+end

app/services/ci/web_hook_service.rb

+module Ci
+  class WebHookService
+    def build_end(build)
+      execute_hooks(build.project, build_data(build))
+    end
+
+    def execute_hooks(project, data)
+      project.web_hooks.each do |web_hook|
+        async_execute_hook(web_hook, data)
+      end
+    end
+
+    def async_execute_hook(hook, data)
+      Sidekiq::Client.enqueue(Ci::WebHookWorker, hook.id, data)
+    end
+
+    def build_data(build)
+      project = build.project
+      data = {}
+      data.merge!({
+        build_id: build.id,
+        build_name: build.name,
+        build_status: build.status,
+        build_started_at: build.started_at,
+        build_finished_at: build.finished_at,
+        project_id: project.id,
+        project_name: project.name,
+        gitlab_url: project.gitlab_url,
+        ref: build.ref,
+        sha: build.sha,
+        before_sha: build.before_sha,
+        push_data: build.commit.push_data
+      })
+    end
+  end
+end

app/views/ci/admin/application_settings/_form.html.haml

+= form_for @application_setting, url: ci_admin_application_settings_path, html: { class: 'form-horizontal fieldset-form' } do |f|
+  - if @application_setting.errors.any?
+    #error_explanation
+      .alert.alert-danger
+        - @application_setting.errors.full_messages.each do |msg|
+          %p= msg
+
+  %fieldset
+    %legend Default Project Settings
+    .form-group
+      .col-sm-offset-2.col-sm-10
+        .checkbox
+          = f.label :all_broken_builds do
+            = f.check_box :all_broken_builds
+            Send emails only on broken builds
+    .form-group
+      .col-sm-offset-2.col-sm-10
+        .checkbox
+          = f.label :add_pusher do
+            = f.check_box :add_pusher
+            Add pusher to recipients list
+
+  .form-actions
+    = f.submit 'Save', class: 'btn btn-primary'

app/views/ci/admin/application_settings/show.html.haml

+%h3.page-title Settings
+%hr
+= render 'form'

app/views/ci/admin/builds/_build.html.haml

+- if build.commit && build.project
+  %tr.build.alert{class: build_status_alert_class(build)}
+    %td.build-link
+      = link_to ci_project_build_url(build.project, build) do
+        %strong #{build.id}
+
+    %td.status
+      = build.status
+
+    %td.commit-link
+      = commit_link(build.commit)
+
+    %td.runner
+      - if build.runner
+        = link_to build.runner.id, ci_admin_runner_path(build.runner)
+
+    %td.build-project
+      = truncate build.project.name, length: 30
+
+    %td.build-message
+      %span= truncate(build.commit.git_commit_message, length: 30)
+
+    %td.build-branch
+      %span= truncate(build.ref, length: 25)
+
+    %td.duration
+      - if build.duration
+        #{duration_in_words(build.finished_at, build.started_at)}
+
+    %td.timestamp
+      - if build.finished_at
+        %span #{time_ago_in_words build.finished_at} ago

app/views/ci/admin/builds/index.html.haml

+%ul.nav.nav-tabs.append-bottom-20
+  %li{class: ("active" if @scope.nil?)}
+    = link_to 'All builds', ci_admin_builds_path
+  
+  %li{class: ("active" if @scope == "pending")}
+    = link_to "Pending", ci_admin_builds_path(scope: :pending)
+
+  %li{class: ("active" if @scope == "running")}
+    = link_to "Running", ci_admin_builds_path(scope: :running)
+
+
+%table.builds
+  %thead
+    %tr
+      %th Build
+      %th Status
+      %th Commit
+      %th Runner
+      %th Project
+      %th Message
+      %th Branch
+      %th Duration
+      %th Finished at
+
+  - @builds.each do |build|
+    = render "ci/admin/builds/build", build: build
+
+= paginate @builds

app/views/ci/admin/events/index.html.haml

+%table.table
+  %thead
+    %tr
+      %th User ID
+      %th Description
+      %th When
+  - @events.each do |event|
+    %tr
+      %td
+        = event.user_id
+      %td
+        = event.description
+      %td.light
+        = time_ago_in_words event.updated_at
+        ago
+
+= paginate @events
\ No newline at end of file

app/views/ci/admin/projects/_project.html.haml

+- last_commit = project.last_commit
+%tr.alert{class: commit_status_alert_class(last_commit) }
+  %td
+    = project.id
+  %td
+    = link_to [:ci, project] do
+      %strong= project.name
+  %td
+    - if last_commit
+      #{last_commit.status} (#{commit_link(last_commit)})
+      - if project.last_commit_date
+        = time_ago_in_words project.last_commit_date
+        ago
+    - else
+      No builds yet
+  %td
+    - if project.public
+      %i.fa.fa-globe
+      Public
+    - else
+      %i.fa.fa-lock
+      Private
+  %td
+    = project.commits.count
+  %td
+    = link_to [:ci, :admin, project], method: :delete, class: 'btn btn-danger btn-sm' do
+      %i.fa.fa-remove
+      Remove

app/views/ci/admin/projects/index.html.haml

+%table.table
+  %thead
+    %tr
+      %th ID
+      %th Name
+      %th Last build
+      %th Access
+      %th Builds
+      %th
+
+  - @projects.each do |project|
+    = render "ci/admin/projects/project", project: project
+
+= paginate @projects
+

app/views/ci/admin/runner_projects/index.html.haml

+%p.lead
+  To register new runner visit #{link_to 'this page ', ci_runners_path}
+
+.row
+  .col-md-8
+    %h5 Activated:
+    %table.table
+      %tr
+        %th Runner ID
+        %th Runner Description
+        %th Last build
+        %th Builds Stats
+        %th Registered
+        %th
+
+      - @runner_projects.each do |runner_project|
+        - runner = runner_project.runner
+        - builds = runner.builds.where(project_id: @project.id)
+        %tr
+          %td
+            %span.badge.badge-info= runner.id
+          %td
+            = runner.display_name
+          %td
+            - last_build = builds.last
+            - if last_build
+              = link_to last_build.short_sha, [last_build.project, last_build]
+            - else
+              unknown
+          %td
+            %span.badge.badge-success
+              #{builds.success.count}
+            %span /
+            %span.badge.badge-important
+              #{builds.failed.count}
+          %td
+            #{time_ago_in_words(runner_project.created_at)} ago
+          %td
+            = link_to 'Disable', [:ci, @project, runner_project], data: { confirm: "Are you sure?" }, method: :delete, class: 'btn btn-danger btn-sm right'
+  .col-md-4
+    %h5 Available
+    %table.table
+      %tr
+        %th ID
+        %th Token
+        %th
+
+      - (Ci::Runner.all - @project.runners).each do |runner|
+        %tr
+          %td
+            = runner.id
+          %td
+            = runner.token
+          %td
+            = form_for [:ci, @project, @runner_project] do |f|
+              = f.hidden_field :runner_id, value: runner.id
+              = f.submit 'Add', class: 'btn btn-sm'

app/views/ci/admin/runners/_runner.html.haml

+%tr{id: dom_id(runner)}
+  %td
+    - if runner.shared?
+      %span.label.label-success shared
+    - else
+      %span.label.label-info specific
+    - unless runner.active?
+      %span.label.label-danger paused
+
+  %td
+    = link_to ci_admin_runner_path(runner) do
+      = runner.short_sha
+  %td
+    .runner-description
+      = runner.description
+      %span (#{link_to 'edit', '#', class: 'edit-runner-link'})
+    .runner-description-form.hide
+      = form_for [:ci, :admin, runner], remote: true, html: { class: 'form-inline' } do |f|
+        .form-group
+          = f.text_field :description, class: 'form-control'
+        = f.submit 'Save', class: 'btn'
+        %span (#{link_to 'cancel', '#', class: 'cancel'})
+  %td
+    - if runner.shared?
+      \-
+    - else
+      = runner.projects.count(:all)
+  %td
+    #{runner.builds.count(:all)}
+  %td
+    - runner.tag_list.each do |tag|
+      %span.label.label-primary
+        = tag
+  %td
+    - if runner.contacted_at
+      #{time_ago_in_words(runner.contacted_at)} ago
+    - else
+      Never
+  %td
+    .pull-right
+      = link_to 'Edit', ci_admin_runner_path(runner), class: 'btn btn-sm'
+       
+      - if runner.active?
+        = link_to 'Pause', [:pause, :ci, :admin, runner], data: { confirm: "Are you sure?" }, method: :get, class: 'btn btn-danger btn-sm'
+      - else
+        = link_to 'Resume', [:resume, :ci, :admin, runner], method: :get, class: 'btn btn-success btn-sm'
+      = link_to 'Remove', [:ci, :admin, runner], data: { confirm: "Are you sure?" }, method: :delete, class: 'btn btn-danger btn-sm'
+

app/views/ci/admin/runners/index.html.haml

+%p.lead
+  %span To register new runner you should enter the following registration token. With this token the runner will request a unique runner token and use that for future communication.
+  %code #{GitlabCi::REGISTRATION_TOKEN}
+
+.bs-callout
+  %p
+    A 'runner' is a process which runs a build.
+    You can setup as many runners as you need.
+    %br
+    Runners can be placed on separate users, servers, and even on your local machine.
+    %br
+
+  %div
+    %span Each runner can be in one of the following states:
+    %ul
+      %li
+        %span.label.label-success shared
+        \- run builds from all unassigned projects
+      %li
+        %span.label.label-info specific
+        \- run builds from assigned projects
+      %li
+        %span.label.label-danger paused
+        \- runner will not receive any new build
+
+.append-bottom-20.clearfix
+  .pull-left
+    = form_tag ci_admin_runners_path, id: 'runners-search', class: 'form-inline', method: :get do
+      .form-group
+        = search_field_tag :search, params[:search], class: 'form-control', placeholder: 'Runner description or token'
+      = submit_tag 'Search', class: 'btn'
+
+  .pull-right.light
+    Runners with last contact less than a minute ago: #{@active_runners_cnt}
+
+%br
+
+%table.table
+  %thead
+    %tr
+      %th Type
+      %th Runner token
+      %th Description
+      %th Projects
+      %th Builds
+      %th Tags
+      %th Last contact
+      %th
+
+  - @runners.each do |runner|
+    = render "ci/admin/runners/runner", runner: runner
+= paginate @runners