Merge branch 'with-pipeline-view' into 'master'

Add pipeline view

This is continuation of https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/3653

cc @DouweM @grzesiek 

Fixes https://gitlab.com/gitlab-org/gitlab-ce/issues/17551
Fixes https://gitlab.com/gitlab-org/gitlab-ce/issues/15625


See merge request !3703

app/assets/stylesheets/pages/pipelines.scss

+.pipeline-stage {
+  overflow: hidden;
+  text-overflow: ellipsis;
+}

app/controllers/projects/pipelines_controller.rb

+class Projects::PipelinesController < Projects::ApplicationController
+  before_action :pipeline, except: [:index, :new, :create]
+  before_action :commit, only: [:show]
+  before_action :authorize_read_pipeline!
+  before_action :authorize_create_pipeline!, only: [:new, :create]
+  before_action :authorize_update_pipeline!, only: [:retry, :cancel]
+
+  def index
+    @scope = params[:scope]
+    all_pipelines = project.ci_commits
+    @pipelines_count = all_pipelines.count
+    @running_or_pending_count = all_pipelines.running_or_pending.count
+    @pipelines = PipelinesFinder.new(project).execute(all_pipelines, @scope)
+    @pipelines = @pipelines.order(id: :desc).page(params[:page]).per(30)
+  end
+
+  def new
+    @pipeline = project.ci_commits.new(ref: @project.default_branch)
+  end
+
+  def create
+    @pipeline = Ci::CreatePipelineService.new(project, current_user, create_params).execute
+    unless @pipeline.persisted?
+      render 'new'
+      return
+    end
+
+    redirect_to namespace_project_pipeline_path(project.namespace, project, @pipeline)
+  end
+
+  def show
+  end
+
+  def retry
+    pipeline.retry_failed
+
+    redirect_back_or_default default: namespace_project_pipelines_path(project.namespace, project)
+  end
+
+  def cancel
+    pipeline.cancel_running
+
+    redirect_back_or_default default: namespace_project_pipelines_path(project.namespace, project)
+  end
+
+  private
+
+  def create_params
+    params.require(:pipeline).permit(:ref)
+  end
+
+  def pipeline
+    @pipeline ||= project.ci_commits.find_by!(id: params[:id])
+  end
+
+  def commit
+    @commit ||= @pipeline.commit_data
+  end
+end

app/finders/pipelines_finder.rb

+class PipelinesFinder
+  attr_reader :project
+
+  def initialize(project)
+    @project = project
+  end
+
+  def execute(pipelines, scope)
+    case scope
+    when 'running'
+      pipelines.running_or_pending
+    when 'branches'
+      from_ids(pipelines, ids_for_ref(pipelines, branches))
+    when 'tags'
+      from_ids(pipelines, ids_for_ref(pipelines, tags))
+    else
+      pipelines
+    end
+  end
+
+  private
+
+  def ids_for_ref(pipelines, refs)
+    pipelines.where(ref: refs).group(:ref).select('max(id)')
+  end
+
+  def from_ids(pipelines, ids)
+    pipelines.unscoped.where(id: ids)
+  end
+
+  def branches
+    project.repository.branches.map(&:name)
+  end
+
+  def tags
+    project.repository.tags.map(&:name)
+  end
+end

app/helpers/ci_status_helper.rb

@@ -38,19 +38,30 @@ module CiStatusHelper
     icon(icon_name + ' fw')
   end
-  def render_ci_status(ci_commit, tooltip_placement: 'auto left')
-    # TODO: split this method into
-    # - render_commit_status
-    # - render_pipeline_status
-    link_to ci_icon_for_status(ci_commit.status),
-      ci_status_path(ci_commit),
-      class: "ci-status-link ci-status-icon-#{ci_commit.status.dasherize}",
-      title: "Build #{ci_label_for_status(ci_commit.status)}",
-      data: { toggle: 'tooltip', placement: tooltip_placement }
+  def render_commit_status(commit, tooltip_placement: 'auto left')
+    project = commit.project
+    path = builds_namespace_project_commit_path(project.namespace, project, commit)
+    render_status_with_link('commit', commit.status, path, tooltip_placement)
+  end
+
+  def render_pipeline_status(pipeline, tooltip_placement: 'auto left')
+    project = pipeline.project
+    path = namespace_project_pipeline_path(project.namespace, project, pipeline)
+    render_status_with_link('pipeline', pipeline.status, path, tooltip_placement)
   end
   def no_runners_for_project?(project)
     project.runners.blank? &&
       Ci::Runner.shared.blank?
   end
+
+  private
+
+  def render_status_with_link(type, status, path, tooltip_placement)
+    link_to ci_icon_for_status(status),
+            path,
+            class: "ci-status-link ci-status-icon-#{status.dasherize}",
+            title: "#{type.titleize}: #{ci_label_for_status(status)}",
+            data: { toggle: 'tooltip', placement: tooltip_placement }
+  end
 end

app/models/ability.rb

@@ -205,6 +205,7 @@ class Ability
         :read_commit_status,
         :read_build,
         :read_container_image,
+        :read_pipeline,
       ]
     end
@@ -216,6 +217,8 @@ class Ability
         :update_commit_status,
         :create_build,
         :update_build,
+        :create_pipeline,
+        :update_pipeline,
         :create_merge_request,
         :create_wiki,
         :push_code,
@@ -248,6 +251,7 @@ class Ability
         :admin_commit_status,
         :admin_build,
         :admin_container_image,
+        :admin_pipeline
       ]
     end
@@ -290,6 +294,7 @@ class Ability
       unless project.builds_enabled
         rules += named_abilities('build')
+        rules += named_abilities('pipeline')
       end
       unless project.container_registry_enabled

app/models/ci/commit.rb

@@ -8,8 +8,6 @@ module Ci
     has_many :builds, class_name: 'Ci::Build'
     has_many :trigger_requests, dependent: :destroy, class_name: 'Ci::TriggerRequest'
-    delegate :stages, to: :statuses
-
     validates_presence_of :sha
     validates_presence_of :status
     validate :valid_commit_sha
@@ -22,7 +20,8 @@ module Ci
     end
     def self.stages
-      CommitStatus.where(commit: all).stages
+      # We use pluck here due to problems with MySQL which doesn't allow LIMIT/OFFSET in queries
+      CommitStatus.where(commit: pluck(:id)).stages
     end
     def project_id
@@ -67,6 +66,25 @@ module Ci
       end
     end
+    def cancel_running
+      builds.running_or_pending.each(&:cancel)
+    end
+
+    def retry_failed
+      builds.latest.failed.select(&:retryable?).each(&:retry)
+    end
+
+    def latest?
+      return false unless ref
+      commit = project.commit(ref)
+      return false unless commit
+      commit.sha == sha
+    end
+
+    def triggered?
+      trigger_requests.any?
+    end
+
     def create_builds(user, trigger_request = nil)
       return unless config_processor
       config_processor.stages.any? do |stage|

app/models/commit_status.rb

@@ -14,7 +14,8 @@ class CommitStatus < ActiveRecord::Base
   alias_attribute :author, :user
   scope :latest, -> { where(id: unscope(:select).select('max(id)').group(:name, :commit_id)) }
-  scope :ordered, -> { order(:ref, :stage_idx, :name) }
+  scope :retried, -> { where.not(id: latest) }
+  scope :ordered, -> { order(:name) }
   scope :ignored, -> { where(allow_failure: true, status: [:failed, :canceled]) }
   state_machine :status, initial: :pending do
@@ -54,13 +55,15 @@ class CommitStatus < ActiveRecord::Base
   end
   def self.stages
-    order_by = 'max(stage_idx)'
-    group('stage').order(order_by).pluck(:stage, order_by).map(&:first).compact
+    # We group by stage name, but order stages by theirs' index
+    unscoped.from(all, :sg).group('stage').order('max(stage_idx)', 'stage').pluck('sg.stage')
   end
   def self.stages_status
-    all.stages.inject({}) do |h, stage|
-      h[stage] = all.where(stage: stage).status
+    # We execute subquery for each stage to calculate a stage status
+    statuses = unscoped.from(all, :sg).group('stage').pluck('sg.stage', all.where('stage=sg.stage').status_sql)
+    statuses.inject({}) do |h, k|
+      h[k.first] = k.last
       h
     end
   end

app/services/ci/create_pipeline_service.rb

+module Ci
+  class CreatePipelineService < BaseService
+    def execute
+      pipeline = project.ci_commits.new(params)
+
+      unless ref_names.include?(params[:ref])
+        pipeline.errors.add(:base, 'Reference not found')
+        return pipeline
+      end
+
+      unless commit
+        pipeline.errors.add(:base, 'Commit not found')
+        return pipeline
+      end
+
+      unless can?(current_user, :create_pipeline, project)
+        pipeline.errors.add(:base, 'Insufficient permissions to create a new pipeline')
+        return pipeline
+      end
+
+      begin
+        Ci::Commit.transaction do
+          pipeline.sha = commit.id
+
+          unless pipeline.config_processor
+            pipeline.errors.add(:base, pipeline.yaml_errors || 'Missing .gitlab-ci.yml file')
+            raise ActiveRecord::Rollback
+          end
+
+          pipeline.save!
+          pipeline.create_builds(current_user)
+        end
+      rescue
+        pipeline.errors.add(:base, 'The pipeline could not be created. Please try again.')
+      end
+
+      pipeline
+    end
+
+    private
+
+    def ref_names
+      @ref_names ||= project.repository.ref_names
+    end
+
+    def commit
+      @commit ||= project.commit(params[:ref])
+    end
+  end
+end

app/views/layouts/nav/_project.html.haml

@@ -39,6 +39,13 @@
           Commits
   - if project_nav_tab? :builds
+    = nav_link(controller: :pipelines) do
+      = link_to project_pipelines_path(@project), title: 'Pipelines', class: 'shortcuts-pipelines' do
+        = icon('ship fw')
+        %span
+          Pipelines
+          %span.count.ci_counter= number_with_delimiter(@project.ci_commits.running_or_pending.count)
+
     = nav_link(controller: %w(builds)) do
       = link_to project_builds_path(@project), title: 'Builds', class: 'shortcuts-builds' do
         = icon('cubes fw')

app/views/projects/ci/builds/_build.html.haml

@@ -13,7 +13,9 @@
       %strong ##{build.id}
     - if build.stuck?
-      %i.fa.fa-warning.text-warning
+      = icon('warning', class: 'text-warning has-tooltip', title: 'Build is stuck. Check runners.')
+    - if defined?(retried) && retried
+      = icon('warning', class: 'text-warning has-tooltip', title: 'Build was retried.')
   - if defined?(commit_sha) && commit_sha
     %td
@@ -40,25 +42,29 @@
   %td
     = build.name
-  %td
-    .label-container
-      - if build.tags.any?
-        - build.tags.each do |tag|
-          %span.label.label-primary
-            = tag
-      - if build.try(:trigger_request)
-        %span.label.label-info triggered
-      - if build.try(:allow_failure)
-        %span.label.label-danger allowed to fail
-      - if defined?(retried) && retried
-        %span.label.label-warning retried
+    .pull-right
+      .label-container
+        - if build.tags.any?
+          - build.tags.each do |tag|
+            %span.label.label-primary
+              = tag
+        - if build.try(:trigger_request)
+          %span.label.label-info triggered
+        - if build.try(:allow_failure)
+          %span.label.label-danger allowed to fail
+        - if defined?(retried) && retried
+          %span.label.label-warning retried
   %td.duration
     - if build.duration
+      = icon("clock-o")
+       
       #{duration_in_words(build.finished_at, build.started_at)}
   %td.timestamp
     - if build.finished_at
+      = icon("calendar")
+       
       %span #{time_ago_with_tooltip(build.finished_at)}
   - if defined?(coverage) && coverage
@@ -70,11 +76,11 @@
     .pull-right
       - if can?(current_user, :read_build, build) && build.artifacts?
         = link_to download_namespace_project_build_artifacts_path(build.project.namespace, build.project, build), title: 'Download artifacts', class: 'btn btn-build' do
-          %i.fa.fa-download
+          = icon('download')
       - if can?(current_user, :update_build, build)
         - if build.active?
           = link_to cancel_namespace_project_build_path(build.project.namespace, build.project, build, return_to: request.original_url), method: :post, title: 'Cancel', class: 'btn btn-build' do
-            %i.fa.fa-remove.cred
+            = icon('remove', class: 'cred')
         - elsif defined?(allow_retry) && allow_retry && build.retryable?
           = link_to retry_namespace_project_build_path(build.project.namespace, build.project, build, return_to: request.original_url), method: :post, title: 'Retry', class: 'btn btn-build' do
-            %i.fa.fa-refresh
+            = icon('refresh')

app/views/projects/ci/commits/_commit.html.haml

+- status = commit.status
+%tr.commit
+  %td.commit-link
+    = link_to namespace_project_pipeline_path(@project.namespace, @project, commit.id), class: "ci-status ci-#{status}" do
+      = ci_icon_for_status(status)
+      %strong ##{commit.id}
+
+  %td
+    %div.branch-commit
+      - if commit.ref
+        = link_to commit.ref, namespace_project_commits_path(@project.namespace, @project, commit.ref), class: "monospace"
+        ·
+      = link_to commit.short_sha, namespace_project_commit_path(@project.namespace, @project, commit.sha), class: "commit-id monospace"
+       
+      - if commit.latest?
+        %span.label.label-success latest
+      - if commit.tag?
+        %span.label.label-primary tag
+      - if commit.triggered?
+        %span.label.label-primary triggered
+      - if commit.yaml_errors.present?
+        %span.label.label-danger.has-tooltip{ title: "#{commit.yaml_errors}" } yaml invalid
+      - if commit.builds.any?(&:stuck?)
+        %span.label.label-warning stuck
+
+      %p
+        %span
+          - if commit_data = commit.commit_data
+            = link_to_gfm commit_data.title, namespace_project_commit_path(@project.namespace, @project, commit_data.id), class: "commit-row-message"
+          - else
+            Cant find HEAD commit for this branch
+
+
+    - stages_status = commit.statuses.stages_status
+    - stages.each do |stage|
+      %td
+        - if status = stages_status[stage]
+          - tooltip = "#{stage.titleize}: #{status}"
+          %span.has-tooltip{ title: "#{tooltip}", class: "ci-status-icon-#{status}" }
+            = ci_icon_for_status(status)
+
+  %td
+    - if commit.started_at && commit.finished_at
+      %p
+        = icon("clock-o")
+         
+        #{duration_in_words(commit.finished_at, commit.started_at)}
+    - if commit.finished_at
+      %p
+        = icon("calendar")
+         
+        #{time_ago_with_tooltip(commit.finished_at)}
+
+  %td
+    .controls.hidden-xs.pull-right
+      - artifacts = commit.builds.latest.select { |b| b.artifacts? }
+      - if artifacts.present?
+        .dropdown.inline.build-artifacts
+          %button.dropdown-toggle.btn{type: 'button', 'data-toggle' => 'dropdown'}
+            = icon('download')
+            %b.caret
+          %ul.dropdown-menu.dropdown-menu-align-right
+            - artifacts.each do |build|
+              %li
+                = link_to download_namespace_project_build_artifacts_path(@project.namespace, @project, build), rel: 'nofollow' do
+                  = icon("download")
+                  %span #{build.name}
+
+      - if can?(current_user, :update_pipeline, @project)
+         
+        - if commit.retryable? && commit.builds.failed.any?
+          = link_to retry_namespace_project_pipeline_path(@project.namespace, @project, commit.id), class: 'btn has-tooltip', title: "Retry", method: :post do
+            = icon("repeat")
+         
+        - if commit.active?
+          = link_to cancel_namespace_project_pipeline_path(@project.namespace, @project, commit.id), class: 'btn btn-remove has-tooltip', title: "Cancel", method: :post do
+            = icon("remove")

app/views/projects/commit/_builds.html.haml

 - @ci_commits.each do |ci_commit|
-  = render "ci_commit", ci_commit: ci_commit
+  = render "ci_commit", ci_commit: ci_commit, pipeline_details: true

app/views/projects/commit/_ci_commit.html.haml

 .row-content-block.build-content.middle-block
   .pull-right
-    - if can?(current_user, :update_build, @project)
+    - if can?(current_user, :update_pipeline, @project)
       - if ci_commit.builds.latest.failed.any?(&:retryable?)
-        = link_to "Retry failed", retry_builds_namespace_project_commit_path(@project.namespace, @project, ci_commit.sha), class: 'btn btn-grouped btn-primary', method: :post
+        = link_to "Retry failed", retry_namespace_project_pipeline_path(@project.namespace, @project, ci_commit.id), class: 'btn btn-grouped btn-primary', method: :post
       - if ci_commit.builds.running_or_pending.any?
-        = link_to "Cancel running", cancel_builds_namespace_project_commit_path(@project.namespace, @project, ci_commit.sha), data: { confirm: 'Are you sure?' }, class: 'btn btn-grouped btn-danger', method: :post
-  .oneline
-    = pluralize ci_commit.statuses.count(:id), "build"
-    - if ci_commit.ref
-      for
-      %span.label.label-info
-        = ci_commit.ref
-    - if defined?(link_to_commit) && link_to_commit
-      for commit
-      = link_to ci_commit.short_sha, namespace_project_commit_path(@project.namespace, @project, ci_commit.sha), class: "monospace"
-    - if ci_commit.duration
-      in
-      = time_interval_in_words ci_commit.duration
+        = link_to "Cancel running", cancel_namespace_project_pipeline_path(@project.namespace, @project, ci_commit.id), data: { confirm: 'Are you sure?' }, class: 'btn btn-grouped btn-danger', method: :post
+  .oneline.clearfix
+    - if defined?(pipeline_details) && pipeline_details
+      Pipeline
+      = link_to "##{ci_commit.id}", namespace_project_pipeline_path(@project.namespace, @project, ci_commit.id), class: "monospace"
+      with
+      = pluralize ci_commit.statuses.count(:id), "build"
+      - if ci_commit.ref
+        for
+        = link_to ci_commit.ref, namespace_project_commits_path(@project.namespace, @project, ci_commit.ref), class: "monospace"
+      - if defined?(link_to_commit) && link_to_commit
+        for commit
+        = link_to ci_commit.short_sha, namespace_project_commit_path(@project.namespace, @project, ci_commit.sha), class: "monospace"
+      - if ci_commit.duration
+        in
+        = time_interval_in_words ci_commit.duration
 - if ci_commit.yaml_errors.present?
   .bs-callout.bs-callout-danger
@@ -34,38 +37,5 @@
 .table-holder
   %table.table.builds
-    %thead
-      %tr
-        %th Status
-        %th Build ID
-        %th Stage
-        %th Name
-        %th Tags
-        %th Duration
-        %th Finished at
-        - if @project.build_coverage_enabled?
-          %th Coverage
-        %th
-    - builds = ci_commit.statuses.latest.ordered
-    = render builds, coverage: @project.build_coverage_enabled?, stage: true, ref: false, allow_retry: true
-
-- if ci_commit.retried.any?
-  .row-content-block.second-block
-    Retried builds
-
-  .table-holder
-    %table.table.builds
-      %thead
-        %tr
-          %th Status
-          %th Build ID
-          %th Ref
-          %th Stage
-          %th Name
-          %th Tags
-          %th Duration
-          %th Finished at
-          - if @project.build_coverage_enabled?
-            %th Coverage
-          %th
-      = render ci_commit.retried, coverage: @project.build_coverage_enabled?, stage: true, ref: false
+    - ci_commit.statuses.stages.each do |stage|
+      = render 'projects/commit/ci_stage', stage: stage, statuses: ci_commit.statuses.where(stage: stage)

app/views/projects/commit/_ci_stage.html.haml

+%tr
+  %th{colspan: 10}
+    %strong
+      - status = statuses.latest.status
+      %span{class: "ci-status-link ci-status-icon-#{status}"}
+        = ci_icon_for_status(status)
+      - if stage
+         
+        = stage.titleize.pluralize
+  = render statuses.latest.ordered, coverage: @project.build_coverage_enabled?, stage: false, ref: false, allow_retry: true
+  = render statuses.retried.ordered, coverage: @project.build_coverage_enabled?, stage: false, ref: false, retried: true
+  %tr
+    %td{colspan: 10}
+       

app/views/projects/commit/_commit_box.html.haml

 .pull-right.commit-action-buttons
   %div
-    - if @notes_count > 0
+    - if defined?(@notes_count) && @notes_count > 0
       %span.btn.disabled.btn-grouped
         %i.fa.fa-comment
         = @notes_count
@@ -23,11 +23,6 @@
 %p
 .commit-info-row
-  - if @commit.status
-    = link_to builds_namespace_project_commit_path(@project.namespace, @project, @commit.id), class: "ci-status ci-#{@commit.status}" do
-      = ci_icon_for_status(@commit.status)
-      build:
-      = ci_label_for_status(@commit.status)
   %span.light Authored by
   %strong
     = commit_author_link(@commit, avatar: true, size: 24)
@@ -51,6 +46,17 @@
   %span.commit-info.branches
     %i.fa.fa-spinner.fa-spin
+- if @commit.status
+  .commit-info-row
+    Builds for
+    = pluralize(@commit.ci_commits.count, 'pipeline')
+    = link_to builds_namespace_project_commit_path(@project.namespace, @project, @commit.id), class: "ci-status-link ci-status-icon-#{@commit.status}" do
+      = ci_icon_for_status(@commit.status)
+      = ci_label_for_status(@commit.status)
+    - if @commit.ci_commits.duration
+      in
+      = time_interval_in_words @commit.ci_commits.duration
+
 .commit-box.content-block
   %h3.commit-title
     = markdown escape_once(@commit.title), pipeline: :single_line

app/views/projects/commits/_commit.html.haml

@@ -17,7 +17,7 @@
       .pull-right
         - if commit.status
-          = render_ci_status(commit)
+          = render_commit_status(commit)
         = clipboard_button(clipboard_text: commit.id)
         = link_to commit.short_id, namespace_project_commit_path(project.namespace, project, commit), class: "commit_short_id"

app/views/projects/generic_commit_statuses/_generic_commit_status.html.haml

@@ -12,6 +12,9 @@
     - else
       %strong ##{generic_commit_status.id}
+    - if defined?(retried) && retried
+      = icon('warning', class: 'text-warning has-tooltip', title: 'Status was retried.')
+
   - if defined?(commit_sha) && commit_sha
     %td
       = link_to generic_commit_status.short_sha, namespace_project_commit_path(generic_commit_status.project.namespace, generic_commit_status.project, generic_commit_status.sha), class: "monospace"
@@ -42,13 +45,19 @@
         - generic_commit_status.tags.each do |tag|
           %span.label.label-primary
             = tag
+      - if defined?(retried) && retried
+        %span.label.label-warning retried
   %td.duration
     - if generic_commit_status.duration
+      = icon("clock-o")
+       
       #{duration_in_words(generic_commit_status.finished_at, generic_commit_status.started_at)}
   %td.timestamp
     - if generic_commit_status.finished_at
+      = icon("calendar")
+       
       %span #{time_ago_with_tooltip(generic_commit_status.finished_at)}
   - if defined?(coverage) && coverage

app/views/projects/issues/_merge_requests.html.haml

@@ -7,7 +7,7 @@
       %li
         %span.merge-request-ci-status
           - if merge_request.ci_commit
-            = render_ci_status(merge_request.ci_commit)
+            = render_pipeline_status(merge_request.ci_commit)
           - elsif has_any_ci
             = icon('blank fw')
         %span.merge-request-id

app/views/projects/issues/_related_branches.html.haml

@@ -8,7 +8,7 @@
         - ci_commit = @project.ci_commit(sha, branch) if sha
         - if ci_commit
           %span.related-branch-ci-status
-            = render_ci_status(ci_commit)
+            = render_pipeline_status(ci_commit)
         %span.related-branch-info
           %strong
             = link_to namespace_project_compare_path(@project.namespace, @project, from: @project.default_branch, to: branch), class: "label-branch" do

app/views/projects/merge_requests/_merge_request.html.haml

@@ -13,7 +13,7 @@
       - if merge_request.ci_commit
         %li
-          = render_ci_status(merge_request.ci_commit)
+          = render_pipeline_status(merge_request.ci_commit)
       - if merge_request.open? && merge_request.broken?
         %li