Add latest changes from gitlab-org/gitlab@master

PROCESS.md

@@ -79,7 +79,7 @@ star, smile, etc.). Some good tips about code reviews can be found in our
 Overview and details of feature flag processes in development of GitLab itself is described in [feature flags process documentation](https://docs.gitlab.com/ee/development/feature_flags/process.html).
-Guides on how to include feature flags in your backend/frontend code while developing GitLab are described in [developing with feature flags documentation](https://docs.gitlab.com/ee/development/feature_flags/developing.html).
+Guides on how to include feature flags in your backend/frontend code while developing GitLab are described in [developing with feature flags documentation](https://docs.gitlab.com/ee/development/feature_flags/development.html).
 Getting access and how to expose the feature to users is detailed in [controlling feature flags documentation](https://docs.gitlab.com/ee/development/feature_flags/controls.html).

app/assets/javascripts/vue_shared/components/markdown/header.vue

@@ -124,7 +124,7 @@ export default {
               :cursor-offset="4"
               :tag-content="lineContent"
               icon="doc-code"
-              class="qa-suggestion-btn js-suggestion-btn"
+              class="js-suggestion-btn"
               @click="handleSuggestDismissed"
             />
             <gl-popover

app/assets/javascripts/vue_shared/components/markdown/suggestion_diff_header.vue

@@ -55,7 +55,7 @@ export default {
     <gl-button
       v-else-if="canApply"
       v-gl-tooltip.viewport="__('This also resolves the discussion')"
-      class="btn-inverted qa-apply-btn js-apply-btn"
+      class="btn-inverted js-apply-btn"
       :disabled="isApplying"
       variant="success"
       @click="applySuggestion"

app/graphql/types/base_enum.rb

@@ -2,5 +2,18 @@
 module Types
   class BaseEnum < GraphQL::Schema::Enum
+    class << self
+      def value(*args, **kwargs, &block)
+        enum[args[0].downcase] = kwargs[:value] || args[0]
+
+        super(*args, **kwargs, &block)
+      end
+
+      # Returns an indifferent access hash with the key being the downcased name of the attribute
+      # and the value being the Ruby value (either the explicit `value` passed or the same as the value attr).
+      def enum
+        @enum_values ||= {}.with_indifferent_access
+      end
+    end
   end
 end

app/models/service.rb

@@ -40,6 +40,7 @@ class Service < ApplicationRecord
   scope :external_wikis, -> { where(type: 'ExternalWikiService').active }
   scope :active, -> { where(active: true) }
   scope :without_defaults, -> { where(default: false) }
+  scope :by_type, -> (type) { where(type: type) }
   scope :push_hooks, -> { where(push_events: true, active: true) }
   scope :tag_push_hooks, -> { where(tag_push_events: true, active: true) }

changelogs/unreleased/31134-add-usage-ping-data-for-project-services.yml

+---
+title: Add usage ping data for project services
+merge_request: 19687
+author:
+type: added

changelogs/unreleased/ab-projects-api-indexes.yml

+---
+title: Add index for unauthenticated requests to projects API default endpoint
+merge_request: 19989
+author:
+type: performance

db/migrate/20191112214305_add_indexes_for_projects_api_default_params.rb

+# frozen_string_literal: true
+
+class AddIndexesForProjectsApiDefaultParams < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :projects, %i(visibility_level created_at id)
+    remove_concurrent_index_by_name :projects, 'index_projects_on_visibility_level'
+  end
+
+  def down
+    add_concurrent_index :projects, :visibility_level
+    remove_concurrent_index :projects, %i(visibility_level created_at id)
+  end
+end

db/post_migrate/20191017045817_schedule_fix_gitlab_com_pages_access_level.rb

@@ -3,34 +3,13 @@
 # See http://doc.gitlab.com/ce/development/migration_style_guide.html
 # for more information on how to write migrations for GitLab.
-# TODO: remove this migration after execution on gitlab.com https://gitlab.com/gitlab-org/gitlab/issues/34018
+# Code of this migration was removed after execution on gitlab.com
+# https://gitlab.com/gitlab-org/gitlab/issues/34018
+# Empty migration is left here to avoid any problems with rolling back
 class ScheduleFixGitlabComPagesAccessLevel < ActiveRecord::Migration[5.2]
-  include Gitlab::Database::MigrationHelpers
-
   DOWNTIME = false
-  MIGRATION = 'FixGitlabComPagesAccessLevel'
-  BATCH_SIZE = 20_000
-  BATCH_TIME = 2.minutes
-
-  # Project
-  class Project < ActiveRecord::Base
-    include EachBatch
-
-    self.table_name = 'projects'
-    self.inheritance_column = :_type_disabled
-  end
-
-  disable_ddl_transaction!
-
   def up
-    return unless ::Gitlab.com?
-
-    queue_background_migration_jobs_by_range_at_intervals(
-      Project,
-      MIGRATION,
-      BATCH_TIME,
-      batch_size: BATCH_SIZE)
   end
   def down

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema.define(version: 2019_11_12_115317) do
+ActiveRecord::Schema.define(version: 2019_11_12_214305) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "pg_trgm"
@@ -3145,7 +3145,7 @@ ActiveRecord::Schema.define(version: 2019_11_12_115317) do
     t.index ["runners_token"], name: "index_projects_on_runners_token"
     t.index ["runners_token_encrypted"], name: "index_projects_on_runners_token_encrypted"
     t.index ["star_count"], name: "index_projects_on_star_count"
-    t.index ["visibility_level"], name: "index_projects_on_visibility_level"
+    t.index ["visibility_level", "created_at", "id"], name: "index_projects_on_visibility_level_and_created_at_and_id"
   end
   create_table "prometheus_alert_events", force: :cascade do |t|

lib/gitlab/background_migration/fix_gitlab_com_pages_access_level.rb

-# frozen_string_literal: true
-
-module Gitlab
-  module BackgroundMigration
-    # Fixes https://gitlab.com/gitlab-org/gitlab/issues/32961
-    class FixGitlabComPagesAccessLevel
-      # Copy routable here to avoid relying on application logic
-      module Routable
-        def build_full_path
-          if parent && path
-            parent.build_full_path + '/' + path
-          else
-            path
-          end
-        end
-      end
-
-      # Namespace
-      class Namespace < ActiveRecord::Base
-        self.table_name = 'namespaces'
-        self.inheritance_column = :_type_disabled
-
-        include Routable
-
-        belongs_to :parent, class_name: "Namespace"
-      end
-
-      # ProjectPagesMetadatum
-      class ProjectPagesMetadatum < ActiveRecord::Base
-        self.primary_key = :project_id
-
-        belongs_to :project, inverse_of: :pages_metadatum
-
-        scope :deployed, -> { where(deployed: true) }
-      end
-
-      # Project
-      class Project < ActiveRecord::Base
-        self.table_name = 'projects'
-        self.inheritance_column = :_type_disabled
-
-        include Routable
-
-        belongs_to :namespace
-        alias_method :parent, :namespace
-        alias_attribute :parent_id, :namespace_id
-
-        has_one :project_feature, inverse_of: :project
-
-        has_one :pages_metadatum, class_name: 'ProjectPagesMetadatum', inverse_of: :project
-
-        scope :with_pages_deployed, -> do
-          joins(:pages_metadatum).merge(ProjectPagesMetadatum.deployed)
-        end
-
-        PRIVATE = 0
-        INTERNAL = 10
-        PUBLIC = 20
-
-        delegate :public_pages?, to: :project_feature
-
-        def public_pages_path
-          File.join(pages_path, 'public')
-        end
-
-        def pages_path
-          File.join(Settings.pages.path, build_full_path)
-        end
-
-        def public?
-          visibility_level == PUBLIC
-        end
-      end
-
-      # ProjectFeature
-      class ProjectFeature < ActiveRecord::Base
-        self.table_name = 'project_features'
-
-        belongs_to :project
-
-        DISABLED = 0
-        PRIVATE  = 10
-        ENABLED  = 20
-        PUBLIC   = 30
-      end
-
-      def perform(start_id, stop_id)
-        logger = Gitlab::BackgroundMigration::Logger.build
-
-        Project.where(id: start_id..stop_id).with_pages_deployed.includes(:project_feature).find_each do |project|
-          config_path = File.join(project.pages_path, 'config.json')
-          ac_is_enabled_in_config = JSON.parse(File.read(config_path))["access_control"]
-
-          next if ac_is_enabled_in_config # we already made site private and don't want to surprise the user
-
-          next if project.project_feature.pages_access_level == ProjectFeature::DISABLED
-
-          new_access_level = project.public? ? ProjectFeature::ENABLED : ProjectFeature::PUBLIC
-
-          next if project.project_feature.pages_access_level == new_access_level
-
-          logger.info(
-            message: "Changing pages access control level",
-            project_id: project.id,
-            access_level_before: project.project_feature.pages_access_level,
-            access_level_after: new_access_level
-          )
-
-          project.project_feature.update_column(:pages_access_level, new_access_level)
-        rescue => e
-          Gitlab::Sentry.track_exception(e, extra: { project_id: project.id })
-        end
-      end
-    end
-  end
-end

lib/gitlab/usage_data.rb

@@ -13,7 +13,8 @@ module Gitlab
       end
       def uncached_data
-        license_usage_data.merge(system_usage_data)
+        license_usage_data
+          .merge(system_usage_data)
           .merge(features_usage_data)
           .merge(components_usage_data)
           .merge(cycle_analytics_usage_data)
@@ -170,10 +171,13 @@ module Gitlab
         types = {
           SlackService: :projects_slack_notifications_active,
           SlackSlashCommandsService: :projects_slack_slash_active,
-          PrometheusService: :projects_prometheus_active
+          PrometheusService: :projects_prometheus_active,
+          CustomIssueTrackerService: :projects_custom_issue_tracker_active,
+          JenkinsService: :projects_jenkins_active,
+          MattermostService: :projects_mattermost_active
         }
-        results = count(Service.unscoped.where(type: types.keys, active: true).group(:type), fallback: Hash.new(-1))
+        results = count(Service.active.by_type(types.keys).group(:type), fallback: Hash.new(-1))
         types.each_with_object({}) { |(klass, key), response| response[key] = results[klass.to_s] || 0 }
           .merge(jira_usage)
       end
@@ -188,8 +192,8 @@ module Gitlab
           projects_jira_active: -1
         }
-        Service.unscoped
-          .where(type: :JiraService, active: true)
+        Service.active
+          .by_type(:JiraService)
           .includes(:jira_tracker_data)
           .find_in_batches(batch_size: BATCH_SIZE) do |services|

spec/controllers/projects/pipelines_controller_spec.rb

@@ -149,7 +149,7 @@ describe Projects::PipelinesController do
   end
   describe 'GET show.json' do
-    let(:pipeline) { create(:ci_pipeline_with_one_job, project: project) }
+    let(:pipeline) { create(:ci_pipeline, project: project) }
     it 'returns the pipeline' do
       get_pipeline_json

spec/factories/ci/pipelines.rb

 # frozen_string_literal: true
 FactoryBot.define do
+  # TODO: we can remove this factory in favour of :ci_pipeline
   factory :ci_empty_pipeline, class: Ci::Pipeline do
     source { :push }
     ref { 'master' }
@@ -10,20 +11,6 @@ FactoryBot.define do
     project
-    factory :ci_pipeline_without_jobs do
-      after(:build) do |pipeline|
-        pipeline.instance_variable_set(:@ci_yaml_file, YAML.dump({}))
-      end
-    end
-
-    factory :ci_pipeline_with_one_job do
-      after(:build) do |pipeline|
-        allow(pipeline).to receive(:ci_yaml_file) do
-          pipeline.instance_variable_set(:@ci_yaml_file, YAML.dump({ rspec: { script: "ls" } }))
-        end
-      end
-    end
-
     # Persist merge request head_pipeline_id
     # on pipeline factories to avoid circular references
     transient { head_pipeline_of { nil } }
@@ -34,24 +21,8 @@ FactoryBot.define do
     end
     factory :ci_pipeline do
-      transient { config { nil } }
-
-      after(:build) do |pipeline, evaluator|
-        if evaluator.config
-          pipeline.instance_variable_set(:@ci_yaml_file, YAML.dump(evaluator.config))
-
-          # Populates pipeline with errors
-          pipeline.config_processor if evaluator.config
-        else
-          pipeline.instance_variable_set(:@ci_yaml_file, File.read(Rails.root.join('spec/support/gitlab_stubs/gitlab_ci.yml')))
-        end
-      end
-
       trait :invalid do
-        config do
-          { rspec: nil }
-        end
-
+        yaml_errors { 'invalid YAML' }
         failure_reason { :config_error }
       end

spec/factories/commit_statuses.rb

@@ -7,7 +7,7 @@ FactoryBot.define do
     stage_idx { 0 }
     status { 'success' }
     description { 'commit status'}
-    pipeline factory: :ci_pipeline_with_one_job
+    pipeline factory: :ci_pipeline
     started_at { 'Tue, 26 Jan 2016 08:21:42 +0100'}
     finished_at { 'Tue, 26 Jan 2016 08:23:42 +0100'}

spec/features/merge_request/user_sees_deployment_widget_spec.rb

@@ -11,7 +11,7 @@ describe 'Merge request > User sees deployment widget', :js do
     let(:role) { :developer }
     let(:ref) { merge_request.target_branch }
     let(:sha) { project.commit(ref).id }
-    let(:pipeline) { create(:ci_pipeline_without_jobs, sha: sha, project: project, ref: ref) }
+    let(:pipeline) { create(:ci_pipeline, sha: sha, project: project, ref: ref) }
     let!(:manual) { }
     before do
@@ -33,7 +33,7 @@ describe 'Merge request > User sees deployment widget', :js do
       end
       context 'when a user created a new merge request with the same SHA' do
-        let(:pipeline2) { create(:ci_pipeline_without_jobs, sha: sha, project: project, ref: 'new-patch-1') }
+        let(:pipeline2) { create(:ci_pipeline, sha: sha, project: project, ref: 'new-patch-1') }
         let(:build2) { create(:ci_build, :success, pipeline: pipeline2) }
         let(:environment2) { create(:environment, project: project) }
         let!(:deployment2) { create(:deployment, environment: environment2, sha: sha, ref: 'new-patch-1', deployable: build2) }

spec/features/merge_request/user_sees_merge_widget_spec.rb

@@ -44,7 +44,7 @@ describe 'Merge request > User sees merge widget', :js do
   context 'view merge request' do
     let!(:environment) { create(:environment, project: project) }
     let(:sha)          { project.commit(merge_request.source_branch).sha }
-    let(:pipeline)     { create(:ci_pipeline_without_jobs, status: 'success', sha: sha, project: project, ref: merge_request.source_branch) }
+    let(:pipeline)     { create(:ci_pipeline, status: 'success', sha: sha, project: project, ref: merge_request.source_branch) }
     let(:build)        { create(:ci_build, :success, pipeline: pipeline) }
     let!(:deployment) do
@@ -745,7 +745,7 @@ describe 'Merge request > User sees merge widget', :js do
   context 'when MR has pipeline but user does not have permission' do
     let(:sha) { project.commit(merge_request.source_branch).sha }
-    let!(:pipeline) { create(:ci_pipeline_without_jobs, status: 'success', sha: sha, project: project, ref: merge_request.source_branch) }
+    let!(:pipeline) { create(:ci_pipeline, status: 'success', sha: sha, project: project, ref: merge_request.source_branch) }
     before do
       project.update(

spec/graphql/types/base_enum_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Types::BaseEnum do
+  describe '#enum' do
+    let(:enum) do
+      Class.new(described_class) do
+        value 'TEST', value: 3
+        value 'other'
+        value 'NORMAL'
+      end
+    end
+
+    it 'adds all enum values to #enum' do
+      expect(enum.enum.keys).to contain_exactly('test', 'other', 'normal')
+      expect(enum.enum.values).to contain_exactly(3, 'other', 'NORMAL')
+    end
+
+    it 'is a HashWithIndefferentAccess' do
+      expect(enum.enum).to be_a(HashWithIndifferentAccess)
+    end
+  end
+end

spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb

@@ -7,9 +7,7 @@ describe Gitlab::Ci::Pipeline::Chain::Populate do
   set(:user) { create(:user) }
   let(:pipeline) do
-    build(:ci_pipeline_with_one_job, project: project,
-                                     ref: 'master',
-                                     user: user)
+    build(:ci_pipeline, project: project, ref: 'master', user: user)
   end
   let(:command) do
@@ -22,6 +20,14 @@ describe Gitlab::Ci::Pipeline::Chain::Populate do
   let(:step) { described_class.new(pipeline, command) }
+  let(:config) do
+    { rspec: { script: 'rspec' } }
+  end
+
+  before do
+    stub_ci_pipeline_yaml_file(YAML.dump(config))
+  end
+
   context 'when pipeline doesn not have seeds block' do
     before do
       step.perform!
@@ -59,10 +65,6 @@ describe Gitlab::Ci::Pipeline::Chain::Populate do
       } }
     end
-    let(:pipeline) do
-      build(:ci_pipeline, project: project, config: config)
-    end
-
     before do
       step.perform!
     end
@@ -202,10 +204,6 @@ describe Gitlab::Ci::Pipeline::Chain::Populate do
           prod: { script: 'cap prod', stage: 'deploy', only: ['tags'] } }
       end
-      let(:pipeline) do
-        build(:ci_pipeline, ref: 'master', project: project, config: config)
-      end
-
       it_behaves_like 'a correct pipeline'
       context 'when variables expression is specified' do

spec/lib/gitlab/ci/pipeline/chain/remove_unwanted_chat_jobs_spec.rb

@@ -6,13 +6,17 @@ describe Gitlab::Ci::Pipeline::Chain::RemoveUnwantedChatJobs do
   let(:project) { create(:project, :repository) }
   let(:pipeline) do
-    build(:ci_pipeline_with_one_job, project: project, ref: 'master')
+    build(:ci_pipeline, project: project)
   end
   let(:command) do
     double(:command, project: project, chat_data: { command: 'echo' })
   end
+  before do
+    stub_ci_pipeline_yaml_file(YAML.dump(rspec: { script: 'rspec' }))
+  end
+
   describe '#perform!' do
     it 'removes unwanted jobs for chat pipelines' do
       allow(pipeline).to receive(:chat?).and_return(true)