Add latest changes from gitlab-org/gitlab@master

app/controllers/projects/prometheus/metrics_controller.rb

@@ -23,7 +23,7 @@ module Projects
       private
       def prometheus_adapter
-        @prometheus_adapter ||= ::Prometheus::AdapterService.new(project).prometheus_adapter
+        @prometheus_adapter ||= ::Prometheus::AdapterService.new(project, project.deployment_platform&.cluster).prometheus_adapter
       end
       def require_prometheus_metrics!

app/models/ci/build.rb

@@ -719,8 +719,8 @@ module Ci
         end
     end
-    def has_expiring_archive_artifacts?
-      has_expiring_artifacts? && artifacts_file&.exists?
+    def has_expiring_artifacts?
+      artifacts_expire_at.present? && artifacts_expire_at > Time.now
     end
     def keep_artifacts!
@@ -935,10 +935,6 @@ module Ci
         value.with_indifferent_access
       end
     end
-
-    def has_expiring_artifacts?
-      artifacts_expire_at.present? && artifacts_expire_at > Time.now
-    end
   end
 end

app/models/ci/pipeline.rb

@@ -62,6 +62,8 @@ module Ci
     has_one :triggered_by_pipeline, through: :source_pipeline, source: :source_pipeline
     has_one :source_job, through: :source_pipeline, source: :source_job
+    has_one :pipeline_config, class_name: 'Ci::PipelineConfig', inverse_of: :pipeline
+
     accepts_nested_attributes_for :variables, reject_if: :persisted?
     delegate :id, to: :project, prefix: true

app/models/ci/pipeline_config.rb

+# frozen_string_literal: true
+
+module Ci
+  class PipelineConfig < ApplicationRecord
+    extend Gitlab::Ci::Model
+
+    self.table_name = 'ci_pipelines_config'
+    self.primary_key = :pipeline_id
+
+    belongs_to :pipeline, class_name: "Ci::Pipeline", inverse_of: :pipeline_config
+    validates :pipeline, presence: true
+    validates :content, presence: true
+  end
+end

app/models/environment.rb

@@ -223,7 +223,7 @@ class Environment < ApplicationRecord
   # rubocop: disable CodeReuse/ServiceClass
   def prometheus_adapter
-    @prometheus_adapter ||= Prometheus::AdapterService.new(project, deployment_platform).prometheus_adapter
+    @prometheus_adapter ||= Prometheus::AdapterService.new(project, deployment_platform&.cluster).prometheus_adapter
   end
   # rubocop: enable CodeReuse/ServiceClass

app/serializers/build_artifact_entity.rb

@@ -14,7 +14,7 @@ class BuildArtifactEntity < Grape::Entity
     download_project_job_artifacts_path(project, job)
   end
-  expose :keep_path, if: -> (*) { job.has_expiring_archive_artifacts? } do |job|
+  expose :keep_path, if: -> (*) { job.has_expiring_artifacts? } do |job|
     keep_project_job_artifacts_path(project, job)
   end

app/serializers/build_details_entity.rb

@@ -31,7 +31,7 @@ class BuildDetailsEntity < JobEntity
       browse_project_job_artifacts_path(project, build)
     end
-    expose :keep_path, if: -> (*) { build.has_expiring_archive_artifacts? && can?(current_user, :update_build, build) } do |build|
+    expose :keep_path, if: -> (*) { build.has_expiring_artifacts? && can?(current_user, :update_build, build) } do |build|
       keep_project_job_artifacts_path(project, build)
     end

app/services/prometheus/adapter_service.rb

@@ -2,18 +2,13 @@
 module Prometheus
   class AdapterService
-    def initialize(project, deployment_platform = nil)
-      @project = project
-      @deployment_platform = if deployment_platform
-                               deployment_platform
-                             else
-                               project.deployment_platform
-                             end
+    attr_reader :project, :cluster
+    def initialize(project, cluster)
+      @project = project
+      @cluster = cluster
     end
-    attr_reader :deployment_platform, :project
-
     def prometheus_adapter
       @prometheus_adapter ||= if service_prometheus_adapter.can_query?
                                 service_prometheus_adapter
@@ -27,7 +22,7 @@ module Prometheus
     end
     def cluster_prometheus_adapter
-      application = deployment_platform&.cluster&.application_prometheus
+      application = cluster&.application_prometheus
       application if application&.available?
     end

changelogs/unreleased/118640_add_gitlab_version_and_revision_to_export.yml

+---
+title: Add Gitlab version and revision to export
+merge_request: 22108
+author:
+type: added

changelogs/unreleased/38242-persisting-pipeline-config_content.yml

+---
+title: Implement saving config content for pipelines in a new table 'ci_pipelines_config'
+merge_request: 21827
+author:
+type: other

changelogs/unreleased/eb-no-keep-report-artifacts.yml

----
-title: Remove keep button for non archive artifacts
-merge_request: 21553
-author:
-type: fixed

db/migrate/20191213143656_create_ci_pipelines_config.rb

+# frozen_string_literal: true
+
+class CreateCiPipelinesConfig < ActiveRecord::Migration[5.2]
+  DOWNTIME = false
+
+  def change
+    create_table :ci_pipelines_config, id: false do |t|
+      t.references :pipeline,
+                   primary_key: true,
+                   foreign_key: { to_table: :ci_pipelines, on_delete: :cascade }
+      t.text :content, null: false
+    end
+  end
+end

db/schema.rb

@@ -876,6 +876,11 @@ ActiveRecord::Schema.define(version: 2019_12_18_225624) do
     t.index ["user_id"], name: "index_ci_pipelines_on_user_id"
   end
+  create_table "ci_pipelines_config", primary_key: "pipeline_id", force: :cascade do |t|
+    t.text "content", null: false
+    t.index ["pipeline_id"], name: "index_ci_pipelines_config_on_pipeline_id"
+  end
+
   create_table "ci_resource_groups", force: :cascade do |t|
     t.datetime_with_timezone "created_at", null: false
     t.datetime_with_timezone "updated_at", null: false
@@ -4438,6 +4443,7 @@ ActiveRecord::Schema.define(version: 2019_12_18_225624) do
   add_foreign_key "ci_pipelines", "external_pull_requests", name: "fk_190998ef09", on_delete: :nullify
   add_foreign_key "ci_pipelines", "merge_requests", name: "fk_a23be95014", on_delete: :cascade
   add_foreign_key "ci_pipelines", "projects", name: "fk_86635dbd80", on_delete: :cascade
+  add_foreign_key "ci_pipelines_config", "ci_pipelines", column: "pipeline_id", on_delete: :cascade
   add_foreign_key "ci_resource_groups", "projects", name: "fk_774722d144", on_delete: :cascade
   add_foreign_key "ci_resources", "ci_builds", column: "build_id", name: "fk_e169a8e3d5", on_delete: :nullify
   add_foreign_key "ci_resources", "ci_resource_groups", column: "resource_group_id", on_delete: :cascade

doc/integration/elasticsearch.md

@@ -200,8 +200,7 @@ To backfill existing data, you can use one of the methods below to index it in b
 To index via the admin area:
-1. Navigate to the **Admin Area** (wrench icon), then **Settings > Integrations** and expand the **Elasticsearch** section.
-1. [Enable **Elasticsearch indexing** and configure your host and port](#enabling-elasticsearch).
+1. [Configure your Elasticsearch host and port](#enabling-elasticsearch).
 1. Create empty indexes using one of the following commands:
    ```sh
@@ -212,7 +211,8 @@ To index via the admin area:
    bundle exec rake gitlab:elastic:create_empty_index RAILS_ENV=production
    ```
-1. Click **Index all projects**.
+1. [Enable **Elasticsearch indexing**](#enabling-elasticsearch).
+1. Click **Index all projects** in **Admin Area > Settings > Integrations > Elasticsearch**.
 1. Click **Check progress** in the confirmation message to see the status of the background jobs.
 1. Personal snippets need to be indexed manually by running one of these commands:
@@ -255,7 +255,7 @@ Performing asynchronous indexing will generate a lot of Sidekiq jobs.
 Make sure to prepare for this task by either [Horizontally Scaling](../administration/high_availability/README.md#basic-scaling)
 or creating [extra Sidekiq processes](../administration/operations/extra_sidekiq_processes.md)
-1. [Enable **Elasticsearch indexing** and configure your host and port](#enabling-elasticsearch).
+1. [Configure your Elasticsearch host and port](#enabling-elasticsearch).
 1. Create empty indexes using one of the following commands:
    ```sh
@@ -266,6 +266,7 @@ or creating [extra Sidekiq processes](../administration/operations/extra_sidekiq
    bundle exec rake gitlab:elastic:create_empty_index RAILS_ENV=production
    ```
+1. [Enable **Elasticsearch indexing**](#enabling-elasticsearch).
 1. Indexing large Git repositories can take a while. To speed up the process, you
    can temporarily disable auto-refreshing and replicating. In our experience, you can expect a 20%
    decrease in indexing time. We'll enable them when indexing is done. This step is optional!

lib/gitlab/ci/pipeline/chain/config/content.rb

@@ -24,8 +24,7 @@ module Gitlab
             def perform!
               if config = find_config
-                # TODO: we should persist config_content
-                # @pipeline.config_content = config.content
+                @pipeline.build_pipeline_config(content: config.content) if ci_root_config_content_enabled?
                 @command.config_content = config.content
                 @pipeline.config_source = config.source
               else
@@ -49,11 +48,11 @@ module Gitlab
             end
             def sources
-              if Feature.enabled?(:ci_root_config_content, @command.project, default_enabled: true)
-                SOURCES
-              else
-                LEGACY_SOURCES
-              end
+              ci_root_config_content_enabled? ? SOURCES : LEGACY_SOURCES
+            end
+
+            def ci_root_config_content_enabled?
+              Feature.enabled?(:ci_root_config_content, @command.project, default_enabled: true)
             end
           end
         end

lib/gitlab/import_export.rb

@@ -50,6 +50,14 @@ module Gitlab
       'VERSION'
     end
+    def gitlab_version_filename
+      'GITLAB_VERSION'
+    end
+
+    def gitlab_revision_filename
+      'GITLAB_REVISION'
+    end
+
     def export_filename(exportable:)
       basename = "#{Time.now.strftime('%Y-%m-%d_%H-%M-%3N')}_#{exportable.full_path.tr('/', '_')}"

lib/gitlab/import_export/group_tree_saver.rb

@@ -3,7 +3,7 @@
 module Gitlab
   module ImportExport
     class GroupTreeSaver
-      attr_reader :full_path
+      attr_reader :full_path, :shared
       def initialize(group:, current_user:, shared:, params: {})
         @params       = params

lib/gitlab/import_export/version_saver.rb

@@ -13,6 +13,8 @@ module Gitlab
         mkdir_p(@shared.export_path)
         File.write(version_file, Gitlab::ImportExport.version, mode: 'w')
+        File.write(gitlab_version_file, Gitlab::VERSION, mode: 'w')
+        File.write(gitlab_revision_file, Gitlab.revision, mode: 'w')
       rescue => e
         @shared.error(e)
         false
@@ -20,6 +22,14 @@ module Gitlab
       private
+      def gitlab_version_file
+        File.join(@shared.export_path, Gitlab::ImportExport.gitlab_version_filename)
+      end
+
+      def gitlab_revision_file
+        File.join(@shared.export_path, Gitlab::ImportExport.gitlab_revision_filename)
+      end
+
       def version_file
         File.join(@shared.export_path, Gitlab::ImportExport.version_filename)
       end

lib/gitlab/kubernetes/kube_client.rb

@@ -17,6 +17,7 @@ module Gitlab
         core: { group: 'api', version: 'v1' },
         rbac: { group: 'apis/rbac.authorization.k8s.io', version: 'v1' },
         extensions: { group: 'apis/extensions', version: 'v1beta1' },
+        istio: { group: 'apis/networking.istio.io', version: 'v1alpha3' },
         knative: { group: 'apis/serving.knative.dev', version: 'v1alpha1' }
       }.freeze
@@ -91,6 +92,13 @@ module Gitlab
         :watch_pod_log,
         to: :core_client
+      # Gateway methods delegate to the apis/networking.istio.io api
+      # group client
+      delegate :create_gateway,
+        :get_gateway,
+        :update_gateway,
+        to: :istio_client
+
       attr_reader :api_prefix, :kubeclient_options
       # We disable redirects through 'http_max_redirects: 0',

spec/lib/gitlab/ci/pipeline/chain/config/content_spec.rb

@@ -29,6 +29,7 @@ describe Gitlab::Ci::Pipeline::Chain::Config::Content do
           subject.perform!
           expect(pipeline.config_source).to eq 'repository_source'
+          expect(pipeline.pipeline_config).to be_nil
           expect(command.config_content).to eq('the-content')
         end
       end
@@ -40,6 +41,7 @@ describe Gitlab::Ci::Pipeline::Chain::Config::Content do
           subject.perform!
           expect(pipeline.config_source).to eq 'auto_devops_source'
+          expect(pipeline.pipeline_config).to be_nil
           template = Gitlab::Template::GitlabCiYmlTemplate.find('Beta/Auto-DevOps')
           expect(command.config_content).to eq(template.content)
         end
@@ -52,6 +54,7 @@ describe Gitlab::Ci::Pipeline::Chain::Config::Content do
           subject.perform!
           expect(pipeline.config_source).to eq 'auto_devops_source'
+          expect(pipeline.pipeline_config).to be_nil
           template = Gitlab::Template::GitlabCiYmlTemplate.find('Beta/Auto-DevOps')
           expect(command.config_content).to eq(template.content)
         end
@@ -71,6 +74,7 @@ describe Gitlab::Ci::Pipeline::Chain::Config::Content do
           subject.perform!
           expect(pipeline.config_source).to eq 'repository_source'
+          expect(pipeline.pipeline_config).to be_nil
           expect(command.config_content).to eq('the-content')
         end
       end
@@ -91,6 +95,7 @@ describe Gitlab::Ci::Pipeline::Chain::Config::Content do
             subject.perform!
             expect(pipeline.config_source).to eq 'auto_devops_source'
+            expect(pipeline.pipeline_config).to be_nil
             template = Gitlab::Template::GitlabCiYmlTemplate.find('Beta/Auto-DevOps')
             expect(command.config_content).to eq(template.content)
           end
@@ -105,6 +110,7 @@ describe Gitlab::Ci::Pipeline::Chain::Config::Content do
             subject.perform!
             expect(pipeline.config_source).to eq 'auto_devops_source'
+            expect(pipeline.pipeline_config).to be_nil
             template = Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps')
             expect(command.config_content).to eq(template.content)
           end
@@ -122,6 +128,7 @@ describe Gitlab::Ci::Pipeline::Chain::Config::Content do
           subject.perform!
           expect(pipeline.config_source).to eq('unknown_source')
+          expect(pipeline.pipeline_config).to be_nil
           expect(command.config_content).to be_nil
           expect(pipeline.errors.full_messages).to include('Missing CI config file')
         end
@@ -130,6 +137,13 @@ describe Gitlab::Ci::Pipeline::Chain::Config::Content do
     context 'when config is defined in a custom path in the repository' do
       let(:ci_config_path) { 'path/to/config.yml' }
+      let(:config_content_result) do
+        <<~EOY
+          ---
+          include:
+          - local: #{ci_config_path}
+        EOY
+      end
       before do
         expect(project.repository)
@@ -142,47 +156,59 @@ describe Gitlab::Ci::Pipeline::Chain::Config::Content do
         subject.perform!
         expect(pipeline.config_source).to eq 'repository_source'
-        expect(command.config_content).to eq(<<~EOY)
-          ---
-          include:
-          - local: #{ci_config_path}
-        EOY
+        expect(pipeline.pipeline_config.content).to eq(config_content_result)
+        expect(command.config_content).to eq(config_content_result)
       end
     end
     context 'when config is defined remotely' do
       let(:ci_config_path) { 'http://example.com/path/to/ci/config.yml' }
+      let(:config_content_result) do
+        <<~EOY
+          ---
+          include:
+          - remote: #{ci_config_path}
+        EOY
+      end
       it 'builds root config including the remote config' do
         subject.perform!
         expect(pipeline.config_source).to eq 'remote_source'
-        expect(command.config_content).to eq(<<~EOY)
-          ---
-          include:
-          - remote: #{ci_config_path}
-        EOY
+        expect(pipeline.pipeline_config.content).to eq(config_content_result)
+        expect(command.config_content).to eq(config_content_result)
       end
     end
     context 'when config is defined in a separate repository' do
       let(:ci_config_path) { 'path/to/.gitlab-ci.yml@another-group/another-repo' }
-
-      it 'builds root config including the path to another repository' do
-        subject.perform!
-
-        expect(pipeline.config_source).to eq 'external_project_source'
-        expect(command.config_content).to eq(<<~EOY)
+      let(:config_content_result) do
+        <<~EOY
           ---
           include:
           - project: another-group/another-repo
             file: path/to/.gitlab-ci.yml
         EOY
       end
+
+      it 'builds root config including the path to another repository' do
+        subject.perform!
+
+        expect(pipeline.config_source).to eq 'external_project_source'
+        expect(pipeline.pipeline_config.content).to eq(config_content_result)
+        expect(command.config_content).to eq(config_content_result)
+      end
     end
     context 'when config is defined in the default .gitlab-ci.yml' do
       let(:ci_config_path) { nil }
+      let(:config_content_result) do
+        <<~EOY
+          ---
+          include:
+          - local: ".gitlab-ci.yml"
+        EOY
+      end
       before do
         expect(project.repository)
@@ -195,16 +221,20 @@ describe Gitlab::Ci::Pipeline::Chain::Config::Content do
         subject.perform!
         expect(pipeline.config_source).to eq 'repository_source'
-        expect(command.config_content).to eq(<<~EOY)
-          ---
-          include:
-          - local: ".gitlab-ci.yml"
-        EOY
+        expect(pipeline.pipeline_config.content).to eq(config_content_result)
+        expect(command.config_content).to eq(config_content_result)
       end
     end
     context 'when config is the Auto-Devops template' do
       let(:ci_config_path) { nil }
+      let(:config_content_result) do
+        <<~EOY
+          ---
+          include:
+          - template: Beta/Auto-DevOps.gitlab-ci.yml
+        EOY
+      end
       before do
         expect(project).to receive(:auto_devops_enabled?).and_return(true)
@@ -219,11 +249,8 @@ describe Gitlab::Ci::Pipeline::Chain::Config::Content do
           subject.perform!
           expect(pipeline.config_source).to eq 'auto_devops_source'
-          expect(command.config_content).to eq(<<~EOY)
-            ---
-            include:
-            - template: Beta/Auto-DevOps.gitlab-ci.yml
-          EOY
+          expect(pipeline.pipeline_config.content).to eq(config_content_result)
+          expect(command.config_content).to eq(config_content_result)
         end
       end
@@ -232,16 +259,21 @@ describe Gitlab::Ci::Pipeline::Chain::Config::Content do
           stub_feature_flags(auto_devops_beta: false)
         end
-        it 'builds root config including the auto-devops template' do
-          subject.perform!
-
-          expect(pipeline.config_source).to eq 'auto_devops_source'
-          expect(command.config_content).to eq(<<~EOY)
+        let(:config_content_result) do
+          <<~EOY
             ---
             include:
             - template: Auto-DevOps.gitlab-ci.yml
           EOY
         end
+
+        it 'builds root config including the auto-devops template' do
+          subject.perform!
+
+          expect(pipeline.config_source).to eq 'auto_devops_source'
+          expect(pipeline.pipeline_config.content).to eq(config_content_result)
+          expect(command.config_content).to eq(config_content_result)
+        end
       end
     end
@@ -256,6 +288,7 @@ describe Gitlab::Ci::Pipeline::Chain::Config::Content do
         subject.perform!
         expect(pipeline.config_source).to eq('unknown_source')
+        expect(pipeline.pipeline_config).to be_nil
         expect(command.config_content).to be_nil
         expect(pipeline.errors.full_messages).to include('Missing CI config file')
       end