Add latest changes from gitlab-org/gitlab@master

app/workers/propagate_instance_level_service_worker.rb → app/workers/propagate_service_template_worker.rb

 # frozen_string_literal: true
 # Worker for updating any project specific caches.
-class PropagateInstanceLevelServiceWorker
+class PropagateServiceTemplateWorker
   include ApplicationWorker
   feature_category :source_code_management
@@ -9,18 +9,18 @@ class PropagateInstanceLevelServiceWorker
   LEASE_TIMEOUT = 4.hours.to_i
   # rubocop: disable CodeReuse/ActiveRecord
-  def perform(instance_level_service_id)
-    return unless try_obtain_lease_for(instance_level_service_id)
-    Projects::PropagateInstanceLevelService.propagate(Service.find_by(id: instance_level_service_id))
+  def perform(template_id)
+    return unless try_obtain_lease_for(template_id)
+    Projects::PropagateServiceTemplate.propagate(Service.find_by(id: template_id))
   end
   # rubocop: enable CodeReuse/ActiveRecord
   private
-  def try_obtain_lease_for(instance_level_service_id)
+  def try_obtain_lease_for(template_id)
     Gitlab::ExclusiveLease
-      .new("propagate_instance_level_service_worker:#{instance_level_service_id}", timeout: LEASE_TIMEOUT)
+      .new("propagate_service_template_worker:#{template_id}", timeout: LEASE_TIMEOUT)
       .try_obtain
   end
 end

changelogs/unreleased/fj-add-plain-data-field-to-snippet-blob-type-endpoint.yml

+---
+title: Add plain_highlighted_data field to SnippetBlobType
+merge_request: 24856
+author:
+type: changed

changelogs/unreleased/rename_services_template_to_instance.yml

----
-title: 'Service model: Rename template attribute to instance'
-merge_request: 23595
-author:
-type: other

config/sidekiq_queues.yml

@@ -194,7 +194,7 @@
   - 1
 - - project_update_repository_storage
   - 1
-- - propagate_instance_level_service
+- - propagate_service_template
   - 1
 - - reactive_caching
   - 1

db/migrate/20200123092602_rename_services_template_to_instance.rb

-# frozen_string_literal: true
-
-class RenameServicesTemplateToInstance < ActiveRecord::Migration[5.2]
-  include Gitlab::Database::MigrationHelpers
-
-  DOWNTIME = false
-
-  disable_ddl_transaction!
-
-  def up
-    rename_column_concurrently :services, :template, :instance
-  end
-
-  def down
-    undo_rename_column_concurrently :services, :template, :instance
-  end
-end

db/post_migrate/20191021101942_remove_empty_github_service_templates.rb

@@ -23,10 +23,6 @@ class RemoveEmptyGithubServiceTemplates < ActiveRecord::Migration[5.2]
   private
   def relationship
-    # The column `template` was renamed to `instance`. Column information needs
-    # to be resetted to avoid cache problems after migrating down.
-    RemoveEmptyGithubServiceTemplates::Service.reset_column_information
-
     RemoveEmptyGithubServiceTemplates::Service.where(template: true, type: 'GithubService')
   end
 end

db/post_migrate/20200123101859_cleanup_rename_services_template_to_instance.rb

-# frozen_string_literal: true
-
-class CleanupRenameServicesTemplateToInstance < ActiveRecord::Migration[5.2]
-  include Gitlab::Database::MigrationHelpers
-
-  DOWNTIME = false
-
-  disable_ddl_transaction!
-
-  def up
-    cleanup_concurrent_column_rename :services, :template, :instance
-  end
-
-  def down
-    undo_cleanup_concurrent_column_rename :services, :template, :instance
-  end
-end

db/post_migrate/20200206111847_migrate_propagate_service_template_sidekiq_queue.rb

-# frozen_string_literal: true
-
-class MigratePropagateServiceTemplateSidekiqQueue < ActiveRecord::Migration[6.0]
-  include Gitlab::Database::MigrationHelpers
-
-  DOWNTIME = false
-
-  def up
-    sidekiq_queue_migrate 'propagate_service_template', to: 'propagate_instance_level_service'
-  end
-
-  def down
-    sidekiq_queue_migrate 'propagate_instance_level_service', to: 'propagate_service_template'
-  end
-end

db/schema.rb

@@ -3858,9 +3858,9 @@ ActiveRecord::Schema.define(version: 2020_02_07_151640) do
     t.boolean "deployment_events", default: false, null: false
     t.string "description", limit: 500
     t.boolean "comment_on_event_enabled", default: true, null: false
-    t.boolean "instance", default: false
-    t.index ["instance"], name: "index_services_on_instance"
+    t.boolean "template", default: false
     t.index ["project_id"], name: "index_services_on_project_id"
+    t.index ["template"], name: "index_services_on_template"
     t.index ["type"], name: "index_services_on_type"
   end

doc/api/graphql/reference/gitlab_schema.graphql

@@ -6777,6 +6777,11 @@ type SnippetBlob {
   """
   path: String
+  """
+  Blob plain highlighted data
+  """
+  plainHighlightedData: String
+
   """
   Blob raw content endpoint path
   """

doc/api/graphql/reference/gitlab_schema.json

@@ -7612,6 +7612,20 @@
               "isDeprecated": false,
               "deprecationReason": null
             },
+            {
+              "name": "plainHighlightedData",
+              "description": "Blob plain highlighted data",
+              "args": [
+
+              ],
+              "type": {
+                "kind": "SCALAR",
+                "name": "String",
+                "ofType": null
+              },
+              "isDeprecated": false,
+              "deprecationReason": null
+            },
             {
               "name": "rawPath",
               "description": "Blob raw content endpoint path",

doc/api/graphql/reference/index.md

@@ -1071,6 +1071,7 @@ Represents the snippet blob
 | `mode` | String | Blob mode |
 | `name` | String | Blob name |
 | `path` | String | Blob path |
+| `plainHighlightedData` | String | Blob plain highlighted data |
 | `rawPath` | String! | Blob raw content endpoint path |
 | `richViewer` | SnippetBlobViewer | Blob content rich viewer |
 | `simpleViewer` | SnippetBlobViewer! | Blob content simple viewer |

lib/api/services.rb

@@ -132,7 +132,7 @@ module API
       helpers do
         # rubocop: disable CodeReuse/ActiveRecord
         def slash_command_service(project, service_slug, params)
-          project.services.active.where(instance: false).find do |service|
+          project.services.active.where(template: false).find do |service|
             service.try(:token) == params[:token] && service.to_param == service_slug.underscore
           end
         end

lib/gitlab/import_export/import_export.yml

@@ -257,7 +257,7 @@ excluded_attributes:
     - :token
     - :token_encrypted
   services:
-    - :instance
+    - :template
   error_tracking_setting:
     - :encrypted_token
     - :encrypted_token_iv

lib/gitlab/serverless/service.rb

+# frozen_string_literal: true
+
+class Gitlab::Serverless::Service
+  include Gitlab::Utils::StrongMemoize
+
+  def initialize(attributes)
+    @attributes = attributes
+  end
+
+  def name
+    @attributes.dig('metadata', 'name')
+  end
+
+  def namespace
+    @attributes.dig('metadata', 'namespace')
+  end
+
+  def environment_scope
+    @attributes.dig('environment_scope')
+  end
+
+  def environment
+    @attributes.dig('environment')
+  end
+
+  def podcount
+    @attributes.dig('podcount')
+  end
+
+  def created_at
+    strong_memoize(:created_at) do
+      timestamp = @attributes.dig('metadata', 'creationTimestamp')
+      DateTime.parse(timestamp) if timestamp
+    end
+  end
+
+  def image
+    @attributes.dig(
+      'spec',
+      'runLatest',
+      'configuration',
+      'build',
+      'template',
+      'name')
+  end
+
+  def description
+    knative_07_description || knative_05_06_description
+  end
+
+  def cluster
+    @attributes.dig('cluster')
+  end
+
+  def url
+    proxy_url || knative_06_07_url || knative_05_url
+  end
+
+  private
+
+  def proxy_url
+    if cluster&.serverless_domain
+      Gitlab::Serverless::FunctionURI.new(function: name, cluster: cluster.serverless_domain, environment: environment)
+    end
+  end
+
+  def knative_07_description
+    @attributes.dig(
+      'spec',
+      'template',
+      'metadata',
+      'annotations',
+      'Description'
+    )
+  end
+
+  def knative_05_06_description
+    @attributes.dig(
+      'spec',
+      'runLatest',
+      'configuration',
+      'revisionTemplate',
+      'metadata',
+      'annotations',
+      'Description')
+  end
+
+  def knative_05_url
+    domain = @attributes.dig('status', 'domain')
+    return unless domain
+
+    "http://#{domain}"
+  end
+
+  def knative_06_07_url
+    @attributes.dig('status', 'url')
+  end
+end

lib/gitlab/usage_data.rb

@@ -179,7 +179,7 @@ module Gitlab
       # rubocop: disable CodeReuse/ActiveRecord
       def services_usage
-        service_counts = count(Service.active.where(instance: false).where.not(type: 'JiraService').group(:type), fallback: Hash.new(-1))
+        service_counts = count(Service.active.where(template: false).where.not(type: 'JiraService').group(:type), fallback: Hash.new(-1))
         results = Service.available_services_names.each_with_object({}) do |service_name, response|
           response["projects_#{service_name}_active".to_sym] = service_counts["#{service_name}_service".camelize] || 0

qa/qa.rb

@@ -469,6 +469,7 @@ module QA
         autoload :Configure, 'qa/vendor/jenkins/page/configure'
         autoload :NewCredentials, 'qa/vendor/jenkins/page/new_credentials'
         autoload :NewJob, 'qa/vendor/jenkins/page/new_job'
+        autoload :Job, 'qa/vendor/jenkins/page/job'
         autoload :ConfigureJob, 'qa/vendor/jenkins/page/configure_job'
       end
     end

qa/qa/vendor/jenkins/page/job.rb

+# frozen_string_literal: true
+
+require 'capybara/dsl'
+
+module QA
+  module Vendor
+    module Jenkins
+      module Page
+        class Job < Page::Base
+          attr_accessor :job_name
+
+          def path
+            "/job/#{@job_name}"
+          end
+
+          def has_successful_build?
+            page.has_text?("Last successful build")
+          end
+        end
+      end
+    end
+  end
+end

spec/controllers/admin/services_controller_spec.rb

@@ -15,11 +15,11 @@ describe Admin::ServicesController do
     Service.available_services_names.each do |service_name|
       context "#{service_name}" do
         let!(:service) do
-          service_instance = "#{service_name}_service".camelize.constantize
-          service_instance.where(instance: true).first_or_create
+          service_template = "#{service_name}_service".camelize.constantize
+          service_template.where(template: true).first_or_create
         end
-        it 'successfully displays the service' do
+        it 'successfully displays the template' do
           get :edit, params: { id: service.id }
           expect(response).to have_gitlab_http_status(:ok)
@@ -34,7 +34,7 @@ describe Admin::ServicesController do
       RedmineService.create(
         project: project,
         active: false,
-        instance: true,
+        template: true,
         properties: {
           project_url: 'http://abc',
           issues_url: 'http://abc',
@@ -44,7 +44,7 @@ describe Admin::ServicesController do
     end
     it 'calls the propagation worker when service is active' do
-      expect(PropagateInstanceLevelServiceWorker).to receive(:perform_async).with(service.id)
+      expect(PropagateServiceTemplateWorker).to receive(:perform_async).with(service.id)
       put :update, params: { id: service.id, service: { active: true } }
@@ -52,7 +52,7 @@ describe Admin::ServicesController do
     end
     it 'does not call the propagation worker when service is not active' do
-      expect(PropagateInstanceLevelServiceWorker).not_to receive(:perform_async)
+      expect(PropagateServiceTemplateWorker).not_to receive(:perform_async)
       put :update, params: { id: service.id, service: { properties: {} } }

spec/controllers/projects/serverless/functions_controller_spec.rb

@@ -14,9 +14,11 @@ describe Projects::Serverless::FunctionsController do
   let!(:deployment) { create(:deployment, :success, environment: environment, cluster: cluster) }
   let(:knative_services_finder) { environment.knative_services_finder }
   let(:function_description) { 'A serverless function' }
+  let(:function_name) { 'some-function-name' }
   let(:knative_stub_options) do
-    { namespace: namespace.namespace, name: cluster.project.name, description: function_description }
+    { namespace: namespace.namespace, name: function_name, description: function_description }
   end
+  let(:knative) { create(:clusters_applications_knative, :installed, cluster: cluster) }
   let(:namespace) do
     create(:cluster_kubernetes_namespace,
@@ -87,25 +89,65 @@ describe Projects::Serverless::FunctionsController do
       end
       context 'when functions were found' do
-        let(:functions) { ["asdf"] }
+        let(:functions) { [{}, {}] }
         before do
-          stub_kubeclient_knative_services(namespace: namespace.namespace)
-          get :index, params: params({ format: :json })
+          stub_kubeclient_knative_services(namespace: namespace.namespace, cluster_id: cluster.id, name: function_name)
         end
         it 'returns functions' do
+          get :index, params: params({ format: :json })
           expect(json_response["functions"]).not_to be_empty
         end
-        it { expect(response).to have_gitlab_http_status(:ok) }
+        it 'filters out the functions whose cluster the user does not have permission to read' do
+          allow(controller).to receive(:can?).and_return(true)
+          expect(controller).to receive(:can?).with(user, :read_cluster, cluster).and_return(false)
+
+          get :index, params: params({ format: :json })
+
+          expect(json_response["functions"]).to be_empty
+        end
+
+        it 'returns a successful response status' do
+          get :index, params: params({ format: :json })
+          expect(response).to have_gitlab_http_status(:ok)
+        end
+
+        context 'when there is serverless domain for a cluster' do
+          let!(:serverless_domain_cluster) do
+            create(:serverless_domain_cluster, clusters_applications_knative_id: knative.id)
+          end
+
+          it 'returns JSON with function details with serverless domain URL' do
+            get :index, params: params({ format: :json })
+            expect(response).to have_gitlab_http_status(:ok)
+
+            expect(json_response["functions"]).not_to be_empty
+
+            expect(json_response["functions"]).to all(
+              include(
+                'url' => "https://#{function_name}-#{serverless_domain_cluster.uuid[0..1]}a1#{serverless_domain_cluster.uuid[2..-3]}f2#{serverless_domain_cluster.uuid[-2..-1]}#{"%x" % environment.id}-#{environment.slug}.#{serverless_domain_cluster.domain}"
+              )
+            )
+          end
+        end
+
+        context 'when there is no serverless domain for a cluster' do
+          it 'keeps function URL as it was' do
+            expect(Gitlab::Serverless::Domain).not_to receive(:new)
+
+            get :index, params: params({ format: :json })
+            expect(response).to have_gitlab_http_status(:ok)
+          end
+        end
       end
     end
   end
   describe 'GET #show' do
-    context 'invalid data' do
-      it 'has a bad function name' do
+    context 'with function that does not exist' do
+      it 'returns 404' do
         get :show, params: params({ format: :json, environment_id: "*", id: "foo" })
         expect(response).to have_gitlab_http_status(:not_found)
       end
@@ -113,15 +155,50 @@ describe Projects::Serverless::FunctionsController do
     context 'with valid data', :use_clean_rails_memory_store_caching do
       shared_examples 'GET #show with valid data' do
-        it 'has a valid function name' do
-          get :show, params: params({ format: :json, environment_id: "*", id: cluster.project.name })
+        context 'when there is serverless domain for a cluster' do
+          let!(:serverless_domain_cluster) do
+            create(:serverless_domain_cluster, clusters_applications_knative_id: knative.id)
+          end
+
+          it 'returns JSON with function details with serverless domain URL' do
+            get :show, params: params({ format: :json, environment_id: "*", id: function_name })
+            expect(response).to have_gitlab_http_status(:ok)
+
+            expect(json_response).to include(
+              'url' => "https://#{function_name}-#{serverless_domain_cluster.uuid[0..1]}a1#{serverless_domain_cluster.uuid[2..-3]}f2#{serverless_domain_cluster.uuid[-2..-1]}#{"%x" % environment.id}-#{environment.slug}.#{serverless_domain_cluster.domain}"
+            )
+          end
+
+          it 'returns 404 when user does not have permission to read the cluster' do
+            allow(controller).to receive(:can?).and_return(true)
+            expect(controller).to receive(:can?).with(user, :read_cluster, cluster).and_return(false)
+
+            get :show, params: params({ format: :json, environment_id: "*", id: function_name })
+
+            expect(response).to have_gitlab_http_status(:not_found)
+          end
+        end
+
+        context 'when there is no serverless domain for a cluster' do
+          it 'keeps function URL as it was' do
+            get :show, params: params({ format: :json, environment_id: "*", id: function_name })
+            expect(response).to have_gitlab_http_status(:ok)
+
+            expect(json_response).to include(
+              'url' => "http://#{function_name}.#{namespace.namespace}.example.com"
+            )
+          end
+        end
+
+        it 'return json with function details' do
+          get :show, params: params({ format: :json, environment_id: "*", id: function_name })
           expect(response).to have_gitlab_http_status(:ok)
           expect(json_response).to include(
-            'name' => project.name,
-            'url' => "http://#{project.name}.#{namespace.namespace}.example.com",
+            'name' => function_name,
+            'url' => "http://#{function_name}.#{namespace.namespace}.example.com",
             'description' => function_description,
-            'podcount' => 1
+            'podcount' => 0
           )
         end
       end
@@ -180,8 +257,8 @@ describe Projects::Serverless::FunctionsController do
                                          'knative_installed' => 'checking',
                                          'functions' => [
                                            a_hash_including(
-                                             'name' => project.name,
-                                             'url' => "http://#{project.name}.#{namespace.namespace}.example.com",
+                                             'name' => function_name,
+                                             'url' => "http://#{function_name}.#{namespace.namespace}.example.com",
                                              'description' => function_description
                                            )
                                          ]