Merge branch 'master' into 'update-kubeclient'

 Conflicts:
   Gemfile.lock

app/models/clusters/platforms/kubernetes.rb

@@ -56,19 +56,19 @@ module Clusters
       def predefined_variables
         config = YAML.dump(kubeconfig)
-        variables = [
-          { key: 'KUBE_URL', value: api_url, public: true },
-          { key: 'KUBE_TOKEN', value: token, public: false },
-          { key: 'KUBE_NAMESPACE', value: actual_namespace, public: true },
-          { key: 'KUBECONFIG', value: config, public: false, file: true }
-        ]
-
-        if ca_pem.present?
-          variables << { key: 'KUBE_CA_PEM', value: ca_pem, public: true }
-          variables << { key: 'KUBE_CA_PEM_FILE', value: ca_pem, public: true, file: true }
+        Gitlab::Ci::Variables::Collection.new.tap do |variables|
+          variables
+            .append(key: 'KUBE_URL', value: api_url)
+            .append(key: 'KUBE_TOKEN', value: token, public: false)
+            .append(key: 'KUBE_NAMESPACE', value: actual_namespace)
+            .append(key: 'KUBECONFIG', value: config, public: false, file: true)
+
+          if ca_pem.present?
+            variables
+              .append(key: 'KUBE_CA_PEM', value: ca_pem)
+              .append(key: 'KUBE_CA_PEM_FILE', value: ca_pem, file: true)
+          end
         end
-
-        variables
       end
       # Constructs a list of terminals from the reactive cache

app/models/concerns/storage/legacy_namespace.rb

@@ -7,29 +7,24 @@ module Storage
         raise Gitlab::UpdatePathError.new('Namespace cannot be moved, because at least one project has tags in container registry')
       end
-      expires_full_path_cache
-
-      # Move the namespace directory in all storage paths used by member projects
-      repository_storage_paths.each do |repository_storage_path|
-        # Ensure old directory exists before moving it
-        gitlab_shell.add_namespace(repository_storage_path, full_path_was)
-
-        # Ensure new directory exists before moving it (if there's a parent)
-        gitlab_shell.add_namespace(repository_storage_path, parent.full_path) if parent
-        unless gitlab_shell.mv_namespace(repository_storage_path, full_path_was, full_path)
-          Rails.logger.error "Exception moving path #{repository_storage_path} from #{full_path_was} to #{full_path}"
-          # if we cannot move namespace directory we should rollback
-          # db changes in order to prevent out of sync between db and fs
-          raise Gitlab::UpdatePathError.new('namespace directory cannot be moved')
-        end
+      parent_was = if parent_changed? && parent_id_was.present?
+                     Namespace.find(parent_id_was) # raise NotFound early if needed
+                   end
+      expires_full_path_cache
+      move_repositories
+      if parent_changed?
+        former_parent_full_path = parent_was&.full_path
+        parent_full_path = parent&.full_path
+        Gitlab::UploadsTransfer.new.move_namespace(path, former_parent_full_path, parent_full_path)
+        Gitlab::PagesTransfer.new.move_namespace(path, former_parent_full_path, parent_full_path)
+      else
+        Gitlab::UploadsTransfer.new.rename_namespace(full_path_was, full_path)
+        Gitlab::PagesTransfer.new.rename_namespace(full_path_was, full_path)
       end
-      Gitlab::UploadsTransfer.new.rename_namespace(full_path_was, full_path)
-      Gitlab::PagesTransfer.new.rename_namespace(full_path_was, full_path)
-
       remove_exports!
       # If repositories moved successfully we need to
@@ -57,6 +52,26 @@ module Storage
     private
+    def move_repositories
+      # Move the namespace directory in all storage paths used by member projects
+      repository_storage_paths.each do |repository_storage_path|
+        # Ensure old directory exists before moving it
+        gitlab_shell.add_namespace(repository_storage_path, full_path_was)
+
+        # Ensure new directory exists before moving it (if there's a parent)
+        gitlab_shell.add_namespace(repository_storage_path, parent.full_path) if parent
+
+        unless gitlab_shell.mv_namespace(repository_storage_path, full_path_was, full_path)
+
+          Rails.logger.error "Exception moving path #{repository_storage_path} from #{full_path_was} to #{full_path}"
+
+          # if we cannot move namespace directory we should rollback
+          # db changes in order to prevent out of sync between db and fs
+          raise Gitlab::UpdatePathError.new('namespace directory cannot be moved')
+        end
+      end
+    end
+
     def old_repository_storage_paths
       @old_repository_storage_paths ||= repository_storage_paths
     end

app/models/environment.rb

@@ -65,10 +65,9 @@ class Environment < ActiveRecord::Base
   end
   def predefined_variables
-    [
-      { key: 'CI_ENVIRONMENT_NAME', value: name, public: true },
-      { key: 'CI_ENVIRONMENT_SLUG', value: slug, public: true }
-    ]
+    Gitlab::Ci::Variables::Collection.new
+      .append(key: 'CI_ENVIRONMENT_NAME', value: name)
+      .append(key: 'CI_ENVIRONMENT_SLUG', value: slug)
   end
   def recently_updated_on_branch?(ref)

app/models/member.rb

@@ -55,7 +55,7 @@ class Member < ActiveRecord::Base
   scope :active_without_invites, -> do
     left_join_users
       .where(users: { state: 'active' })
-      .where(requested_at: nil)
+      .non_request
       .reorder(nil)
   end

app/models/merge_request.rb

@@ -579,9 +579,10 @@ class MergeRequest < ActiveRecord::Base
     return unless open?
     old_diff_refs = self.diff_refs
-    create_merge_request_diff
-    MergeRequests::MergeRequestDiffCacheService.new.execute(self)
+    new_diff = create_merge_request_diff
+
+    MergeRequests::MergeRequestDiffCacheService.new.execute(self, new_diff)
     new_diff_refs = self.diff_refs
     update_diff_discussion_positions(

app/models/project.rb

@@ -1083,7 +1083,7 @@ class Project < ActiveRecord::Base
     # Forked import is handled asynchronously
     return if forked? && !force
-    if gitlab_shell.add_repository(repository_storage, disk_path)
+    if gitlab_shell.create_repository(repository_storage, disk_path)
       repository.after_create
       true
     else
@@ -1519,8 +1519,8 @@ class Project < ActiveRecord::Base
     @errors = original_errors
   end
-  def add_export_job(current_user:)
-    job_id = ProjectExportWorker.perform_async(current_user.id, self.id)
+  def add_export_job(current_user:, params: {})
+    job_id = ProjectExportWorker.perform_async(current_user.id, self.id, params)
     if job_id
       Rails.logger.info "Export job started for project ID #{self.id} with job ID #{job_id}"
@@ -1572,29 +1572,30 @@ class Project < ActiveRecord::Base
   end
   def predefined_variables
-    [
-      { key: 'CI_PROJECT_ID', value: id.to_s, public: true },
-      { key: 'CI_PROJECT_NAME', value: path, public: true },
-      { key: 'CI_PROJECT_PATH', value: full_path, public: true },
-      { key: 'CI_PROJECT_PATH_SLUG', value: full_path_slug, public: true },
-      { key: 'CI_PROJECT_NAMESPACE', value: namespace.full_path, public: true },
-      { key: 'CI_PROJECT_URL', value: web_url, public: true },
-      { key: 'CI_PROJECT_VISIBILITY', value: Gitlab::VisibilityLevel.string_level(visibility_level), public: true }
-    ]
+    visibility = Gitlab::VisibilityLevel.string_level(visibility_level)
+
+    Gitlab::Ci::Variables::Collection.new
+      .append(key: 'CI_PROJECT_ID', value: id.to_s)
+      .append(key: 'CI_PROJECT_NAME', value: path)
+      .append(key: 'CI_PROJECT_PATH', value: full_path)
+      .append(key: 'CI_PROJECT_PATH_SLUG', value: full_path_slug)
+      .append(key: 'CI_PROJECT_NAMESPACE', value: namespace.full_path)
+      .append(key: 'CI_PROJECT_URL', value: web_url)
+      .append(key: 'CI_PROJECT_VISIBILITY', value: visibility)
+      .concat(container_registry_variables)
+      .concat(auto_devops_variables)
   end
   def container_registry_variables
-    return [] unless Gitlab.config.registry.enabled
-    variables = [
-      { key: 'CI_REGISTRY', value: Gitlab.config.registry.host_port, public: true }
-    ]
-    if container_registry_enabled?
-      variables << { key: 'CI_REGISTRY_IMAGE', value: container_registry_url, public: true }
+    Gitlab::Ci::Variables::Collection.new.tap do |variables|
+      return variables unless Gitlab.config.registry.enabled
+      variables.append(key: 'CI_REGISTRY', value: Gitlab.config.registry.host_port)
+      if container_registry_enabled?
+        variables.append(key: 'CI_REGISTRY_IMAGE', value: container_registry_url)
+      end
     end
-
-    variables
   end
   def secret_variables_for(ref:, environment: nil)
@@ -1614,16 +1615,14 @@ class Project < ActiveRecord::Base
     end
   end
-  def deployment_variables
-    return [] unless deployment_platform
-
-    deployment_platform.predefined_variables
+  def deployment_variables(environment: nil)
+    deployment_platform(environment: environment)&.predefined_variables || []
   end
   def auto_devops_variables
     return [] unless auto_devops_enabled?
-    (auto_devops || build_auto_devops)&.variables
+    (auto_devops || build_auto_devops)&.predefined_variables
   end
   def append_or_update_attribute(name, value)

app/models/project_auto_devops.rb

@@ -14,9 +14,12 @@ class ProjectAutoDevops < ActiveRecord::Base
     domain.present? || instance_domain.present?
   end
-  def variables
-    variables = []
-    variables << { key: 'AUTO_DEVOPS_DOMAIN', value: domain.presence || instance_domain, public: true } if has_domain?
-    variables
+  def predefined_variables
+    Gitlab::Ci::Variables::Collection.new.tap do |variables|
+      if has_domain?
+        variables.append(key: 'AUTO_DEVOPS_DOMAIN',
+                         value: domain.presence || instance_domain)
+      end
+    end
   end
 end

app/models/project_services/jira_service.rb

@@ -161,11 +161,6 @@ class JiraService < IssueTrackerService
     add_comment(data, jira_issue)
   end
-  # reason why service cannot be tested
-  def disabled_title
-    "Please fill in Password and Username."
-  end
-
   def test(_)
     result = test_settings
     success = result.present?

app/models/project_services/kubernetes_service.rb

@@ -105,19 +105,19 @@ class KubernetesService < DeploymentService
   def predefined_variables
     config = YAML.dump(kubeconfig)
-    variables = [
-      { key: 'KUBE_URL', value: api_url, public: true },
-      { key: 'KUBE_TOKEN', value: token, public: false },
-      { key: 'KUBE_NAMESPACE', value: actual_namespace, public: true },
-      { key: 'KUBECONFIG', value: config, public: false, file: true }
-    ]
-
-    if ca_pem.present?
-      variables << { key: 'KUBE_CA_PEM', value: ca_pem, public: true }
-      variables << { key: 'KUBE_CA_PEM_FILE', value: ca_pem, public: true, file: true }
+    Gitlab::Ci::Variables::Collection.new.tap do |variables|
+      variables
+        .append(key: 'KUBE_URL', value: api_url)
+        .append(key: 'KUBE_TOKEN', value: token, public: false)
+        .append(key: 'KUBE_NAMESPACE', value: actual_namespace)
+        .append(key: 'KUBECONFIG', value: config, public: false, file: true)
+
+      if ca_pem.present?
+        variables
+          .append(key: 'KUBE_CA_PEM', value: ca_pem)
+          .append(key: 'KUBE_CA_PEM_FILE', value: ca_pem, file: true)
+      end
     end
-
-    variables
   end
   # Constructs a list of terminals from the reactive cache

app/models/project_services/pipelines_email_service.rb

@@ -39,10 +39,6 @@ class PipelinesEmailService < Service
     project.pipelines.any?
   end
-  def disabled_title
-    'Please setup a pipeline on your repository.'
-  end
-
   def test_data(project, user)
     data = Gitlab::DataBuilder::Pipeline.build(project.pipelines.last)
     data[:user] = user.hook_attrs

app/models/project_wiki.rb

@@ -169,7 +169,7 @@ class ProjectWiki
   private
   def create_repo!(raw_repository)
-    gitlab_shell.add_repository(project.repository_storage, disk_path)
+    gitlab_shell.create_repository(project.repository_storage, disk_path)
     raise CouldNotCreateWikiError unless raw_repository.exists?

app/models/service.rb

@@ -162,11 +162,6 @@ class Service < ActiveRecord::Base
     true
   end
-  # reason why service cannot be tested
-  def disabled_title
-    "Please setup a project repository."
-  end
-
   # Provide convenient accessor methods
   # for each serialized property.
   # Also keep track of updated properties in a similar way as ActiveModel::Dirty

app/services/files/create_service.rb

 module Files
   class CreateService < Files::BaseService
     def create_commit!
-      handler = Lfs::FileModificationHandler.new(project, @branch_name)
-      handler.new_file(@file_path, @file_content) do |content_or_lfs_pointer|
-        create_transformed_commit(content_or_lfs_pointer)
-      end
+      transformer = Lfs::FileTransformer.new(project, @branch_name)
+      result = transformer.new_file(@file_path, @file_content)
+
+      create_transformed_commit(result.content)
     end
     private

app/services/files/multi_service.rb

@@ -3,11 +3,33 @@ module Files
     UPDATE_FILE_ACTIONS = %w(update move delete).freeze
     def create_commit!
+      transformer = Lfs::FileTransformer.new(project, @branch_name)
+
+      actions = actions_after_lfs_transformation(transformer, params[:actions])
+
+      commit_actions!(actions)
+    end
+
+    private
+
+    def actions_after_lfs_transformation(transformer, actions)
+      actions.map do |action|
+        if action[:action] == 'create'
+          result = transformer.new_file(action[:file_path], action[:content], encoding: action[:encoding])
+          action[:content] = result.content
+          action[:encoding] = result.encoding
+        end
+
+        action
+      end
+    end
+
+    def commit_actions!(actions)
       repository.multi_action(
         current_user,
         message: @commit_message,
         branch_name: @branch_name,
-        actions: params[:actions],
+        actions: actions,
         author_email: @author_email,
         author_name: @author_name,
         start_project: @start_project,
@@ -17,8 +39,6 @@ module Files
       raise_error(e)
     end
-    private
-
     def validate!
       super

app/services/lfs/file_modification_handler.rb → app/services/lfs/file_transformer.rb

 module Lfs
-  class FileModificationHandler
+  # Usage: Calling `new_file` check to see if a file should be in LFS and
+  #        return a transformed result with `content` and `encoding` to commit.
+  #
+  #        For LFS an LfsObject linked to the project is stored and an LFS
+  #        pointer returned. If the file isn't in LFS the untransformed content
+  #        is returned to save in the commit.
+  #
+  # transformer = Lfs::FileTransformer.new(project, @branch_name)
+  # content_or_lfs_pointer = transformer.new_file(file_path, content).content
+  # create_transformed_commit(content_or_lfs_pointer)
+  #
+  class FileTransformer
     attr_reader :project, :branch_name
     delegate :repository, to: :project
@@ -9,24 +20,37 @@ module Lfs
       @branch_name = branch_name
     end
-    def new_file(file_path, file_content)
+    def new_file(file_path, file_content, encoding: nil)
       if project.lfs_enabled? && lfs_file?(file_path)
+        file_content = Base64.decode64(file_content) if encoding == 'base64'
         lfs_pointer_file = Gitlab::Git::LfsPointerFile.new(file_content)
         lfs_object = create_lfs_object!(lfs_pointer_file, file_content)
-        content = lfs_pointer_file.pointer
-        success = yield(content)
-        link_lfs_object!(lfs_object) if success
+        link_lfs_object!(lfs_object)
+        Result.new(content: lfs_pointer_file.pointer, encoding: 'text')
       else
-        yield(file_content)
+        Result.new(content: file_content, encoding: encoding)
+      end
+    end
+
+    class Result
+      attr_reader :content, :encoding
+
+      def initialize(content:, encoding:)
+        @content = content
+        @encoding = encoding
       end
     end
     private
     def lfs_file?(file_path)
-      repository.attributes_at(branch_name, file_path)['filter'] == 'lfs'
+      cached_attributes.attributes(file_path)['filter'] == 'lfs'
+    end
+
+    def cached_attributes
+      @cached_attributes ||= Gitlab::Git::AttributesAtRefParser.new(repository, branch_name)
     end
     def create_lfs_object!(lfs_pointer_file, file_content)

app/services/merge_requests/merge_request_diff_cache_service.rb

 module MergeRequests
   class MergeRequestDiffCacheService
-    def execute(merge_request)
+    def execute(merge_request, new_diff)
       # Executing the iteration we cache all the highlighted diff information
       merge_request.diffs.diff_files.to_a
+
+      # Remove cache for all diffs on this MR. Do not use the association on the
+      # model, as that will interfere with other actions happening when
+      # reloading the diff.
+      MergeRequestDiff.where(merge_request: merge_request).each do |merge_request_diff|
+        next if merge_request_diff == new_diff
+
+        merge_request_diff.diffs.clear_cache!
+      end
     end
   end
 end

app/services/notification_service.rb

@@ -208,9 +208,9 @@ class NotificationService
   def new_access_request(member)
     return true unless member.notifiable?(:subscription)
-    recipients = member.source.members.owners_and_masters
+    recipients = member.source.members.active_without_invites.owners_and_masters
     if fallback_to_group_owners_masters?(recipients, member)
-      recipients = member.source.group.members.owners_and_masters
+      recipients = member.source.group.members.active_without_invites.owners_and_masters
     end
     recipients.each { |recipient| deliver_access_request_email(recipient, member) }

app/services/projects/import_export/export_service.rb

@@ -26,7 +26,7 @@ module Projects
       end
       def project_tree_saver
-        Gitlab::ImportExport::ProjectTreeSaver.new(project: project, current_user: @current_user, shared: @shared)
+        Gitlab::ImportExport::ProjectTreeSaver.new(project: project, current_user: @current_user, shared: @shared, params: @params)
       end
       def uploads_saver

app/views/admin/runners/show.html.haml

@@ -9,6 +9,10 @@
         %span.runner-state.runner-state-specific
           Specific
+- add_to_breadcrumbs _("Runners"), admin_runners_path
+- breadcrumb_title "##{@runner.id}"
+- @no_container = true
+
 - if @runner.shared?
   .bs-callout.bs-callout-success
     %h4 This Runner will process jobs from ALL UNASSIGNED projects

app/views/discussions/_diff_with_notes.html.haml

@@ -2,8 +2,12 @@
 - blob = discussion.blob
 - discussions = { discussion.original_line_code => [discussion] }
 - diff_file_class = diff_file.text? ? 'text-file' : 'js-image-file'
-.diff-file.file-holder{ class: diff_file_class }
+- diff_data = {}
+- expanded = discussion.expanded? || local_assigns.fetch(:expanded, nil)
+- unless expanded
+  - diff_data = { lines_path: project_merge_request_discussion_path(discussion.project, discussion.noteable, discussion) }
+.diff-file.file-holder{ class: diff_file_class, data: diff_data }
   .js-file-title.file-title.file-title-flex-parent
     .file-header-content
       = render "projects/diffs/file_header", diff_file: diff_file, url: discussion_path(discussion), show_toggle: false
@@ -11,17 +15,24 @@
   - if diff_file.text?
     .diff-content.code.js-syntax-highlight
       %table
-        = render partial: "projects/diffs/line",
-          collection: discussion.truncated_diff_lines,
-          as: :line,
-          locals: { diff_file: diff_file,
-            discussions: discussions,
-            discussion_expanded: true,
-            plain: true }
+        - if expanded
+          - discussions = { discussion.original_line_code => [discussion] }
+          = render partial: "projects/diffs/line",
+            collection: discussion.truncated_diff_lines,
+            as: :line,
+            locals: { diff_file: diff_file,
+              discussions: discussions,
+              discussion_expanded: true,
+              plain: true }
+        - else
+          %tr.line_holder.line-holder-placeholder
+            %td.old_line.diff-line-num
+            %td.new_line.diff-line-num
+            %td.line_content
+              .js-code-placeholder
+          = render "discussions/diff_discussion", discussions: [discussion], expanded: true
   - else
     - partial = (diff_file.new_file? || diff_file.deleted_file?) ? 'single_image_diff' : 'replaced_image_diff'
-
     = render partial: "projects/diffs/#{partial}", locals: { diff_file: diff_file, position: discussion.position.to_json, click_to_comment: false }
-
     .note-container
       = render partial: "discussions/notes", locals: { discussion: discussion, show_toggle: false, show_image_comment_badge: true, disable_collapse_class: true }