Add latest changes from gitlab-org/gitlab@master

doc/user/application_security/sast/index.md

@@ -25,7 +25,7 @@ that is provided by [Auto DevOps](../../../topics/autodevops/index.md).
 GitLab checks the SAST report, compares the found vulnerabilities between the
 source and target branches, and shows the information right on the merge request.
-![SAST Widget](img/sast.png)
+![SAST Widget](img/sast_v12_9.png)
 The results are sorted by the priority of the vulnerability:

lib/gitlab/bitbucket_import/importer.rb

@@ -3,8 +3,6 @@
 module Gitlab
   module BitbucketImport
     class Importer
-      include Gitlab::ShellAdapter
-
       LABELS = [{ title: 'bug', color: '#FF0000' },
                 { title: 'enhancement', color: '#428BCA' },
                 { title: 'proposal', color: '#69D100' },
@@ -80,7 +78,7 @@ module Gitlab
         wiki = WikiFormatter.new(project)
-        gitlab_shell.import_wiki_repository(project, wiki)
+        project.wiki.repository.import_repository(wiki.import_url)
       rescue StandardError => e
         errors << { type: :wiki, errors: e.message }
       end

lib/gitlab/git/repository.rb

@@ -793,6 +793,14 @@ module Gitlab
         end
       end
+      def import_repository(url)
+        raise ArgumentError, "don't use disk paths with import_repository: #{url.inspect}" if url.start_with?('.', '/')
+
+        wrapped_gitaly_errors do
+          gitaly_repository_client.import_repository(url)
+        end
+      end
+
       def blob_at(sha, path)
         Gitlab::Git::Blob.find(self, sha, path) unless Gitlab::Git.blank_ref?(sha)
       end

lib/gitlab/github_import/importer/repository_importer.rb

@@ -4,7 +4,6 @@ module Gitlab
   module GithubImport
     module Importer
       class RepositoryImporter
-        include Gitlab::ShellAdapter
         include Gitlab::Utils::StrongMemoize
         attr_reader :project, :client, :wiki_formatter
@@ -65,10 +64,10 @@ module Gitlab
         end
         def import_wiki_repository
-          gitlab_shell.import_wiki_repository(project, wiki_formatter)
+          project.wiki.repository.import_repository(wiki_formatter.import_url)
           true
-        rescue Gitlab::Shell::Error => e
+        rescue ::Gitlab::Git::CommandError => e
           if e.message !~ /repository not exported/
             project.create_wiki
             fail_import("Failed to import the wiki: #{e.message}")

lib/gitlab/legacy_github_import/importer.rb

@@ -3,8 +3,6 @@
 module Gitlab
   module LegacyGithubImport
     class Importer
-      include Gitlab::ShellAdapter
-
       def self.refmap
         Gitlab::GithubImport.refmap
       end
@@ -264,11 +262,11 @@ module Gitlab
       end
       def import_wiki
-        unless project.wiki.repository_exists?
-          wiki = WikiFormatter.new(project)
-          gitlab_shell.import_wiki_repository(project, wiki)
-        end
-      rescue Gitlab::Shell::Error => e
+        return if project.wiki.repository_exists?
+
+        wiki = WikiFormatter.new(project)
+        project.wiki.repository.import_repository(wiki.import_url)
+      rescue ::Gitlab::Git::CommandError => e
         # GitHub error message when the wiki repo has not been created,
         # this means that repo has wiki enabled, but have no pages. So,
         # we can skip the import.

lib/gitlab/redacted_search_results_logger.rb

+# frozen_string_literal: true
+
+module Gitlab
+  class RedactedSearchResultsLogger < ::Gitlab::JsonLogger
+    def self.file_name_noext
+      'redacted_search_results'
+    end
+  end
+end

lib/gitlab/shell.rb

 # frozen_string_literal: true
-# Gitaly note: SSH key operations are not part of Gitaly so will never be migrated.
-
 require 'securerandom'
 module Gitlab
+  # This class is an artifact of a time when common repository operations were
+  # performed by calling out to scripts in the gitlab-shell project. Now, these
+  # operations are all performed by Gitaly, and are mostly accessible through
+  # the Repository class. Prefer using a Repository to functionality here.
+  #
+  # Legacy code relating to namespaces still relies on Gitlab::Shell; it can be
+  # converted to a module once https://gitlab.com/groups/gitlab-org/-/epics/2320
+  # is completed. https://gitlab.com/gitlab-org/gitlab/-/issues/25095 tracks it.
   class Shell
     Error = Class.new(StandardError)
@@ -77,47 +83,6 @@ module Gitlab
       end
     end
-    # Import wiki repository from external service
-    #
-    # @param [Project] project
-    # @param [Gitlab::LegacyGithubImport::WikiFormatter, Gitlab::BitbucketImport::WikiFormatter] wiki_formatter
-    # @return [Boolean] whether repository could be imported
-    def import_wiki_repository(project, wiki_formatter)
-      import_repository(project.repository_storage, wiki_formatter.disk_path, wiki_formatter.import_url, project.wiki.full_path)
-    end
-
-    # Import project repository from external service
-    #
-    # @param [Project] project
-    # @return [Boolean] whether repository could be imported
-    def import_project_repository(project)
-      import_repository(project.repository_storage, project.disk_path, project.import_url, project.full_path)
-    end
-
-    # Import repository
-    #
-    # @example Import a repository
-    #   import_repository("nfs-file06", "gitlab/gitlab-ci", "https://gitlab.com/gitlab-org/gitlab-test.git", "gitlab/gitlab-ci")
-    #
-    # @param [String] storage  project's storage name
-    # @param [String] disk_path project path on disk
-    # @param [String] url from external resource to import from
-    # @param [String] gl_project_path project name
-    # @return [Boolean] whether repository could be imported
-    def import_repository(storage, disk_path, url, gl_project_path)
-      if url.start_with?('.', '/')
-        raise Error.new("don't use disk paths with import_repository: #{url.inspect}")
-      end
-
-      relative_path = "#{disk_path}.git"
-      cmd = GitalyGitlabProjects.new(storage, relative_path, gl_project_path)
-
-      success = cmd.import_project(url, git_timeout)
-      raise Error, cmd.output unless success
-
-      success
-    end
-
     # Move or rename a repository
     #
     # @example Move/rename a repository
@@ -127,6 +92,8 @@ module Gitlab
     # @param [String] disk_path current project path on disk
     # @param [String] new_disk_path new project path on disk
     # @return [Boolean] whether repository could be moved/renamed on disk
+    #
+    # @deprecated
     def mv_repository(storage, disk_path, new_disk_path)
       return false if disk_path.empty? || new_disk_path.empty?
@@ -139,17 +106,6 @@ module Gitlab
       false
     end
-    # Fork repository to new path
-    #
-    # @param [Project] source_project forked-from Project
-    # @param [Project] target_project forked-to Project
-    def fork_repository(source_project, target_project)
-      forked_from_relative_path = "#{source_project.disk_path}.git"
-      fork_args = [target_project.repository_storage, "#{target_project.disk_path}.git", target_project.full_path]
-
-      GitalyGitlabProjects.new(source_project.repository_storage, forked_from_relative_path, source_project.full_path).fork_repository(*fork_args)
-    end
-
     # Removes a repository from file system, using rm_diretory which is an alias
     # for rm_namespace. Given the underlying implementation removes the name
     # passed as second argument on the passed storage.
@@ -159,6 +115,8 @@ module Gitlab
     #
     # @param [String] storage project's storage path
     # @param [String] disk_path current project path on disk
+    #
+    # @deprecated
     def remove_repository(storage, disk_path)
       return false if disk_path.empty?
@@ -179,6 +137,8 @@ module Gitlab
     #
     # @param [String] storage project's storage path
     # @param [String] name namespace name
+    #
+    # @deprecated
     def add_namespace(storage, name)
       Gitlab::GitalyClient.allow_n_plus_1_calls do
         Gitlab::GitalyClient::NamespaceService.new(storage).add(name)
@@ -195,6 +155,8 @@ module Gitlab
     #
     # @param [String] storage project's storage path
     # @param [String] name namespace name
+    #
+    # @deprecated
     def rm_namespace(storage, name)
       Gitlab::GitalyClient::NamespaceService.new(storage).remove(name)
     rescue GRPC::InvalidArgument => e
@@ -210,6 +172,8 @@ module Gitlab
     # @param [String] storage project's storage path
     # @param [String] old_name current namespace name
     # @param [String] new_name new namespace name
+    #
+    # @deprecated
     def mv_namespace(storage, old_name, new_name)
       Gitlab::GitalyClient::NamespaceService.new(storage).rename(old_name, new_name)
     rescue GRPC::InvalidArgument => e
@@ -226,67 +190,12 @@ module Gitlab
     # @return [Boolean] whether repository exists or not
     # @param [String] storage project's storage path
     # @param [Object] dir_name repository dir name
+    #
+    # @deprecated
     def repository_exists?(storage, dir_name)
       Gitlab::Git::Repository.new(storage, dir_name, nil, nil).exists?
     rescue GRPC::Internal
       false
     end
-
-    protected
-
-    def full_path(storage, dir_name)
-      raise ArgumentError.new("Directory name can't be blank") if dir_name.blank?
-
-      File.join(Gitlab.config.repositories.storages[storage].legacy_disk_path, dir_name)
-    end
-
-    private
-
-    def git_timeout
-      Gitlab.config.gitlab_shell.git_timeout
-    end
-
-    def wrapped_gitaly_errors
-      yield
-    rescue GRPC::NotFound, GRPC::BadStatus => e
-      # Old Popen code returns [Error, output] to the caller, so we
-      # need to do the same here...
-      raise Error, e
-    end
-
-    class GitalyGitlabProjects
-      attr_reader :shard_name, :repository_relative_path, :output, :gl_project_path
-
-      def initialize(shard_name, repository_relative_path, gl_project_path)
-        @shard_name = shard_name
-        @repository_relative_path = repository_relative_path
-        @output = ''
-        @gl_project_path = gl_project_path
-      end
-
-      def import_project(source, _timeout)
-        raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil, gl_project_path)
-
-        Gitlab::GitalyClient::RepositoryService.new(raw_repository).import_repository(source)
-        true
-      rescue GRPC::BadStatus => e
-        @output = e.message
-        false
-      end
-
-      def fork_repository(new_shard_name, new_repository_relative_path, new_project_name)
-        target_repository = Gitlab::Git::Repository.new(new_shard_name, new_repository_relative_path, nil, new_project_name)
-        raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil, gl_project_path)
-
-        Gitlab::GitalyClient::RepositoryService.new(target_repository).fork_repository(raw_repository)
-      rescue GRPC::BadStatus => e
-        logger.error "fork-repository failed: #{e.message}"
-        false
-      end
-
-      def logger
-        Rails.logger # rubocop:disable Gitlab/RailsLogger
-      end
-    end
   end
 end

lib/quality/test_level.rb

@@ -7,6 +7,8 @@ module Quality
     TEST_LEVEL_FOLDERS = {
       migration: %w[
         migrations
+      ],
+      background_migration: %w[
         lib/gitlab/background_migration
         lib/ee/gitlab/background_migration
       ],
@@ -70,7 +72,7 @@ module Quality
       case file_path
       # Detect migration first since some background migration tests are under
       # spec/lib/gitlab/background_migration and tests under spec/lib are unit by default
-      when regexp(:migration)
+      when regexp(:migration), regexp(:background_migration)
         :migration
       when regexp(:unit)
         :unit
@@ -83,6 +85,10 @@ module Quality
       end
     end
+    def background_migration?(file_path)
+      !!(file_path =~ regexp(:background_migration))
+    end
+
     private
     def folders_pattern(level)

package.json

@@ -39,8 +39,8 @@
     "@babel/plugin-syntax-import-meta": "^7.8.3",
     "@babel/preset-env": "^7.8.4",
     "@gitlab/at.js": "^1.5.5",
-    "@gitlab/svgs": "^1.111.0",
-    "@gitlab/ui": "^9.28.0",
+    "@gitlab/svgs": "^1.113.0",
+    "@gitlab/ui": "^9.29.0",
     "@gitlab/visual-review-tools": "1.5.1",
     "@sentry/browser": "^5.10.2",
     "@sourcegraph/code-host-integration": "0.0.31",

scripts/security-harness

@@ -28,7 +28,7 @@ HOOK_DATA = <<~HOOK
   if [ -e "$harness" ]
   then
-    if [["$url" != *"gitlab-org/security/"*]]
+    if [[ "$url" != *"gitlab-org/security/"* ]]
     then
       echo "Pushing to remotes other than gitlab.com/gitlab-org/security has been disabled!"
       echo "Run scripts/security-harness to disable this check."

spec/controllers/projects/pipelines_controller_spec.rb

@@ -634,7 +634,8 @@ describe Projects::PipelinesController do
       it 'does not persist a pipeline' do
         expect { post_request }.not_to change { project.ci_pipelines.count }
-        expect(response).to have_gitlab_http_status(:ok)
+        expect(response).to have_gitlab_http_status(:bad_request)
+        expect(response).to render_template('new')
       end
     end

spec/frontend/ci_variable_list/components/ci_variable_modal_spec.js

 import Vuex from 'vuex';
 import { createLocalVue, shallowMount } from '@vue/test-utils';
-import { GlModal } from '@gitlab/ui';
+import { GlButton } from '@gitlab/ui';
 import CiVariableModal from '~/ci_variable_list/components/ci_variable_modal.vue';
 import createStore from '~/ci_variable_list/store';
 import mockData from '../services/mock_data';
+import ModalStub from '../stubs';
 const localVue = createLocalVue();
 localVue.use(Vuex);
@@ -15,12 +16,23 @@ describe('Ci variable modal', () => {
   const createComponent = () => {
     store = createStore();
     wrapper = shallowMount(CiVariableModal, {
+      stubs: {
+        GlModal: ModalStub,
+      },
       localVue,
       store,
     });
   };
-  const findModal = () => wrapper.find(GlModal);
+  const findModal = () => wrapper.find(ModalStub);
+  const addOrUpdateButton = index =>
+    findModal()
+      .findAll(GlButton)
+      .at(index);
+  const deleteVariableButton = () =>
+    findModal()
+      .findAll(GlButton)
+      .at(1);
   beforeEach(() => {
     createComponent();
@@ -32,7 +44,7 @@ describe('Ci variable modal', () => {
   });
   it('button is disabled when no key/value pair are present', () => {
-    expect(findModal().props('actionPrimary').attributes.disabled).toBeTruthy();
+    expect(addOrUpdateButton(1).attributes('disabled')).toBeTruthy();
   });
   describe('Adding a new variable', () => {
@@ -42,11 +54,11 @@ describe('Ci variable modal', () => {
     });
     it('button is enabled when key/value pair are present', () => {
-      expect(findModal().props('actionPrimary').attributes.disabled).toBeFalsy();
+      expect(addOrUpdateButton(1).attributes('disabled')).toBeFalsy();
     });
     it('Add variable button dispatches addVariable action', () => {
-      findModal().vm.$emit('ok');
+      addOrUpdateButton(1).vm.$emit('click');
       expect(store.dispatch).toHaveBeenCalledWith('addVariable');
     });
@@ -63,11 +75,11 @@ describe('Ci variable modal', () => {
     });
     it('button text is Update variable when updating', () => {
-      expect(wrapper.vm.modalActionText).toBe('Update variable');
+      expect(addOrUpdateButton(2).text()).toBe('Update variable');
     });
     it('Update variable button dispatches updateVariable with correct variable', () => {
-      findModal().vm.$emit('ok');
+      addOrUpdateButton(2).vm.$emit('click');
       expect(store.dispatch).toHaveBeenCalledWith(
         'updateVariable',
         store.state.variableBeingEdited,
@@ -80,7 +92,7 @@ describe('Ci variable modal', () => {
     });
     it('dispatches deleteVariable with correct variable to delete', () => {
-      findModal().vm.$emit('secondary');
+      deleteVariableButton().vm.$emit('click');
       expect(store.dispatch).toHaveBeenCalledWith('deleteVariable', mockData.mockVariables[0]);
     });
   });

spec/frontend/ci_variable_list/stubs.js

+const ModalStub = {
+  name: 'glmodal-stub',
+  template: `
+    <div>
+      <slot></slot>
+      <slot name="modal-footer"></slot>
+    </div>
+  `,
+  methods: {
+    hide: jest.fn(),
+  },
+};
+
+export default ModalStub;

spec/lib/gitlab/background_migration/backfill_project_repositories_spec.rb

@@ -3,7 +3,7 @@
 require 'spec_helper'
 # rubocop:disable RSpec/FactoriesInMigrationSpecs
-describe Gitlab::BackgroundMigration::BackfillProjectRepositories, schema: :latest do
+describe Gitlab::BackgroundMigration::BackfillProjectRepositories do
   let(:group) { create(:group, name: 'foo', path: 'foo') }
   describe described_class::ShardFinder do

spec/lib/gitlab/background_migration/legacy_upload_mover_spec.rb

@@ -2,7 +2,7 @@
 require 'spec_helper'
 # rubocop: disable RSpec/FactoriesInMigrationSpecs
-describe Gitlab::BackgroundMigration::LegacyUploadMover, schema: :latest do
+describe Gitlab::BackgroundMigration::LegacyUploadMover do
   let(:test_dir) { FileUploader.options['storage_path'] }
   let(:filename) { 'image.png' }

spec/lib/gitlab/background_migration/legacy_uploads_migrator_spec.rb

@@ -2,7 +2,7 @@
 require 'spec_helper'
 # rubocop: disable RSpec/FactoriesInMigrationSpecs
-describe Gitlab::BackgroundMigration::LegacyUploadsMigrator, schema: :latest do
+describe Gitlab::BackgroundMigration::LegacyUploadsMigrator do
   let(:test_dir) { FileUploader.options['storage_path'] }
   let!(:hashed_project) { create(:project) }

spec/lib/gitlab/background_migration/merge_request_assignees_migration_progress_check_spec.rb

@@ -2,7 +2,7 @@
 require 'spec_helper'
-describe Gitlab::BackgroundMigration::MergeRequestAssigneesMigrationProgressCheck, schema: :latest do
+describe Gitlab::BackgroundMigration::MergeRequestAssigneesMigrationProgressCheck do
   context 'rescheduling' do
     context 'when there are ongoing and no dead jobs' do
       it 'reschedules check' do

spec/lib/gitlab/background_migration_spec.rb

@@ -2,7 +2,7 @@
 require 'spec_helper'
-describe Gitlab::BackgroundMigration, schema: :latest do
+describe Gitlab::BackgroundMigration do
   describe '.queue' do
     it 'returns background migration worker queue' do
       expect(described_class.queue)

spec/lib/gitlab/bitbucket_import/importer_spec.rb

@@ -80,8 +80,7 @@ describe Gitlab::BitbucketImport::Importer do
   end
   let(:importer) { described_class.new(project) }
-  let(:gitlab_shell) { double }
-
+  let(:sample) { RepoHelpers.sample_compare }
   let(:issues_statuses_sample_data) do
     {
       count: sample_issues_statuses.count,
@@ -89,12 +88,6 @@ describe Gitlab::BitbucketImport::Importer do
     }
   end
-  let(:sample) { RepoHelpers.sample_compare }
-
-  before do
-    allow(importer).to receive(:gitlab_shell) { gitlab_shell }
-  end
-
   subject { described_class.new(project) }
   describe '#import_pull_requests' do
@@ -316,7 +309,7 @@ describe Gitlab::BitbucketImport::Importer do
     describe 'wiki import' do
       it 'is skipped when the wiki exists' do
         expect(project.wiki).to receive(:repository_exists?) { true }
-        expect(importer.gitlab_shell).not_to receive(:import_wiki_repository)
+        expect(project.wiki.repository).not_to receive(:import_repository)
         importer.execute
@@ -325,7 +318,7 @@ describe Gitlab::BitbucketImport::Importer do
       it 'imports to the project disk_path' do
         expect(project.wiki).to receive(:repository_exists?) { false }
-        expect(importer.gitlab_shell).to receive(:import_wiki_repository)
+        expect(project.wiki.repository).to receive(:import_repository)
         importer.execute

spec/lib/gitlab/git/repository_spec.rb

@@ -2138,6 +2138,33 @@ describe Gitlab::Git::Repository, :seed_helper do
     end
   end
+  describe '#import_repository' do
+    let_it_be(:project) { create(:project) }
+
+    let(:repository) { project.repository }
+    let(:url) { 'http://invalid.invalid' }
+
+    it 'raises an error if a relative path is provided' do
+      expect { repository.import_repository('/foo') }.to raise_error(ArgumentError, /disk path/)
+    end
+
+    it 'raises an error if an absolute path is provided' do
+      expect { repository.import_repository('./foo') }.to raise_error(ArgumentError, /disk path/)
+    end
+
+    it 'delegates to Gitaly' do
+      expect_next_instance_of(Gitlab::GitalyClient::RepositoryService) do |svc|
+        expect(svc).to receive(:import_repository).with(url).and_return(nil)
+      end
+
+      repository.import_repository(url)
+    end
+
+    it_behaves_like 'wrapping gRPC errors', Gitlab::GitalyClient::RepositoryService, :import_repository do
+      subject { repository.import_repository('http://invalid.invalid') }
+    end
+  end
+
   describe '#replicate' do
     let(:new_repository) do
       Gitlab::Git::Repository.new('test_second_storage', TEST_REPO_PATH, '', 'group/project')