Add latest changes from gitlab-org/gitlab@master

doc/user/project/pages/introduction.md

@@ -297,7 +297,7 @@ For a list of known issues, visit GitLab's [public issue tracker].
 [staticgen]: https://www.staticgen.com/
 [pages-jekyll]: https://gitlab.com/pages/jekyll
 [metarefresh]: https://en.wikipedia.org/wiki/Meta_refresh
-[public issue tracker]: https://gitlab.com/gitlab-org/gitlab-foss/issues?label_name=pages
+[public issue tracker]: https://gitlab.com/gitlab-org/gitlab/-/issues?label_name[]=Category%3APages
 [quick start guide]: ../../../ci/quick_start/README.md
 [pages-index-guide]: index.md
 [pages-quick]: getting_started_part_one.md

lib/gitlab/danger/helper.rb

@@ -133,6 +133,7 @@ module Gitlab
         %r{\A(\.gitlab-ci\.yml\z|\.gitlab\/ci)} => :engineering_productivity,
         %r{\A\.overcommit\.yml\.example\z} => :engineering_productivity,
         %r{\Atooling/overcommit/} => :engineering_productivity,
+        %r{\A.editorconfig\z} => :engineering_productivity,
         %r{Dangerfile\z} => :engineering_productivity,
         %r{\A(ee/)?(danger/|lib/gitlab/danger/)} => :engineering_productivity,
         %r{\A(ee/)?scripts/} => :engineering_productivity,

lib/gitlab/diff/highlight_cache.rb

@@ -101,6 +101,8 @@ module Gitlab
             #
             redis.expire(key, EXPIRATION)
           end
+
+          record_memory_usage(fetch_memory_usage(redis, key))
         end
         # Subsequent read_file calls would need the latest cache.
@@ -109,6 +111,23 @@ module Gitlab
         clear_memoization(:cacheable_files)
       end
+      def record_memory_usage(memory_usage)
+        if memory_usage
+          self.class.gitlab_redis_diff_caching_memory_usage_bytes.observe({}, memory_usage)
+        end
+      end
+
+      def fetch_memory_usage(redis, key)
+        # Redis versions prior to 4.0.0 do not support memory usage reporting
+        #   for a specific key. As of 11-March-2020 we support Redis 3.x, so
+        #   need to account for this. We can remove this check once we
+        #   officially cease supporting versions <4.0.0.
+        #
+        return if Gem::Version.new(redis.info["redis_version"]) < Gem::Version.new("4")
+
+        redis.memory("USAGE", key)
+      end
+
       def file_paths
         strong_memoize(:file_paths) do
           diff_files.collect(&:file_path)

lib/gitlab/repository_cache_adapter.rb

@@ -237,7 +237,7 @@ module Gitlab
     end
     def expire_redis_set_method_caches(methods)
-      methods.each { |name| redis_set_cache.expire(name) }
+      redis_set_cache.expire(*methods)
     end
     def expire_redis_hash_method_caches(methods)

lib/gitlab/set_cache.rb

@@ -14,8 +14,11 @@ module Gitlab
       "#{key}:set"
     end
-    def expire(key)
-      with { |redis| redis.del(cache_key(key)) }
+    def expire(*keys)
+      with do |redis|
+        keys = keys.map { |key| cache_key(key) }
+        unlink_or_delete(redis, keys)
+      end
     end
     def exist?(key)
@@ -51,5 +54,15 @@ module Gitlab
     def with(&blk)
       Gitlab::Redis::Cache.with(&blk) # rubocop:disable CodeReuse/ActiveRecord
     end
+
+    def unlink_or_delete(redis, keys)
+      if Feature.enabled?(:repository_set_cache_unlink, default_enabled: true)
+        redis.unlink(*keys)
+      else
+        redis.del(*keys)
+      end
+    rescue ::Redis::CommandError
+      redis.del(*keys)
+    end
   end
 end

lib/gitlab/usage_data.rb

@@ -66,8 +66,8 @@ module Gitlab
             clusters_disabled: count(::Clusters::Cluster.disabled),
             project_clusters_disabled: count(::Clusters::Cluster.disabled.project_type),
             group_clusters_disabled: count(::Clusters::Cluster.disabled.group_type),
-            clusters_platforms_eks: count(::Clusters::Cluster.aws_installed.enabled, batch: false),
-            clusters_platforms_gke: count(::Clusters::Cluster.gcp_installed.enabled, batch: false),
+            clusters_platforms_eks: count(::Clusters::Cluster.aws_installed.enabled),
+            clusters_platforms_gke: count(::Clusters::Cluster.gcp_installed.enabled),
             clusters_platforms_user: count(::Clusters::Cluster.user_provided.enabled),
             clusters_applications_helm: count(::Clusters::Applications::Helm.available),
             clusters_applications_ingress: count(::Clusters::Applications::Ingress.available),

locale/gitlab.pot

@@ -363,6 +363,9 @@ msgstr ""
 msgid "%{name} found %{resultsString}"
 msgstr ""
+msgid "%{name} is scheduled for %{action}"
+msgstr ""
+
 msgid "%{name}'s avatar"
 msgstr ""
@@ -1585,6 +1588,9 @@ msgstr ""
 msgid "All"
 msgstr ""
+msgid "All %{replicableType} are being scheduled for %{action}"
+msgstr ""
+
 msgid "All Members"
 msgstr ""
@@ -12795,7 +12801,13 @@ msgstr ""
 msgid "Missing commit signatures endpoint!"
 msgstr ""
-msgid "MissingSSHKeyWarningLink|add an SSH key"
+msgid "MissingSSHKeyWarningLink|Add SSH key"
+msgstr ""
+
+msgid "MissingSSHKeyWarningLink|Don't show again"
+msgstr ""
+
+msgid "MissingSSHKeyWarningLink|You won't be able to pull or push project code via SSH until you add an SSH key to your profile"
 msgstr ""
 msgid "Modal|Cancel"
@@ -13139,7 +13151,7 @@ msgstr ""
 msgid "No %{providerTitle} repositories found"
 msgstr ""
-msgid "No Design Repositories match this filter"
+msgid "No %{replicableType} match this filter"
 msgstr ""
 msgid "No Epic"
@@ -16896,7 +16908,7 @@ msgstr ""
 msgid "Resync"
 msgstr ""
-msgid "Resync all designs"
+msgid "Resync all %{replicableType}"
 msgstr ""
 msgid "Retry"
@@ -20013,10 +20025,10 @@ msgstr ""
 msgid "There was an error fetching median data for stages"
 msgstr ""
-msgid "There was an error fetching the Node's Groups"
+msgid "There was an error fetching the %{replicableType}"
 msgstr ""
-msgid "There was an error fetching the designs"
+msgid "There was an error fetching the Node's Groups"
 msgstr ""
 msgid "There was an error fetching the environments information."
@@ -20061,7 +20073,10 @@ msgstr ""
 msgid "There was an error subscribing to this label."
 msgstr ""
-msgid "There was an error syncing the designs."
+msgid "There was an error syncing project %{name}"
+msgstr ""
+
+msgid "There was an error syncing the %{replicableType}"
 msgstr ""
 msgid "There was an error trying to validate your query"
@@ -23102,9 +23117,6 @@ msgstr ""
 msgid "You won't be able to pull or push project code via %{protocol} until you %{set_password_link} on your account"
 msgstr ""
-msgid "You won't be able to pull or push project code via SSH until you %{add_ssh_key_link} to your profile"
-msgstr ""
-
 msgid "You won't be able to pull or push project code via SSH until you add an SSH key to your profile"
 msgstr ""

qa/Dockerfile

 FROM ruby:2.6-stretch
-LABEL maintainer "Grzegorz Bizon <grzegorz@gitlab.com>"
+LABEL maintainer="GitLab Quality Department <quality@gitlab.com>"
 ENV DEBIAN_FRONTEND noninteractive
 ##

qa/README.md

@@ -44,6 +44,14 @@ Note: GitLab QA uses [Selenium WebDriver](https://www.seleniumhq.org/) via
 the browser to use. You will need to have Chrome (or Chromium) and
 [chromedriver](https://chromedriver.chromium.org/) installed / in your `$PATH`.
+### Writing tests
+
+- [Writing tests from scratch tutorial](../doc/development/testing_guide/end_to_end/quick_start_guide.md)
+    - [Best practices](../doc/development/testing_guide/best_practices.md)
+    - [Using page objects](../doc/development/testing_guide/end_to_end/page_objects.md)
+    - [Guidelines](../doc/development/testing_guide/index.md)
+    - [Tests with special setup for local environemnts](../doc/development/testing_guide/end_to_end/running_tests_that_require_special_setup.md)
+
 ### Run the end-to-end tests in a local development environment
 Follow the GDK instructions to [prepare](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/master/doc/prepare.md)
@@ -77,13 +85,6 @@ Once you have the license file you can export it as an environment variable and
 export EE_LICENSE=$(cat /path/to/gitlab_license)
 ```
-### Writing tests
-
-- [Writing tests from scratch tutorial](../doc/development/testing_guide/end_to_end/quick_start_guide.md)
-    - [Best practices](../doc/development/testing_guide/best_practices.md)
-    - [Using page objects](../doc/development/testing_guide/end_to_end/page_objects.md)
-    - [Guidelines](../doc/development/testing_guide/index.md)
-
 ### Running specific tests
 You can also supply specific tests to run as another parameter. For example, to

spec/helpers/submodule_helper_spec.rb

@@ -81,6 +81,33 @@ describe SubmoduleHelper do
       end
     end
+    context 'submodule on gist.github.com' do
+      it 'detects ssh' do
+        stub_url('git@gist.github.com:gitlab-org/gitlab-foss.git')
+        is_expected.to eq(['https://gist.github.com/gitlab-org/gitlab-foss', 'https://gist.github.com/gitlab-org/gitlab-foss/hash'])
+      end
+
+      it 'detects http' do
+        stub_url('http://gist.github.com/gitlab-org/gitlab-foss.git')
+        is_expected.to eq(['https://gist.github.com/gitlab-org/gitlab-foss', 'https://gist.github.com/gitlab-org/gitlab-foss/hash'])
+      end
+
+      it 'detects https' do
+        stub_url('https://gist.github.com/gitlab-org/gitlab-foss.git')
+        is_expected.to eq(['https://gist.github.com/gitlab-org/gitlab-foss', 'https://gist.github.com/gitlab-org/gitlab-foss/hash'])
+      end
+
+      it 'handles urls with no .git on the end' do
+        stub_url('http://gist.github.com/gitlab-org/gitlab-foss')
+        is_expected.to eq(['https://gist.github.com/gitlab-org/gitlab-foss', 'https://gist.github.com/gitlab-org/gitlab-foss/hash'])
+      end
+
+      it 'returns original with non-standard url' do
+        stub_url('http://gist.github.com/another/gitlab-org/gitlab-foss.git')
+        is_expected.to eq([repo.submodule_url_for, nil])
+      end
+    end
+
     context 'submodule on github.com' do
       it 'detects ssh' do
         stub_url('git@github.com:gitlab-org/gitlab-foss.git')

spec/lib/gitlab/danger/helper_spec.rb

@@ -222,6 +222,7 @@ describe Gitlab::Danger::Helper do
       'lib/gitlab/danger/foo'                                 | :engineering_productivity
       'ee/lib/gitlab/danger/foo'                              | :engineering_productivity
       '.overcommit.yml.example'                               | :engineering_productivity
+      '.editorconfig'                                         | :engineering_productivity
       'tooling/overcommit/foo'                                | :engineering_productivity
       'lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml'   | :backend

spec/lib/gitlab/diff/highlight_cache_spec.rb

@@ -97,6 +97,28 @@ describe Gitlab::Diff::HighlightCache, :clean_gitlab_redis_cache do
       let(:paths) { merge_request.diffs.raw_diff_files.select(&:text?).map(&:file_path) }
     end
+    it 'updates memory usage metrics if Redis version >= 4' do
+      allow_next_instance_of(Redis) do |redis|
+        allow(redis).to receive(:info).and_return({ "redis_version" => "4.0.0" })
+
+        expect(described_class.gitlab_redis_diff_caching_memory_usage_bytes)
+          .to receive(:observe).and_call_original
+
+        cache.send(:write_to_redis_hash, diff_hash)
+      end
+    end
+
+    it 'does not update memory usage metrics if Redis version < 4' do
+      allow_next_instance_of(Redis) do |redis|
+        allow(redis).to receive(:info).and_return({ "redis_version" => "3.0.0" })
+
+        expect(described_class.gitlab_redis_diff_caching_memory_usage_bytes)
+          .not_to receive(:observe).and_call_original
+
+        cache.send(:write_to_redis_hash, diff_hash)
+      end
+    end
+
     context 'different diff_collections for the same diffable' do
       before do
         cache.write_if_empty

spec/lib/gitlab/repository_cache_adapter_spec.rb

@@ -211,8 +211,7 @@ describe Gitlab::RepositoryCacheAdapter do
     it 'expires the caches of the given methods' do
       expect(cache).to receive(:expire).with(:rendered_readme)
       expect(cache).to receive(:expire).with(:branch_names)
-      expect(redis_set_cache).to receive(:expire).with(:rendered_readme)
-      expect(redis_set_cache).to receive(:expire).with(:branch_names)
+      expect(redis_set_cache).to receive(:expire).with(:rendered_readme, :branch_names)
       expect(redis_hash_cache).to receive(:delete).with(:rendered_readme)
       expect(redis_hash_cache).to receive(:delete).with(:branch_names)

spec/lib/gitlab/repository_set_cache_spec.rb

@@ -51,12 +51,52 @@ describe Gitlab::RepositorySetCache, :clean_gitlab_redis_cache do
   end
   describe '#expire' do
-    it 'expires the given key from the cache' do
+    subject { cache.expire(*keys) }
+
+    before do
       cache.write(:foo, ['value'])
+      cache.write(:bar, ['value2'])
+    end
+    it 'actually wrote the values' do
       expect(cache.read(:foo)).to contain_exactly('value')
-      expect(cache.expire(:foo)).to eq(1)
-      expect(cache.read(:foo)).to be_empty
+      expect(cache.read(:bar)).to contain_exactly('value2')
+    end
+
+    context 'single key' do
+      let(:keys) { %w(foo) }
+
+      it { is_expected.to eq(1) }
+
+      it 'deletes the given key from the cache' do
+        subject
+
+        expect(cache.read(:foo)).to be_empty
+      end
+    end
+
+    context 'multiple keys' do
+      let(:keys) { %w(foo bar) }
+
+      it { is_expected.to eq(2) }
+
+      it 'deletes the given keys from the cache' do
+        subject
+
+        expect(cache.read(:foo)).to be_empty
+        expect(cache.read(:bar)).to be_empty
+      end
+    end
+
+    context "unlink isn't supported" do
+      before do
+        allow_any_instance_of(Redis).to receive(:unlink) { raise ::Redis::CommandError }
+      end
+
+      it 'still deletes the given key' do
+        expect(cache.expire(:foo)).to eq(1)
+        expect(cache.read(:foo)).to be_empty
+      end
     end
   end

spec/services/commits/cherry_pick_service_spec.rb

@@ -61,18 +61,6 @@ describe Commits::CherryPickService do
         expect(mr_notes.length).to eq(1)
         expect(mr_notes[0].commit_id).to eq(result[:result])
       end
-
-      context 'when :track_mr_picking feature flag is disabled' do
-        before do
-          stub_feature_flags(track_mr_picking: false)
-        end
-
-        it 'does not add system notes' do
-          expect do
-            cherry_pick(merge_commit_sha, branch_name)
-          end.not_to change { Note.count }
-        end
-      end
     end
     def find_cherry_pick_notes(noteable)

spec/services/deployments/link_merge_requests_service_spec.rb

@@ -160,53 +160,6 @@ describe Deployments::LinkMergeRequestsService do
       expect(deploy.merge_requests).to be_empty
     end
-
-    context 'when :track_mr_picking feature flag is disabled' do
-      before do
-        stub_feature_flags(track_mr_picking: false)
-      end
-
-      it 'does not link picked merge requests' do
-        environment = create(:environment, project: project)
-        deploy =
-          create(:deployment, :success, project: project, environment: environment)
-
-        picked_mr = create(
-          :merge_request,
-          :merged,
-          merge_commit_sha: '123abc',
-          source_project: project,
-          target_project: project
-        )
-
-        mr1 = create(
-          :merge_request,
-          :merged,
-          merge_commit_sha: mr1_merge_commit_sha,
-          source_project: project,
-          target_project: project
-        )
-
-        # mr1 includes c1c67abba which is a cherry-pick of the fake picked_mr merge request
-        create(:track_mr_picking_note, noteable: picked_mr, project: project, commit_id: 'c1c67abbaf91f624347bb3ae96eabe3a1b742478')
-
-        mr2 = create(
-          :merge_request,
-          :merged,
-          merge_commit_sha: mr2_merge_commit_sha,
-          source_project: project,
-          target_project: project
-        )
-
-        described_class.new(deploy).link_merge_requests_for_range(
-          first_deployment_sha,
-          mr2_merge_commit_sha
-        )
-
-        expect(deploy.merge_requests).to include(mr1, mr2)
-        expect(deploy.merge_requests).not_to include(picked_mr)
-      end
-    end
   end
   describe '#link_all_merged_merge_requests' do

spec/services/system_notes/merge_requests_service_spec.rb

@@ -253,7 +253,7 @@ describe ::SystemNotes::MergeRequestsService do
     end
     it "posts the 'picked merge request' system note" do
-      expect(subject.note).to eq("picked this merge request into branch [`#{branch_name}`](/#{project.full_path}/-/tree/#{branch_name}) with commit #{commit_sha}")
+      expect(subject.note).to eq("picked the changes into the branch [`#{branch_name}`](/#{project.full_path}/-/tree/#{branch_name}) with commit #{commit_sha}")
     end
     it 'links the merge request and the cherry-pick commit' do