Merge branch '14-10-stable-ee-patch-3' into '14-10-stable-ee'

Prepare 14.10.3-ee release

See merge request gitlab-org/gitlab!87729

app/assets/javascripts/issues/list/constants.js

@@ -56,6 +56,7 @@ export const ISSUE_REFERENCE = /^#\d+$/;
 export const MAX_LIST_SIZE = 10;
 export const PAGE_SIZE = 20;
 export const PAGE_SIZE_MANUAL = 100;
+export const PARAM_ASSIGNEE_ID = 'assignee_id';
 export const PARAM_PAGE_AFTER = 'page_after';
 export const PARAM_PAGE_BEFORE = 'page_before';
 export const PARAM_STATE = 'state';
@@ -112,7 +113,8 @@ export const URL_PARAM = 'urlParam';
 export const NORMAL_FILTER = 'normalFilter';
 export const SPECIAL_FILTER = 'specialFilter';
 export const ALTERNATIVE_FILTER = 'alternativeFilter';
-export const SPECIAL_FILTER_VALUES = [
+
+export const specialFilterValues = [
   FILTER_NONE,
   FILTER_ANY,
   FILTER_CURRENT,

app/assets/javascripts/issues/list/utils.js

 import { isPositiveInteger } from '~/lib/utils/number_utils';
+import { getParameterByName } from '~/lib/utils/url_utility';
 import { __ } from '~/locale';
 import {
   FILTERED_SEARCH_TERM,
@@ -20,13 +21,14 @@ import {
   NORMAL_FILTER,
   PAGE_SIZE,
   PAGE_SIZE_MANUAL,
+  PARAM_ASSIGNEE_ID,
   POPULARITY_ASC,
   POPULARITY_DESC,
   PRIORITY_ASC,
   PRIORITY_DESC,
   RELATIVE_POSITION_ASC,
   SPECIAL_FILTER,
-  SPECIAL_FILTER_VALUES,
+  specialFilterValues,
   TITLE_ASC,
   TITLE_DESC,
   TOKEN_TYPE_ASSIGNEE,
@@ -202,16 +204,19 @@ export const getFilterTokens = (locationSearch) => {
   return filterTokens.concat(searchTokens);
 };
-const getFilterType = (data, tokenType = '') =>
-  SPECIAL_FILTER_VALUES.includes(data) ||
-  (tokenType === TOKEN_TYPE_ASSIGNEE && isPositiveInteger(data))
-    ? SPECIAL_FILTER
-    : NORMAL_FILTER;
+const getFilterType = (data, tokenType = '') => {
+  const isAssigneeIdParam =
+    tokenType === TOKEN_TYPE_ASSIGNEE &&
+    isPositiveInteger(data) &&
+    getParameterByName(PARAM_ASSIGNEE_ID) === data;
+
+  return specialFilterValues.includes(data) || isAssigneeIdParam ? SPECIAL_FILTER : NORMAL_FILTER;
+};
 const wildcardTokens = [TOKEN_TYPE_ITERATION, TOKEN_TYPE_MILESTONE, TOKEN_TYPE_RELEASE];
 const isWildcardValue = (tokenType, value) =>
-  wildcardTokens.includes(tokenType) && SPECIAL_FILTER_VALUES.includes(value);
+  wildcardTokens.includes(tokenType) && specialFilterValues.includes(value);
 const requiresUpperCaseValue = (tokenType, value) =>
   tokenType === TOKEN_TYPE_TYPE || isWildcardValue(tokenType, value);

doc/security/rate_limits.md

@@ -70,6 +70,21 @@ For configuration information, see
 ## Non-configurable limits
+### Git operations using SSH
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/78373) in GitLab 14.7.
+
+GitLab rate limits Git operations by user account and project. If a request from a user for a Git operation
+on a project exceeds the rate limit, GitLab drops further connection requests from that user for the project.
+
+The rate limit applies at the Git command ([plumbing](https://git-scm.com/book/en/v2/Git-Internals-Plumbing-and-Porcelain)) level. 
+Each command has a rate limit of 600 per minute. For example:
+
+- `git push` has a rate limit of 600 per minute.
+- `git pull` has its own rate limit of 600 per minute.
+
+Because the same commands are shared by `git-upload-pack`, `git pull`, and `git clone`, they share a rate limit.
+
 ### Repository archives
 > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/25750) in GitLab 12.9.

doc/update/index.md

@@ -402,6 +402,12 @@ NOTE:
 Specific information that follow related to Ruby and Git versions do not apply to [Omnibus installations](https://docs.gitlab.com/omnibus/)
 and [Helm Chart deployments](https://docs.gitlab.com/charts/). They come with appropriate Ruby and Git versions and are not using system binaries for Ruby and Git. There is no need to install Ruby or Git when utilizing these two approaches.
+### 14.10.0
+
+- The upgrade to GitLab 14.10 executes a [concurrent index drop](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/84308) of unneeded
+  entries from the `ci_job_artifacts` database table. This could potentially run for multiple minutes, especially if the table has a lot of
+  traffic and the migration is unable to acquire a lock. It is advised to let this process finish as restarting may result in data loss.
+
 ### 14.9.0
 - Database changes made by the upgrade to GitLab 14.9 can take hours or days to complete on larger GitLab instances. 

lib/gitlab/database/migration_helpers.rb

@@ -943,7 +943,7 @@ module Gitlab
         execute("DELETE FROM batched_background_migrations WHERE #{conditions}")
       end
-      def ensure_batched_background_migration_is_finished(job_class_name:, table_name:, column_name:, job_arguments:)
+      def ensure_batched_background_migration_is_finished(job_class_name:, table_name:, column_name:, job_arguments:, finalize: true)
         migration = Gitlab::Database::BackgroundMigration::BatchedMigration
           .for_configuration(job_class_name, table_name, column_name, job_arguments).first
@@ -954,9 +954,13 @@ module Gitlab
           job_arguments: job_arguments
         }
-        if migration.nil?
-          Gitlab::AppLogger.warn "Could not find batched background migration for the given configuration: #{configuration}"
-        elsif !migration.finished?
+        return Gitlab::AppLogger.warn "Could not find batched background migration for the given configuration: #{configuration}" if migration.nil?
+
+        return if migration.finished?
+
+        Gitlab::Database::BackgroundMigration::BatchedMigrationRunner.finalize(job_class_name, table_name, column_name, job_arguments, connection: connection) if finalize
+
+        unless migration.reload.finished? # rubocop:disable Cop/ActiveRecordAssociationReload
           raise "Expected batched background migration for the given configuration to be marked as 'finished', " \
             "but it is '#{migration.status_name}':" \
             "\t#{configuration}" \

lib/gitlab/database/migrations/batched_background_migration_helpers.rb

@@ -122,6 +122,14 @@ module Gitlab
           migration.save!
           migration
         end
+
+        def finalize_batched_background_migration(job_class_name:, table_name:, column_name:, job_arguments:)
+          migration = Gitlab::Database::BackgroundMigration::BatchedMigration.find_for_configuration(job_class_name, table_name, column_name, job_arguments)
+
+          raise 'Could not find batched background migration' if migration.nil?
+
+          Gitlab::Database::BackgroundMigration::BatchedMigrationRunner.finalize(job_class_name, table_name, column_name, job_arguments, connection: connection)
+        end
       end
     end
   end

spec/frontend/issues/list/mock_data.js

@@ -117,6 +117,7 @@ export const locationSearch = [
   'not[author_username]=marge',
   'assignee_username[]=bart',
   'assignee_username[]=lisa',
+  'assignee_username[]=5',
   'not[assignee_username][]=patty',
   'not[assignee_username][]=selma',
   'milestone_title=season+3',
@@ -165,6 +166,7 @@ export const filteredTokens = [
   { type: 'author_username', value: { data: 'marge', operator: OPERATOR_IS_NOT } },
   { type: 'assignee_username', value: { data: 'bart', operator: OPERATOR_IS } },
   { type: 'assignee_username', value: { data: 'lisa', operator: OPERATOR_IS } },
+  { type: 'assignee_username', value: { data: '5', operator: OPERATOR_IS } },
   { type: 'assignee_username', value: { data: 'patty', operator: OPERATOR_IS_NOT } },
   { type: 'assignee_username', value: { data: 'selma', operator: OPERATOR_IS_NOT } },
   { type: 'milestone', value: { data: 'season 3', operator: OPERATOR_IS } },
@@ -212,7 +214,7 @@ export const filteredTokensWithSpecialValues = [
 export const apiParams = {
   authorUsername: 'homer',
-  assigneeUsernames: ['bart', 'lisa'],
+  assigneeUsernames: ['bart', 'lisa', '5'],
   milestoneTitle: ['season 3', 'season 4'],
   labelName: ['cartoon', 'tv'],
   releaseTag: ['v3', 'v4'],
@@ -251,7 +253,7 @@ export const apiParamsWithSpecialValues = {
 export const urlParams = {
   author_username: 'homer',
   'not[author_username]': 'marge',
-  'assignee_username[]': ['bart', 'lisa'],
+  'assignee_username[]': ['bart', 'lisa', '5'],
   'not[assignee_username][]': ['patty', 'selma'],
   milestone_title: ['season 3', 'season 4'],
   'not[milestone_title]': ['season 20', 'season 30'],

spec/frontend/issues/list/utils_spec.js

+import setWindowLocation from 'helpers/set_window_location_helper';
+import { TEST_HOST } from 'helpers/test_constants';
 import {
   apiParams,
   apiParamsWithSpecialValues,
@@ -127,21 +129,33 @@ describe('getFilterTokens', () => {
 });
 describe('convertToApiParams', () => {
+  beforeEach(() => {
+    setWindowLocation(TEST_HOST);
+  });
+
   it('returns api params given filtered tokens', () => {
     expect(convertToApiParams(filteredTokens)).toEqual(apiParams);
   });
   it('returns api params given filtered tokens with special values', () => {
+    setWindowLocation('?assignee_id=123');
+
     expect(convertToApiParams(filteredTokensWithSpecialValues)).toEqual(apiParamsWithSpecialValues);
   });
 });
 describe('convertToUrlParams', () => {
+  beforeEach(() => {
+    setWindowLocation(TEST_HOST);
+  });
+
   it('returns url params given filtered tokens', () => {
     expect(convertToUrlParams(filteredTokens)).toEqual(urlParams);
   });
   it('returns url params given filtered tokens with special values', () => {
+    setWindowLocation('?assignee_id=123');
+
     expect(convertToUrlParams(filteredTokensWithSpecialValues)).toEqual(urlParamsWithSpecialValues);
   });
 });

spec/lib/gitlab/database/migration_helpers_spec.rb

@@ -2210,12 +2210,17 @@ RSpec.describe Gitlab::Database::MigrationHelpers do
   end
   describe '#ensure_batched_background_migration_is_finished' do
+    let(:job_class_name) { 'CopyColumnUsingBackgroundMigrationJob' }
+    let(:table) { :events }
+    let(:column_name) { :id }
+    let(:job_arguments) { [["id"], ["id_convert_to_bigint"], nil] }
+
     let(:configuration) do
       {
-        job_class_name: 'CopyColumnUsingBackgroundMigrationJob',
-        table_name: :events,
-        column_name: :id,
-        job_arguments: [["id"], ["id_convert_to_bigint"], nil]
+        job_class_name: job_class_name,
+        table_name: table,
+        column_name: column_name,
+        job_arguments: job_arguments
       }
     end
@@ -2224,6 +2229,10 @@ RSpec.describe Gitlab::Database::MigrationHelpers do
     it 'raises an error when migration exists and is not marked as finished' do
       create(:batched_background_migration, :active, configuration)
+      allow_next_instance_of(Gitlab::Database::BackgroundMigration::BatchedMigrationRunner) do |runner|
+        allow(runner).to receive(:finalize).with(job_class_name, table, column_name, job_arguments).and_return(false)
+      end
+
       expect { ensure_batched_background_migration_is_finished }
         .to raise_error "Expected batched background migration for the given configuration to be marked as 'finished', but it is 'active':" \
             "\t#{configuration}" \
@@ -2251,6 +2260,28 @@ RSpec.describe Gitlab::Database::MigrationHelpers do
       expect { ensure_batched_background_migration_is_finished }
         .not_to raise_error
     end
+
+    it 'finalizes the migration' do
+      migration = create(:batched_background_migration, :active, configuration)
+
+      allow_next_instance_of(Gitlab::Database::BackgroundMigration::BatchedMigrationRunner) do |runner|
+        expect(runner).to receive(:finalize).with(job_class_name, table, column_name, job_arguments).and_return(migration.finish!)
+      end
+
+      ensure_batched_background_migration_is_finished
+    end
+
+    context 'when the flag finalize is false' do
+      it 'does not finalize the migration' do
+        create(:batched_background_migration, :active, configuration)
+
+        allow_next_instance_of(Gitlab::Database::BackgroundMigration::BatchedMigrationRunner) do |runner|
+          expect(runner).not_to receive(:finalize).with(job_class_name, table, column_name, job_arguments)
+        end
+
+        expect { model.ensure_batched_background_migration_is_finished(**configuration.merge(finalize: false)) }.to raise_error(RuntimeError)
+      end
+    end
   end
   describe '#index_exists_by_name?' do

spec/lib/gitlab/database/migrations/batched_background_migration_helpers_spec.rb

@@ -163,4 +163,24 @@ RSpec.describe Gitlab::Database::Migrations::BatchedBackgroundMigrationHelpers d
       end
     end
   end
+
+  describe '#finalize_batched_background_migration' do
+    let!(:batched_migration) { create(:batched_background_migration, job_class_name: 'MyClass', table_name: :projects, column_name: :id, job_arguments: []) }
+
+    it 'finalizes the migration' do
+      allow_next_instance_of(Gitlab::Database::BackgroundMigration::BatchedMigrationRunner) do |runner|
+        expect(runner).to receive(:finalize).with('MyClass', :projects, :id, [])
+      end
+
+      migration.finalize_batched_background_migration(job_class_name: 'MyClass', table_name: :projects, column_name: :id, job_arguments: [])
+    end
+
+    context 'when the migration does not exist' do
+      it 'raises an exception' do
+        expect do
+          migration.finalize_batched_background_migration(job_class_name: 'MyJobClass', table_name: :projects, column_name: :id, job_arguments: [])
+        end.to raise_error(RuntimeError, 'Could not find batched background migration')
+      end
+    end
+  end
 end

spec/migrations/finalize_project_namespaces_backfill_spec.rb

@@ -9,9 +9,11 @@ RSpec.describe FinalizeProjectNamespacesBackfill, :migration do
   let_it_be(:migration) { described_class::MIGRATION }
   describe '#up' do
-    shared_examples 'raises migration not finished exception' do
-      it 'raises exception' do
-        expect { migrate! }.to raise_error(/Expected batched background migration for the given configuration to be marked as 'finished'/)
+    shared_examples 'finalizes the migration' do
+      it 'finalizes the migration' do
+        allow_next_instance_of(Gitlab::Database::BackgroundMigration::BatchedMigrationRunner) do |runner|
+          expect(runner).to receive(:finalize).with('"ProjectNamespaces::BackfillProjectNamespaces"', :projects, :id, [nil, "up"])
+        end
       end
     end
@@ -61,7 +63,7 @@ RSpec.describe FinalizeProjectNamespacesBackfill, :migration do
             project_namespace_backfill.update!(status: status)
           end
-          it_behaves_like 'raises migration not finished exception'
+          it_behaves_like 'finalizes the migration'
         end
       end
     end