Add latest changes from gitlab-org/gitlab@master

doc/raketasks/README.md

@@ -14,5 +14,5 @@ comments: false
 - [Webhooks](web_hooks.md)
 - [Import](import.md) of Git repositories in bulk
 - [Rebuild authorized_keys file](../administration/raketasks/maintenance.md#rebuild-authorized_keys-file) task for administrators
-- [Migrate Uploads](../administration/raketasks/uploads/migrate.md)
-- [Sanitize Uploads](../administration/raketasks/uploads/sanitize.md)
+- [Uploads Migrate](../administration/raketasks/uploads/migrate.md)
+- [Uploads Sanitize](../administration/raketasks/uploads/sanitize.md)

doc/update/README.md

@@ -134,8 +134,8 @@ Please follow the [Upgrade Recommendations](../policy/maintenance.md#upgrade-rec
 to identify the ideal upgrade path.
 Before upgrading to a new major version, you should ensure that any background
-migration jobs from previous releases have been completed. The number of remaining
-migrations jobs can be found by running the following command:
+migration jobs from previous releases have been completed. To see the current size
+of the `background_migration` queue, [check for background migrations before upgrading](#checking-for-background-migrations-before-upgrading).
 ## Upgrading between editions

lib/gitlab/auth/unique_ips_limiter.rb

@@ -8,7 +8,7 @@ module Gitlab
       class << self
         def limit_user_id!(user_id)
           if config.unique_ips_limit_enabled
-            ip = RequestContext.client_ip
+            ip = RequestContext.instance.client_ip
             unique_ips = update_and_return_ips_count(user_id, ip)
             raise TooManyIps.new(user_id, ip, unique_ips) if unique_ips > config.unique_ips_limit_per_user

lib/gitlab/email/receiver.rb

@@ -66,7 +66,8 @@ module Gitlab
       def key_from_additional_headers(mail)
         find_key_from_references(mail) ||
-          find_key_from_delivered_to_header(mail)
+          find_key_from_delivered_to_header(mail) ||
+          find_key_from_envelope_to_header(mail)
       end
       def ensure_references_array(references)
@@ -96,6 +97,13 @@ module Gitlab
         end
       end
+      def find_key_from_envelope_to_header(mail)
+        Array(mail[:envelope_to]).find do |header|
+          key = Gitlab::IncomingEmail.key_from_address(header.value)
+          break key if key
+        end
+      end
+
       def ignore_auto_reply!(mail)
         if auto_submitted?(mail) || auto_replied?(mail)
           raise AutoGeneratedEmailError

lib/gitlab/gitaly_client.rb

@@ -160,6 +160,7 @@ module Gitlab
     def self.execute(storage, service, rpc, request, remote_storage:, timeout:)
       enforce_gitaly_request_limits(:call)
+      Gitlab::RequestContext.instance.ensure_deadline_not_exceeded!
       kwargs = request_kwargs(storage, timeout: timeout.to_f, remote_storage: remote_storage)
       kwargs = yield(kwargs) if block_given?
@@ -234,12 +235,28 @@ module Gitlab
       metadata['gitaly-session-id'] = session_id
       metadata.merge!(Feature::Gitaly.server_feature_flags)
-      result = { metadata: metadata }
-      result[:deadline] = real_time + timeout if timeout > 0
-      result
+      deadline_info = request_deadline(timeout)
+      metadata.merge!(deadline_info.slice(:deadline_type))
+      { metadata: metadata, deadline: deadline_info[:deadline] }
     end
+    def self.request_deadline(timeout)
+      # timeout being 0 means the request is allowed to run indefinitely.
+      # We can't allow that inside a request, but this won't count towards Gitaly
+      # error budgets
+      regular_deadline = real_time.to_i + timeout if timeout > 0
+
+      return { deadline: regular_deadline } if Sidekiq.server?
+      return { deadline: regular_deadline } unless Gitlab::RequestContext.instance.request_deadline
+
+      limited_deadline = [regular_deadline, Gitlab::RequestContext.instance.request_deadline].compact.min
+      limited = limited_deadline < regular_deadline
+
+      { deadline: limited_deadline, deadline_type: limited ? "limited" : "regular" }
+    end
+    private_class_method :request_deadline
+
     def self.session_id
       Gitlab::SafeRequestStore[:gitaly_session_id] ||= SecureRandom.uuid
     end

lib/gitlab/graphql/authorize/authorize_resource.rb

@@ -40,7 +40,7 @@ module Gitlab
         def authorize!(object)
           unless authorized_resource?(object)
-            raise_resource_not_avaiable_error!
+            raise_resource_not_available_error!
           end
         end
@@ -63,7 +63,7 @@ module Gitlab
           end
         end
-        def raise_resource_not_avaiable_error!
+        def raise_resource_not_available_error!
           raise Gitlab::Graphql::Errors::ResourceNotAvailable, RESOURCE_ACCESS_ERROR
         end
       end

lib/gitlab/middleware/request_context.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Middleware
+    class RequestContext
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        # We should be using ActionDispatch::Request instead of
+        # Rack::Request to be consistent with Rails, but due to a Rails
+        # bug described in
+        # https://gitlab.com/gitlab-org/gitlab-foss/issues/58573#note_149799010
+        # hosts behind a load balancer will only see 127.0.0.1 for the
+        # load balancer's IP.
+        req = Rack::Request.new(env)
+
+        Gitlab::RequestContext.instance.client_ip = req.ip
+        Gitlab::RequestContext.instance.start_thread_cpu_time = Gitlab::Metrics::System.thread_cpu_time
+        Gitlab::RequestContext.instance.request_start_time = Gitlab::Metrics::System.real_time
+
+        @app.call(env)
+      end
+    end
+  end
+end

lib/gitlab/request_context.rb

@@ -2,34 +2,37 @@
 module Gitlab
   class RequestContext
-    class << self
-      def client_ip
-        Gitlab::SafeRequestStore[:client_ip]
-      end
-      def start_thread_cpu_time
-        Gitlab::SafeRequestStore[:start_thread_cpu_time]
+    include Singleton
+
+    RequestDeadlineExceeded = Class.new(StandardError)
+
+    attr_accessor :client_ip, :start_thread_cpu_time, :request_start_time
+    class << self
+      def instance
+        Gitlab::SafeRequestStore[:request_context] ||= new
       end
     end
-    def initialize(app)
-      @app = app
+    def request_deadline
+      return unless request_start_time
+      return unless Feature.enabled?(:request_deadline)
+
+      @request_deadline ||= request_start_time + max_request_duration_seconds
     end
-    def call(env)
-      # We should be using ActionDispatch::Request instead of
-      # Rack::Request to be consistent with Rails, but due to a Rails
-      # bug described in
-      # https://gitlab.com/gitlab-org/gitlab-foss/issues/58573#note_149799010
-      # hosts behind a load balancer will only see 127.0.0.1 for the
-      # load balancer's IP.
-      req = Rack::Request.new(env)
-      Gitlab::SafeRequestStore[:client_ip] = req.ip
-      Gitlab::SafeRequestStore[:start_thread_cpu_time] = Gitlab::Metrics::System.thread_cpu_time
-      @app.call(env)
+    def ensure_deadline_not_exceeded!
+      return unless request_deadline
+      return if Gitlab::Metrics::System.real_time < request_deadline
+      raise RequestDeadlineExceeded,
+            "Request takes longer than #{max_request_duration_seconds}"
+    end
+    private
+    def max_request_duration_seconds
+      Settings.gitlab.max_request_duration_seconds
     end
   end
 end

locale/gitlab.pot

@@ -1068,9 +1068,6 @@ msgstr ""
 msgid "Add new directory"
 msgstr ""
-msgid "Add new member to %{strong_start}%{group_name}%{strong_end}"
-msgstr ""
-
 msgid "Add or subtract spent time"
 msgstr ""
@@ -1095,9 +1092,6 @@ msgstr ""
 msgid "Add to merge train when pipeline succeeds"
 msgstr ""
-msgid "Add to project"
-msgstr ""
-
 msgid "Add to review"
 msgstr ""
@@ -7458,10 +7452,10 @@ msgstr ""
 msgid "Excluding merge commits. Limited to 6,000 commits."
 msgstr ""
-msgid "Existing"
+msgid "Existing members and groups"
 msgstr ""
-msgid "Existing members and groups"
+msgid "Existing shares"
 msgstr ""
 msgid "Expand"
@@ -9122,6 +9116,9 @@ msgstr ""
 msgid "Group maintainers can register group runners in the %{link}"
 msgstr ""
+msgid "Group members"
+msgstr ""
+
 msgid "Group name"
 msgstr ""
@@ -9392,6 +9389,9 @@ msgstr ""
 msgid "Groups can also be nested by creating %{subgroup_docs_link_start}subgroups%{subgroup_docs_link_end}."
 msgstr ""
+msgid "Groups with access to %{strong_start}%{group_name}%{strong_end}"
+msgstr ""
+
 msgid "Groups with access to <strong>%{project_name}</strong>"
 msgstr ""
@@ -10314,9 +10314,6 @@ msgstr ""
 msgid "Job has wrong arguments format."
 msgstr ""
-msgid "Job is in progress"
-msgstr ""
-
 msgid "Job is missing the `model_type` argument."
 msgstr ""
@@ -10326,6 +10323,12 @@ msgstr ""
 msgid "Job logs and artifacts"
 msgstr ""
+msgid "Job to create self-monitoring project is in progress"
+msgstr ""
+
+msgid "Job to delete self-monitoring project is in progress"
+msgstr ""
+
 msgid "Job was retried"
 msgstr ""
@@ -12639,12 +12642,6 @@ msgstr ""
 msgid "OperationsDashboard|The operations dashboard provides a summary of each project's operational health, including pipeline and alert statuses."
 msgstr ""
-msgid "Option 1"
-msgstr ""
-
-msgid "Option 2"
-msgstr ""
-
 msgid "Optional"
 msgstr ""
@@ -15784,9 +15781,6 @@ msgstr ""
 msgid "Save Changes"
 msgstr ""
-msgid "Save Expiration Policy"
-msgstr ""
-
 msgid "Save anyway"
 msgstr ""
@@ -15802,6 +15796,9 @@ msgstr ""
 msgid "Save comment"
 msgstr ""
+msgid "Save expiration policy"
+msgstr ""
+
 msgid "Save password"
 msgstr ""
@@ -16373,6 +16370,12 @@ msgstr ""
 msgid "Self-monitoring project does not exist. Please check logs for any error messages"
 msgstr ""
+msgid "Self-monitoring project has been successfully deleted"
+msgstr ""
+
+msgid "Self-monitoring project was not deleted. Please check logs for any error messages"
+msgstr ""
+
 msgid "Send a separate email notification to Developers."
 msgstr ""

qa/qa/page/group/sub_menus/members.rb

@@ -7,12 +7,9 @@ module QA
         class Members < Page::Base
           include Page::Component::UsersSelect
-          view 'app/views/groups/group_members/_new_group_member.html.haml' do
-            element :add_to_group_button
-          end
-
-          view 'app/helpers/groups/group_members_helper.rb' do
+          view 'app/views/shared/members/_invite_member.html.haml' do
             element :member_select_field
+            element :invite_member_button
           end
           view 'app/views/shared/members/_member.html.haml' do
@@ -24,7 +21,7 @@ module QA
           def add_member(username)
             select_user :member_select_field, username
-            click_element :add_to_group_button
+            click_element :invite_member_button
           end
           def update_access_level(username, access_level)

qa/qa/page/project/settings/members.rb

@@ -8,9 +8,9 @@ module QA
           include Page::Component::UsersSelect
           include QA::Page::Component::Select2
-          view 'app/views/projects/project_members/_new_project_member.html.haml' do
-            element :member_select_input
-            element :add_member_button
+          view 'app/views/shared/members/_invite_member.html.haml' do
+            element :member_select_field
+            element :invite_member_button
           end
           view 'app/views/projects/project_members/_team.html.haml' do
@@ -21,7 +21,7 @@ module QA
             element :invite_group_tab
           end
-          view 'app/views/projects/project_members/_new_project_group.html.haml' do
+          view 'app/views/shared/members/_invite_group.html.haml' do
             element :group_select_field
             element :invite_group_button
           end
@@ -43,8 +43,8 @@ module QA
           end
           def add_member(username)
-            select_user :member_select_input, username
-            click_element :add_member_button
+            select_user :member_select_field, username
+            click_element :invite_member_button
           end
           def remove_group(group_name)

spec/controllers/groups/group_links_controller_spec.rb

@@ -111,4 +111,100 @@ describe Groups::GroupLinksController do
       end
     end
   end
+
+  describe '#update' do
+    let!(:link) do
+      create(:group_group_link, { shared_group: shared_group,
+                                  shared_with_group: shared_with_group })
+    end
+
+    let(:expiry_date) { 1.month.from_now.to_date }
+
+    subject do
+      post(:update, params: { group_id: shared_group,
+                               id: link.id,
+                               group_link: { group_access: Gitlab::Access::GUEST,
+                                             expires_at: expiry_date } })
+    end
+
+    context 'when user has admin access to the shared group' do
+      before do
+        shared_group.add_owner(user)
+      end
+
+      it 'updates existing link' do
+        expect(link.group_access).to eq(Gitlab::Access::DEVELOPER)
+        expect(link.expires_at).to be_nil
+
+        subject
+
+        link.reload
+
+        expect(link.group_access).to eq(Gitlab::Access::GUEST)
+        expect(link.expires_at).to eq(expiry_date)
+      end
+    end
+
+    context 'when user does not have admin access to the shared group' do
+      it 'renders 404' do
+        subject
+
+        expect(response).to have_gitlab_http_status(404)
+      end
+    end
+
+    context 'when feature flag is disabled' do
+      before do
+        stub_feature_flags(share_group_with_group: false)
+      end
+
+      it 'renders 404' do
+        subject
+
+        expect(response).to have_gitlab_http_status(404)
+      end
+    end
+  end
+
+  describe '#destroy' do
+    let!(:link) do
+      create(:group_group_link, { shared_group: shared_group,
+                                  shared_with_group: shared_with_group })
+    end
+
+    subject do
+      post(:destroy, params: { group_id: shared_group,
+                               id: link.id })
+    end
+
+    context 'when user has admin access to the shared group' do
+      before do
+        shared_group.add_owner(user)
+      end
+
+      it 'deletes existing link' do
+        expect { subject }.to change(GroupGroupLink, :count).by(-1)
+      end
+    end
+
+    context 'when user does not have admin access to the shared group' do
+      it 'renders 404' do
+        subject
+
+        expect(response).to have_gitlab_http_status(404)
+      end
+    end
+
+    context 'when feature flag is disabled' do
+      before do
+        stub_feature_flags(share_group_with_group: false)
+      end
+
+      it 'renders 404' do
+        subject
+
+        expect(response).to have_gitlab_http_status(404)
+      end
+    end
+  end
 end

spec/controllers/groups/group_members_controller_spec.rb

@@ -31,6 +31,12 @@ describe Groups::GroupMembersController do
         expect(assigns(:invited_members).map(&:invite_email)).to match_array(invited.map(&:invite_email))
       end
+      it 'assigns skip groups' do
+        get :index, params: { group_id: group }
+
+        expect(assigns(:skip_groups)).to match_array(group.related_group_ids)
+      end
+
       it 'restricts search to one email' do
         get :index, params: { group_id: group, search_invited: invited.first.invite_email }

spec/controllers/health_check_controller_spec.rb

@@ -2,7 +2,7 @@
 require 'spec_helper'
-describe HealthCheckController do
+describe HealthCheckController, :request_store do
   include StubENV
   let(:xml_response) { Hash.from_xml(response.body)['hash'] }
@@ -18,7 +18,7 @@ describe HealthCheckController do
   describe 'GET #index' do
     context 'when services are up but accessed from outside whitelisted ips' do
       before do
-        allow(Gitlab::RequestContext).to receive(:client_ip).and_return(not_whitelisted_ip)
+        allow(Gitlab::RequestContext.instance).to receive(:client_ip).and_return(not_whitelisted_ip)
       end
       it 'returns a not found page' do
@@ -48,7 +48,7 @@ describe HealthCheckController do
     context 'when services are up and accessed from whitelisted ips' do
       before do
-        allow(Gitlab::RequestContext).to receive(:client_ip).and_return(whitelisted_ip)
+        allow(Gitlab::RequestContext.instance).to receive(:client_ip).and_return(whitelisted_ip)
       end
       it 'supports successful plaintext response' do
@@ -95,7 +95,7 @@ describe HealthCheckController do
       before do
         allow(HealthCheck::Utils).to receive(:process_checks).with(['standard']).and_return('The server is on fire')
         allow(HealthCheck::Utils).to receive(:process_checks).with(['email']).and_return('Email is on fire')
-        allow(Gitlab::RequestContext).to receive(:client_ip).and_return(whitelisted_ip)
+        allow(Gitlab::RequestContext.instance).to receive(:client_ip).and_return(whitelisted_ip)
       end
       it 'supports failure plaintext response' do

spec/controllers/metrics_controller_spec.rb

@@ -2,7 +2,7 @@
 require 'spec_helper'
-describe MetricsController do
+describe MetricsController, :request_store do
   include StubENV
   let(:metrics_multiproc_dir) { @metrics_multiproc_dir }
@@ -53,7 +53,7 @@ describe MetricsController do
     context 'accessed from whitelisted ip' do
       before do
-        allow(Gitlab::RequestContext).to receive(:client_ip).and_return(whitelisted_ip)
+        allow(Gitlab::RequestContext.instance).to receive(:client_ip).and_return(whitelisted_ip)
       end
       it_behaves_like 'endpoint providing metrics'
@@ -61,7 +61,7 @@ describe MetricsController do
     context 'accessed from ip in whitelisted range' do
       before do
-        allow(Gitlab::RequestContext).to receive(:client_ip).and_return(ip_in_whitelisted_range)
+        allow(Gitlab::RequestContext.instance).to receive(:client_ip).and_return(ip_in_whitelisted_range)
       end
       it_behaves_like 'endpoint providing metrics'
@@ -69,7 +69,7 @@ describe MetricsController do
     context 'accessed from not whitelisted ip' do
       before do
-        allow(Gitlab::RequestContext).to receive(:client_ip).and_return(not_whitelisted_ip)
+        allow(Gitlab::RequestContext.instance).to receive(:client_ip).and_return(not_whitelisted_ip)
       end
       it 'returns the expected error response' do

spec/features/admin/admin_groups_spec.rb

@@ -167,14 +167,14 @@ describe 'Admin Groups' do
     it 'adds admin a to a group as developer', :js do
       visit group_group_members_path(group)
-      page.within '.users-group-form' do
+      page.within '.invite-users-form' do
         select2(current_user.id, from: '#user_ids', multiple: true)
         select 'Developer', from: 'access_level'
       end
-      click_button 'Add to group'
-      page.within '.content-list' do
+      click_button 'Invite'
+      page.within '[data-qa-selector="members_list"]' do
         expect(page).to have_content(current_user.name)
         expect(page).to have_content('Developer')
       end
@@ -187,7 +187,7 @@ describe 'Admin Groups' do
       visit group_group_members_path(group)
-      page.within '.content-list' do
+      page.within '[data-qa-selector="members_list"]' do
         expect(page).to have_content(current_user.name)
         expect(page).to have_content('Developer')
       end
@@ -196,7 +196,7 @@ describe 'Admin Groups' do
       visit group_group_members_path(group)
-      page.within '.content-list' do
+      page.within '[data-qa-selector="members_list"]' do
         expect(page).not_to have_content(current_user.name)
         expect(page).not_to have_content('Developer')
       end

spec/features/admin/admin_projects_spec.rb

@@ -98,12 +98,12 @@ describe "Admin::Projects" do
     it 'adds admin a to a project as developer', :js do
       visit project_project_members_path(project)
-      page.within '.users-project-form' do
+      page.within '.invite-users-form' do
         select2(current_user.id, from: '#user_ids', multiple: true)
         select 'Developer', from: 'access_level'
       end
-      click_button 'Add to project'
+      click_button 'Invite'
       page.within '.content-list' do
         expect(page).to have_content(current_user.name)

spec/features/groups/members/manage_groups_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe 'Groups > Members > Manage groups', :js do
+  include Select2Helper
+  include Spec::Support::Helpers::Features::ListRowsHelpers
+
+  let(:user) { create(:user) }
+  let(:shared_with_group) { create(:group) }
+  let(:shared_group) { create(:group) }
+
+  before do
+    shared_group.add_owner(user)
+    sign_in(user)
+  end
+
+  context 'with share groups with groups feature flag' do
+    before do
+      stub_feature_flags(shared_with_group: true)
+    end
+
+    it 'add group to group' do
+      visit group_group_members_path(shared_group)
+
+      add_group(shared_with_group.id, 'Reporter')
+
+      page.within(first_row) do
+        expect(page).to have_content(shared_with_group.name)
+        expect(page).to have_content('Reporter')
+      end
+    end
+
+    it 'remove user from group' do
+      create(:group_group_link, shared_group: shared_group,
+        shared_with_group: shared_with_group, group_access: ::Gitlab::Access::DEVELOPER)
+
+      visit group_group_members_path(shared_group)
+
+      expect(page).to have_content(shared_with_group.name)
+
+      accept_confirm do
+        find(:css, '#existing_shares li', text: shared_with_group.name).find(:css, 'a.btn-remove').click
+      end
+
+      wait_for_requests
+
+      expect(page).not_to have_content(shared_with_group.name)
+    end
+
+    it 'update group to owner level' do
+      create(:group_group_link, shared_group: shared_group,
+        shared_with_group: shared_with_group, group_access: ::Gitlab::Access::DEVELOPER)
+
+      visit group_group_members_path(shared_group)
+
+      page.within(first_row) do
+        click_button('Developer')
+        click_link('Maintainer')
+
+        wait_for_requests
+
+        expect(page).to have_button('Maintainer')
+      end
+    end
+
+    def add_group(id, role)
+      page.click_link 'Invite group'
+      page.within ".invite-group-form" do
+        select2(id, from: "#shared_with_group_id")
+        select(role, from: "shared_group_access")
+        click_button "Invite"
+      end
+    end
+  end
+
+  context 'without share groups with groups feature flag' do
+    before do
+      stub_feature_flags(share_group_with_group: false)
+    end
+
+    it 'does not render invitation form and tabs' do
+      visit group_group_members_path(shared_group)
+
+      expect(page).not_to have_link('Invite member')
+      expect(page).not_to have_link('Invite group')
+    end
+  end
+end

spec/features/groups/members/manage_members_spec.rb

@@ -113,7 +113,8 @@ describe 'Groups > Members > Manage members' do
     visit group_group_members_path(group)
-    expect(page).not_to have_button 'Add to group'
+    expect(page).not_to have_selector '.invite-users-form'
+    expect(page).not_to have_selector '.invite-group-form'
     page.within(second_row) do
       # Can not modify user2 role
@@ -125,11 +126,10 @@ describe 'Groups > Members > Manage members' do
   end
   def add_user(id, role)
-    page.within ".users-group-form" do
+    page.within ".invite-users-form" do
       select2(id, from: "#user_ids", multiple: true)
       select(role, from: "access_level")
+      click_button "Invite"
     end
-
-    click_button "Add to group"
   end
 end

spec/features/groups/members/search_members_spec.rb

@@ -24,7 +24,7 @@ describe 'Search group member' do
       find('.user-search-btn').click
     end
-    group_members_list = find(".card .content-list")
+    group_members_list = find('[data-qa-selector="members_list"]')
     expect(group_members_list).to have_content(member.name)
     expect(group_members_list).not_to have_content(user.name)
   end