Merge branch '14-7-stable-ee-patch-6' into '14-7-stable-ee'

Prepare 14.7.6-ee release

See merge request gitlab-org/gitlab!83054

.gitlab/ci/rules.gitlab-ci.yml

@@ -13,6 +13,9 @@
 .if-jh: &if-jh
   if: '$CI_PROJECT_PATH =~ /^gitlab-(jh|cn)\/.*/'
+.if-force-ci: &if-force-ci
+  if: '$FORCE_GITLAB_CI'
+
 .if-default-refs: &if-default-refs
   if: '$CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH || $CI_COMMIT_REF_NAME =~ /^[\d-]+-stable(-ee)?$/ || $CI_COMMIT_REF_NAME =~ /^\d+-\d+-auto-deploy-\d+$/ || $CI_COMMIT_REF_NAME =~ /^security\// || $CI_MERGE_REQUEST_IID || $CI_COMMIT_TAG || $FORCE_GITLAB_CI'
@@ -485,6 +488,7 @@
     - <<: *if-dot-com-gitlab-org-default-branch
       changes: *code-qa-patterns
     - <<: *if-dot-com-gitlab-org-schedule
+    - <<: *if-force-ci
 .build-images:rules:build-assets-image:
   rules:
@@ -781,6 +785,9 @@
       allow_failure: true
     - <<: *if-dot-com-gitlab-org-schedule
       allow_failure: true
+    - <<: *if-force-ci
+      when: manual
+      allow_failure: true
 .qa:rules:package-and-qa:feature-flags:
   rules:

app/services/ci/job_artifacts/destroy_associations_service.rb

@@ -12,7 +12,7 @@ module Ci
       def destroy_records
         @job_artifacts_relation.each_batch(of: BATCH_SIZE) do |relation|
-          service = Ci::JobArtifacts::DestroyBatchService.new(relation, pick_up_at: Time.current)
+          service = Ci::JobArtifacts::DestroyBatchService.new(relation, pick_up_at: Time.current, fix_expire_at: false)
           result  = service.execute(update_stats: false)
           updates = result[:statistics_updates]

app/services/ci/job_artifacts/destroy_batch_service.rb

@@ -17,13 +17,18 @@ module Ci
       # +pick_up_at+:: When to pick up for deletion of files
       # Returns:
       # +Hash+:: A hash with status and destroyed_artifacts_count keys
-      def initialize(job_artifacts, pick_up_at: nil)
+      def initialize(job_artifacts, pick_up_at: nil, fix_expire_at: fix_expire_at?)
         @job_artifacts = job_artifacts.with_destroy_preloads.to_a
         @pick_up_at = pick_up_at
+        @fix_expire_at = fix_expire_at
       end
       # rubocop: disable CodeReuse/ActiveRecord
       def execute(update_stats: true)
+        # Detect and fix artifacts that had `expire_at` wrongly backfilled by migration
+        # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/47723
+        detect_and_fix_wrongly_expired_artifacts
+
         return success(destroyed_artifacts_count: 0, statistics_updates: {}) if @job_artifacts.empty?
         destroy_related_records(@job_artifacts)
@@ -89,6 +94,55 @@ module Ci
           @job_artifacts.sum { |artifact| artifact.try(:size) || 0 }
         end
       end
+
+      # This detects and fixes job artifacts that have `expire_at` wrongly backfilled by the migration
+      # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/47723.
+      # These job artifacts will not be deleted and will have their `expire_at` removed.
+      #
+      # The migration would have backfilled `expire_at`
+      # to midnight on the 22nd of the month of the local timezone,
+      # storing it as UTC time in the database.
+      #
+      # If the timezone setting has changed since the migration,
+      # the `expire_at` stored in the database could have changed to a different local time other than midnight.
+      # For example:
+      # - changing timezone from UTC+02:00 to UTC+02:30 would change the `expire_at` in local time 00:00:00 to 00:30:00.
+      # - changing timezone from UTC+00:00 to UTC-01:00 would change the `expire_at` in local time 00:00:00 to 23:00:00 on the previous day (21st).
+      #
+      # Therefore job artifacts that have `expire_at` exactly on the 00, 30 or 45 minute mark
+      # on the dates 21, 22, 23 of the month will not be deleted.
+      # https://en.wikipedia.org/wiki/List_of_UTC_time_offsets
+      def detect_and_fix_wrongly_expired_artifacts
+        return unless @fix_expire_at
+
+        wrongly_expired_artifacts, @job_artifacts = @job_artifacts.partition { |artifact| wrongly_expired?(artifact) }
+
+        remove_expire_at(wrongly_expired_artifacts)
+      end
+
+      def fix_expire_at?
+        Feature.enabled?(:ci_detect_wrongly_expired_artifacts, default_enabled: :yaml)
+      end
+
+      def wrongly_expired?(artifact)
+        return false unless artifact.expire_at.present?
+
+        match_date?(artifact.expire_at) && match_time?(artifact.expire_at)
+      end
+
+      def match_date?(expire_at)
+        [21, 22, 23].include?(expire_at.day)
+      end
+
+      def match_time?(expire_at)
+        %w[00:00.000 30:00.000 45:00.000].include?(expire_at.strftime('%M:%S.%L'))
+      end
+
+      def remove_expire_at(artifacts)
+        Ci::JobArtifact.id_in(artifacts).update_all(expire_at: nil)
+
+        Gitlab::AppLogger.info(message: "Fixed expire_at from artifacts.", fixed_artifacts_expire_at_count: artifacts.count)
+      end
     end
   end
 end

config/feature_flags/development/ci_destroy_all_expired_service.yml

@@ -5,4 +5,4 @@ rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/348786
 milestone: '14.6'
 type: development
 group: group::pipeline execution
-default_enabled: false
+default_enabled: true

config/feature_flags/development/ci_detect_wrongly_expired_artifacts.yml

+---
+name: ci_detect_wrongly_expired_artifacts
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/82084
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/354955
+milestone: '14.9'
+type: development
+group: group::pipeline insights
+default_enabled: true

ee/app/assets/javascripts/admin/subscriptions/show/components/subscription_activation_form.vue

@@ -156,6 +156,7 @@ export default {
           v-model.trim="form.fields.activationCode.value"
           v-validation:[form.showValidation]
           class="gl-mb-4"
+          data-qa-selector="activation_code"
           :disabled="isLoading"
           :placeholder="$options.i18n.pasteActivationCode"
           :state="form.fields.activationCode.state"
@@ -197,6 +198,7 @@ export default {
         category="primary"
         class="gl-mt-6 js-no-auto-disable"
         data-testid="activate-button"
+        data-qa-selector="activate"
         type="submit"
         variant="confirm"
       >

ee/app/assets/javascripts/admin/subscriptions/show/components/subscription_breakdown.vue

@@ -179,6 +179,7 @@ export default {
           :subscription="subscription"
           :sync-did-fail="syncDidFail"
           data-testid="subscription-details"
+          data-qa-selector="subscription_details"
         >
           <template v-if="shouldShowFooter" #footer>
             <div class="gl-display-flex gl-flex-wrap gl-align-items-flex-start">

qa/lib/gitlab/page/admin/subscription.rb

@@ -6,6 +6,10 @@ module Gitlab
       class Subscription < Chemlab::Page
         path '/admin/subscription'
+        div :subscription_details
+        text_field :activation_code
+        button :activate
+        label :terms_of_services, text: /I agree that/
         p :plan
         p :started
         p :name
@@ -16,6 +20,33 @@ module Gitlab
         h2 :users_in_subscription
         h2 :users_over_subscription
         table :subscription_history
+
+        def accept_terms
+          terms_of_services_element.click # workaround for hidden checkbox
+        end
+
+        # Checks if a subscription record exists in subscription history table
+        #
+        # @param plan [Hash] Name of the plan
+        # @option plan [Hash] Support::Helpers::FREE
+        # @option plan [Hash] Support::Helpers::PREMIUM
+        # @option plan [Hash] Support::Helpers::PREMIUM_SELF_MANAGED
+        # @option plan [Hash] Support::Helpers::ULTIMATE
+        # @option plan [Hash] Support::Helpers::ULTIMATE_SELF_MANAGED
+        # @option plan [Hash] Support::Helpers::CI_MINUTES
+        # @option plan [Hash] Support::Helpers::STORAGE
+        # @param users_in_license [Integer] Number of users in license
+        # @param license_type [Hash] Type of the license
+        # @option license_type [String] 'license file'
+        # @option license_type [String] 'cloud license'
+        # @return [Boolean] True if record exsists, false if not
+        def has_subscription_record?(plan, users_in_license, license_type)
+          # find any records that have a matching plan and seats and type
+          subscription_history_element.hashes.any? do |record|
+            record['Plan'] == plan[:name].capitalize && record['Seats'] == users_in_license.to_s && \
+              record['Type'].strip.downcase == license_type
+          end
+        end
       end
     end
   end

qa/lib/gitlab/page/admin/subscription.stub.rb

@@ -4,6 +4,112 @@ module Gitlab
   module Page
     module Admin
       module Subscription
+        # @note Defined as +h6 :subscription_details+
+        # @return [String] The text content or value of +subscription_details+
+        def subscription_details
+          # This is a stub, used for indexing. The method is dynamically generated.
+        end
+
+        # @example
+        #   Gitlab::Page::Admin::Subscription.perform do |subscription|
+        #     expect(subscription.subscription_details_element).to exist
+        #   end
+        # @return [Watir::H6] The raw +H6+ element
+        def subscription_details_element
+          # This is a stub, used for indexing. The method is dynamically generated.
+        end
+
+        # @example
+        #   Gitlab::Page::Admin::Subscription.perform do |subscription|
+        #     expect(subscription).to be_subscription_details
+        #   end
+        # @return [Boolean] true if the +subscription_details+ element is present on the page
+        def subscription_details?
+          # This is a stub, used for indexing. The method is dynamically generated.
+        end
+
+        # @note Defined as +text_field :activation_code+
+        # @return [String] The text content or value of +activation_code+
+        def activation_code
+          # This is a stub, used for indexing. The method is dynamically generated.
+        end
+
+        # Set the value of activation_code
+        # @example
+        #   Gitlab::Page::Admin::Subscription.perform do |subscription|
+        #     subscription.activation_code = 'value'
+        #   end
+        # @param value [String] The value to set.
+        def activation_code=(value)
+          # This is a stub, used for indexing. The method is dynamically generated.
+        end
+
+        # @example
+        #   Gitlab::Page::Admin::Subscription.perform do |subscription|
+        #     expect(subscription.activation_code_element).to exist
+        #   end
+        # @return [Watir::TextField] The raw +TextField+ element
+        def activation_code_element
+          # This is a stub, used for indexing. The method is dynamically generated.
+        end
+
+        # @example
+        #   Gitlab::Page::Admin::Subscription.perform do |subscription|
+        #     expect(subscription).to be_activation_code
+        #   end
+        # @return [Boolean] true if the +activation_code+ element is present on the page
+        def activation_code?
+          # This is a stub, used for indexing. The method is dynamically generated.
+        end
+
+        # @note Defined as +label :terms_of_services+
+        # @return [String] The text content or value of +terms_of_services+
+        def terms_of_services
+          # This is a stub, used for indexing. The method is dynamically generated.
+        end
+
+        # @example
+        #   Gitlab::Page::Admin::Subscription.perform do |subscription|
+        #     expect(subscription.terms_of_services_element).to exist
+        #   end
+        # @return [Watir::Label] The raw +Label+ element
+        def terms_of_services_element
+          # This is a stub, used for indexing. The method is dynamically generated.
+        end
+
+        # @example
+        #   Gitlab::Page::Admin::Subscription.perform do |subscription|
+        #     expect(subscription).to be_terms_of_services
+        #   end
+        # @return [Boolean] true if the +terms_of_services+ element is present on the page
+        def terms_of_services?
+          # This is a stub, used for indexing. The method is dynamically generated.
+        end
+
+        # @note Defined as +button :activate+
+        # Clicks +activate+
+        def activate
+          # This is a stub, used for indexing. The method is dynamically generated.
+        end
+
+        # @example
+        #   Gitlab::Page::Admin::Subscription.perform do |subscription|
+        #     expect(subscription.activate_element).to exist
+        #   end
+        # @return [Watir::Button] The raw +Button+ element
+        def activate_element
+          # This is a stub, used for indexing. The method is dynamically generated.
+        end
+
+        # @example
+        #   Gitlab::Page::Admin::Subscription.perform do |subscription|
+        #     expect(subscription).to be_activate
+        #   end
+        # @return [Boolean] true if the +activate+ element is present on the page
+        def activate?
+          # This is a stub, used for indexing. The method is dynamically generated.
+        end
+
         # @note Defined as +p :plan+
         # @return [String] The text content or value of +plan+
         def plan

qa/qa/runtime/env.rb

@@ -434,6 +434,10 @@ module QA
         ENV.fetch('QA_TEST_RESOURCES_CREATED_FILEPATH', File.join(Path.qa_root, 'tmp', file_name))
       end
+      def ee_activation_code
+        ENV['QA_EE_ACTIVATION_CODE']
+      end
+
       private
       def remote_grid_credentials

qa/qa/specs/features/browser_ui/1_manage/login/register_spec.rb

@@ -64,7 +64,9 @@ module QA
             Page::Profile::Accounts::Show.perform do |show|
               show.delete_account(user.password)
             end
-            Support::Waiter.wait_until { !user.exists? }
+
+            # TODO: Remove retry_on_exception once https://gitlab.com/gitlab-org/gitlab/-/issues/24294 is resolved
+            Support::Waiter.wait_until(retry_on_exception: true, sleep_interval: 3) { !user.exists? }
           end
           it 'allows recreating with same credentials', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347868' do

qa/qa/specs/features/ee/browser_ui/11_fulfillment/license/cloud_activation_spec.rb

+# frozen_string_literal: true
+
+module QA
+  include QA::Support::Helpers::Plan
+
+  RSpec.describe 'Fulfillment', :requires_admin, :orchestrated, :cloud_activation do
+    let(:user) { 'GitLab QA' }
+    let(:company) { 'QA User' }
+    let(:user_count) { 10_000 }
+    let(:plan) { ULTIMATE_SELF_MANAGED }
+
+    context 'Cloud activation code' do
+      before do
+        Flow::Login.sign_in_as_admin
+        Gitlab::Page::Admin::Subscription.perform(&:visit)
+      end
+
+      it 'activates instance with correct subscription details', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/350294' do
+        Gitlab::Page::Admin::Subscription.perform do |subscription|
+          subscription.activation_code = Runtime::Env.ee_activation_code
+          subscription.accept_terms
+          subscription.activate
+
+          aggregate_failures do
+            expect { subscription.subscription_details?}.to eventually_be_truthy.within(max_duration: 60)
+            expect(subscription.name).to eq(user)
+            expect(subscription.company).to include(company)
+            expect(subscription.plan).to eq(plan[:name].capitalize)
+            expect(subscription.users_in_subscription).to eq(user_count.to_s)
+            expect(subscription).to have_subscription_record(plan, user_count, LICENSE_TYPE[:cloud_license])
+          end
+        end
+      end
+    end
+  end
+end

qa/qa/specs/features/ee/browser_ui/11_fulfillment/license/license_spec.rb

@@ -6,7 +6,7 @@ module QA
   RSpec.describe 'Fulfillment', :requires_admin, :skip_live_env, except: { job: 'review-qa-*' } do
     let(:user) { 'GitLab QA' }
     let(:company) { 'QA User' }
-    let(:user_count) { 10000 }
+    let(:user_count) { 10_000 }
     let(:plan) { ULTIMATE_SELF_MANAGED }
     context 'Active license details' do
@@ -23,34 +23,7 @@ module QA
             expect(subscription.company).to include(company)
             expect(subscription.plan).to eq(plan[:name].capitalize)
             expect(subscription.users_in_subscription).to eq(user_count.to_s)
-            expect(subscription_record_exists(plan, user_count, LICENSE_TYPE[:license_file])).to be(true)
-          end
-        end
-      end
-
-      private
-
-      # Checks if a subscription record exists in subscription history table
-      #
-      # @param plan [Hash] Name of the plan
-      # @option plan [Hash] Support::Helpers::FREE
-      # @option plan [Hash] Support::Helpers::PREMIUM
-      # @option plan [Hash] Support::Helpers::PREMIUM_SELF_MANAGED
-      # @option plan [Hash] Support::Helpers::ULTIMATE
-      # @option plan [Hash] Support::Helpers::ULTIMATE_SELF_MANAGED
-      # @option plan [Hash] Support::Helpers::CI_MINUTES
-      # @option plan [Hash] Support::Helpers::STORAGE
-      # @param users_in_license [Integer] Number of users in license
-      # @param license_type [Hash] Type of the license
-      # @option license_type [String] 'license file'
-      # @option license_type [String] 'cloud license'
-      # @return [Boolean] True if record exsists, false if not
-      def subscription_record_exists(plan, users_in_license, license_type)
-        Gitlab::Page::Admin::Subscription.perform do |subscription|
-          # find any records that have a matching plan and seats and type
-          subscription.subscription_history_element.hashes.any? do |record|
-            record['Plan'] == plan[:name].capitalize && record['Seats'] == users_in_license.to_s && \
-              record['Type'].strip.downcase == license_type
+            expect(subscription).to have_subscription_record(plan, user_count, LICENSE_TYPE[:license_file])
           end
         end
       end

spec/factories/ci/job_artifacts.rb

@@ -7,7 +7,7 @@ FactoryBot.define do
     file_format { :zip }
     trait :expired do
-      expire_at { Date.yesterday }
+      expire_at { Time.current.yesterday.change(minute: 9) }
     end
     trait :locked do

spec/models/ci/deleted_object_spec.rb

@@ -22,7 +22,7 @@ RSpec.describe Ci::DeletedObject, :aggregate_failures do
         expect(deleted_artifact.file_store).to eq(artifact.file_store)
         expect(deleted_artifact.store_dir).to eq(artifact.file.store_dir.to_s)
         expect(deleted_artifact.file_identifier).to eq(artifact.file_identifier)
-        expect(deleted_artifact.pick_up_at).to eq(artifact.expire_at)
+        expect(deleted_artifact.pick_up_at).to be_like_time(artifact.expire_at)
       end
     end

spec/services/ci/job_artifacts/destroy_all_expired_service_spec.rb

@@ -40,7 +40,7 @@ RSpec.describe Ci::JobArtifacts::DestroyAllExpiredService, :clean_gitlab_redis_s
             # COMMIT
             # SELECT next expired ci_job_artifacts
-            expect(log.count).to be_within(1).of(10)
+            expect(log.count).to be_within(1).of(11)
           end
         end
@@ -51,7 +51,7 @@ RSpec.describe Ci::JobArtifacts::DestroyAllExpiredService, :clean_gitlab_redis_s
           it 'performs the smallest number of queries for job_artifacts' do
             log = ActiveRecord::QueryRecorder.new { subject }
-            expect(log.count).to be_within(1).of(8)
+            expect(log.count).to be_within(1).of(9)
           end
         end
       end

spec/services/ci/job_artifacts/destroy_batch_service_spec.rb

@@ -102,5 +102,81 @@ RSpec.describe Ci::JobArtifacts::DestroyBatchService do
         is_expected.to eq(destroyed_artifacts_count: 0, statistics_updates: {}, status: :success)
       end
     end
+
+    context 'with artifacts that has backfilled expire_at' do
+      let!(:created_on_00_30_45_minutes_on_21_22_23) do
+        [
+          create(:ci_job_artifact, expire_at: Time.zone.parse('2022-01-21 00:00:00.000')),
+          create(:ci_job_artifact, expire_at: Time.zone.parse('2022-01-21 01:30:00.000')),
+          create(:ci_job_artifact, expire_at: Time.zone.parse('2022-01-22 12:00:00.000')),
+          create(:ci_job_artifact, expire_at: Time.zone.parse('2022-01-22 12:30:00.000')),
+          create(:ci_job_artifact, expire_at: Time.zone.parse('2022-01-23 23:00:00.000')),
+          create(:ci_job_artifact, expire_at: Time.zone.parse('2022-01-23 23:30:00.000')),
+          create(:ci_job_artifact, expire_at: Time.zone.parse('2022-01-23 06:45:00.000'))
+        ]
+      end
+
+      let!(:created_close_to_00_or_30_minutes) do
+        [
+          create(:ci_job_artifact, expire_at: Time.zone.parse('2022-01-21 00:00:00.001')),
+          create(:ci_job_artifact, expire_at: Time.zone.parse('2022-01-21 00:30:00.999'))
+        ]
+      end
+
+      let!(:created_on_00_or_30_minutes_on_other_dates) do
+        [
+          create(:ci_job_artifact, expire_at: Time.zone.parse('2022-01-01 00:00:00.000')),
+          create(:ci_job_artifact, expire_at: Time.zone.parse('2022-01-19 12:00:00.000')),
+          create(:ci_job_artifact, expire_at: Time.zone.parse('2022-01-24 23:30:00.000'))
+        ]
+      end
+
+      let!(:created_at_other_times) do
+        [
+          create(:ci_job_artifact, expire_at: Time.zone.parse('2022-01-19 00:00:00.000')),
+          create(:ci_job_artifact, expire_at: Time.zone.parse('2022-01-19 00:30:00.000')),
+          create(:ci_job_artifact, expire_at: Time.zone.parse('2022-01-24 00:00:00.000')),
+          create(:ci_job_artifact, expire_at: Time.zone.parse('2022-01-24 00:30:00.000'))
+        ]
+      end
+
+      let(:artifacts_to_keep) { created_on_00_30_45_minutes_on_21_22_23 }
+      let(:artifacts_to_delete) { created_close_to_00_or_30_minutes + created_on_00_or_30_minutes_on_other_dates + created_at_other_times }
+      let(:all_artifacts) { artifacts_to_keep + artifacts_to_delete }
+
+      let(:artifacts) { Ci::JobArtifact.where(id: all_artifacts.map(&:id)) }
+
+      it 'deletes job artifacts that do not have expire_at on 00, 30 or 45 minute of 21, 22, 23 of the month' do
+        expect { subject }.to change { Ci::JobArtifact.count }.by(artifacts_to_delete.size * -1)
+      end
+
+      it 'keeps job artifacts that have expire_at on 00, 30 or 45 minute of 21, 22, 23 of the month' do
+        expect { subject }.not_to change { Ci::JobArtifact.where(id: artifacts_to_keep.map(&:id)).count }
+      end
+
+      it 'removes expire_at on job artifacts that have expire_at on 00, 30 or 45 minute of 21, 22, 23 of the month' do
+        subject
+
+        expect(artifacts_to_keep.all? { |artifact| artifact.reload.expire_at.nil? }).to be(true)
+      end
+
+      context 'when feature flag is disabled' do
+        before do
+          stub_feature_flags(ci_detect_wrongly_expired_artifacts: false)
+        end
+
+        it 'deletes all job artifacts' do
+          expect { subject }.to change { Ci::JobArtifact.count }.by(all_artifacts.size * -1)
+        end
+      end
+
+      context 'when fix_expire_at is false' do
+        let(:service) { described_class.new(artifacts, pick_up_at: Time.current, fix_expire_at: false) }
+
+        it 'deletes all job artifacts' do
+          expect { subject }.to change { Ci::JobArtifact.count }.by(all_artifacts.size * -1)
+        end
+      end
+    end
   end
 end