Add latest changes from gitlab-org/gitlab@master

Gemfile

@@ -87,7 +87,7 @@ gem 'grape-entity', '~> 0.7.1'
 gem 'rack-cors', '~> 1.0.6', require: 'rack/cors'
 # GraphQL API
-gem 'graphql', '~> 1.9.11'
+gem 'graphql', '~> 1.9.12'
 # NOTE: graphiql-rails v1.5+ doesn't work: https://gitlab.com/gitlab-org/gitlab/issues/31771
 # TODO: remove app/views/graphiql/rails/editors/show.html.erb when https://github.com/rmosolgo/graphiql-rails/pull/71 is released:
 # https://gitlab.com/gitlab-org/gitlab/issues/31747

Gemfile.lock

@@ -455,7 +455,7 @@ GEM
     graphiql-rails (1.4.10)
       railties
       sprockets-rails
-    graphql (1.9.11)
+    graphql (1.9.12)
     graphql-docs (1.6.0)
       commonmarker (~> 0.16)
       escape_utils (~> 1.2)
@@ -1252,7 +1252,7 @@ DEPENDENCIES
   grape-path-helpers (~> 1.2)
   grape_logging (~> 1.7)
   graphiql-rails (~> 1.4.10)
-  graphql (~> 1.9.11)
+  graphql (~> 1.9.12)
   graphql-docs (~> 1.6.0)
   grpc (~> 1.24.0)
   gssapi

app/assets/javascripts/notifications_form.js

@@ -26,7 +26,7 @@ export default class NotificationsForm {
       .addClass('is-loading')
       .find('.custom-notification-event-loading')
       .removeClass('fa-check')
-      .addClass('fa-spin fa-spinner')
+      .addClass('spinner align-middle')
       .removeClass('is-done');
   }
@@ -41,12 +41,12 @@ export default class NotificationsForm {
         if (data.saved) {
           $parent
             .find('.custom-notification-event-loading')
-            .toggleClass('fa-spin fa-spinner fa-check is-done');
+            .toggleClass('spinner fa-check is-done align-middle');
           setTimeout(() => {
             $parent
               .removeClass('is-loading')
               .find('.custom-notification-event-loading')
-              .toggleClass('fa-spin fa-spinner fa-check is-done');
+              .toggleClass('spinner fa-check is-done align-middle');
           }, 2000);
         }
       })

app/models/ci/build.rb

@@ -818,7 +818,7 @@ module Ci
       depended_jobs = depends_on_builds
       # find all jobs that are needed
-      if Feature.enabled?(:ci_dag_support, project, default_enabled: true) && needs.exists?
+      if Feature.enabled?(:ci_dag_support, project, default_enabled: true) && scheduling_type_dag?
         depended_jobs = depended_jobs.where(name: needs.artifacts.select(:name))
       end

app/models/ci/processable.rb

@@ -12,6 +12,18 @@ module Ci
     scope :preload_needs, -> { preload(:needs) }
+    scope :with_needs, -> (names = nil) do
+      needs = Ci::BuildNeed.scoped_build.select(1)
+      needs = needs.where(name: names) if names
+      where('EXISTS (?)', needs).preload(:needs)
+    end
+
+    scope :without_needs, -> (names = nil) do
+      needs = Ci::BuildNeed.scoped_build.select(1)
+      needs = needs.where(name: names) if names
+      where('NOT EXISTS (?)', needs)
+    end
+
     def self.select_with_aggregated_needs(project)
       return all unless Feature.enabled?(:ci_dag_support, project, default_enabled: true)
@@ -26,6 +38,18 @@ module Ci
       )
     end
+    # Old processables may have scheduling_type as nil,
+    # so we need to ensure the data exists before using it.
+    def self.populate_scheduling_type!
+      needs = Ci::BuildNeed.scoped_build.select(1)
+      where(scheduling_type: nil).update_all(
+        "scheduling_type = CASE WHEN (EXISTS (#{needs.to_sql}))
+         THEN #{scheduling_types[:dag]}
+         ELSE #{scheduling_types[:stage]}
+         END"
+      )
+    end
+
     validates :type, presence: true
     validates :scheduling_type, presence: true, on: :create, if: :validate_scheduling_type?
@@ -53,6 +77,11 @@ module Ci
       raise NotImplementedError
     end
+    # Overriding scheduling_type enum's method for nil `scheduling_type`s
+    def scheduling_type_dag?
+      super || find_legacy_scheduling_type == :dag
+    end
+
     # scheduling_type column of previous builds/bridges have not been populated,
     # so we calculate this value on runtime when we need it.
     def find_legacy_scheduling_type

app/models/commit_status.rb

@@ -62,18 +62,6 @@ class CommitStatus < ApplicationRecord
     preload(project: :namespace)
   end
-  scope :with_needs, -> (names = nil) do
-    needs = Ci::BuildNeed.scoped_build.select(1)
-    needs = needs.where(name: names) if names
-    where('EXISTS (?)', needs).preload(:needs)
-  end
-
-  scope :without_needs, -> (names = nil) do
-    needs = Ci::BuildNeed.scoped_build.select(1)
-    needs = needs.where(name: names) if names
-    where('NOT EXISTS (?)', needs)
-  end
-
   scope :match_id_and_lock_version, -> (slice) do
     # it expects that items are an array of attributes to match
     # each hash needs to have `id` and `lock_version`

app/models/project_services/youtrack_service.rb

@@ -6,9 +6,9 @@ class YoutrackService < IssueTrackerService
   # {PROJECT-KEY}-{NUMBER} Examples: YT-1, PRJ-1, gl-030
   def self.reference_pattern(only_long: false)
     if only_long
-      /(?<issue>\b[A-Za-z][A-Za-z0-9_]*-\d+)/
+      /(?<issue>\b[A-Za-z][A-Za-z0-9_]*-\d+\b)/
     else
-      /(?<issue>\b[A-Za-z][A-Za-z0-9_]*-\d+)|(#{Issue.reference_prefix}(?<issue>\d+))/
+      /(?<issue>\b[A-Za-z][A-Za-z0-9_]*-\d+\b)|(#{Issue.reference_prefix}(?<issue>\d+))/
     end
   end

app/services/ci/create_pipeline_service.rb

@@ -61,7 +61,7 @@ module Ci
           Ci::ProcessPipelineService
             .new(pipeline)
-            .execute
+            .execute(nil, initial_process: true)
         end
       end

app/services/ci/pipeline_processing/atomic_processing_service.rb

@@ -93,9 +93,9 @@ module Ci
       end
       def processable_status(processable)
-        if needs_names = processable.aggregated_needs_names
+        if Feature.enabled?(:ci_dag_support, project, default_enabled: true) && processable.scheduling_type_dag?
           # Processable uses DAG, get status of all dependent needs
-          @collection.status_for_names(needs_names)
+          @collection.status_for_names(processable.aggregated_needs_names.to_a)
         else
           # Processable uses Stages, get status of prior stage
           @collection.status_for_prior_stage_position(processable.stage_idx.to_i)

app/services/ci/pipeline_processing/legacy_processing_service.rb

@@ -11,12 +11,13 @@ module Ci
         @pipeline = pipeline
       end
-      def execute(trigger_build_ids = nil)
-        success = process_stages_without_needs
+      def execute(trigger_build_ids = nil, initial_process: false)
+        success = process_stages_for_stage_scheduling
         # we evaluate dependent needs,
         # only when the another job has finished
-        success = process_builds_with_needs(trigger_build_ids) || success
+        success = process_dag_builds_without_needs || success if initial_process
+        success = process_dag_builds_with_needs(trigger_build_ids) || success
         @pipeline.update_legacy_status
@@ -25,23 +26,31 @@ module Ci
       private
-      def process_stages_without_needs
-        stage_indexes_of_created_processables_without_needs.flat_map do |index|
-          process_stage_without_needs(index)
+      def process_stages_for_stage_scheduling
+        stage_indexes_of_created_stage_scheduled_processables.flat_map do |index|
+          process_stage_for_stage_scheduling(index)
         end.any?
       end
-      def process_stage_without_needs(index)
+      def process_stage_for_stage_scheduling(index)
         current_status = status_for_prior_stages(index)
         return unless HasStatus::COMPLETED_STATUSES.include?(current_status)
-        created_processables_in_stage_without_needs(index).find_each.select do |build|
+        created_stage_scheduled_processables_in_stage(index).find_each.select do |build|
           process_build(build, current_status)
         end.any?
       end
-      def process_builds_with_needs(trigger_build_ids)
+      def process_dag_builds_without_needs
+        return false unless Feature.enabled?(:ci_dag_support, project, default_enabled: true)
+
+        created_processables.scheduling_type_dag.without_needs.each do |build|
+          process_build(build, 'success')
+        end
+      end
+
+      def process_dag_builds_with_needs(trigger_build_ids)
         return false unless trigger_build_ids.present?
         return false unless Feature.enabled?(:ci_dag_support, project, default_enabled: true)
@@ -56,14 +65,15 @@ module Ci
         # Each found processable is guaranteed here to have completed status
         created_processables
+          .scheduling_type_dag
           .with_needs(trigger_build_names)
           .without_needs(incomplete_build_names)
           .find_each
-          .map(&method(:process_build_with_needs))
+          .map(&method(:process_dag_build_with_needs))
           .any?
       end
-      def process_build_with_needs(build)
+      def process_dag_build_with_needs(build)
         current_status = status_for_build_needs(build.needs.map(&:name))
         return unless HasStatus::COMPLETED_STATUSES.include?(current_status)
@@ -87,23 +97,23 @@ module Ci
       end
       # rubocop: disable CodeReuse/ActiveRecord
-      def stage_indexes_of_created_processables_without_needs
-        created_processables_without_needs.order(:stage_idx)
+      def stage_indexes_of_created_stage_scheduled_processables
+        created_stage_scheduled_processables.order(:stage_idx)
           .pluck(Arel.sql('DISTINCT stage_idx'))
       end
       # rubocop: enable CodeReuse/ActiveRecord
-      def created_processables_in_stage_without_needs(index)
-        created_processables_without_needs
+      def created_stage_scheduled_processables_in_stage(index)
+        created_stage_scheduled_processables
           .with_preloads
           .for_stage(index)
       end
-      def created_processables_without_needs
+      def created_stage_scheduled_processables
         if Feature.enabled?(:ci_dag_support, project, default_enabled: true)
-          pipeline.processables.created.without_needs
+          created_processables.scheduling_type_stage
         else
-          pipeline.processables.created
+          created_processables
         end
       end

app/services/ci/process_pipeline_service.rb

@@ -8,8 +8,9 @@ module Ci
       @pipeline = pipeline
     end
-    def execute(trigger_build_ids = nil)
+    def execute(trigger_build_ids = nil, initial_process: false)
       update_retried
+      ensure_scheduling_type_for_processables
       if Feature.enabled?(:ci_atomic_processing, pipeline.project)
         Ci::PipelineProcessing::AtomicProcessingService
@@ -18,7 +19,7 @@ module Ci
       else
         Ci::PipelineProcessing::LegacyProcessingService
           .new(pipeline)
-          .execute(trigger_build_ids)
+          .execute(trigger_build_ids, initial_process: initial_process)
       end
     end
@@ -43,5 +44,17 @@ module Ci
         .update_all(retried: true) if latest_statuses.any?
     end
     # rubocop: enable CodeReuse/ActiveRecord
+
+    # Set scheduling type of processables if they were created before scheduling_type
+    # data was deployed (https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22246).
+    # Given that this service runs multiple times during the pipeline
+    # life cycle we need to ensure we populate the data once.
+    # See more: https://gitlab.com/gitlab-org/gitlab/issues/205426
+    def ensure_scheduling_type_for_processables
+      lease = Gitlab::ExclusiveLease.new("set-scheduling-types:#{pipeline.id}", timeout: 1.hour.to_i)
+      return unless lease.try_obtain
+
+      pipeline.processables.populate_scheduling_type!
+    end
   end
 end

app/services/ci/retry_pipeline_service.rb

@@ -36,7 +36,7 @@ module Ci
       Ci::ProcessPipelineService
         .new(pipeline)
-        .execute(completed_build_ids)
+        .execute(completed_build_ids, initial_process: true)
     end
   end
 end

app/views/notify/_failed_builds.html.haml

@@ -27,6 +27,6 @@
             - if build.has_trace?
               %td{ colspan: "2", style: "font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; padding: 0 0 16px;" }
                 %pre{ style: "font-family: Monaco,'Lucida Console','Courier New',Courier,monospace; background-color: #fafafa; border-radius: 4px; overflow: hidden; white-space: pre-wrap; word-break: break-all; font-size:13px; line-height: 1.4; padding: 16px 8px; color: #333333; margin: 0;" }
-                  = build.trace.html(last_lines: 10).html_safe
+                  = build.trace.html(last_lines: 30).html_safe
             - else
               %td{ colspan: "2" }

app/views/notify/autodevops_disabled_email.text.erb

@@ -15,6 +15,6 @@ had <%= failed.size %> failed <%= 'build'.pluralize(failed.size) %>.
   Stage: <%= build.stage %>
   Name: <%= build.name %>
   <% if build.has_trace? -%>
-    Trace: <%= build.trace.raw(last_lines: 10) %>
+    Trace: <%= build.trace.raw(last_lines: 30) %>
   <% end -%>
 <% end -%>

app/views/notify/pipeline_failed_email.text.erb

@@ -35,7 +35,7 @@ had <%= failed.size %> failed <%= 'build'.pluralize(failed.size) %>.
 Stage: <%= build.stage %>
 Name: <%= build.name %>
 <% if build.has_trace? -%>
-Trace: <%= build.trace.raw(last_lines: 10) %>
+Trace: <%= build.trace.raw(last_lines: 30) %>
 <% end -%>
 <% end -%>

app/views/shared/notifications/_custom_notifications.html.haml

@@ -30,4 +30,4 @@
                       %label.form-check-label{ for: field_id }
                         %strong
                           = notification_event_name(event)
-                          = icon("spinner spin", class: "custom-notification-event-loading")
+                          .fa.custom-notification-event-loading.spinner

changelogs/unreleased/13679-add-get-endpoint-to-ldap-group-link-api.yml

+---
+title: Add GET endpoint to LDAP group link API
+merge_request: 24216
+author:
+type: added

changelogs/unreleased/23068-ability-to-delete-identities-for-user.yml

+---
+title: Add delete identity endpoint on the users API
+merge_request: 24122
+author:
+type: added

changelogs/unreleased/30631-allow-empty-needs-v4.yml

+---
+title: Implement allowing empty needs for jobs in DAG pipelines
+merge_request: 22246
+author:
+type: added

changelogs/unreleased/georgekoltsov-filter-group-attributes.yml

+---
+title: Extend the list of excluded_attributes for group on Group Import
+merge_request: 25031
+author:
+type: fixed