Merge commit '6d000c9f' into 11-10-stable

app/helpers/visibility_level_helper.rb

@@ -42,11 +42,11 @@ module VisibilityLevelHelper
   def group_visibility_level_description(level)
     case level
     when Gitlab::VisibilityLevel::PRIVATE
-      "The group and its projects can only be viewed by members."
+      _("The group and its projects can only be viewed by members.")
     when Gitlab::VisibilityLevel::INTERNAL
-      "The group and any internal projects can be viewed by any logged in user."
+      _("The group and any internal projects can be viewed by any logged in user.")
     when Gitlab::VisibilityLevel::PUBLIC
-      "The group and any public projects can be viewed without any authentication."
+      _("The group and any public projects can be viewed without any authentication.")
     end
   end
@@ -54,20 +54,20 @@ module VisibilityLevelHelper
     case level
     when Gitlab::VisibilityLevel::PRIVATE
       if snippet.is_a? ProjectSnippet
-        "The snippet is visible only to project members."
+        _("The snippet is visible only to project members.")
       else
-        "The snippet is visible only to me."
+        _("The snippet is visible only to me.")
       end
     when Gitlab::VisibilityLevel::INTERNAL
-      "The snippet is visible to any logged in user."
+      _("The snippet is visible to any logged in user.")
     when Gitlab::VisibilityLevel::PUBLIC
-      "The snippet can be accessed without any authentication."
+      _("The snippet can be accessed without any authentication.")
     end
   end
   def restricted_visibility_level_description(level)
     level_name = Gitlab::VisibilityLevel.level_name(level)
-    "#{level_name.capitalize} visibility has been restricted by the administrator."
+    _("%{level_name} visibility has been restricted by the administrator.") % { level_name: level_name.capitalize }
   end
   def disallowed_visibility_level_description(level, form_model)

app/helpers/wiki_helper.rb

@@ -47,4 +47,24 @@ module WikiHelper
   def wiki_attachment_upload_url
     expose_url(api_v4_projects_wikis_attachments_path(id: @project.id))
   end
+
+  def wiki_sort_controls(project, sort, direction)
+    sort ||= ProjectWiki::TITLE_ORDER
+    link_class = 'btn btn-default has-tooltip reverse-sort-btn qa-reverse-sort'
+    reversed_direction = direction == 'desc' ? 'asc' : 'desc'
+    icon_class = direction == 'desc' ? 'highest' : 'lowest'
+
+    link_to(project_wikis_pages_path(project, sort: sort, direction: reversed_direction),
+      type: 'button', class: link_class, title: _('Sort direction')) do
+      sprite_icon("sort-#{icon_class}", size: 16)
+    end
+  end
+
+  def wiki_sort_title(key)
+    if key == ProjectWiki::CREATED_AT_ORDER
+      s_("Wiki|Created date")
+    else
+      s_("Wiki|Title")
+    end
+  end
 end

app/models/ci/build_runner_session.rb

@@ -6,6 +6,8 @@ module Ci
   class BuildRunnerSession < ApplicationRecord
     extend Gitlab::Ci::Model
+    TERMINAL_SUBPROTOCOL = 'terminal.gitlab.com'.freeze
+
     self.table_name = 'ci_builds_runner_session'
     belongs_to :build, class_name: 'Ci::Build', inverse_of: :runner_session
@@ -14,11 +16,21 @@ module Ci
     validates :url, url: { protocols: %w(https) }
     def terminal_specification
-      return {} unless url.present?
+      wss_url = Gitlab::UrlHelpers.as_wss(self.url)
+      return {} unless wss_url.present?
+
+      wss_url = "#{wss_url}/exec"
+      channel_specification(wss_url, TERMINAL_SUBPROTOCOL)
+    end
+
+    private
+
+    def channel_specification(url, subprotocol)
+      return {} if subprotocol.blank? || url.blank?
       {
-        subprotocols: ['terminal.gitlab.com'].freeze,
-        url: "#{url}/exec".sub("https://", "wss://"),
+        subprotocols: Array(subprotocol),
+        url: url,
         headers: { Authorization: [authorization.presence] }.compact,
         ca_pem: certificate.presence
       }

app/models/concerns/reactive_caching.rb

@@ -29,6 +29,40 @@
 # However, it will enqueue a background worker to call `#calculate_reactive_cache`
 # and set an initial cache lifetime of ten minutes.
 #
+# The background worker needs to find or generate the object on which
+# `with_reactive_cache` was called.
+# The default behaviour can be overridden by defining a custom
+# `reactive_cache_worker_finder`.
+# Otherwise the background worker will use the class name and primary key to get
+# the object using the ActiveRecord find_by method.
+#
+#    class Bar
+#      include ReactiveCaching
+#
+#      self.reactive_cache_key = ->() { ["bar", "thing"] }
+#      self.reactive_cache_worker_finder = ->(_id, *args) { from_cache(*args) }
+#
+#      def self.from_cache(var1, var2)
+#        # This method will be called by the background worker with "bar1" and
+#        # "bar2" as arguments.
+#        new(var1, var2)
+#      end
+#
+#      def initialize(var1, var2)
+#        # ...
+#      end
+#
+#      def calculate_reactive_cache
+#        # Expensive operation here. The return value of this method is cached
+#      end
+#
+#      def result
+#        with_reactive_cache("bar1", "bar2") do |data|
+#          # ...
+#        end
+#      end
+#    end
+#
 # Each time the background job completes, it stores the return value of
 # `#calculate_reactive_cache`. It is also re-enqueued to run again after
 # `reactive_cache_refresh_interval`, so keeping the stored value up to date.
@@ -52,6 +86,7 @@ module ReactiveCaching
     class_attribute :reactive_cache_key
     class_attribute :reactive_cache_lifetime
     class_attribute :reactive_cache_refresh_interval
+    class_attribute :reactive_cache_worker_finder
     # defaults
     self.reactive_cache_lease_timeout = 2.minutes
@@ -59,6 +94,10 @@ module ReactiveCaching
     self.reactive_cache_refresh_interval = 1.minute
     self.reactive_cache_lifetime = 10.minutes
+    self.reactive_cache_worker_finder = ->(id, *_args) do
+      find_by(primary_key => id)
+    end
+
     def calculate_reactive_cache(*args)
       raise NotImplementedError
     end

app/models/project_wiki.rb

@@ -13,6 +13,11 @@ class ProjectWiki
   CouldNotCreateWikiError = Class.new(StandardError)
   SIDEBAR = '_sidebar'
+  TITLE_ORDER = 'title'
+  CREATED_AT_ORDER = 'created_at'
+  DIRECTION_DESC = 'desc'
+  DIRECTION_ASC = 'asc'
+
   # Returns a string describing what went wrong after
   # an operation fails.
   attr_reader :error_message
@@ -82,8 +87,15 @@ class ProjectWiki
   # Returns an Array of GitLab WikiPage instances or an
   # empty Array if this Wiki has no pages.
-  def pages(limit: 0)
-    wiki.pages(limit: limit).map { |page| WikiPage.new(self, page, true) }
+  def pages(limit: 0, sort: nil, direction: DIRECTION_ASC)
+    sort ||= TITLE_ORDER
+    direction_desc = direction == DIRECTION_DESC
+
+    wiki.pages(
+      limit: limit, sort: sort, direction_desc: direction_desc
+    ).map do |page|
+      WikiPage.new(self, page, true)
+    end
   end
   # Finds a page within the repository based on a tile

app/models/wiki_page.rb

@@ -28,16 +28,15 @@ class WikiPage
   def self.group_by_directory(pages)
     return [] if pages.blank?
-    pages.sort_by { |page| [page.directory, page.slug] }
-      .group_by(&:directory)
-      .map do |dir, pages|
-        if dir.present?
-          WikiDirectory.new(dir, pages)
-        else
-          pages
-        end
-      end
-      .flatten
+    pages.each_with_object([]) do |page, grouped_pages|
+      next grouped_pages << page unless page.directory.present?
+
+      directory = grouped_pages.find { |dir| dir.slug == page.directory }
+
+      next directory.pages << page if directory
+
+      grouped_pages << WikiDirectory.new(page.directory, [page])
+    end
   end
   def self.unhyphenize(name)

app/views/projects/wikis/pages.html.haml

@@ -2,6 +2,7 @@
 - add_to_breadcrumbs "Wiki", project_wiki_path(@project, :home)
 - breadcrumb_title s_("Wiki|Pages")
 - page_title s_("Wiki|Pages"), _("Wiki")
+- sort_title = wiki_sort_title(params[:sort])
 %div{ class: container_class }
   .wiki-page-header
@@ -15,6 +16,18 @@
         = icon('cloud-download')
         = _("Clone repository")
+      .dropdown.inline.wiki-sort-dropdown
+        .btn-group{ role: 'group' }
+          .btn-group{ role: 'group' }
+            %button.dropdown-toggle{ type: 'button', data: { toggle: 'dropdown', display: 'static' }, class: 'btn btn-default' }
+              = sort_title
+              = icon('chevron-down')
+            %ul.dropdown-menu.dropdown-menu-right.dropdown-menu-selectable.dropdown-menu-sort
+              %li
+                = sortable_item(s_("Wiki|Title"), project_wikis_pages_path(@project, sort: ProjectWiki::TITLE_ORDER), sort_title)
+                = sortable_item(s_("Wiki|Created date"), project_wikis_pages_path(@project, sort: ProjectWiki::CREATED_AT_ORDER), sort_title)
+          = wiki_sort_controls(@project, params[:sort], params[:direction])
+
   %ul.wiki-pages-list.content-list
     = render @wiki_entries, context: 'pages'

app/workers/reactive_caching_worker.rb

@@ -3,7 +3,6 @@
 class ReactiveCachingWorker
   include ApplicationWorker
-  # rubocop: disable CodeReuse/ActiveRecord
   def perform(class_name, id, *args)
     klass = begin
       class_name.constantize
@@ -12,7 +11,9 @@ class ReactiveCachingWorker
     end
     return unless klass
-    klass.find_by(klass.primary_key => id).try(:exclusively_update_reactive_cache!, *args)
+    klass
+      .reactive_cache_worker_finder
+      .call(id, *args)
+      .try(:exclusively_update_reactive_cache!, *args)
   end
-  # rubocop: enable CodeReuse/ActiveRecord
 end

changelogs/unreleased/58375-reactive-caching-changes.yml

+---
+title: Allow reactive caching to be used in services 
+merge_request: 26839
+author:
+type: added

changelogs/unreleased/58405-basic-limiting-complexity-of-graphql-queries.yml

+---
+title: Add initial complexity limits to GraphQL queries
+merge_request: 26629
+author:
+type: performance

changelogs/unreleased/58839-automatically-set-prometheus-step-interval.yml

+---
+title: Automatically set Prometheus step interval
+merge_request: 26441
+author:
+type: changed

changelogs/unreleased/allow-to-use-untrusted-ruby-syntax.yml

+---
+title: Allow to use untrusted Regexp via feature flag
+merge_request: 26905
+author:
+type: deprecated

changelogs/unreleased/id-51433-sort-wiki-by-date.yml

+---
+title: Allow to sort wiki pages by date and title
+merge_request: 25365
+author:
+type: added

changelogs/unreleased/update-gitlab-shell.yml

+---
+title: Update GitLab Shell to v9.0.0
+merge_request: 27002
+author:
+type: other

changelogs/unreleased/xanf-gitlab-ce-transfer-disables-js.yml

+---
+title: Group transfer now properly redirects to edit on failure
+merge_request: 26837
+author:
+type: fixed

doc/administration/container_registry.md

@@ -362,6 +362,10 @@ configuring a different storage driver. By default the GitLab Container Registry
 is configured to use the filesystem driver, which makes use of [storage path](#container-registry-storage-path)
 configuration.
+NOTE: **Note:** Enabling a storage driver other than `filesystem` would mean
+that your Docker client needs to be able to access the storage backend directly.
+In that case, you must use an address that resolves and is accessible outside GitLab server.
+
 The different supported drivers are:
 | Driver     | Description                         |
@@ -369,20 +373,16 @@ The different supported drivers are:
 | filesystem | Uses a path on the local filesystem |
 | azure      | Microsoft Azure Blob Storage        |
 | gcs        | Google Cloud Storage                |
-| s3         | Amazon Simple Storage Service       |
+| s3         | Amazon Simple Storage Service. Be sure to configure your storage bucket with the correct [S3 Permission Scopes](https://docs.docker.com/registry/storage-drivers/s3/#s3-permission-scopes).       |
 | swift      | OpenStack Swift Object Storage      |
 | oss        | Aliyun OSS                          |
 Read more about the individual driver's config options in the
 [Docker Registry docs][storage-config].
-> **Warning** GitLab will not backup Docker images that are not stored on the
+CAUTION: **Warning:** GitLab will not backup Docker images that are not stored on the
 filesystem. Remember to enable backups with your object storage provider if
 desired.
->
-> **Important** Enabling storage driver other than `filesystem` would mean
-that your Docker client needs to be able to access the storage backend directly.
-So you must use an address that resolves and is accessible outside GitLab server.
 ---

doc/ci/yaml/README.md

@@ -414,6 +414,27 @@ job:
   only: ['branches', 'tags']
 ```
+### Supported `only`/`except` regexp syntax
+
+CAUTION: **Warning:**
+This is a breaking change that was introduced with GitLab 11.9.4.
+
+In GitLab 11.9.4, GitLab begun internally converting regexp used
+in `only` and `except` parameters to [RE2](https://github.com/google/re2/wiki/Syntax).
+
+This means that only subset of features provided by [Ruby Regexp](https://ruby-doc.org/core/Regexp.html)
+is supported. [RE2](https://github.com/google/re2/wiki/Syntax) limits the set of features
+provided due to computational complexity, which means some features became unavailable in GitLab 11.9.4.
+For example, negative lookaheads.
+
+For GitLab versions from 11.9.7 and up to GitLab 12.0, GitLab provides a feature flag that can be
+enabled by administrators that allows users to use unsafe regexp syntax. This brings compatibility
+with previously allowed syntax version and allows users to gracefully migrate to the new syntax.
+
+```ruby
+Feature.enable(:allow_unsafe_ruby_regexp)
+```
+
 ### `only`/`except` (advanced)
 > - `refs` and `kubernetes` policies introduced in GitLab 10.0.

lib/gitlab/ci/build/policy/refs.rb

@@ -35,7 +35,7 @@ module Gitlab
             # patterns can be matched only when branch or tag is used
             # the pattern matching does not work for merge requests pipelines
             if pipeline.branch? || pipeline.tag?
-              if regexp = Gitlab::UntrustedRegexp::RubySyntax.fabricate(pattern)
+              if regexp = Gitlab::UntrustedRegexp::RubySyntax.fabricate(pattern, fallback: true)
                 regexp.match?(pipeline.ref)
               else
                 pattern == pipeline.ref

lib/gitlab/ci/config/entry/policy.rb

@@ -17,7 +17,7 @@ module Gitlab
             include ::Gitlab::Config::Entry::Validatable
             validations do
-              validates :config, array_of_strings_or_regexps: true
+              validates :config, array_of_strings_or_regexps_with_fallback: true
             end
             def value
@@ -38,7 +38,7 @@ module Gitlab
               validate :variables_expressions_syntax
               with_options allow_nil: true do
-                validates :refs, array_of_strings_or_regexps: true
+                validates :refs, array_of_strings_or_regexps_with_fallback: true
                 validates :kubernetes, allowed_values: %w[active]
                 validates :variables, array_of_strings: true
                 validates :changes, array_of_strings: true

lib/gitlab/config/entry/validators.rb

@@ -129,6 +129,12 @@ module Gitlab
             end
           end
+          protected
+
+          def fallback
+            false
+          end
+
           private
           def matches_syntax?(value)
@@ -137,7 +143,7 @@ module Gitlab
           def validate_regexp(value)
             matches_syntax?(value) &&
-              Gitlab::UntrustedRegexp::RubySyntax.valid?(value)
+              Gitlab::UntrustedRegexp::RubySyntax.valid?(value, fallback: fallback)
           end
         end
@@ -162,6 +168,14 @@ module Gitlab
           end
         end
+        class ArrayOfStringsOrRegexpsWithFallbackValidator < ArrayOfStringsOrRegexpsValidator
+          protected
+
+          def fallback
+            true
+          end
+        end
+
         class ArrayOfStringsOrStringValidator < RegexpValidator
           def validate_each(record, attribute, value)
             unless validate_array_of_strings_or_string(value)