Merge remote-tracking branch 'origin' into 12-3-auto-deploy-20190911

CHANGELOG.md

@@ -2,6 +2,13 @@
 documentation](doc/development/changelog.md) for instructions on adding your own
 entry.
+## 12.2.5
+
+### Security (1 change)
+
+- Upgrade pages to 1.7.2.
+
+
 ## 12.2.4
 ### Fixed (7 changes)

GITLAB_PAGES_VERSION

-1.8.0
+1.8.1

app/controllers/concerns/issuable_collections.rb

@@ -2,6 +2,7 @@
 module IssuableCollections
   extend ActiveSupport::Concern
+  include PaginatedCollection
   include SortingHelper
   include SortingPreference
   include Gitlab::IssuableMetadata
@@ -17,8 +18,11 @@ module IssuableCollections
   def set_issuables_index
     @issuables = issuables_collection
-    set_pagination
-    return if redirect_out_of_range(@total_pages)
+    unless pagination_disabled?
+      set_pagination
+
+      return if redirect_out_of_range(@issuables, @total_pages)
+    end
     if params[:label_name].present? && @project
       labels_params = { project_id: @project.id, title: params[:label_name] }
@@ -38,12 +42,10 @@ module IssuableCollections
   end
   def set_pagination
-    return if pagination_disabled?
-
     @issuables          = @issuables.page(params[:page])
     @issuables          = per_page_for_relative_position if params[:sort] == 'relative_position'
     @issuable_meta_data = issuable_meta_data(@issuables, collection_type, current_user)
-    @total_pages        = issuable_page_count
+    @total_pages        = issuable_page_count(@issuables)
   end
   # rubocop:enable Gitlab/ModuleWithInstanceVariables
@@ -57,20 +59,8 @@ module IssuableCollections
   end
   # rubocop: enable CodeReuse/ActiveRecord
-  def redirect_out_of_range(total_pages)
-    return false if total_pages.nil? || total_pages.zero?
-
-    out_of_range = @issuables.current_page > total_pages # rubocop:disable Gitlab/ModuleWithInstanceVariables
-
-    if out_of_range
-      redirect_to(url_for(safe_params.merge(page: total_pages, only_path: true)))
-    end
-
-    out_of_range
-  end
-
-  def issuable_page_count
-    page_count_for_relation(@issuables, finder.row_count) # rubocop:disable Gitlab/ModuleWithInstanceVariables
+  def issuable_page_count(relation)
+    page_count_for_relation(relation, finder.row_count)
   end
   def page_count_for_relation(relation, row_count)

app/controllers/concerns/paginated_collection.rb

+# frozen_string_literal: true
+
+module PaginatedCollection
+  extend ActiveSupport::Concern
+
+  private
+
+  def redirect_out_of_range(collection, total_pages = collection.total_pages)
+    return false if total_pages.zero?
+
+    out_of_range = collection.current_page > total_pages
+
+    if out_of_range
+      redirect_to(url_for(safe_params.merge(page: total_pages, only_path: true)))
+    end
+
+    out_of_range
+  end
+end

app/controllers/dashboard/snippets_controller.rb

 # frozen_string_literal: true
 class Dashboard::SnippetsController < Dashboard::ApplicationController
+  include PaginatedCollection
+  include Gitlab::NoteableMetadata
+
   skip_cross_project_access_check :index
   def index
-    @snippets = SnippetsFinder.new(
-      current_user,
-      author: current_user,
-      scope: params[:scope]
-    ).execute
-    @snippets = @snippets.page(params[:page])
+    @snippets = SnippetsFinder.new(current_user, author: current_user, scope: params[:scope])
+      .execute
+      .page(params[:page])
+      .inc_author
+
+    return if redirect_out_of_range(@snippets)
+
+    @noteable_meta_data = noteable_meta_data(@snippets, 'Snippet')
   end
 end

app/controllers/dashboard/todos_controller.rb

@@ -2,6 +2,7 @@
 class Dashboard::TodosController < Dashboard::ApplicationController
   include ActionView::Helpers::NumberHelper
+  include PaginatedCollection
   before_action :authorize_read_project!, only: :index
   before_action :authorize_read_group!, only: :index
@@ -12,7 +13,7 @@ class Dashboard::TodosController < Dashboard::ApplicationController
     @todos = @todos.page(params[:page])
     @todos = @todos.with_entity_associations
-    return if redirect_out_of_range(@todos)
+    return if redirect_out_of_range(@todos, todos_page_count(@todos))
   end
   def destroy
@@ -82,28 +83,15 @@ class Dashboard::TodosController < Dashboard::ApplicationController
     }
   end
-  def todo_params
-    params.permit(:action_id, :author_id, :project_id, :type, :sort, :state, :group_id)
-  end
-
-  # rubocop: disable CodeReuse/ActiveRecord
-  def redirect_out_of_range(todos)
-    total_pages =
-      if todo_params.except(:sort, :page).empty?
-        (current_user.todos_pending_count.to_f / todos.limit_value).ceil
-      else
-        todos.total_pages
-      end
-
-    return false if total_pages.zero?
-
-    out_of_range = todos.current_page > total_pages
-
-    if out_of_range
-      redirect_to url_for(safe_params.merge(page: total_pages, only_path: true))
+  def todos_page_count(todos)
+    if todo_params.except(:sort, :page).empty? # rubocop: disable CodeReuse/ActiveRecord
+      (current_user.todos_pending_count.to_f / todos.limit_value).ceil
+    else
+      todos.total_pages
     end
-    out_of_range
+  end
+  def todo_params
+    params.permit(:action_id, :author_id, :project_id, :type, :sort, :state, :group_id)
   end
-  # rubocop: enable CodeReuse/ActiveRecord
 end

app/controllers/explore/snippets_controller.rb

 # frozen_string_literal: true
 class Explore::SnippetsController < Explore::ApplicationController
+  include PaginatedCollection
+  include Gitlab::NoteableMetadata
+
   def index
-    @snippets = SnippetsFinder.new(current_user).execute
-    @snippets = @snippets.page(params[:page])
+    @snippets = SnippetsFinder.new(current_user)
+      .execute
+      .page(params[:page])
+      .inc_author
+
+    return if redirect_out_of_range(@snippets)
+
+    @noteable_meta_data = noteable_meta_data(@snippets, 'Snippet')
   end
 end

app/controllers/projects/forks_controller.rb

@@ -2,6 +2,7 @@
 class Projects::ForksController < Projects::ApplicationController
   include ContinueParams
+  include RendersMemberAccess
   # Authorize
   before_action :whitelist_query_limiting, only: [:create]
@@ -11,14 +12,16 @@ class Projects::ForksController < Projects::ApplicationController
   # rubocop: disable CodeReuse/ActiveRecord
   def index
-    base_query = project.forks.includes(:creator)
-    forks                = ForkProjectsFinder.new(project, params: params.merge(search: params[:filter_projects]), current_user: current_user).execute
-    @total_forks_count   = base_query.size
-    @private_forks_count = @total_forks_count - forks.size
-    @public_forks_count  = @total_forks_count - @private_forks_count
-    @forks = forks.page(params[:page])
+    @total_forks_count    = project.forks.size
+    @public_forks_count   = project.forks.public_only.size
+    @private_forks_count  = @total_forks_count - project.forks.public_and_internal_only.size
+    @internal_forks_count = @total_forks_count - @public_forks_count - @private_forks_count
+    @forks = ForkProjectsFinder.new(project, params: params.merge(search: params[:filter_projects]), current_user: current_user).execute
+    @forks = @forks.includes(:route, :creator, :group, namespace: [:route, :owner])
+                   .page(params[:page])
+    prepare_projects_for_rendering(@forks)
     respond_to do |format|
       format.html

app/controllers/projects/snippets_controller.rb

@@ -6,6 +6,8 @@ class Projects::SnippetsController < Projects::ApplicationController
   include SpammableActions
   include SnippetsActions
   include RendersBlob
+  include PaginatedCollection
+  include Gitlab::NoteableMetadata
   skip_before_action :verify_authenticity_token,
     if: -> { action_name == 'show' && js_request? }
@@ -28,15 +30,14 @@ class Projects::SnippetsController < Projects::ApplicationController
   respond_to :html
   def index
-    @snippets = SnippetsFinder.new(
-      current_user,
-      project: @project,
-      scope: params[:scope]
-    ).execute
-    @snippets = @snippets.page(params[:page])
-    if @snippets.out_of_range? && @snippets.total_pages != 0
-      redirect_to project_snippets_path(@project, page: @snippets.total_pages)
-    end
+    @snippets = SnippetsFinder.new(current_user, project: @project, scope: params[:scope])
+      .execute
+      .page(params[:page])
+      .inc_author
+
+    return if redirect_out_of_range(@snippets)
+
+    @noteable_meta_data = noteable_meta_data(@snippets, 'Snippet')
   end
   def new

app/controllers/snippets_controller.rb

@@ -7,6 +7,8 @@ class SnippetsController < ApplicationController
   include SnippetsActions
   include RendersBlob
   include PreviewMarkdown
+  include PaginatedCollection
+  include Gitlab::NoteableMetadata
   skip_before_action :verify_authenticity_token,
     if: -> { action_name == 'show' && js_request? }
@@ -32,7 +34,13 @@ class SnippetsController < ApplicationController
       @user = UserFinder.new(params[:username]).find_by_username!
       @snippets = SnippetsFinder.new(current_user, author: @user, scope: params[:scope])
-        .execute.page(params[:page])
+        .execute
+        .page(params[:page])
+        .inc_author
+
+      return if redirect_out_of_range(@snippets)
+
+      @noteable_meta_data = noteable_meta_data(@snippets, 'Snippet')
       render 'index'
     else

app/controllers/users_controller.rb

@@ -4,6 +4,7 @@ class UsersController < ApplicationController
   include RoutableActions
   include RendersMemberAccess
   include ControllerWithCrossProjectAccessCheck
+  include Gitlab::NoteableMetadata
   requires_cross_project_access show: false,
                                 groups: false,
@@ -165,11 +166,12 @@ class UsersController < ApplicationController
   end
   def load_snippets
-    @snippets = SnippetsFinder.new(
-      current_user,
-      author: user,
-      scope: params[:scope]
-    ).execute.page(params[:page])
+    @snippets = SnippetsFinder.new(current_user, author: user, scope: params[:scope])
+      .execute
+      .page(params[:page])
+      .inc_author
+
+    @noteable_meta_data = noteable_meta_data(@snippets, 'Snippet')
   end
   def build_canonical_path(user)

app/models/concerns/noteable.rb

@@ -3,6 +3,10 @@
 module Noteable
   extend ActiveSupport::Concern
+  # This object is used to gather noteable meta data for list displays
+  # avoiding n+1 queries and improving performance.
+  NoteableMeta = Struct.new(:user_notes_count)
+
   class_methods do
     # `Noteable` class names that support replying to individual notes.
     def replyable_types

app/models/snippet.rb

@@ -55,6 +55,7 @@ class Snippet < ApplicationRecord
   scope :are_public, -> { where(visibility_level: Snippet::PUBLIC) }
   scope :public_and_internal, -> { where(visibility_level: [Snippet::PUBLIC, Snippet::INTERNAL]) }
   scope :fresh, -> { order("created_at DESC") }
+  scope :inc_author, -> { includes(:author) }
   scope :inc_relations_for_view, -> { includes(author: :status) }
   participant :author

app/views/projects/forks/index.html.haml

 .top-area
   .nav-text
-    - full_count_title = "#{@public_forks_count} public and #{@private_forks_count} private"
+    - full_count_title = "#{@public_forks_count} public, #{@internal_forks_count} internal, and #{@private_forks_count} private"
     #{pluralize(@total_forks_count, 'fork')}: #{full_count_title}
   .nav-controls

app/views/shared/projects/_list.html.haml

@@ -42,12 +42,6 @@
           avatar: avatar, stars: stars, css_class: css_class, ci: ci, use_creator_avatar: use_creator_avatar,
           forks: forks, show_last_commit_as_description: show_last_commit_as_description, user: user, merge_requests: merge_requests,
           issues: issues, pipeline_status: pipeline_status, compact_mode: compact_mode
-
-      - if @private_forks_count && @private_forks_count > 0
-        %li.project-row.private-forks-notice
-          = icon('lock fw', base: 'circle', class: 'fa-lg private-fork-icon')
-          %strong= pluralize(@private_forks_count, 'private fork')
-          %span  you have no access to.
     = paginate_collection(projects, remote: remote) unless skip_pagination
   - else
     - if @contributed_projects

app/views/shared/snippets/_list.html.haml

 - remote = local_assigns.fetch(:remote, false)
 - link_project = local_assigns.fetch(:link_project, false)
-- if @snippets.exists?
+- if @snippets.to_a.empty?
+  .nothing-here-block= s_("SnippetsEmptyState|No snippets found")
+- else
   .snippets-list-holder
     %ul.content-list
       = render partial: 'shared/snippets/snippet', collection: @snippets, locals: { link_project: link_project }
     = paginate @snippets, theme: 'gitlab', remote: remote
-
-- else
-  .nothing-here-block= s_("SnippetsEmptyState|No snippets found")

app/views/shared/snippets/_snippet.html.haml

 - link_project = local_assigns.fetch(:link_project, false)
+- notes_count = @noteable_meta_data[snippet.id].user_notes_count
 %li.snippet-row
   = image_tag avatar_icon_for_user(snippet.author), class: "avatar s40 d-none d-sm-block", alt: ''
@@ -12,10 +13,9 @@
     %ul.controls
       %li
-        - note_count = snippet.notes.user.count
-        = link_to reliable_snippet_path(snippet, anchor: 'notes'), class: ('no-comments' if note_count.zero?) do
+        = link_to reliable_snippet_path(snippet, anchor: 'notes'), class: ('no-comments' if notes_count.zero?) do
           = icon('comments')
-          = note_count
+          = notes_count
       %li
         %span.sr-only
           = visibility_level_label(snippet.visibility_level)

changelogs/unreleased/65988-optimize-snippet-listings.yml

+---
+title: Optimize queries for snippet listings
+merge_request: 32576
+author:
+type: performance

changelogs/unreleased/security-12-3-bump-pages.yml

+---
+title: Upgrade pages to 1.8.1
+merge_request:
+author:
+type: security

doc/administration/geo/replication/docker_registry.md

 # Docker Registry for a secondary node **(PREMIUM ONLY)**
-You can set up a [Docker Registry] on your
+You can set up a [Docker Registry](https://docs.docker.com/registry/) on your
 **secondary** Geo node that mirrors the one on the **primary** Geo node.
 ## Storage support
-CAUTION: **Warning:**
-If you use [local storage][registry-storage]
-for the Container Registry you **cannot** replicate it to a **secondary** node.
-
 Docker Registry currently supports a few types of storages. If you choose a
 distributed storage (`azure`, `gcs`, `s3`, `swift`, or `oss`) for your Docker
 Registry on the **primary** node, you can use the same storage for a **secondary**
 Docker Registry as well. For more information, read the
-[Load balancing considerations][registry-load-balancing]
+[Load balancing considerations](https://docs.docker.com/registry/deploying/#load-balancing-considerations)
 when deploying the Registry, and how to set up the storage driver for GitLab's
-integrated [Container Registry][registry-storage].
-[ee]: https://about.gitlab.com/pricing/
-[Docker Registry]: https://docs.docker.com/registry/
-[registry-storage]: ../../container_registry.md#container-registry-storage-driver
-[registry-load-balancing]: https://docs.docker.com/registry/deploying/#load-balancing-considerations
+integrated [Container Registry](../../container_registry.md#container-registry-storage-driver).
+
+## Replicating Docker Registry
+
+You can enable a storage-agnostic replication so it
+can be used for cloud or local storages. Whenever a new image is pushed to the
+primary node, each **secondary** node will pull it to its own container
+repository.
+
+To configure Docker Registry replication:
+
+1. Configure the [**primary** node](#configure-primary-node).
+1. Configure the [**secondary** node](#configure-secondary-node).
+1. Verify Docker Registry [replication](#verify-replication).
+
+### Configure **primary** node
+
+Make sure that you have Container Registry set up and working on
+the **primary** node before following the next steps.
+
+We need to make Docker Registry send notification events to the
+**primary** node.
+
+1. SSH into your GitLab **primary** server and login as root:
+
+   ```sh
+   sudo -i
+   ```
+
+1. Edit `/etc/gitlab/gitlab.rb`:
+
+   ```ruby
+   registry['notifications'] = [
+     {
+       'name' => 'geo_event',
+       'url' => 'https://example.com/api/v4/container_registry_event/events',
+       'timeout' => '500ms',
+       'threshold' => 5,
+       'backoff' => '1s',
+       'headers' => {
+         'Authorization' => ['<replace_with_a_secret_token>']
+       }
+     }
+   ]
+   ```
+
+   NOTE: **Note:**
+   If you use an external Registry (not the one integrated with GitLab), you must add
+   these settings to its configuration. In this case, you will also have to specify
+   notification secret in `registry.notification_secret` section of
+   `/etc/gitlab/gitlab.rb` file.
+
+1. Reconfigure the **primary** node for the change to take effect:
+
+   ```sh
+   gitlab-ctl reconfigure
+   ```
+
+### Configure **secondary** node
+
+Make sure you have Container Registry set up and working on
+the **secondary** node before following the next steps.
+
+The following steps should be done on each **secondary** node you're
+expecting to see the Docker images replicated.
+
+Because we need to allow the **secondary** node to communicate securely with
+the **primary** node Container Registry, we need to have a single key
+pair for all the nodes. The **secondary** node will use this key to
+generate a short-lived JWT that is pull-only-capable to access the
+**primary** node Container Registry.
+
+1. SSH into the **secondary** node and login as the `root` user:
+
+   ```sh
+   sudo -i
+   ```
+
+1. Copy `/var/opt/gitlab/gitlab-rails/etc/gitlab-registry.key` from the **primary** to the **secondary** node.
+
+1. Edit `/etc/gitlab/gitlab.rb`:
+
+   ```ruby
+   gitlab_rails['registry_replication'] = {
+     enabled: true,
+     primary_api_url: 'http://primary.example.com:5000/' # internal address to the primary registry, will be used by GitLab to directly communicate with primary registry API
+   }
+   ```
+
+1. Reconfigure the **secondary** node for the change to take effect:
+
+   ```sh
+   gitlab-ctl reconfigure
+   ```
+
+### Verify replication
+To verify Container Registry replication is working, go to **Admin Area > Geo** (`/admin/geo/nodes`) on the **secondary** node.
+The initial replication, or "backfill", will probably still be in progress.
+You can monitor the synchronization process on each Geo node from the **primary** node's **Geo Nodes** dashboard in your browser.