Add latest changes from gitlab-org/gitlab@master

app/controllers/boards/issues_controller.rb

@@ -27,17 +27,7 @@ module Boards
       issues = list_service.execute
       issues = issues.page(params[:page]).per(params[:per] || 20).without_count
       Issue.move_nulls_to_end(issues) if Gitlab::Database.read_write?
-      issues = issues.preload(:milestone,
-                              :assignees,
-                              project: [
-                                  :route,
-                                  {
-                                      namespace: [:route]
-                                  }
-                              ],
-                              labels: [:priorities],
-                              notes: [:award_emoji, :author]
-                             )
+      issues = issues.preload(associations_to_preload)
       render_issues(issues, list_service.metadata)
     end
@@ -74,6 +64,21 @@ module Boards
     private
+    def associations_to_preload
+      [
+        :milestone,
+        :assignees,
+        project: [
+            :route,
+            {
+                namespace: [:route]
+            }
+        ],
+        labels: [:priorities],
+        notes: [:award_emoji, :author]
+      ]
+    end
+
     def can_move_issues?
       head(:forbidden) unless can?(current_user, :admin_issue, board)
     end
@@ -139,3 +144,5 @@ module Boards
     end
   end
 end
+
+Boards::IssuesController.prepend_if_ee('EE::Boards::IssuesController')

changelogs/unreleased/ab-projects-api-indexes-authenticated-calls.yml

+---
+title: Add indexes for authenticated Project API calls
+merge_request: 22886
+author:
+type: performance

db/migrate/20200113133352_add_indexes_for_projects_api_authenticated.rb

+# frozen_string_literal: true
+
+class AddIndexesForProjectsApiAuthenticated < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  COLUMNS = %i(updated_at name)
+
+  def up
+    add_concurrent_index :projects, %i(created_at id), order: { id: :desc }, name: 'index_projects_api_created_at_id_desc'
+
+    add_concurrent_index :projects, %i(last_activity_at id), name: 'index_projects_on_last_activity_at_and_id'
+    remove_concurrent_index :projects, :last_activity_at
+    add_concurrent_index :projects, %i(last_activity_at id), order: { id: :desc }, name: 'index_projects_api_last_activity_at_id_desc'
+
+    add_concurrent_index :projects, %i(path id), name: 'index_projects_on_path_and_id'
+    remove_concurrent_index_by_name :projects, 'index_projects_on_path'
+    add_concurrent_index :projects, %i(path id), order: { id: :desc }, name: 'index_projects_api_path_id_desc'
+
+    COLUMNS.each do |column|
+      add_concurrent_index :projects, [column, :id], name: "index_projects_on_#{column}_and_id"
+      add_concurrent_index :projects, [column, :id], order: { id: :desc }, name: "index_projects_api_#{column}_id_desc"
+    end
+  end
+
+  def down
+    remove_concurrent_index_by_name :projects, 'index_projects_api_created_at_id_desc'
+
+    remove_concurrent_index_by_name :projects, 'index_projects_on_last_activity_at_and_id'
+    add_concurrent_index :projects, :last_activity_at, name: 'index_projects_on_last_activity_at'
+    remove_concurrent_index_by_name :projects, 'index_projects_api_last_activity_at_id_desc'
+
+    remove_concurrent_index_by_name :projects, 'index_projects_on_path_and_id'
+    add_concurrent_index :projects, :path, name: 'index_projects_on_path'
+    remove_concurrent_index_by_name :projects, 'index_projects_api_path_id_desc'
+
+    COLUMNS.each do |column|
+      remove_concurrent_index_by_name :projects, "index_projects_on_#{column}_and_id"
+      remove_concurrent_index_by_name :projects, "index_projects_api_#{column}_id_desc"
+    end
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema.define(version: 2020_01_10_144316) do
+ActiveRecord::Schema.define(version: 2020_01_13_133352) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "pg_trgm"
@@ -3353,6 +3353,7 @@ ActiveRecord::Schema.define(version: 2020_01_10_144316) do
     t.boolean "autoclose_referenced_issues"
     t.string "suggestion_commit_message", limit: 255
     t.index "lower((name)::text)", name: "index_projects_on_lower_name"
+    t.index ["created_at", "id"], name: "index_projects_api_created_at_id_desc", order: { id: :desc }
     t.index ["created_at", "id"], name: "index_projects_api_vis20_created_at", where: "(visibility_level = 20)"
     t.index ["created_at", "id"], name: "index_projects_api_vis20_created_at_id_desc", order: { id: :desc }, where: "(visibility_level = 20)"
     t.index ["created_at", "id"], name: "index_projects_on_created_at_and_id"
@@ -3362,9 +3363,10 @@ ActiveRecord::Schema.define(version: 2020_01_10_144316) do
     t.index ["id"], name: "index_on_id_partial_with_legacy_storage", where: "((storage_version < 2) OR (storage_version IS NULL))"
     t.index ["id"], name: "index_projects_on_id_partial_for_visibility", unique: true, where: "(visibility_level = ANY (ARRAY[10, 20]))"
     t.index ["id"], name: "index_projects_on_mirror_and_mirror_trigger_builds_both_true", where: "((mirror IS TRUE) AND (mirror_trigger_builds IS TRUE))"
+    t.index ["last_activity_at", "id"], name: "index_projects_api_last_activity_at_id_desc", order: { id: :desc }
     t.index ["last_activity_at", "id"], name: "index_projects_api_vis20_last_activity_at", where: "(visibility_level = 20)"
     t.index ["last_activity_at", "id"], name: "index_projects_api_vis20_last_activity_at_id_desc", order: { id: :desc }, where: "(visibility_level = 20)"
-    t.index ["last_activity_at"], name: "index_projects_on_last_activity_at"
+    t.index ["last_activity_at", "id"], name: "index_projects_on_last_activity_at_and_id"
     t.index ["last_repository_check_at"], name: "index_projects_on_last_repository_check_at", where: "(last_repository_check_at IS NOT NULL)"
     t.index ["last_repository_check_failed"], name: "index_projects_on_last_repository_check_failed"
     t.index ["last_repository_updated_at"], name: "index_projects_on_last_repository_updated_at"
@@ -3372,13 +3374,16 @@ ActiveRecord::Schema.define(version: 2020_01_10_144316) do
     t.index ["marked_for_deletion_by_user_id"], name: "index_projects_on_marked_for_deletion_by_user_id", where: "(marked_for_deletion_by_user_id IS NOT NULL)"
     t.index ["mirror_last_successful_update_at"], name: "index_projects_on_mirror_last_successful_update_at"
     t.index ["mirror_user_id"], name: "index_projects_on_mirror_user_id"
+    t.index ["name", "id"], name: "index_projects_api_name_id_desc", order: { id: :desc }
     t.index ["name", "id"], name: "index_projects_api_vis20_name", where: "(visibility_level = 20)"
     t.index ["name", "id"], name: "index_projects_api_vis20_name_id_desc", order: { id: :desc }, where: "(visibility_level = 20)"
+    t.index ["name", "id"], name: "index_projects_on_name_and_id"
     t.index ["name"], name: "index_projects_on_name_trigram", opclass: :gin_trgm_ops, using: :gin
     t.index ["namespace_id"], name: "index_projects_on_namespace_id"
+    t.index ["path", "id"], name: "index_projects_api_path_id_desc", order: { id: :desc }
     t.index ["path", "id"], name: "index_projects_api_vis20_path", where: "(visibility_level = 20)"
     t.index ["path", "id"], name: "index_projects_api_vis20_path_id_desc", order: { id: :desc }, where: "(visibility_level = 20)"
-    t.index ["path"], name: "index_projects_on_path"
+    t.index ["path", "id"], name: "index_projects_on_path_and_id"
     t.index ["path"], name: "index_projects_on_path_trigram", opclass: :gin_trgm_ops, using: :gin
     t.index ["pending_delete"], name: "index_projects_on_pending_delete"
     t.index ["pool_repository_id"], name: "index_projects_on_pool_repository_id", where: "(pool_repository_id IS NOT NULL)"
@@ -3387,8 +3392,10 @@ ActiveRecord::Schema.define(version: 2020_01_10_144316) do
     t.index ["runners_token"], name: "index_projects_on_runners_token"
     t.index ["runners_token_encrypted"], name: "index_projects_on_runners_token_encrypted"
     t.index ["star_count"], name: "index_projects_on_star_count"
+    t.index ["updated_at", "id"], name: "index_projects_api_updated_at_id_desc", order: { id: :desc }
     t.index ["updated_at", "id"], name: "index_projects_api_vis20_updated_at", where: "(visibility_level = 20)"
     t.index ["updated_at", "id"], name: "index_projects_api_vis20_updated_at_id_desc", order: { id: :desc }, where: "(visibility_level = 20)"
+    t.index ["updated_at", "id"], name: "index_projects_on_updated_at_and_id"
   end
   create_table "prometheus_alert_events", force: :cascade do |t|

doc/development/logging.md

@@ -209,6 +209,36 @@ I, [2020-01-13T19:01:17.091Z #11056]  INFO -- : {"message"=>"Message", "project_
 {:severity=>"INFO", :time=>"2020-01-13T11:06:09.851Z", :correlation_id=>"d7e0886f096db9a8526a4f89da0e45f6", :message=>"This is my message", :project_id=>123}
 ```
+### Logging context metadata (through Rails or Grape requests)
+
+`Gitlab::ApplicationContext` stores metadata in a request
+lifecycle, which can then be added to the web request
+or Sidekiq logs.
+
+Entry points can be seen at:
+
+- [`ApplicationController`](https://gitlab.com/gitlab-org/gitlab/blob/master/app/controllers/application_controller.rb)
+- [External API](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/api/api.rb)
+- [Internal API](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/api/internal/base.rb)
+
+#### Adding attributes
+
+When adding new attributes, make sure they're exposed within the context of the entry points above and:
+
+- Pass them within the hash to the `with_context` (or `push`) method (make sure to pass a Proc if the
+method or variable shouldn't be evaluated right away)
+- Change `Gitlab::ApplicationContext` to accept these new values
+- Make sure the new attributes are accepted at [`Labkit::Context`](https://gitlab.com/gitlab-org/labkit-ruby/blob/master/lib/labkit/context.rb)
+
+See our [HOWTO: Use Sidekiq metadata logs](https://www.youtube.com/watch?v=_wDllvO_IY0) for further knowledge on
+creating visualizations in Kibana.
+
+**Note:**
+The fields of the context are currently only logged for Sidekiq jobs triggered
+through web requests. See the
+[follow-up work](https://gitlab.com/gitlab-com/gl-infra/scalability/issues/68)
+for more information.
+
 ## Exception Handling
 It often happens that you catch the exception and want to track it.

qa/load/artillery.yml

-config:
-  target: "{{ $processEnvironment.HOST_URL }}"
-  http:
-    pool: 10 # All HTTP requests from all virtual users will be sent over the same <pool> connections.
-             # This also means that there is a limit on the number of requests sent per second.
-  phases:
-    - duration: 30
-      arrivalRate: 10
-      name: "Warm up"
-    - duration: 90
-      arrivalRate: 10
-      rampTo: 100
-      name: "Gradual ramp up"
-    - duration: 90
-      arrivalRate: 100
-      name: "Sustained max load"
-scenarios:
-  - name: "Visit large issue url"
-    flow:
-      - get:
-          url: "{{ $processEnvironment.LARGE_ISSUE_URL }}"
-  - name: "Visit large MR url"
-    flow:
-      - get:
-          url: "{{ $processEnvironment.LARGE_MR_URL }}"

qa/qa/page/base.rb

@@ -26,8 +26,8 @@ module QA
         wait_for_requests
       end
-      def wait(max: 60, interval: 0.1, reload: true, raise_on_failure: false)
-        Support::Waiter.wait_until(max_duration: max, sleep_interval: interval, raise_on_failure: raise_on_failure) do
+      def wait_until(max_duration: 60, sleep_interval: 0.1, reload: true, raise_on_failure: false)
+        Support::Waiter.wait_until(max_duration: max_duration, sleep_interval: sleep_interval, raise_on_failure: raise_on_failure) do
           yield || (reload && refresh && false)
         end
       end
@@ -71,7 +71,7 @@ module QA
           xhr.send();
         JS
-        return false unless wait(interval: 0.5, max: 60, reload: false) do
+        return false unless wait_until(sleep_interval: 0.5, max_duration: 60, reload: false) do
           page.evaluate_script('xhr.readyState == XMLHttpRequest.DONE')
         end
@@ -115,8 +115,8 @@ module QA
       end
       # replace with (..., page = self.class)
-      def click_element(name, page = nil, text: nil)
-        find_element(name, text: text).click
+      def click_element(name, page = nil, text: nil, wait: Capybara.default_max_wait_time)
+        find_element(name, text: text, wait: wait).click
         page.validate_elements_present! if page
       end
@@ -161,10 +161,10 @@ module QA
         page.has_text?(text, wait: wait)
       end
-      def has_no_text?(text)
+      def has_no_text?(text, wait: Capybara.default_max_wait_time)
         wait_for_requests
-        page.has_no_text? text
+        page.has_no_text?(text, wait: wait)
       end
       def has_normalized_ws_text?(text, wait: Capybara.default_max_wait_time)
@@ -191,7 +191,7 @@ module QA
         # This loop gives time for the img tags to be rendered and for
         # images to start loading.
         previous_total_images = 0
-        wait(interval: 1) do
+        wait_until(sleep_interval: 1) do
           current_total_images = all("img").size
           result = previous_total_images == current_total_images
           previous_total_images = current_total_images

qa/qa/page/component/ci_badge_link.rb

@@ -26,7 +26,7 @@ module QA
         private
         def completed?(timeout: 60)
-          wait(reload: false, max: timeout) do
+          wait_until(reload: false, max_duration: timeout) do
             COMPLETED_STATUSES.include?(status_badge)
           end
         end

qa/qa/page/component/clone_panel.rb

@@ -24,7 +24,7 @@ module QA
         private
         def repository_clone_location(kind)
-          wait(reload: false) do
+          wait_until(reload: false) do
             click_element :clone_dropdown
             within_element :clone_options do

qa/qa/page/component/dropdown_filter.rb

@@ -5,9 +5,7 @@ module QA
     module Component
       module DropdownFilter
         def filter_and_select(item)
-          wait(reload: false) do
-            page.has_css?('.dropdown-input-field')
-          end
+          page.has_css?('.dropdown-input-field', wait: QA::Support::Repeater::DEFAULT_MAX_WAIT_TIME)
           find('.dropdown-input-field').set(item)
           click_link item

qa/qa/page/component/dropzone.rb

@@ -23,7 +23,7 @@ module QA
           page.attach_file(attachment, class: 'dz-hidden-input', make_visible: field_style)
           # Wait for link to be appended to dropzone text
-          page.wait(reload: false) do
+          page.wait_until(reload: false) do
             page.find("#{container} textarea").value.match(filename)
           end
         end

qa/qa/page/component/groups_filter.rb

@@ -23,9 +23,7 @@ module QA
           # Since we submitted after filtering, the presence of
           # groups_list_tree_container means we have the complete filtered list
           # of groups
-          wait(reload: false) do
-            page.has_css?(element_selector_css(:groups_list_tree_container))
-          end
+          has_element?(:groups_list_tree_container, wait: QA::Support::Repeater::DEFAULT_MAX_WAIT_TIME)
           # If there are no groups we'll know immediately because we filtered the list
           return false if page.has_text?('No groups or projects matched your search', wait: 0)

qa/qa/page/component/legacy_clone_panel.rb

@@ -30,7 +30,7 @@ module QA
         private
         def choose_repository_clone(kind, detect_text)
-          wait(reload: false) do
+          wait_until(reload: false) do
             click_element :clone_dropdown
             page.within('.clone-options-dropdown') do

qa/qa/page/file/shared/commit_button.rb

@@ -14,7 +14,7 @@ module QA
           def commit_changes
             click_element(:commit_button)
-            wait(reload: false, max: 60) do
+            wait_until(reload: false, max_duration: 60) do
               finished_loading?
             end
           end

qa/qa/page/group/show.rb

@@ -58,7 +58,7 @@ module QA
           QA::Support::Retrier.retry_on_exception(sleep_interval: 1.0) do
             within_element(:new_project_or_subgroup_dropdown) do
               # May need to click again because it is possible to click the button quicker than the JS is bound
-              wait(reload: false) do
+              wait_until(reload: false) do
                 click_element :new_project_or_subgroup_dropdown_toggle
                 has_element?(kind)

qa/qa/page/merge_request/show.rb

@@ -73,7 +73,7 @@ module QA
         end
         def add_comment_to_diff(text)
-          wait(interval: 5) do
+          wait_until(sleep_interval: 5) do
             has_text?("No newline at end of file")
           end
           all_elements(:new_diff_line, minimum: 1).first.hover
@@ -140,12 +140,12 @@ module QA
         def mark_to_squash
           # The squash checkbox is disabled on load
-          wait do
+          wait_until do
             has_element?(:squash_checkbox)
           end
           # The squash checkbox is enabled via JS
-          wait(reload: false) do
+          wait_until(reload: false) do
             !find_element(:squash_checkbox).disabled?
           end
@@ -164,30 +164,30 @@ module QA
         def ready_to_merge?
           # The merge button is disabled on load
-          wait do
+          wait_until do
             has_element?(:merge_button)
           end
           # The merge button is enabled via JS
-          wait(reload: false) do
+          wait_until(reload: false) do
             !find_element(:merge_button).disabled?
           end
         end
         def rebase!
           # The rebase button is disabled on load
-          wait do
+          wait_until do
             has_element?(:mr_rebase_button)
           end
           # The rebase button is enabled via JS
-          wait(reload: false) do
+          wait_until(reload: false) do
             !find_element(:mr_rebase_button).disabled?
           end
           click_element :mr_rebase_button
-          success = wait do
+          success = wait_until do
             has_text?('Fast-forward merge without a merge commit')
           end
@@ -209,7 +209,7 @@ module QA
         end
         def wait_for_merge_request_error_message
-          wait(max: 30, reload: false) do
+          wait_until(max_duration: 30, reload: false) do
             has_element?(:merge_request_error_content)
           end
         end

qa/qa/page/project/branches/show.rb

@@ -29,7 +29,7 @@ module QA
           end
           def has_no_branch?(branch_name, reload: false)
-            wait(reload: reload) do
+            wait_until(reload: reload) do
               within_element(:all_branches) do
                 has_no_element?(:branch_name, text: branch_name)
               end

qa/qa/page/project/import/github.rb

@@ -35,7 +35,7 @@ module QA
           private
           def within_repo_path(full_path)
-            wait(reload: false) do
+            wait_until(reload: false) do
               has_element?(:project_import_row, text: full_path)
             end
@@ -67,7 +67,7 @@ module QA
           end
           def wait_for_success
-            wait(max: 60, interval: 1.0, reload: false) do
+            wait_until(max_duration: 60, sleep_interval: 1.0, reload: false) do
               page.has_content?('Done', wait: 1.0)
             end
           end

qa/qa/page/project/issue/show.rb

@@ -89,9 +89,7 @@ module QA
           end
           def has_comment?(comment_text)
-            wait(reload: false) do
-              has_element?(:noteable_note_item, text: comment_text)
-            end
+            has_element?(:noteable_note_item, text: comment_text, wait: QA::Support::Repeater::DEFAULT_MAX_WAIT_TIME)
           end
           def more_assignees_link
@@ -155,7 +153,7 @@ module QA
           def wait_assignees_block_finish_loading
             within_element(:assignee_block) do
-              wait(reload: false, max: 10, interval: 1) do
+              wait_until(reload: false, max_duration: 10, sleep_interval: 1) do
                 finished_loading_block?
                 yield
               end

qa/qa/page/project/job/show.rb

@@ -24,7 +24,7 @@ module QA::Page
       def output(wait: 5)
         result = ''
-        wait(reload: false, max: wait, interval: 1) do
+        wait_until(reload: false, max_duration: wait, sleep_interval: 1) do
           result = find_element(:job_log_content).text
           result.include?('Job')
@@ -36,7 +36,7 @@ module QA::Page
       private
       def loaded?(wait: 60)
-        wait(reload: true, max: wait, interval: 1) do
+        wait_until(reload: true, max_duration: wait, sleep_interval: 1) do
           has_element?(:job_log_content, wait: 1)
         end
       end