Add latest changes from gitlab-org/gitlab@master

changelogs/unreleased/24072-user-profile-add-job-title.yml

+---
+title: Add "Job Title" field in user settings and display on profile
+merge_request: 25155
+author:
+type: added

changelogs/unreleased/27880-clearing-release-note-from-the-tags-page-deletes-release.yml

+---
+title: 27880 Make release notes optional and do not delete release when they are removed
+merge_request: 26231
+author: Pavlo Dudchenko
+type: changed

changelogs/unreleased/fj-fix-bug-hook-env.yml

+---
+title: Fix bug setting hook env with personal snippets
+merge_request: 27235
+author:
+type: fixed

changelogs/unreleased/mermaid-fix.yml

+---
+title: Add functionality to render individual mermaids
+merge_request: 26564
+author:
+type: changed

config/environments/development.rb

@@ -11,6 +11,9 @@ Rails.application.configure do
   config.consider_all_requests_local       = true
   config.action_controller.perform_caching = false
+  # Show a warning when a large data set is loaded into memory
+  config.active_record.warn_on_records_fetched_greater_than = 1000
+
   # Print deprecation notices to the Rails logger
   config.active_support.deprecation = :log

config/initializers/5_backend.rb

 unless Rails.env.test?
   required_version = Gitlab::VersionInfo.parse(Gitlab::Shell.version_required)
-  current_version = Gitlab::VersionInfo.parse(Gitlab::Shell.new.version)
+  current_version = Gitlab::VersionInfo.parse(Gitlab::Shell.version)
   unless current_version.valid? && required_version <= current_version
     warn "WARNING: This version of GitLab depends on gitlab-shell #{required_version}, but you're running #{current_version}. Please update gitlab-shell."

db/migrate/20200306192548_add_index_on_project_id_and_type_to_services.rb

+# frozen_string_literal: true
+
+class AddIndexOnProjectIdAndTypeToServices < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  INDEX_NAME = 'index_services_on_project_id'
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :services, [:project_id, :type]
+
+    remove_concurrent_index_by_name :services, INDEX_NAME
+  end
+
+  def down
+    add_concurrent_index :services, :project_id, name: INDEX_NAME
+
+    remove_concurrent_index :services, [:project_id, :type]
+  end
+end

db/migrate/20200306193236_add_index_on_creator_id_and_created_at_to_projects.rb

+# frozen_string_literal: true
+
+class AddIndexOnCreatorIdAndCreatedAtToProjects < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  INDEX_NAME = 'index_projects_on_creator_id'
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :projects, [:creator_id, :created_at]
+
+    remove_concurrent_index_by_name :projects, INDEX_NAME
+  end
+
+  def down
+    add_concurrent_index :projects, :creator_id, name: INDEX_NAME
+
+    remove_concurrent_index :projects, [:creator_id, :created_at]
+  end
+end

db/schema.rb

@@ -3486,8 +3486,8 @@ ActiveRecord::Schema.define(version: 2020_03_12_163407) do
     t.index ["created_at", "id"], name: "index_projects_api_created_at_id_desc", order: { id: :desc }
     t.index ["created_at", "id"], name: "index_projects_api_vis20_created_at", where: "(visibility_level = 20)"
     t.index ["created_at", "id"], name: "index_projects_on_created_at_and_id"
+    t.index ["creator_id", "created_at"], name: "index_projects_on_creator_id_and_created_at"
     t.index ["creator_id", "created_at"], name: "index_projects_on_mirror_creator_id_created_at", where: "((mirror = true) AND (mirror_trigger_builds = true))"
-    t.index ["creator_id"], name: "index_projects_on_creator_id"
     t.index ["description"], name: "index_projects_on_description_trigram", opclass: :gin_trgm_ops, using: :gin
     t.index ["id", "repository_storage", "last_repository_updated_at"], name: "idx_projects_on_repository_storage_last_repository_updated_at"
     t.index ["id"], name: "index_on_id_partial_with_legacy_storage", where: "((storage_version < 2) OR (storage_version IS NULL))"
@@ -3961,7 +3961,7 @@ ActiveRecord::Schema.define(version: 2020_03_12_163407) do
     t.boolean "comment_on_event_enabled", default: true, null: false
     t.boolean "template", default: false
     t.boolean "instance", default: false, null: false
-    t.index ["project_id"], name: "index_services_on_project_id"
+    t.index ["project_id", "type"], name: "index_services_on_project_id_and_type"
     t.index ["template"], name: "index_services_on_template"
     t.index ["type", "instance"], name: "index_services_on_type_and_instance", unique: true, where: "(instance IS TRUE)"
     t.index ["type", "template"], name: "index_services_on_type_and_template", unique: true, where: "(template IS TRUE)"

doc/api/releases/index.md

@@ -328,7 +328,7 @@ POST /projects/:id/releases
 | `id`               | integer/string  | yes                         | The ID or [URL-encoded path of the project](../README.md#namespaced-path-encoding).                                              |
 | `name`             | string          | no                          | The release name.                                                                                                                |
 | `tag_name`         | string          | yes                         | The tag where the release will be created from.                                                                                  |
-| `description`      | string          | yes                         | The description of the release. You can use [Markdown](../../user/markdown.md).                                                  |
+| `description`      | string          | no                          | The description of the release. You can use [Markdown](../../user/markdown.md).                                                  |
 | `ref`              | string          | yes, if `tag_name` doesn't exist | If `tag_name` doesn't exist, the release will be created from `ref`. It can be a commit SHA, another tag name, or a branch name. |
 | `milestones`       | array of string | no                          | The title of each milestone the release is associated with.                                                                      |
 | `assets:links`     | array of hash   | no                          | An array of assets links.                                                                                                        |

doc/ci/cloud_deployment/index.md

@@ -61,3 +61,61 @@ To do so, please make sure to [push your image into your ECR
 repository](https://docs.aws.amazon.com/AmazonECR/latest/userguide/docker-push-ecr-image.html)
 before referencing it in your `.gitlab-ci.yml` file and replace the `image`
 path to point to your ECR.
+
+### Deploy your application to AWS Elastic Container Service (ECS)
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/207962) in GitLab 12.9.
+
+GitLab provides a series of [CI templates that you can include in your project](../yaml/README.md#include).
+To automate deployments of your application to your [Amazon Elastic Container Service](https://aws.amazon.com/ecs/) (AWS ECS)
+cluster, you can `include` the `Deploy-ECS.gitlab-ci.yml` template in your `.gitlab-ci.yml` file.
+
+Before getting started with this process, you need a cluster on AWS ECS, as well as related
+components, like an ECS service, ECS task definition, a database on AWS RDS, etc.
+[Read more about AWS ECS](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/Welcome.html).
+
+After you're all set up on AWS ECS, follow these steps:
+
+1. Make sure your AWS credentials are set up as environment variables for your
+   project. You can follow [the steps above](#aws) to complete this setup.
+1. Add these variables to your project's `.gitlab-ci.yml` file:
+
+   ```yaml
+   variables:
+     CI_AWS_ECS_CLUSTER: my-cluster
+     CI_AWS_ECS_SERVICE: my-service
+     CI_AWS_ECS_TASK_DEFINITION: my-task-definition
+   ```
+
+   Three variables are defined in this snippet:
+
+   - `CI_AWS_ECS_CLUSTER`: The name of your AWS ECS cluster that you're
+   targeting for your deployments.
+   - `CI_AWS_ECS_SERVICE`: The name of the targeted service tied to
+   your AWS ECS cluster.
+   - `CI_AWS_ECS_TASK_DEFINITION`: The name of the task definition tied
+   to the service mentioned above.
+
+   You can find these names after selecting the targeted cluster on your [AWS ECS dashboard](https://console.aws.amazon.com/ecs/home):
+
+   ![AWS ECS dashboard](../img/ecs_dashboard_v12_9.png)
+
+1. Include this template in `.gitlab-ci.yml`:
+
+   ```yaml
+   include:
+     - template: Deploy-ECS.gitlab-ci.yml
+   ```
+
+   The `Deploy-ECS` template ships with GitLab and is available [on
+   GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml).
+
+1. Commit and push your updated `.gitlab-ci.yml` to your project's repository, and you're done!
+
+   Your application Docker image will be rebuilt and pushed to the GitLab registry.
+   Then the targeted task definition will be updated with the location of the new
+   Docker image, and a new revision will be created in ECS as result.
+
+   Finally, your AWS ECS service will be updated with the new revision of the
+   task definition, making the cluster pull the newest version of your
+   application.

doc/development/elasticsearch.md

@@ -36,11 +36,7 @@ Additionally, if you need large repos or multiple forks for testing, please cons
 The Elasticsearch integration depends on an external indexer. We ship an [indexer written in Go](https://gitlab.com/gitlab-org/gitlab-elasticsearch-indexer). The user must trigger the initial indexing via a rake task but, after this is done, GitLab itself will trigger reindexing when required via `after_` callbacks on create, update, and destroy that are inherited from [/ee/app/models/concerns/elastic/application_versioned_search.rb](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/app/models/concerns/elastic/application_versioned_search.rb).
-After initial indexing is complete, updates proceed in one of two ways, depending on the `:elastic_bulk_incremental_updates` feature flag.
-
-If disabled, every create, update, or delete operation on an Elasticsearch-tracked model enqueues a new `ElasticIndexerWorker` Sidekiq job which takes care of updating just that document. This is quite inefficient.
-
-If the feature flag is enabled, create, update, and delete operations for all models except projects (see [#207494](https://gitlab.com/gitlab-org/gitlab/issues/207494)) are tracked in a Redis [`ZSET`](https://redis.io/topics/data-types#sorted-sets) instead. A regular `sidekiq-cron` `ElasticIndexBulkCronWorker` processes this queue, updating many Elasticsearch documents at a time with the [Bulk Request API](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html).
+After initial indexing is complete, create, update, and delete operations for all models except projects (see [#207494](https://gitlab.com/gitlab-org/gitlab/issues/207494)) are tracked in a Redis [`ZSET`](https://redis.io/topics/data-types#sorted-sets). A regular `sidekiq-cron` `ElasticIndexBulkCronWorker` processes this queue, updating many Elasticsearch documents at a time with the [Bulk Request API](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html).
 Search queries are generated by the concerns found in [ee/app/models/concerns/elastic](https://gitlab.com/gitlab-org/gitlab/tree/master/ee/app/models/concerns/elastic). These concerns are also in charge of access control, and have been a historic source of security bugs so please pay close attention to them!

lib/api/internal/base.rb

@@ -40,7 +40,7 @@ module API
           # Stores some Git-specific env thread-safely
           env = parse_env
-          Gitlab::Git::HookEnv.set(gl_repository, env) if project
+          Gitlab::Git::HookEnv.set(gl_repository, env) if container
           actor.update_last_used_at!
           access_checker = access_checker_for(actor, params[:protocol])

lib/api/releases.rb

@@ -46,7 +46,7 @@ module API
       params do
         requires :tag_name,    type: String, desc: 'The name of the tag', as: :tag
         optional :name,        type: String, desc: 'The name of the release'
-        requires :description, type: String, desc: 'The release notes'
+        optional :description, type: String, desc: 'The release notes'
         optional :ref,         type: String, desc: 'The commit sha or branch name'
         optional :assets, type: Hash do
           optional :links, type: Array do

lib/backup/repository.rb

@@ -4,7 +4,6 @@ require 'yaml'
 module Backup
   class Repository
-    include Gitlab::ShellAdapter
     attr_reader :progress
     def initialize(progress)
@@ -71,23 +70,14 @@ module Backup
     def restore
       Project.find_each(batch_size: 1000) do |project|
         progress.print " * #{project.full_path} ... "
-        path_to_project_bundle = path_to_bundle(project)
-        project.repository.remove rescue nil
-        restore_repo_success = nil
-
-        if File.exist?(path_to_project_bundle)
+        restore_repo_success =
           begin
-            project.repository.create_from_bundle(path_to_project_bundle)
-            restore_custom_hooks(project)
-            restore_repo_success = true
-          rescue => e
-            restore_repo_success = false
-            progress.puts "Error: #{e}".color(:red)
+            try_restore_repository(project)
+          rescue => err
+            progress.puts "Error: #{err}".color(:red)
+            false
           end
-        else
-          restore_repo_success = gitlab_shell.create_project_repository(project)
-        end
         if restore_repo_success
           progress.puts "[DONE]".color(:green)
@@ -118,6 +108,20 @@ module Backup
     protected
+    def try_restore_repository(project)
+      path_to_project_bundle = path_to_bundle(project)
+      project.repository.remove rescue nil
+
+      if File.exist?(path_to_project_bundle)
+        project.repository.create_from_bundle(path_to_project_bundle)
+        restore_custom_hooks(project)
+      else
+        project.repository.create_repository
+      end
+
+      true
+    end
+
     def path_to_bundle(project)
       File.join(backup_repos_path, project.disk_path + '.bundle')
     end

lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml

+stages:
+  - build
+  - test
+  - review
+  - deploy
+  - production
+
+include:
+  - template: Jobs/Build.gitlab-ci.yml
+
+.deploy_to_ecs:
+  image: registry.gitlab.com/gitlab-org/cloud-deploy:latest
+  script:
+    - ecs update-task-definition
+
+review:
+  extends: .deploy_to_ecs
+  stage: review
+  environment:
+    name: review/$CI_COMMIT_REF_NAME
+  only:
+    refs:
+      - branches
+      - tags
+  except:
+    refs:
+      - master
+
+production:
+  extends: .deploy_to_ecs
+  stage: production
+  environment:
+    name: production
+  only:
+    refs:
+      - master

lib/gitlab/shell.rb

@@ -6,8 +6,6 @@ require 'securerandom'
 module Gitlab
   class Shell
-    GITLAB_SHELL_ENV_VARS = %w(GIT_TERMINAL_PROMPT).freeze
-
     Error = Class.new(StandardError)
     class << self
@@ -36,8 +34,31 @@ module Gitlab
                                         .join('GITLAB_SHELL_VERSION')).strip
       end
+      # Return GitLab shell version
+      #
+      # @return [String] version
+      def version
+        @version ||= File.read(gitlab_shell_version_file).chomp if File.readable?(gitlab_shell_version_file)
+      end
+
+      # Return a SSH url for a given project path
+      #
+      # @param [String] full_path project path (URL)
+      # @return [String] SSH URL
+      def url_to_repo(full_path)
+        Gitlab.config.gitlab_shell.ssh_path_prefix + "#{full_path}.git"
+      end
+
       private
+      def gitlab_shell_path
+        File.expand_path(Gitlab.config.gitlab_shell.path)
+      end
+
+      def gitlab_shell_version_file
+        File.join(gitlab_shell_path, 'VERSION')
+      end
+
       # Create (if necessary) and link the secret token file
       def generate_and_link_secret_token
         secret_file = Gitlab.config.gitlab_shell.secret_file
@@ -56,47 +77,6 @@ module Gitlab
       end
     end
-    # Initialize a new project repository using a Project model
-    #
-    # @param [Project] project
-    # @return [Boolean] whether repository could be created
-    def create_project_repository(project)
-      create_repository(project.repository_storage, project.disk_path, project.full_path)
-    end
-
-    # Initialize a new wiki repository using a Project model
-    #
-    # @param [Project] project
-    # @return [Boolean] whether repository could be created
-    def create_wiki_repository(project)
-      create_repository(project.repository_storage, project.wiki.disk_path, project.wiki.full_path)
-    end
-
-    # Init new repository
-    #
-    # @example Create a repository
-    #   create_repository("default", "path/to/gitlab-ci", "gitlab/gitlab-ci")
-    #
-    # @param [String] storage the shard key
-    # @param [String] disk_path project path on disk
-    # @param [String] gl_project_path project name
-    # @return [Boolean] whether repository could be created
-    def create_repository(storage, disk_path, gl_project_path)
-      relative_path = disk_path.dup
-      relative_path << '.git' unless relative_path.end_with?('.git')
-
-      # During creation of a repository, gl_repository may not be known
-      # because that depends on a yet-to-be assigned project ID in the
-      # database (e.g. project-1234), so for now it is blank.
-      repository = Gitlab::Git::Repository.new(storage, relative_path, '', gl_project_path)
-      wrapped_gitaly_errors { repository.gitaly_repository_client.create_repository }
-
-      true
-    rescue => err # Once the Rugged codes gets removes this can be improved
-      Rails.logger.error("Failed to add repository #{storage}/#{disk_path}: #{err}") # rubocop:disable Gitlab/RailsLogger
-      false
-    end
-
     # Import wiki repository from external service
     #
     # @param [Project] project
@@ -238,25 +218,6 @@ module Gitlab
       false
     end
-    # Return a SSH url for a given project path
-    #
-    # @param [String] full_path project path (URL)
-    # @return [String] SSH URL
-    def url_to_repo(full_path)
-      Gitlab.config.gitlab_shell.ssh_path_prefix + "#{full_path}.git"
-    end
-
-    # Return GitLab shell version
-    #
-    # @return [String] version
-    def version
-      gitlab_shell_version_file = "#{gitlab_shell_path}/VERSION"
-
-      if File.readable?(gitlab_shell_version_file)
-        File.read(gitlab_shell_version_file).chomp
-      end
-    end
-
     # Check if repository exists on disk
     #
     # @example Check if repository exists
@@ -271,23 +232,8 @@ module Gitlab
       false
     end
-    # Return hooks folder path used by projects
-    #
-    # @return [String] path
-    def hooks_path
-      File.join(gitlab_shell_path, 'hooks')
-    end
-
     protected
-    def gitlab_shell_path
-      File.expand_path(Gitlab.config.gitlab_shell.path)
-    end
-
-    def gitlab_shell_user_home
-      File.expand_path("~#{Gitlab.config.gitlab_shell.ssh_user}")
-    end
-
     def full_path(storage, dir_name)
       raise ArgumentError.new("Directory name can't be blank") if dir_name.blank?

lib/system_check/gitlab_shell_check.rb

@@ -50,7 +50,7 @@ module SystemCheck
     end
     def gitlab_shell_version
-      Gitlab::Shell.new.version
+      Gitlab::Shell.version
     end
   end
 end

lib/tasks/gitlab/info.rake

@@ -82,15 +82,10 @@ namespace :gitlab do
       puts "Using Omniauth:\t#{Gitlab::Auth.omniauth_enabled? ? "yes".color(:green) : "no"}"
       puts "Omniauth Providers: #{omniauth_providers.join(', ')}" if Gitlab::Auth.omniauth_enabled?
-      # check Gitolite version
-      gitlab_shell_version_file = "#{Gitlab.config.gitlab_shell.path}/VERSION"
-      if File.readable?(gitlab_shell_version_file)
-        gitlab_shell_version = File.read(gitlab_shell_version_file)
-      end
-
+      # check Gitlab Shell version
       puts ""
       puts "GitLab Shell".color(:yellow)
-      puts "Version:\t#{gitlab_shell_version || "unknown".color(:red)}"
+      puts "Version:\t#{Gitlab::Shell.version || "unknown".color(:red)}"
       puts "Repository storage paths:"
       Gitlab::GitalyClient::StorageSettings.allow_disk_access do
         Gitlab.config.repositories.storages.each do |name, repository_storage|