Add latest changes from gitlab-org/gitlab@master

app/models/milestone.rb

@@ -17,6 +17,7 @@ class Milestone < ApplicationRecord
   include StripAttribute
   include Milestoneish
   include FromUnion
+  include Importable
   include Gitlab::SQL::Pattern
   prepend_if_ee('::EE::Milestone') # rubocop: disable Cop/InjectEnterpriseEditionModule
@@ -30,8 +31,8 @@ class Milestone < ApplicationRecord
   has_many :milestone_releases
   has_many :releases, through: :milestone_releases
-  has_internal_id :iid, scope: :project, init: ->(s) { s&.project&.milestones&.maximum(:iid) }
-  has_internal_id :iid, scope: :group, init: ->(s) { s&.group&.milestones&.maximum(:iid) }
+  has_internal_id :iid, scope: :project, track_if: -> { !importing? }, init: ->(s) { s&.project&.milestones&.maximum(:iid) }
+  has_internal_id :iid, scope: :group, track_if: -> { !importing? }, init: ->(s) { s&.group&.milestones&.maximum(:iid) }
   has_many :issues
   has_many :labels, -> { distinct.reorder('labels.title') }, through: :issues

app/models/release.rb

@@ -3,6 +3,7 @@
 class Release < ApplicationRecord
   include Presentable
   include CacheMarkdownField
+  include Importable
   include Gitlab::Utils::StrongMemoize
   cache_markdown_field :description
@@ -33,8 +34,8 @@ class Release < ApplicationRecord
   delegate :repository, to: :project
-  after_commit :create_evidence!, on: :create
-  after_commit :notify_new_release, on: :create
+  after_commit :create_evidence!, on: :create, unless: :importing?
+  after_commit :notify_new_release, on: :create, unless: :importing?
   MAX_NUMBER_TO_DISPLAY = 3

app/models/user.rb

@@ -20,6 +20,7 @@ class User < ApplicationRecord
   include WithUploads
   include OptionallySearch
   include FromUnion
+  include BatchDestroyDependentAssociations
   DEFAULT_NOTIFICATION_LEVEL = :participating

app/services/users/destroy_service.rb

@@ -56,6 +56,13 @@ module Users
       MigrateToGhostUserService.new(user).execute unless options[:hard_delete]
+      if Feature.enabled?(:destroy_user_associations_in_batches)
+        # Rails attempts to load all related records into memory before
+        # destroying: https://github.com/rails/rails/issues/22510
+        # This ensures we delete records in batches.
+        user.destroy_dependent_associations_in_batches
+      end
+
       # Destroy the namespace after destroying the user since certain methods may depend on the namespace existing
       user_data = user.destroy
       namespace.destroy

changelogs/unreleased/195625-wiki-support-for-asciidoc-include-directive.yml

+---
+title: Fix error in Wiki when rendering the AsciiDoc include directive
+merge_request: 22565
+author:
+type: fixed

changelogs/unreleased/196172-issue-create-iid-conflict.yml

+---
+title: 'Fix Issue API: creating with manual IID returns conflict when IID already
+  in use'
+merge_request: 22788
+author: Mara Sophie Grosch
+type: fixed

changelogs/unreleased/add_comment_on_event_enabled_to_services_api.yml

+---
+title: Add comment_on_event_enabled to services API
+merge_request:
+author:
+type: added

changelogs/unreleased/mark-some-as-not-required-during-import.yml

+---
+title: Add `importing?` to disable some callbacks
+merge_request:
+author:
+type: performance

changelogs/unreleased/sh-update-octokit.yml

+---
+title: Upgrade octokit and its dependencies
+merge_request: 22946
+author:
+type: other

changelogs/unreleased/standardize-timestamp-format-in-app-logs.yml

+---
+title: 'Use IS08601.3 format for app level logging of timestamps'
+merge_request: 22793
+author:
+type: other

doc/api/services.md

@@ -32,7 +32,8 @@ Example response:
     "confidential_note_events": true,
     "pipeline_events": true,
     "wiki_page_events": true,
-    "job_events": true
+    "job_events": true,
+    "comment_on_event_enabled": true
   }
   {
     "id": 76,
@@ -50,7 +51,8 @@ Example response:
     "confidential_note_events": true,
     "pipeline_events": true,
     "wiki_page_events": true,
-    "job_events": true
+    "job_events": true,
+    "comment_on_event_enabled": true
   }
 ]
 ```
@@ -723,6 +725,7 @@ Parameters:
 | `jira_issue_transition_id` | string | no | The ID of a transition that moves issues to a closed state. You can find this number under the Jira workflow administration (**Administration > Issues > Workflows**) by selecting **View** under **Operations** of the desired workflow of your project. The ID of each state can be found inside the parenthesis of each transition name under the **Transitions (id)** column. By default, this ID is set to `2`. |
 | `commit_events` | boolean | false | Enable notifications for commit events |
 | `merge_requests_events` | boolean | false | Enable notifications for merge request events |
+| `comment_on_event_enabled` | boolean | false | Enable comments inside Jira issues on each GitLab event (commit / merge request) |
 ### Delete Jira service
@@ -761,6 +764,7 @@ Example response:
   "note_events": true,
   "job_events": true,
   "pipeline_events": true,
+  "comment_on_event_enabled": false,
   "properties": {
     "token": "<your_access_token>"
   }

doc/development/logging.md

@@ -191,7 +191,7 @@ Here are some examples of how messages would be handled by both the loggers.
 FancyMultiLogger.info("Information")
 # UnstructuredLogger
-I, [2020-01-13T12:02:41.566219 #6652]  INFO -- : Information
+I, [2020-01-13T18:48:49.201Z #5647]  INFO -- : Information
 # StructuredLogger
 {:severity=>"INFO", :time=>"2020-01-13T11:02:41.559Z", :correlation_id=>"b1701f7ecc4be4bcd4c2d123b214e65a", :message=>"Information"}
@@ -203,7 +203,7 @@ I, [2020-01-13T12:02:41.566219 #6652]  INFO -- : Information
 FancyMultiLogger.info({:message=>"This is my message", :project_id=>123})
 # UnstructuredLogger
-I, [2020-01-13T12:06:09.856766 #8049]  INFO -- : {:message=>"This is my message", :project_id=>123}
+I, [2020-01-13T19:01:17.091Z #11056]  INFO -- : {"message"=>"Message", "project_id"=>"123"}
 # StructuredLogger
 {:severity=>"INFO", :time=>"2020-01-13T11:06:09.851Z", :correlation_id=>"d7e0886f096db9a8526a4f89da0e45f6", :message=>"This is my message", :project_id=>123}

doc/install/requirements.md

@@ -38,14 +38,40 @@ GitLab is developed for Linux-based operating systems.
 It does **not** run on Microsoft Windows, and we have no plans to support it in the near future. For the latest development status view this [issue](https://gitlab.com/gitlab-org/gitlab/issues/22337).
 Please consider using a virtual machine to run GitLab.
-## Ruby versions
-GitLab requires Ruby (MRI) 2.6. Support for Ruby versions below 2.6 (2.4, 2.5) will stop with GitLab 12.2.
-You will have to use the standard MRI implementation of Ruby.
-We love [JRuby](https://www.jruby.org/) and [Rubinius](https://rubinius.com) but GitLab
+## Software requirements
+### Ruby versions
+GitLab requires Ruby (MRI) 2.6. Beginning in GitLab 12.2, we  no longer support Ruby 2.5 and lower.
+
+You must use the standard MRI implementation of Ruby.
+We love [JRuby](https://www.jruby.org/) and [Rubinius](https://rubinius.com), but GitLab
 needs several Gems that have native extensions.
+### Go versions
+
+The minimum required Go version is 1.12.
+
+### Git versions
+
+GitLab 11.11 and higher only supports Git 2.21.x and newer, and
+[dropped support for older versions](https://gitlab.com/gitlab-org/gitlab-foss/issues/54255).
+
+### Node.js versions
+
+Beginning in GitLab 11.8, we only support Node.js 8.10.0 or higher, and dropped
+support for Node.js 6.
+
+We recommend Node 12.x, as it is faster.
+
+GitLab uses [webpack](https://webpack.js.org/) to compile frontend assets, which requires a minimum
+version of Node.js 8.10.0.
+
+You can check which version you are running with `node -v`. If you are running
+a version older than `v8.10.0`, you need to update to a newer version. You
+can find instructions to install from community maintained packages or compile
+from source at the [Node.js website](https://nodejs.org/en/download).
+
 ## Hardware requirements
 ### Storage

doc/update/upgrading_from_source.md

@@ -82,24 +82,15 @@ Install Bundler:
 sudo gem install bundler --no-document --version '< 2'
 ```
-### 4. Update Node
-NOTE: Beginning in GitLab 11.8, we only support node 8 or higher, and dropped
-support for node 6. Be sure to upgrade if necessary.
-GitLab utilizes [webpack](https://webpack.js.org/) to compile frontend assets.
-This requires a minimum version of node v8.10.0.
-
-You can check which version you are running with `node -v`. If you are running
-a version older than `v8.10.0` you will need to update to a newer version. You
-can find instructions to install from community maintained packages or compile
-from source at the nodejs.org website.
-
-<https://nodejs.org/en/download/>
-
-GitLab also requires the use of yarn `>= v1.10.0` to manage JavaScript
+### 4. Update Node.js
+NOTE: To check the minimum required Node.js version, see [Node.js versions](../install/requirements.md#nodejs-versions).
+GitLab also requires the use of Yarn `>= v1.10.0` to manage JavaScript
 dependencies.
+In Debian or Ubuntu:
+
 ```bash
 curl --silent --show-error https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add -
 echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list
@@ -107,34 +98,33 @@ sudo apt-get update
 sudo apt-get install yarn
 ```
-More information can be found on the [yarn website](https://yarnpkg.com/en/docs/install).
+More information can be found on the [Yarn website](https://yarnpkg.com/en/docs/install).
 ### 5. Update Go
-NOTE: GitLab 11.4 and higher only supports Go 1.10.x and newer, and dropped support for Go
-1.9.x. Be sure to upgrade your installation if necessary.
+NOTE: To check the minimum required Go version, see [Go versions](../install/requirements.md#go-versions).
 You can check which version you are running with `go version`.
-Download and install Go:
+Download and install Go (for Linux, 64-bit):
 ```bash
 # Remove former Go installation folder
 sudo rm -rf /usr/local/go
-curl --remote-name --progress https://dl.google.com/go/go1.11.10.linux-amd64.tar.gz
-echo 'aefaa228b68641e266d1f23f1d95dba33f17552ba132878b65bb798ffa37e6d0  go1.11.10.linux-amd64.tar.gz' | shasum -a256 -c - && \
-  sudo tar -C /usr/local -xzf go1.11.10.linux-amd64.tar.gz
+curl --remote-name --progress https://dl.google.com/go/go1.13.5.linux-amd64.tar.gz
+echo '512103d7ad296467814a6e3f635631bd35574cab3369a97a323c9a585ccaa569  go1.13.5.linux-amd64.tar.gz' | shasum -a256 -c - && \
+  sudo tar -C /usr/local -xzf go1.13.5.linux-amd64.tar.gz
 sudo ln -sf /usr/local/go/bin/{go,godoc,gofmt} /usr/local/bin/
-rm go1.11.10.linux-amd64.tar.gz
+rm go1.13.5.linux-amd64.tar.gz
+
 ```
 ### 6. Update Git
-NOTE: **Note:**
-GitLab 11.11 and higher only supports Git 2.21.x and newer, and
-[dropped support for older versions](https://gitlab.com/gitlab-org/gitlab-foss/issues/54255).
-Be sure to upgrade your installation if necessary.
+NOTE: To check the minimum required Git version, see [Git versions](../install/requirements.md#git-versions).
+
+In Debian or Ubuntu:
 ```bash
 # Make sure Git is version 2.21.0 or higher
@@ -254,9 +244,8 @@ sudo -u git -H make
 #### New configuration options for `gitlab.yml`
-There might be configuration options available for [`gitlab.yml`][yaml]. View
-them with the command below and apply them manually to your current
-`gitlab.yml`:
+There might be configuration options available for [`gitlab.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/config/gitlab.yml.example)).
+View them with the command below and apply them manually to your current `gitlab.yml`:
 ```sh
 cd /home/git/gitlab
@@ -282,7 +271,7 @@ If you are using Strict-Transport-Security in your installation to continue
 using it you must enable it in your NGINX configuration as GitLab application no
 longer handles setting it.
-If you are using Apache instead of NGINX please see the updated [Apache templates].
+If you are using Apache instead of NGINX see the updated [Apache templates](https://gitlab.com/gitlab-org/gitlab-recipes/tree/master/web-server/apache).
 Also note that because Apache does not support upstreams behind Unix sockets you
 will need to let GitLab Workhorse listen on a TCP port. You can do this
 via [`/etc/default/gitlab`](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/support/init.d/gitlab.default.example#L38).
@@ -296,13 +285,13 @@ add the following line to `config/initializers/smtp_settings.rb`:
 ActionMailer::Base.delivery_method = :smtp
 ```
-See [smtp_settings.rb.sample] as an example.
+See [smtp_settings.rb.sample](https://gitlab.com/gitlab-org/gitlab/blob/master/config/initializers/smtp_settings.rb.sample#L13) as an example.
 #### Init script
 There might be new configuration options available for
-[`gitlab.default.example`][gl-example]. View them with the command below and
-apply them manually to your current `/etc/default/gitlab`:
+[`gitlab.default.example`](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/support/init.d/gitlab.default.example).
+View them with the command below and apply them manually to your current `/etc/default/gitlab`:
 ```sh
 cd /home/git/gitlab
@@ -389,17 +378,19 @@ Example:
 Additional instructions here.
 -->
-## Things went south? Revert to previous version
+## Troubleshooting
 ### 1. Revert the code to the previous version
-To revert to a previous version, you'll need to following the upgrading guides
-for the previous version. If you upgraded to 11.8 and want to revert back to
-11.7, you'll need to follow the guides for upgrading from 11.6 to 11.7. You can
+To revert to a previous version, you need to follow the upgrading guides
+for the previous version.
+
+For example, if you have upgraded to GitLab 12.6 and want to revert back to
+12.5, you need to follow the guides for upgrading from 12.4 to 12.5. You can
 use the version dropdown at the top of the page to select the right version.
-When reverting, you should _not_ follow the database migration guides, as the
-backup is already migrated to the previous version.
+When reverting, you should **not** follow the database migration guides, as the
+backup has already been migrated to the previous version.
 ### 2. Restore from the backup
@@ -409,9 +400,4 @@ cd /home/git/gitlab
 sudo -u git -H bundle exec rake gitlab:backup:restore RAILS_ENV=production
 ```
-If you have more than one backup `*.tar` file(s) please add `BACKUP=timestamp_of_backup` to the command above.
-
-[yaml]: https://gitlab.com/gitlab-org/gitlab/blob/master/config/gitlab.yml.example
-[gl-example]: https://gitlab.com/gitlab-org/gitlab/blob/master/lib/support/init.d/gitlab.default.example
-[smtp_settings.rb.sample]: https://gitlab.com/gitlab-org/gitlab/blob/master/config/initializers/smtp_settings.rb.sample#L13
-[Apache templates]: https://gitlab.com/gitlab-org/gitlab-recipes/tree/master/web-server/apache
+If you have more than one backup `*.tar` file, add `BACKUP=timestamp_of_backup` to the above.

doc/user/clusters/applications.md

@@ -464,6 +464,11 @@ chart is used to install this application with a
 [`values.yaml`](https://gitlab.com/gitlab-org/gitlab/blob/master/vendor/elastic_stack/values.yaml)
 file.
+NOTE: **Note:**
+The chart will deploy 4 Elasticsearch nodes: 2 masters, 1 data and 1 client node,
+with resource requests totalling 0.1 CPU and 3GB RAM. Each data node requests 1.5GB of memory,
+which makes it incompatible with clusters of `f1-micro` and `g1-small` instance types.
+
 ## Install using GitLab CI (alpha)
 > [Introduced](https://gitlab.com/gitlab-org/gitlab/merge_requests/20822) in GitLab 12.6.

lib/api/entities.rb

@@ -1132,7 +1132,7 @@ module API
       expose :commit_events, :push_events, :issues_events, :confidential_issues_events
       expose :merge_requests_events, :tag_push_events, :note_events
       expose :confidential_note_events, :pipeline_events, :wiki_page_events
-      expose :job_events
+      expose :job_events, :comment_on_event_enabled
     end
     class ProjectService < ProjectServiceBasic

lib/api/issues.rb

@@ -220,18 +220,22 @@ module API
         issue_params = convert_parameters_from_legacy_format(issue_params)
-        issue = ::Issues::CreateService.new(user_project,
-                                            current_user,
-                                            issue_params.merge(request: request, api: true)).execute
-
-        if issue.spam?
-          render_api_error!({ error: 'Spam detected' }, 400)
-        end
-
-        if issue.valid?
-          present issue, with: Entities::Issue, current_user: current_user, project: user_project
-        else
-          render_validation_error!(issue)
+        begin
+          issue = ::Issues::CreateService.new(user_project,
+                                              current_user,
+                                              issue_params.merge(request: request, api: true)).execute
+
+          if issue.spam?
+            render_api_error!({ error: 'Spam detected' }, 400)
+          end
+
+          if issue.valid?
+            present issue, with: Entities::Issue, current_user: current_user, project: user_project
+          else
+            render_validation_error!(issue)
+          end
+        rescue ::ActiveRecord::RecordNotUnique
+          render_api_error!('Duplicated issue', 409)
         end
       end

lib/gitlab/app_text_logger.rb

@@ -7,7 +7,7 @@ module Gitlab
     end
     def format_message(severity, timestamp, progname, msg)
-      "#{timestamp.to_s(:long)}: #{msg}\n"
+      "#{timestamp.utc.iso8601(3)}: #{msg}\n"
     end
   end
 end

lib/gitlab/asciidoc/include_processor.rb

@@ -13,7 +13,7 @@ module Gitlab
         super(logger: Gitlab::AppLogger)
         @context = context
-        @repository = context[:project].try(:repository)
+        @repository = context[:repository] || context[:project].try(:repository)
         # Note: Asciidoctor calls #freeze on extensions, so we can't set new
         # instance variables after initialization.
@@ -111,7 +111,7 @@ module Gitlab
       end
       def ref
-        context[:ref] || context[:project].default_branch
+        context[:ref] || repository&.root_ref
       end
       def requested_path

lib/gitlab/github_import/client.rb

@@ -189,7 +189,7 @@ module Gitlab
       end
       def default_api_endpoint
-        OmniAuth::Strategies::GitHub.default_options[:client_options][:site]
+        OmniAuth::Strategies::GitHub.default_options[:client_options][:site] || ::Octokit::Default.api_endpoint
       end
       def verify_ssl