Add latest changes from gitlab-org/gitlab@master

.gitlab/ci/frontend.gitlab-ci.yml

@@ -47,7 +47,7 @@
     - rm -f /etc/apt/sources.list.d/google*.list  # We don't need to update Chrome here
   only:
     variables:
-      - $CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org"
+      - $CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE =~ /^gitlab-org($|\/)/  # Matches the gitlab-org group and its subgroups
       - $CI_SERVER_HOST == "dev.gitlab.org"
   tags:
     - gitlab-org

.gitlab/ci/global.gitlab-ci.yml

@@ -33,6 +33,7 @@
       - master
       - /^[\d-]+-stable(-ee)?$/
       - /^\d+-\d+-auto-deploy-\d+$/
+      - /^security\//
       - merge_requests
       - tags

.gitlab/ci/qa.gitlab-ci.yml

@@ -12,7 +12,7 @@
     - ./scripts/trigger-build omnibus
   only:
     variables:
-      - $CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org"
+      - $CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE =~ /^gitlab-org($|\/)/  # Matches the gitlab-org group or its subgroups
 package-and-qa-manual:
   extends:

CHANGELOG-EE.md

@@ -252,6 +252,10 @@ Please view this file on the master branch, on stable branches it's out of date.
 - Fix alignment of activity dropdown in epic tabs; add counter to discussion tab.
+## 12.1.10
+
+- No changes.
+
 ## 12.1.5
 - No changes.

CHANGELOG.md

@@ -320,10 +320,6 @@ entry.
 - Update Packer.gitlab-ci.yml to use latest image. (Kelly Hair)
-## 12.1.11
-
-- No changes.
-
 ## 12.1.10
 - No changes.

README.md

@@ -2,9 +2,9 @@
 ## Canonical source
-The canonical source of GitLab Community Edition is [hosted on GitLab.com](https://gitlab.com/gitlab-org/gitlab-ce/).
-The source of GitLab Enterprise Edition is [hosted on GitLab.com](https://gitlab.com/gitlab-org/gitlab-ee).
+The canonical source of GitLab where development takes place is [hosted on GitLab.com](https://gitlab.com/gitlab-org/gitlab).
+A FOSS source of GitLab is hosted on a [mirrored, read-only project on GitLab.com](https://gitlab.com/gitlab-org/gitlab-foss/).
 ## Free trial

app/assets/javascripts/jobs/components/log/collapsible_section.vue

+<script>
+import LogLine from './line.vue';
+import LogLineHeader from './line_header.vue';
+
+export default {
+  name: 'CollpasibleLogSection',
+  components: {
+    LogLine,
+    LogLineHeader,
+  },
+  props: {
+    section: {
+      type: Object,
+      required: true,
+    },
+    traceEndpoint: {
+      type: String,
+      required: true,
+    },
+  },
+  computed: {
+    badgeDuration() {
+      return this.section.line && this.section.line.section_duration;
+    },
+  },
+  methods: {
+    handleOnClickCollapsibleLine(section) {
+      this.$emit('onClickCollapsibleLine', section);
+    },
+  },
+};
+</script>
+<template>
+  <div>
+    <log-line-header
+      :line="section.line"
+      :duration="badgeDuration"
+      :path="traceEndpoint"
+      :is-closed="section.isClosed"
+      @toggleLine="handleOnClickCollapsibleLine(section)"
+    />
+    <template v-if="!section.isClosed">
+      <template v-for="line in section.lines">
+        <collpasible-log-section
+          v-if="line.isHeader"
+          :key="`collapsible-nested-${line.offset}`"
+          :section="line"
+          :trace-endpoint="traceEndpoint"
+          @toggleLine="handleOnClickCollapsibleLine"
+        />
+        <log-line v-else :key="line.offset" :line="line" :path="traceEndpoint" />
+      </template>
+    </template>
+  </div>
+</template>

app/assets/javascripts/jobs/components/log/log.vue

 <script>
 import { mapState, mapActions } from 'vuex';
+import CollpasibleLogSection from './collapsible_section.vue';
 import LogLine from './line.vue';
-import LogLineHeader from './line_header.vue';
 export default {
   components: {
+    CollpasibleLogSection,
     LogLine,
-    LogLineHeader,
   },
   computed: {
     ...mapState(['traceEndpoint', 'trace', 'isTraceComplete']),
@@ -22,24 +22,13 @@ export default {
 <template>
   <code class="job-log d-block">
     <template v-for="(section, index) in trace">
-      <template v-if="section.isHeader">
-        <log-line-header
-          :key="`collapsible-${index}`"
-          :line="section.line"
-          :duration="section.section_duration"
-          :path="traceEndpoint"
-          :is-closed="section.isClosed"
-          @toggleLine="handleOnClickCollapsibleLine(section)"
-        />
-        <template v-if="!section.isClosed">
-          <log-line
-            v-for="line in section.lines"
-            :key="line.offset"
-            :line="line"
-            :path="traceEndpoint"
-          />
-        </template>
-      </template>
+      <collpasible-log-section
+        v-if="section.isHeader"
+        :key="`collapsible-${index}`"
+        :section="section"
+        :trace-endpoint="traceEndpoint"
+        @onClickCollapsibleLine="handleOnClickCollapsibleLine"
+      />
       <log-line v-else :key="section.offset" :line="section" :path="traceEndpoint" />
     </template>

app/assets/stylesheets/framework.scss

@@ -28,6 +28,7 @@
 @import 'framework/issue_box';
 @import 'framework/lists';
 @import 'framework/logo';
+@import 'framework/job_log';
 @import 'framework/markdown_area';
 @import 'framework/media_object';
 @import 'framework/modal';

app/views/projects/settings/ci_cd/_autodevops_form.html.haml

@@ -46,12 +46,12 @@
                   = form.radio_button :deploy_strategy, 'timed_incremental', class: 'form-check-input'
                   = form.label :deploy_strategy_timed_incremental, class: 'form-check-label' do
                     = s_('CICD|Continuous deployment to production using timed incremental rollout')
-                    = link_to icon('question-circle'), help_page_path('topics/autodevops/index.md', anchor: 'timed-incremental-rollout-to-production'), target: '_blank'
+                    = link_to icon('question-circle'), help_page_path('topics/autodevops/index.md', anchor: 'timed-incremental-rollout-to-production-premium'), target: '_blank'
                 .form-check
                   = form.radio_button :deploy_strategy, 'manual', class: 'form-check-input'
                   = form.label :deploy_strategy_manual, class: 'form-check-label' do
                     = s_('CICD|Automatic deployment to staging, manual deployment to production')
-                    = link_to icon('question-circle'), help_page_path('topics/autodevops/index.md', anchor: 'incremental-rollout-to-production'), target: '_blank'
+                    = link_to icon('question-circle'), help_page_path('topics/autodevops/index.md', anchor: 'incremental-rollout-to-production-premium'), target: '_blank'
           = f.submit _('Save changes'), class: "btn btn-success prepend-top-15", data: { qa_selector: 'save_changes_button' }

changelogs/unreleased/zj-timeouts-gitaly-rpcs.yml

+---
+title: Add timeouts for each RPC call
+merge_request: 31766
+author:
+type: changed

db/post_migrate/20190918104222_schedule_productivity_analytics_backfill.rb

@@ -5,24 +5,8 @@ class ScheduleProductivityAnalyticsBackfill < ActiveRecord::Migration[5.2]
   DOWNTIME = false
-  BATCH_SIZE = 10_000
-  INTERVAL = 3.minutes
-  MIGRATION = 'Gitlab::BackgroundMigration::RecalculateProductivityAnalytics'.freeze
-
-  disable_ddl_transaction!
-
   def up
-    return unless Gitlab.ee?
-
-    metrics_model = Class.new(ActiveRecord::Base) do
-      self.table_name = 'merge_request_metrics'
-
-      include ::EachBatch
-    end
-
-    scope = metrics_model.where("merged_at >= ?", 3.months.ago)
-
-    queue_background_migration_jobs_by_range_at_intervals(scope, MIGRATION, INTERVAL, batch_size: BATCH_SIZE)
+    # no-op since the scheduling times out on GitLab.com
   end
   def down

doc/user/packages/npm_registry/index.md

@@ -84,6 +84,28 @@ NOTE: **Note:**
 If you encounter an error message with [Yarn](https://yarnpkg.com/en/), see the
 [troubleshooting section](#troubleshooting).
+### Using variables to avoid hard-coding auth token values
+
+To avoid hard-coding the `authToken` value, you may use a variables in its place.
+In your `.npmrc` file, you would add:
+
+```ini
+@foo:registry=https://gitlab.com/api/v4/packages/npm/
+//gitlab.com/api/v4/packages/npm/:_authToken=${NPM_TOKEN}
+//gitlab.com/api/v4/projects/<your_project_id>/packages/npm/:_authToken=${NPM_TOKEN}
+```
+
+Then, you could run `npm publish` either locally or via GitLab CI/CD:
+
+- **Locally:** Export `NPM_TOKEN` before publishing:
+
+  ```sh
+  NPM_TOKEN=<your_token> npm publish
+  ```
+
+- **GitLab CI/CD:** Set an `NPM_TOKEN` [variable](../../../ci/variables/README.md)
+  under your project's **Settings > CI/CD > Variables**.
+
 ## Uploading packages
 Before you will be able to upload a package, you need to specify the registry
@@ -145,3 +167,29 @@ with your with your OAuth or personal access token):
 ```text
 //gitlab.com/api/v4/projects/:_authToken=<your_oauth_token>
 ```
+
+### `npm publish` targets default NPM registry (`registry.npmjs.org`)
+
+Ensure that your package scope is set consistently in your `package.json` and `.npmrc` files.
+
+For example, if your project name in GitLab is `foo/my-package`, then your `package.json` file
+should look like:
+
+```json
+{
+  "name": "@foo/my-package",
+  "version": "1.0.0",
+  "description": "Example package for GitLab NPM registry",
+  "publishConfig": {
+    "@foo:registry":"https://gitlab.com/api/v4/projects/<your_project_id>/packages/npm/"
+  }
+}
+```
+
+And the `.npmrc` file should look like:
+
+```ini
+//gitlab.com/api/v4/projects/<your_project_id>/packages/npm/:_authToken=<your_oauth_token>
+//gitlab.com/api/v4/packages/npm/:_authToken=<your_oauth_token>
+@foo:registry=https://gitlab.com/api/v4/packages/npm/
+```

lib/gitlab/gitaly_client.rb

@@ -142,13 +142,13 @@ module Gitlab
     #   kwargs.merge(deadline: Time.now + 10)
     # end
     #
-    def self.call(storage, service, rpc, request, remote_storage: nil, timeout: nil)
+    def self.call(storage, service, rpc, request, remote_storage: nil, timeout: default_timeout)
       start = Gitlab::Metrics::System.monotonic_time
       request_hash = request.is_a?(Google::Protobuf::MessageExts) ? request.to_h : {}
       enforce_gitaly_request_limits(:call)
-      kwargs = request_kwargs(storage, timeout, remote_storage: remote_storage)
+      kwargs = request_kwargs(storage, timeout: timeout.to_f, remote_storage: remote_storage)
       kwargs = yield(kwargs) if block_given?
       stub(service, storage).__send__(rpc, request, kwargs) # rubocop:disable GitlabSecurity/PublicSend
@@ -200,7 +200,7 @@ module Gitlab
     end
     private_class_method :authorization_token
-    def self.request_kwargs(storage, timeout, remote_storage: nil)
+    def self.request_kwargs(storage, timeout:, remote_storage: nil)
       metadata = {
         'authorization' => "Bearer #{authorization_token(storage)}",
         'client_name' => CLIENT_NAME
@@ -216,14 +216,7 @@ module Gitlab
       result = { metadata: metadata }
-      # nil timeout indicates that we should use the default
-      timeout = default_timeout if timeout.nil?
-
-      return result unless timeout > 0
-
-      deadline = real_time + timeout
-      result[:deadline] = deadline
-
+      result[:deadline] = real_time + timeout if timeout > 0
       result
     end
@@ -357,8 +350,6 @@ module Gitlab
     # The default timeout on all Gitaly calls
     def self.default_timeout
-      return no_timeout if Sidekiq.server?
-
       timeout(:gitaly_timeout_default)
     end
@@ -370,8 +361,12 @@ module Gitlab
       timeout(:gitaly_timeout_medium)
     end
-    def self.no_timeout
-      0
+    def self.long_timeout
+      if Sidekiq.server?
+        6.hours
+      else
+        55.seconds
+      end
     end
     def self.storage_metadata_file_path(storage)

lib/gitlab/gitaly_client/cleanup_service.rb

@@ -18,7 +18,7 @@ module Gitlab
           :cleanup_service,
           :apply_bfg_object_map_stream,
           build_object_map_enum(io),
-          timeout: GitalyClient.no_timeout
+          timeout: GitalyClient.long_timeout
         )
         responses.each(&blk)

lib/gitlab/gitaly_client/commit_service.rb

@@ -254,7 +254,7 @@ module Gitlab
       def languages(ref = nil)
         request = Gitaly::CommitLanguagesRequest.new(repository: @gitaly_repo, revision: ref || '')
-        response = GitalyClient.call(@repository.storage, :commit_service, :commit_languages, request)
+        response = GitalyClient.call(@repository.storage, :commit_service, :commit_languages, request, timeout: GitalyClient.long_timeout)
         response.languages.map { |l| { value: l.share.round(2), label: l.name, color: l.color, highlight: l.color } }
       end
@@ -360,7 +360,7 @@ module Gitlab
       def extract_signature(commit_id)
         request = Gitaly::ExtractCommitSignatureRequest.new(repository: @gitaly_repo, commit_id: commit_id)
-        response = GitalyClient.call(@repository.storage, :commit_service, :extract_commit_signature, request)
+        response = GitalyClient.call(@repository.storage, :commit_service, :extract_commit_signature, request, timeout: GitalyClient.fast_timeout)
         signature = +''.b
         signed_text = +''.b

lib/gitlab/gitaly_client/conflicts_service.rb

@@ -20,7 +20,7 @@ module Gitlab
           our_commit_oid: @our_commit_oid,
           their_commit_oid: @their_commit_oid
         )
-        response = GitalyClient.call(@repository.storage, :conflicts_service, :list_conflict_files, request)
+        response = GitalyClient.call(@repository.storage, :conflicts_service, :list_conflict_files, request, timeout: GitalyClient.long_timeout)
         GitalyClient::ConflictFilesStitcher.new(response)
       end

lib/gitlab/gitaly_client/namespace_service.rb

@@ -22,7 +22,7 @@ module Gitlab
       def remove(name)
         request = Gitaly::RemoveNamespaceRequest.new(storage_name: @storage, name: name)
-        gitaly_client_call(:remove_namespace, request, timeout: nil)
+        gitaly_client_call(:remove_namespace, request, timeout: GitalyClient.long_timeout)
       end
       def rename(from, to)

lib/gitlab/gitaly_client/object_pool_service.rb

@@ -15,13 +15,15 @@ module Gitlab
           object_pool: object_pool,
           origin: repository.gitaly_repository)
-        GitalyClient.call(storage, :object_pool_service, :create_object_pool, request)
+        GitalyClient.call(storage, :object_pool_service, :create_object_pool,
+                          request, timeout: GitalyClient.medium_timeout)
       end
       def delete
         request = Gitaly::DeleteObjectPoolRequest.new(object_pool: object_pool)
-        GitalyClient.call(storage, :object_pool_service, :delete_object_pool, request)
+        GitalyClient.call(storage, :object_pool_service, :delete_object_pool,
+                          request, timeout: GitalyClient.long_timeout)
       end
       def link_repository(repository)
@@ -40,7 +42,8 @@ module Gitlab
           origin: repository.gitaly_repository
         )
-        GitalyClient.call(storage, :object_pool_service, :fetch_into_object_pool, request)
+        GitalyClient.call(storage, :object_pool_service, :fetch_into_object_pool,
+                          request, timeout: GitalyClient.long_timeout)
       end
     end
   end

lib/gitlab/gitaly_client/operation_service.rb

@@ -19,7 +19,7 @@ module Gitlab
           user: Gitlab::Git::User.from_gitlab(user).to_gitaly
         )
-        response = GitalyClient.call(@repository.storage, :operation_service, :user_delete_tag, request, timeout: GitalyClient.medium_timeout)
+        response = GitalyClient.call(@repository.storage, :operation_service, :user_delete_tag, request, timeout: GitalyClient.long_timeout)
         if pre_receive_error = response.pre_receive_error.presence
           raise Gitlab::Git::PreReceiveError, pre_receive_error
@@ -35,7 +35,7 @@ module Gitlab
           message: encode_binary(message.to_s)
         )
-        response = GitalyClient.call(@repository.storage, :operation_service, :user_create_tag, request, timeout: GitalyClient.medium_timeout)
+        response = GitalyClient.call(@repository.storage, :operation_service, :user_create_tag, request, timeout: GitalyClient.long_timeout)
         if pre_receive_error = response.pre_receive_error.presence
           raise Gitlab::Git::PreReceiveError, pre_receive_error
         elsif response.exists
@@ -55,7 +55,7 @@ module Gitlab
           start_point: encode_binary(start_point)
         )
         response = GitalyClient.call(@repository.storage, :operation_service,
-          :user_create_branch, request)
+                                     :user_create_branch, request, timeout: GitalyClient.long_timeout)
         if response.pre_receive_error.present?
           raise Gitlab::Git::PreReceiveError.new(response.pre_receive_error)
@@ -79,7 +79,8 @@ module Gitlab
           oldrev: encode_binary(oldrev)
         )
-        response = GitalyClient.call(@repository.storage, :operation_service, :user_update_branch, request)
+        response = GitalyClient.call(@repository.storage, :operation_service,
+                                     :user_update_branch, request, timeout: GitalyClient.long_timeout)
         if pre_receive_error = response.pre_receive_error.presence
           raise Gitlab::Git::PreReceiveError, pre_receive_error
@@ -93,7 +94,8 @@ module Gitlab
           user: Gitlab::Git::User.from_gitlab(user).to_gitaly
         )
-        response = GitalyClient.call(@repository.storage, :operation_service, :user_delete_branch, request)
+        response = GitalyClient.call(@repository.storage, :operation_service,
+                                     :user_delete_branch, request, timeout: GitalyClient.long_timeout)
         if pre_receive_error = response.pre_receive_error.presence
           raise Gitlab::Git::PreReceiveError, pre_receive_error
@@ -111,7 +113,8 @@ module Gitlab
           first_parent_ref: encode_binary(first_parent_ref)
         )
-        response = GitalyClient.call(@repository.storage, :operation_service, :user_merge_to_ref, request)
+        response = GitalyClient.call(@repository.storage, :operation_service,
+                                     :user_merge_to_ref, request, timeout: GitalyClient.long_timeout)
         if pre_receive_error = response.pre_receive_error.presence
           raise Gitlab::Git::PreReceiveError, pre_receive_error
@@ -126,7 +129,8 @@ module Gitlab
           @repository.storage,
           :operation_service,
           :user_merge_branch,
-          request_enum.each
+          request_enum.each,
+          timeout: GitalyClient.long_timeout
         )
         request_enum.push(
@@ -170,7 +174,8 @@ module Gitlab
           @repository.storage,
           :operation_service,
           :user_ff_branch,
-          request
+          request,
+          timeout: GitalyClient.long_timeout
         )
         Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update)
@@ -215,6 +220,7 @@ module Gitlab
           :operation_service,
           :user_rebase,
           request,
+          timeout: GitalyClient.long_timeout,
           remote_storage: remote_repository.storage
         )
@@ -236,6 +242,7 @@ module Gitlab
           :operation_service,
           :user_rebase_confirmable,
           request_enum.each,
+          timeout: GitalyClient.long_timeout,
           remote_storage: remote_repository.storage
         )
@@ -286,7 +293,8 @@ module Gitlab
           @repository.storage,
           :operation_service,
           :user_squash,
-          request
+          request,
+          timeout: GitalyClient.long_timeout
         )
         if response.git_error.presence
@@ -310,7 +318,8 @@ module Gitlab
           @repository.storage,
           :operation_service,
           :user_update_submodule,
-          request
+          request,
+          timeout: GitalyClient.long_timeout
         )
         if response.pre_receive_error.present?
@@ -352,7 +361,8 @@ module Gitlab
         end
         response = GitalyClient.call(@repository.storage, :operation_service,
-          :user_commit_files, req_enum, remote_storage: start_repository.storage)
+                                     :user_commit_files, req_enum, timeout: GitalyClient.long_timeout,
+                                     remote_storage: start_repository.storage)
         if (pre_receive_error = response.pre_receive_error.presence)
           raise Gitlab::Git::PreReceiveError, pre_receive_error
@@ -384,7 +394,8 @@ module Gitlab
           end
         end
-        response = GitalyClient.call(@repository.storage, :operation_service, :user_apply_patch, chunks)
+        response = GitalyClient.call(@repository.storage, :operation_service,
+                                     :user_apply_patch, chunks, timeout: GitalyClient.long_timeout)
         Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update)
       end
@@ -424,7 +435,7 @@ module Gitlab
           :"user_#{rpc}",
           request,
           remote_storage: start_repository.storage,
-          timeout: GitalyClient.medium_timeout
+          timeout: GitalyClient.long_timeout
         )
         handle_cherry_pick_or_revert_response(response)