Add latest changes from gitlab-org/gitlab@master

app/models/concerns/analytics/cycle_analytics/stage.rb

@@ -7,6 +7,7 @@ module Analytics
       included do
         validates :name, presence: true
+        validates :name, exclusion: { in: Gitlab::Analytics::CycleAnalytics::DefaultStages.names }, if: :custom?
         validates :start_event_identifier, presence: true
         validates :end_event_identifier, presence: true
         validate :validate_stage_event_pairs
@@ -15,6 +16,7 @@ module Analytics
         enum end_event_identifier: Gitlab::Analytics::CycleAnalytics::StageEvents.to_enum, _prefix: :end_event_identifier
         alias_attribute :custom_stage?, :custom
+        scope :default_stages, -> { where(custom: false) }
       end
       def parent=(_)

changelogs/unreleased/33337-fix-insights-only-projects-for-project.yml

+---
+title: Make sure project insights stick on its own
+merge_request: 18082
+author:
+type: fixed

changelogs/unreleased/ignore-autreply.yml

+---
+title: Ignore incoming emails with X-Autoreply header.
+merge_request: 18118
+author:
+type: fixed

changelogs/unreleased/jc-remove-cleanup-rake-tasks.yml

+---
+title: Removing cleanup:repo, cleanup:dirs
+merge_request: 18087
+author:
+type: deprecated

doc/raketasks/cleanup.md

@@ -2,46 +2,6 @@
 ## Remove garbage from filesystem
-DANGER: **Danger:**
-The commands below will remove data permanently from your GitLab instance. Only use
-these commands if you are 100% certain that it is safe to delete this data.
-
-Remove namespaces(dirs) from all repository storage paths if they don't exist in GitLab database.
-
-```
-# omnibus-gitlab
-sudo gitlab-rake gitlab:cleanup:dirs
-
-# installation from source
-bundle exec rake gitlab:cleanup:dirs RAILS_ENV=production
-```
-
-DANGER: **Danger:**
-The following task does not currently work as expected.
-The use will probably mark more existing repositories as orphaned.
-For more information, see the [issue](https://gitlab.com/gitlab-org/gitlab/issues/24633).
-
-Rename repositories from all repository storage paths if they don't exist in GitLab database.
-The repositories get a `+orphaned+TIMESTAMP` suffix so that they cannot block new repositories from being created.
-
-```
-# omnibus-gitlab
-sudo gitlab-rake gitlab:cleanup:repos
-
-# installation from source
-bundle exec rake gitlab:cleanup:repos RAILS_ENV=production
-```
-
-Remove old repository copies from repositories moved to another storage.
-
-```
-# omnibus-gitlab
-sudo gitlab-rake gitlab:cleanup:moved
-
-# installation from source
-bundle exec rake gitlab:cleanup:moved RAILS_ENV=production
-```
-
 Clean up local project upload files if they don't exist in GitLab database. The
 task attempts to fix the file if it can find its project, otherwise it moves the
 file to a lost and found directory.

doc/user/project/merge_requests/merge_request_dependencies.md

@@ -4,7 +4,11 @@ type: reference, concepts
 # Merge Request dependencies **(PREMIUM)**
-> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/9688) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.2.
+> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/9688) in
+[GitLab Premium](https://about.gitlab.com/pricing/) 12.2.
+> - [Renamed](https://gitlab.com/gitlab-org/gitlab/merge_requests/17291) from
+"Cross-project dependencies" to "Merge Requests dependencies" in
+[GitLab Premium](https://about.gitlab.com/pricing/) 12.4.
 Merge request dependencies allows a required order of merging
 between merge requests to be expressed. If a merge request "depends on" another,

lib/gitlab/analytics/cycle_analytics/default_stages.rb

@@ -23,6 +23,10 @@ module Gitlab
           ]
         end
+        def self.names
+          all.map { |stage| stage[:name] }
+        end
+
         def self.params_for_issue_stage
           {
             name: 'issue',

lib/gitlab/email/receiver.rb

@@ -32,7 +32,7 @@ module Gitlab
         mail = build_mail
-        ignore_auto_submitted!(mail)
+        ignore_auto_reply!(mail)
         mail_key = extract_mail_key(mail)
         handler = Handler.for(mail, mail_key)
@@ -96,14 +96,25 @@ module Gitlab
         end
       end
-      def ignore_auto_submitted!(mail)
+      def ignore_auto_reply!(mail)
+        if auto_submitted?(mail) || auto_replied?(mail)
+          raise AutoGeneratedEmailError
+        end
+      end
+
+      def auto_submitted?(mail)
         # Mail::Header#[] is case-insensitive
         auto_submitted = mail.header['Auto-Submitted']&.value
         # Mail::Field#value would strip leading and trailing whitespace
-        raise AutoGeneratedEmailError if
-          # See also https://tools.ietf.org/html/rfc3834
-          auto_submitted && auto_submitted != 'no'
+        # See also https://tools.ietf.org/html/rfc3834
+        auto_submitted && auto_submitted != 'no'
+      end
+
+      def auto_replied?(mail)
+        autoreply = mail.header['X-Autoreply']&.value
+
+        autoreply && autoreply == 'yes'
       end
     end
   end

lib/gitlab/import_export/fast_hash_serializer.rb

@@ -135,10 +135,7 @@ module Gitlab
         data = []
-        # rubocop:disable Cop/InBatches
-        # If we put `rubocop:disable` inline after `do |batch|`,
-        # `Cop/LineBreakAroundConditionalBlock` will fail
-        record.in_batches(of: @batch_size) do |batch|
+        record.in_batches(of: @batch_size) do |batch| # rubocop:disable Cop/InBatches
           if Feature.enabled?(:export_fast_serialize_with_raw_json, default_enabled: true)
             data.append(JSONBatchRelation.new(batch, options, preloads[key]).tap(&:raw_json))
           else
@@ -146,7 +143,6 @@ module Gitlab
             data += batch.as_json(options)
           end
         end
-        # rubocop:enable Cop/InBatches
         data
       end

lib/gitlab/metrics/exporter/base_exporter.rb

@@ -36,10 +36,11 @@ module Gitlab
         end
         def stop_working
-          if server # rubocop:disable Cop/LineBreakAroundConditionalBlock
+          if server
             server.shutdown
             server.listeners.each(&:close)
           end
+
           @server = nil
         end

lib/tasks/gitlab/cleanup.rake

@@ -3,69 +3,6 @@ require 'set'
 namespace :gitlab do
   namespace :cleanup do
-    desc "GitLab | Cleanup | Clean namespaces"
-    task dirs: :gitlab_environment do
-      namespaces = Set.new(Namespace.pluck(:path))
-      namespaces << Storage::HashedProject::REPOSITORY_PATH_PREFIX
-
-      Gitaly::Server.all.each do |server|
-        all_dirs = Gitlab::GitalyClient::StorageService
-          .new(server.storage)
-          .list_directories(depth: 0)
-          .reject { |dir| dir.ends_with?('.git') || namespaces.include?(File.basename(dir)) }
-
-        puts "Looking for directories to remove... "
-        all_dirs.each do |dir_path|
-          if remove?
-            begin
-              Gitlab::GitalyClient::NamespaceService.new(server.storage)
-                .remove(dir_path)
-
-              puts "Removed...#{dir_path}"
-            rescue StandardError => e
-              puts "Cannot remove #{dir_path}: #{e.message}".color(:red)
-            end
-          else
-            puts "Can be removed: #{dir_path}".color(:red)
-          end
-        end
-      end
-
-      unless remove?
-        puts "To cleanup this directories run this command with REMOVE=true".color(:yellow)
-      end
-    end
-
-    desc "GitLab | Cleanup | Clean repositories"
-    task repos: :gitlab_environment do
-      move_suffix = "+orphaned+#{Time.now.to_i}"
-
-      Gitaly::Server.all.each do |server|
-        Gitlab::GitalyClient::StorageService
-          .new(server.storage)
-          .list_directories
-          .each do |path|
-          repo_with_namespace = path.chomp('.git').chomp('.wiki')
-
-          # TODO ignoring hashed repositories for now.  But revisit to fully support
-          # possible orphaned hashed repos
-          next if repo_with_namespace.start_with?(Storage::HashedProject::REPOSITORY_PATH_PREFIX)
-          next if Project.find_by_full_path(repo_with_namespace)
-
-          new_path = path + move_suffix
-          puts path.inspect + ' -> ' + new_path.inspect
-
-          begin
-            Gitlab::GitalyClient::NamespaceService
-              .new(server.storage)
-              .rename(path, new_path)
-          rescue StandardError => e
-            puts "Error occurred while moving the repository: #{e.message}".color(:red)
-          end
-        end
-      end
-    end
-
     desc "GitLab | Cleanup | Block users that have been removed in LDAP"
     task block_removed_ldap_users: :gitlab_environment do
       warn_user_is_not_gitlab

scripts/review_apps/review-apps.sh

@@ -255,25 +255,13 @@ EOF
 }
 function display_deployment_debug() {
-  kubectl get pods -n "$KUBE_NAMESPACE" -lrelease=${CI_ENVIRONMENT_SLUG}
-  migrations_pod=$(get_pod "migrations");
-  if [ -z "${migrations_pod}" ]; then
-    echoerr "Migrations pod not found."
-  else
-    echoinfo "Logs tail of the ${migrations_pod} pod..."
-
-    kubectl logs -n "$KUBE_NAMESPACE" "${migrations_pod}" | sed "s/${REVIEW_APPS_ROOT_PASSWORD}/[REDACTED]/g"
-  fi
-
-  unicorn_pod=$(get_pod "unicorn");
-  if [ -z "${unicorn_pod}" ]; then
-    echoerr "Unicorn pod not found."
-  else
-    echoinfo "Logs tail of the ${unicorn_pod} pod..."
-
-    kubectl logs -n "$KUBE_NAMESPACE" -c unicorn "${unicorn_pod}" | sed "s/${REVIEW_APPS_ROOT_PASSWORD}/[REDACTED]/g"
-  fi
+  # Get all pods that are not ready (this will return completed pods for minio and migrations jobs)
+  echoinfo "Unready Pods for release ${CI_ENVIRONMENT_SLUG}"
+  kubectl get pods -n "$KUBE_NAMESPACE" -lrelease=${CI_ENVIRONMENT_SLUG} --field-selector=status.phase!=Running
+  # Get all non-completed jobs
+  echoinfo "Unsuccessful Jobs for release ${CI_ENVIRONMENT_SLUG}"
+  kubectl get jobs -n "$KUBE_NAMESPACE" -lrelease=${CI_ENVIRONMENT_SLUG} --field-selector=status.successful!=1
 }
 function add_license() {

spec/fixtures/emails/auto_reply.eml

@@ -12,7 +12,7 @@ Mime-Version: 1.0
 Content-Type: text/plain;
  charset=ISO-8859-1
 Content-Transfer-Encoding: 7bit
-Auto-Submitted: auto-generated
+X-Autoreply: yes
 X-Sieve: CMU Sieve 2.2
 X-Received: by 10.0.0.1 with SMTP id n7mr11234144ipb.85.1371157428600; Thu,
  13 Jun 2013 14:03:48 -0700 (PDT)

spec/fixtures/emails/auto_submitted.eml

+Return-Path: <jake@adventuretime.ooo>
+Received: from iceking.adventuretime.ooo ([unix socket]) by iceking (Cyrus v2.2.13-Debian-2.2.13-19+squeeze3) with LMTPA; Thu, 13 Jun 2013 17:03:50 -0400
+Received: from mail-ie0-x234.google.com (mail-ie0-x234.google.com [IPv6:2607:f8b0:4001:c03::234]) by iceking.adventuretime.ooo (8.14.3/8.14.3/Debian-9.4) with ESMTP id r5DL3nFJ016967 (version=TLSv1/SSLv3 cipher=RC4-SHA bits=128 verify=NOT) for <reply+59d8df8370b7e95c5a49fbf86aeb2c93@discourse.example.com>; Thu, 13 Jun 2013 17:03:50 -0400
+Received: by mail-ie0-f180.google.com with SMTP id f4so21977375iea.25 for <reply+59d8df8370b7e95c5a49fbf86aeb2c93@discourse.example.com>; Thu, 13 Jun 2013 14:03:48 -0700
+Received: by 10.0.0.1 with HTTP; Thu, 13 Jun 2013 14:03:48 -0700
+Date: Thu, 13 Jun 2013 17:03:48 -0400
+From: Jake the Dog <jake@adventuretime.ooo>
+To: reply+636ca428858779856c226bb145ef4fad@appmail.adventuretime.ooo
+Message-ID: <CADkmRc+rNGAGGbV2iE5p918UVy4UyJqVcXRO2=otppgzduJSg@mail.gmail.com>
+Subject: re: [Discourse Meta] eviltrout posted in 'Adventure Time Sux'
+Mime-Version: 1.0
+Content-Type: text/plain;
+ charset=ISO-8859-1
+Content-Transfer-Encoding: 7bit
+Auto-Submitted: auto-generated
+X-Sieve: CMU Sieve 2.2
+X-Received: by 10.0.0.1 with SMTP id n7mr11234144ipb.85.1371157428600; Thu,
+ 13 Jun 2013 14:03:48 -0700 (PDT)
+X-Scanned-By: MIMEDefang 2.69 on IPv6:2001:470:1d:165::1
+
+Test reply to Discourse email digest

spec/lib/gitlab/email/receiver_spec.rb

@@ -40,7 +40,15 @@ describe Gitlab::Email::Receiver do
     end
   end
-  context "when the email was auto generated" do
+  context "when the email was auto generated with Auto-Submitted header" do
+    let(:email_raw) { fixture_file("emails/auto_submitted.eml") }
+
+    it "raises an AutoGeneratedEmailError" do
+      expect { receiver.execute }.to raise_error(Gitlab::Email::AutoGeneratedEmailError)
+    end
+  end
+
+  context "when the email was auto generated with X-Autoreply header" do
     let(:email_raw) { fixture_file("emails/auto_reply.eml") }
     it "raises an AutoGeneratedEmailError" do

spec/support/shared_examples/cycle_analytics_stage_examples.rb

@@ -46,6 +46,13 @@ shared_examples_for 'cycle analytics stage' do
       expect(stage).not_to be_valid
       expect(stage.errors.details[:end_event]).to eq([{ error: :not_allowed_for_the_given_start_event }])
     end
+
+    context 'disallows default stage names when creating custom stage' do
+      let(:invalid_params) { valid_params.merge(name: Gitlab::Analytics::CycleAnalytics::DefaultStages.names.first, custom: true) }
+      let(:stage) { described_class.new(invalid_params) }
+
+      it { expect(stage).not_to be_valid }
+    end
   end
   describe '#subject_model' do

spec/tasks/gitlab/cleanup_rake_spec.rb

@@ -5,74 +5,6 @@ describe 'gitlab:cleanup rake tasks' do
     Rake.application.rake_require 'tasks/gitlab/cleanup'
   end
-  describe 'cleanup namespaces and repos' do
-    let(:gitlab_shell) { Gitlab::Shell.new }
-    let(:storage) { storages.keys.first }
-    let(:storages) do
-      {
-        'default' => Gitlab::GitalyClient::StorageSettings.new(@default_storage_hash.merge('path' => 'tmp/tests/default_storage'))
-      }
-    end
-
-    before(:all) do
-      @default_storage_hash = Gitlab.config.repositories.storages.default.to_h
-    end
-
-    before do
-      allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
-    end
-
-    after do
-      Gitlab::GitalyClient::StorageService.new(storage).delete_all_repositories
-    end
-
-    describe 'cleanup:repos' do
-      before do
-        gitlab_shell.add_namespace(storage, 'broken/project.git')
-        gitlab_shell.add_namespace(storage, '@hashed/12/34/5678.git')
-      end
-
-      it 'moves it to an orphaned path' do
-        now = Time.now
-
-        Timecop.freeze(now) do
-          run_rake_task('gitlab:cleanup:repos')
-          repo_list = Gitlab::GitalyClient::StorageService.new(storage).list_directories(depth: 0)
-
-          expect(repo_list.last).to include("broken+orphaned+#{now.to_i}")
-        end
-      end
-
-      it 'ignores @hashed repos' do
-        run_rake_task('gitlab:cleanup:repos')
-
-        expect(gitlab_shell.exists?(storage, '@hashed/12/34/5678.git')).to be(true)
-      end
-    end
-
-    describe 'cleanup:dirs' do
-      it 'removes missing namespaces' do
-        gitlab_shell.add_namespace(storage, "namespace_1/project.git")
-        gitlab_shell.add_namespace(storage, "namespace_2/project.git")
-        allow(Namespace).to receive(:pluck).and_return(['namespace_1'])
-
-        stub_env('REMOVE', 'true')
-        run_rake_task('gitlab:cleanup:dirs')
-
-        expect(gitlab_shell.exists?(storage, 'namespace_1')).to be(true)
-        expect(gitlab_shell.exists?(storage, 'namespace_2')).to be(false)
-      end
-
-      it 'ignores @hashed directory' do
-        gitlab_shell.add_namespace(storage, '@hashed/12/34/5678.git')
-
-        run_rake_task('gitlab:cleanup:dirs')
-
-        expect(gitlab_shell.exists?(storage, '@hashed/12/34/5678.git')).to be(true)
-      end
-    end
-  end
-
   # A single integration test that is redundant with one part of the
   # Gitlab::Cleanup::ProjectUploads spec.
   #