Merge remote-tracking branch 'upstream/master' into qa-allow-setting-sandbox-group

* upstream/master: (27 commits)
  Set initial password for instance in LDAP QA test
  Backport EE changes to some hashed storage documentation to CE
  Remove allow_n_plus_1 from Git::Repository#branches_filter
  Bumps Gitlab Shell version to 6.0.3
  Make resetting column information overridable in EE
  Added 'clear' button to ci lint editor
  Issues and merge requests in subgroups docs
  Update docs labels CE
  Refactored merge_requests/show path in dispatcher.js
  wording
  don't check against a hardcoded user name
  10.5 Update the dependencies license list
  10.5 Update the .gitignore, .gitlab-ci.yml, and Dockerfile templates
  Create update guide for 10.5
  Update 10.5 source install guide
  Add docs for MR link in commit page
  Add groups to OpenID Connect claims
  Replaced $.get with axois.get
  Memoize MergeRequest#rebase_in_progress? to prevent N+1 queries in Gitaly
  [docs] Info rescheduling background migrations
  ...

spec/models/project_spec.rb

@@ -2504,6 +2504,7 @@ describe Project do
   end
   describe '#remove_exports' do
+    let(:legacy_project) { create(:project, :legacy_storage, :with_export) }
     let(:project) { create(:project, :with_export) }
     it 'removes the exports directory for the project' do
@@ -2516,15 +2517,29 @@ describe Project do
       expect(File.exist?(project.export_path)).to be_falsy
     end
-    it 'is a no-op when there is no namespace' do
+    it 'is a no-op on legacy projects when there is no namespace' do
+      export_path = legacy_project.export_path
+
+      legacy_project.update_column(:namespace_id, nil)
+
+      expect(FileUtils).not_to receive(:rm_rf).with(export_path)
+
+      legacy_project.remove_exports
+
+      expect(File.exist?(export_path)).to be_truthy
+    end
+
+    it 'runs on hashed storage projects when there is no namespace' do
       export_path = project.export_path
+
       project.update_column(:namespace_id, nil)
-      expect(FileUtils).not_to receive(:rm_rf).with(export_path)
+      allow(FileUtils).to receive(:rm_rf).and_call_original
+      expect(FileUtils).to receive(:rm_rf).with(export_path).and_call_original
       project.remove_exports
-      expect(File.exist?(export_path)).to be_truthy
+      expect(File.exist?(export_path)).to be_falsy
     end
     it 'is run when the project is destroyed' do
@@ -2545,7 +2560,7 @@ describe Project do
   end
   context 'legacy storage' do
-    let(:project) { create(:project, :repository) }
+    let(:project) { create(:project, :repository, :legacy_storage) }
     let(:gitlab_shell) { Gitlab::Shell.new }
     let(:project_storage) { project.send(:storage) }
@@ -2719,6 +2734,8 @@ describe Project do
     let(:project) { create(:project, :repository, skip_disk_validation: true) }
     let(:gitlab_shell) { Gitlab::Shell.new }
     let(:hash) { Digest::SHA2.hexdigest(project.id.to_s) }
+    let(:hashed_prefix) { File.join('@hashed', hash[0..1], hash[2..3]) }
+    let(:hashed_path) { File.join(hashed_prefix, hash) }
     before do
       stub_application_setting(hashed_storage_enabled: true)
@@ -2744,14 +2761,12 @@ describe Project do
     describe '#base_dir' do
       it 'returns base_dir based on hash of project id' do
-        expect(project.base_dir).to eq("@hashed/#{hash[0..1]}/#{hash[2..3]}")
+        expect(project.base_dir).to eq(hashed_prefix)
       end
     end
     describe '#disk_path' do
       it 'returns disk_path based on hash of project id' do
-        hashed_path = "@hashed/#{hash[0..1]}/#{hash[2..3]}/#{hash}"
-
         expect(project.disk_path).to eq(hashed_path)
       end
     end
@@ -2760,7 +2775,7 @@ describe Project do
       it 'delegates to gitlab_shell to ensure namespace is created' do
         allow(project).to receive(:gitlab_shell).and_return(gitlab_shell)
-        expect(gitlab_shell).to receive(:add_namespace).with(project.repository_storage_path, "@hashed/#{hash[0..1]}/#{hash[2..3]}")
+        expect(gitlab_shell).to receive(:add_namespace).with(project.repository_storage_path, hashed_prefix)
         project.ensure_storage_path_exists
       end

spec/models/user_spec.rb

@@ -1586,14 +1586,37 @@ describe User do
   describe '#authorized_groups' do
     let!(:user) { create(:user) }
     let!(:private_group) { create(:group) }
+    let!(:child_group) { create(:group, parent: private_group) }
+
+    let!(:project_group) { create(:group) }
+    let!(:project) { create(:project, group: project_group) }
     before do
       private_group.add_user(user, Gitlab::Access::MASTER)
+      project.add_master(user)
     end
     subject { user.authorized_groups }
-    it { is_expected.to eq([private_group]) }
+    it { is_expected.to contain_exactly private_group, project_group }
+  end
+
+  describe '#membership_groups' do
+    let!(:user) { create(:user) }
+    let!(:parent_group) { create(:group) }
+    let!(:child_group) { create(:group, parent: parent_group) }
+
+    before do
+      parent_group.add_user(user, Gitlab::Access::MASTER)
+    end
+
+    subject { user.membership_groups }
+
+    if Group.supports_nested_groups?
+      it { is_expected.to contain_exactly parent_group, child_group }
+    else
+      it { is_expected.to contain_exactly parent_group }
+    end
   end
   describe '#authorized_projects', :delete do

spec/requests/api/internal_spec.rb

@@ -366,20 +366,9 @@ describe API::Internal do
           end
         end
-        context 'project as /namespace/project' do
-          it do
-            push(key, project_with_repo_path('/' + project.full_path))
-
-            expect(response).to have_gitlab_http_status(200)
-            expect(json_response["status"]).to be_truthy
-            expect(json_response["repository_path"]).to eq(project.repository.path_to_repo)
-            expect(json_response["gl_repository"]).to eq("project-#{project.id}")
-          end
-        end
-
         context 'project as namespace/project' do
           it do
-            push(key, project_with_repo_path(project.full_path))
+            push(key, project)
             expect(response).to have_gitlab_http_status(200)
             expect(json_response["status"]).to be_truthy
@@ -496,8 +485,10 @@ describe API::Internal do
     end
     context 'project does not exist' do
-      it do
-        pull(key, project_with_repo_path('gitlab/notexist'))
+      it 'returns a 200 response with status: false' do
+        project.destroy
+
+        pull(key, project)
         expect(response).to have_gitlab_http_status(200)
         expect(json_response["status"]).to be_falsey
@@ -569,6 +560,7 @@ describe API::Internal do
     end
     context 'the project path was changed' do
+      let(:project) { create(:project, :repository, :legacy_storage) }
       let!(:old_path_to_repo) { project.repository.path_to_repo }
       let!(:repository) { project.repository }
@@ -858,9 +850,14 @@ describe API::Internal do
     end
   end
-  def project_with_repo_path(path)
-    double().tap do |fake_project|
-      allow(fake_project).to receive_message_chain('repository.path_to_repo' => path)
+  def gl_repository_for(project_or_wiki)
+    case project_or_wiki
+    when ProjectWiki
+      project_or_wiki.project.gl_repository(is_wiki: true)
+    when Project
+      project_or_wiki.gl_repository(is_wiki: false)
+    else
+      nil
     end
   end
@@ -868,18 +865,8 @@ describe API::Internal do
     post(
       api("/internal/allowed"),
       key_id: key.id,
-      project: project.repository.path_to_repo,
-      action: 'git-upload-pack',
-      secret_token: secret_token,
-      protocol: protocol
-    )
-  end
-
-  def pull_with_path(key, path_to_repo, protocol = 'ssh')
-    post(
-      api("/internal/allowed"),
-      key_id: key.id,
-      project: path_to_repo,
+      project: project.full_path,
+      gl_repository: gl_repository_for(project),
       action: 'git-upload-pack',
       secret_token: secret_token,
       protocol: protocol
@@ -891,20 +878,8 @@ describe API::Internal do
       api("/internal/allowed"),
       changes: 'd14d6c0abdd253381df51a723d58691b2ee1ab08 570e7b2abdd848b95f2f578043fc23bd6f6fd24d refs/heads/master',
       key_id: key.id,
-      project: project.repository.path_to_repo,
-      action: 'git-receive-pack',
-      secret_token: secret_token,
-      protocol: protocol,
-      env: env
-    )
-  end
-
-  def push_with_path(key, path_to_repo, protocol = 'ssh', env: nil)
-    post(
-      api("/internal/allowed"),
-      changes: 'd14d6c0abdd253381df51a723d58691b2ee1ab08 570e7b2abdd848b95f2f578043fc23bd6f6fd24d refs/heads/master',
-      key_id: key.id,
-      project: path_to_repo,
+      project: project.full_path,
+      gl_repository: gl_repository_for(project),
       action: 'git-receive-pack',
       secret_token: secret_token,
       protocol: protocol,
@@ -917,7 +892,8 @@ describe API::Internal do
       api("/internal/allowed"),
       ref: 'master',
       key_id: key.id,
-      project: project.repository.path_to_repo,
+      project: project.full_path,
+      gl_repository: gl_repository_for(project),
       action: 'git-upload-archive',
       secret_token: secret_token,
       protocol: 'ssh'
@@ -929,7 +905,7 @@ describe API::Internal do
       api("/internal/lfs_authenticate"),
       key_id: key_id,
       secret_token: secret_token,
-      project: project.repository.path_to_repo
+      project: project.full_path
     )
   end
 end

spec/requests/api/projects_spec.rb

@@ -460,7 +460,7 @@ describe API::Projects do
       expect(response).to have_gitlab_http_status(201)
       project.each_pair do |k, v|
-        next if %i[has_external_issue_tracker issues_enabled merge_requests_enabled wiki_enabled].include?(k)
+        next if %i[has_external_issue_tracker issues_enabled merge_requests_enabled wiki_enabled storage_version].include?(k)
         expect(json_response[k.to_s]).to eq(v)
       end
@@ -622,12 +622,8 @@ describe API::Projects do
   end
   describe 'POST /projects/user/:id' do
-    before do
-      expect(project).to be_persisted
-    end
-
     it 'creates new project without path but with name and return 201' do
-      expect { post api("/projects/user/#{user.id}", admin), name: 'Foo Project' }.to change {Project.count}.by(1)
+      expect { post api("/projects/user/#{user.id}", admin), name: 'Foo Project' }.to change { Project.count }.by(1)
       expect(response).to have_gitlab_http_status(201)
       project = Project.last
@@ -666,8 +662,9 @@ describe API::Projects do
       post api("/projects/user/#{user.id}", admin), project
       expect(response).to have_gitlab_http_status(201)
+
       project.each_pair do |k, v|
-        next if %i[has_external_issue_tracker path].include?(k)
+        next if %i[has_external_issue_tracker path storage_version].include?(k)
         expect(json_response[k.to_s]).to eq(v)
       end

spec/requests/api/v3/projects_spec.rb

@@ -401,7 +401,7 @@ describe API::V3::Projects do
       post v3_api('/projects', user), project
       project.each_pair do |k, v|
-        next if %i[has_external_issue_tracker issues_enabled merge_requests_enabled wiki_enabled].include?(k)
+        next if %i[storage_version has_external_issue_tracker issues_enabled merge_requests_enabled wiki_enabled].include?(k)
         expect(json_response[k.to_s]).to eq(v)
       end
@@ -545,7 +545,7 @@ describe API::V3::Projects do
       expect(response).to have_gitlab_http_status(201)
       project.each_pair do |k, v|
-        next if %i[has_external_issue_tracker path].include?(k)
+        next if %i[storage_version has_external_issue_tracker path].include?(k)
         expect(json_response[k.to_s]).to eq(v)
       end

spec/requests/git_http_spec.rb

@@ -163,7 +163,7 @@ describe 'Git HTTP requests' do
             download(path) do |response|
               json_body = ActiveSupport::JSON.decode(response.body)
-              expect(json_body['RepoPath']).to include(wiki.repository.full_path)
+              expect(json_body['RepoPath']).to include(wiki.repository.disk_path)
             end
           end
         end

spec/requests/openid_connect_spec.rb

@@ -65,10 +65,20 @@ describe 'OpenID Connect requests' do
         )
       end
-      let(:public_email) { build :email, email: 'public@example.com' }
-      let(:private_email) { build :email, email: 'private@example.com' }
-      it 'includes all user information' do
+      let!(:public_email) { build :email, email: 'public@example.com' }
+      let!(:private_email) { build :email, email: 'private@example.com' }
+      let!(:group1) { create :group, path: 'group1' }
+      let!(:group2) { create :group, path: 'group2' }
+      let!(:group3) { create :group, path: 'group3', parent: group2 }
+      let!(:group4) { create :group, path: 'group4', parent: group3 }
+
+      before do
+        group1.add_user(user, GroupMember::OWNER)
+        group3.add_user(user, Gitlab::Access::DEVELOPER)
+      end
+
+      it 'includes all user information and group memberships' do
         request_user_info
         expect(json_response).to eq({
@@ -79,7 +89,13 @@ describe 'OpenID Connect requests' do
           'email_verified' => true,
           'website'        => 'https://example.com',
           'profile'        => 'http://localhost/alice',
-          'picture'        => "http://localhost/uploads/-/system/user/avatar/#{user.id}/dk.png"
+          'picture'        => "http://localhost/uploads/-/system/user/avatar/#{user.id}/dk.png",
+          'groups'         =>
+            if Group.supports_nested_groups?
+              ['group1', 'group2/group3', 'group2/group3/group4']
+            else
+              ['group1', 'group2/group3']
+            end
         })
       end
     end

spec/services/groups/destroy_service_spec.rb

@@ -6,7 +6,7 @@ describe Groups::DestroyService do
   let!(:user)         { create(:user) }
   let!(:group)        { create(:group) }
   let!(:nested_group) { create(:group, parent: group) }
-  let!(:project)      { create(:project, namespace: group) }
+  let!(:project)      { create(:project, :legacy_storage, namespace: group) }
   let!(:notification_setting) { create(:notification_setting, source: group)}
   let(:gitlab_shell) { Gitlab::Shell.new }
   let(:remove_path)  { group.path + "+#{group.id}+deleted" }
@@ -141,7 +141,7 @@ describe Groups::DestroyService do
     end
     context 'legacy storage' do
-      let!(:project) { create(:project, :empty_repo, namespace: group) }
+      let!(:project) { create(:project, :legacy_storage, :empty_repo, namespace: group) }
       it 'removes repository' do
         expect(gitlab_shell.exists?(project.repository_storage_path, "#{project.disk_path}.git")).to be_falsey
@@ -149,7 +149,7 @@ describe Groups::DestroyService do
     end
     context 'hashed storage' do
-      let!(:project) { create(:project, :hashed, :empty_repo, namespace: group) }
+      let!(:project) { create(:project, :empty_repo, namespace: group) }
       it 'removes repository' do
         expect(gitlab_shell.exists?(project.repository_storage_path, "#{project.disk_path}.git")).to be_falsey

spec/services/projects/hashed_storage/migrate_attachments_service_spec.rb

@@ -2,7 +2,7 @@ require 'spec_helper'
 describe Projects::HashedStorage::MigrateAttachmentsService do
   subject(:service) { described_class.new(project) }
-  let(:project) { create(:project) }
+  let(:project) { create(:project, :legacy_storage) }
   let(:legacy_storage) { Storage::LegacyProject.new(project) }
   let(:hashed_storage) { Storage::HashedProject.new(project) }

spec/services/projects/hashed_storage/migrate_repository_service_spec.rb

@@ -2,7 +2,7 @@ require 'spec_helper'
 describe Projects::HashedStorage::MigrateRepositoryService do
   let(:gitlab_shell) { Gitlab::Shell.new }
-  let(:project) { create(:project, :repository, :wiki_repo) }
+  let(:project) { create(:project, :legacy_storage, :repository, :wiki_repo) }
   let(:service) { described_class.new(project) }
   let(:legacy_storage) { Storage::LegacyProject.new(project) }
   let(:hashed_storage) { Storage::HashedProject.new(project) }

spec/services/projects/hashed_storage_migration_service_spec.rb

 require 'spec_helper'
 describe Projects::HashedStorageMigrationService do
-  let(:project) { create(:project, :empty_repo, :wiki_repo) }
+  let(:project) { create(:project, :empty_repo, :wiki_repo, :legacy_storage) }
   subject(:service) { described_class.new(project) }
   describe '#execute' do

spec/services/projects/transfer_service_spec.rb

@@ -4,7 +4,7 @@ describe Projects::TransferService do
   let(:gitlab_shell) { Gitlab::Shell.new }
   let(:user) { create(:user) }
   let(:group) { create(:group) }
-  let(:project) { create(:project, :repository, namespace: user.namespace) }
+  let(:project) { create(:project, :repository, :legacy_storage, namespace: user.namespace) }
   context 'namespace -> namespace' do
     before do
@@ -214,7 +214,7 @@ describe Projects::TransferService do
   end
   context 'when hashed storage in use' do
-    let(:hashed_project) { create(:project, :repository, :hashed, namespace: user.namespace) }
+    let(:hashed_project) { create(:project, :repository, namespace: user.namespace) }
     before do
       group.add_owner(user)

spec/services/projects/update_service_spec.rb

@@ -150,6 +150,8 @@ describe Projects::UpdateService do
       let(:repository_storage_path) { Gitlab.config.repositories.storages[repository_storage]['path'] }
       context 'with legacy storage' do
+        let(:project) { create(:project, :legacy_storage, :repository, creator: user, namespace: user.namespace) }
+
         before do
           gitlab_shell.add_repository(repository_storage, "#{user.namespace.full_path}/existing")
         end

spec/services/users/destroy_service_spec.rb

@@ -173,7 +173,7 @@ describe Users::DestroyService do
       end
       context 'legacy storage' do
-        let!(:project) { create(:project, :empty_repo, namespace: user.namespace) }
+        let!(:project) { create(:project, :empty_repo, :legacy_storage, namespace: user.namespace) }
         it 'removes repository' do
           expect(gitlab_shell.exists?(project.repository_storage_path, "#{project.disk_path}.git")).to be_falsey
@@ -181,7 +181,7 @@ describe Users::DestroyService do
       end
       context 'hashed storage' do
-        let!(:project) { create(:project, :empty_repo, :hashed, namespace: user.namespace) }
+        let!(:project) { create(:project, :empty_repo, namespace: user.namespace) }
         it 'removes repository' do
           expect(gitlab_shell.exists?(project.repository_storage_path, "#{project.disk_path}.git")).to be_falsey

spec/support/migrations_helpers.rb

@@ -25,14 +25,19 @@ module MigrationsHelpers
     clear_schema_cache!
     # Reset column information for the most offending classes **after** we
-    # migrated the schema up, otherwise, column information could be outdated
-    ActiveRecord::Base.descendants.each { |klass| klass.reset_column_information }
+    # migrated the schema up, otherwise, column information could be
+    # outdated. We have a separate method for this so we can override it in EE.
+    ActiveRecord::Base.descendants.each(&method(:reset_column_information))
     # Without that, we get errors because of missing attributes, e.g.
     # super: no superclass method `elasticsearch_indexing' for #<ApplicationSetting:0x00007f85628508d8>
     ApplicationSetting.define_attribute_methods
   end
+  def reset_column_information(klass)
+    klass.reset_column_information
+  end
+
   def previous_migration
     migrations.each_cons(2) do |previous, migration|
       break previous if migration.name == described_class.name

spec/uploaders/file_uploader_spec.rb

@@ -2,7 +2,7 @@ require 'spec_helper'
 describe FileUploader do
   let(:group) { create(:group, name: 'awesome') }
-  let(:project) { create(:project, namespace: group, name: 'project') }
+  let(:project) { create(:project, :legacy_storage, namespace: group, name: 'project') }
   let(:uploader) { described_class.new(project) }
   let(:upload)  { double(model: project, path: 'secret/foo.jpg') }
@@ -16,12 +16,12 @@ describe FileUploader do
   shared_examples 'uses hashed storage' do
     context 'when rolled out attachments' do
+      let(:project) { build_stubbed(:project, namespace: group, name: 'project') }
+
       before do
         allow(project).to receive(:disk_path).and_return('ca/fe/fe/ed')
       end
-      let(:project) { build_stubbed(:project, :hashed, namespace: group, name: 'project') }
-
       it_behaves_like 'builds correct paths',
                       store_dir: %r{ca/fe/fe/ed/\h+},
                       absolute_path: %r{#{described_class.root}/ca/fe/fe/ed/secret/foo.jpg}

spec/workers/repository_fork_worker_spec.rb

@@ -68,7 +68,7 @@ describe RepositoryForkWorker do
     end
     it "handles bad fork" do
-      error_message = "Unable to fork project #{fork_project.id} for repository #{project.full_path} -> #{fork_project.full_path}"
+      error_message = "Unable to fork project #{fork_project.id} for repository #{project.disk_path} -> #{fork_project.disk_path}"
       expect_fork_repository.and_return(false)

spec/workers/storage_migrator_worker_spec.rb

@@ -2,7 +2,7 @@ require 'spec_helper'
 describe StorageMigratorWorker do
   subject(:worker) { described_class.new }
-  let(:projects) { create_list(:project, 2) }
+  let(:projects) { create_list(:project, 2, :legacy_storage) }
   describe '#perform' do
     let(:ids) { projects.map(&:id) }

vendor/gitignore/Android.gitignore

@@ -54,3 +54,10 @@ google-services.json
 freeline.py
 freeline/
 freeline_project_description.json
+
+# fastlane
+fastlane/report.xml
+fastlane/Preview.html
+fastlane/screenshots
+fastlane/test_output
+fastlane/readme.md

vendor/gitignore/Dart.gitignore

 # See https://www.dartlang.org/tools/private-files.html
 # Files and directories created by pub
+.dart_tool/
 .packages
 .pub/
 build/