Merge remote-tracking branch 'upstream/master' into qa-allow-setting-sandbox-group

* upstream/master: (27 commits)
  Set initial password for instance in LDAP QA test
  Backport EE changes to some hashed storage documentation to CE
  Remove allow_n_plus_1 from Git::Repository#branches_filter
  Bumps Gitlab Shell version to 6.0.3
  Make resetting column information overridable in EE
  Added 'clear' button to ci lint editor
  Issues and merge requests in subgroups docs
  Update docs labels CE
  Refactored merge_requests/show path in dispatcher.js
  wording
  don't check against a hardcoded user name
  10.5 Update the dependencies license list
  10.5 Update the .gitignore, .gitlab-ci.yml, and Dockerfile templates
  Create update guide for 10.5
  Update 10.5 source install guide
  Add docs for MR link in commit page
  Add groups to OpenID Connect claims
  Replaced $.get with axois.get
  Memoize MergeRequest#rebase_in_progress? to prevent N+1 queries in Gitaly
  [docs] Info rescheduling background migrations
  ...

GITLAB_SHELL_VERSION

-6.0.2
+6.0.3

app/assets/javascripts/boards/components/new_list_dropdown.js

 /* eslint-disable func-names, no-new, space-before-function-paren, one-var,
    promise/catch-or-return */
+import axios from '~/lib/utils/axios_utils';
 import _ from 'underscore';
 import CreateLabelDropdown from '../../create_label';
@@ -28,9 +29,9 @@ gl.issueBoards.newListDropdownInit = () => {
     $this.glDropdown({
       data(term, callback) {
-        $.get($this.attr('data-list-labels-path'))
-          .then((resp) => {
-            callback(resp);
+        axios.get($this.attr('data-list-labels-path'))
+          .then(({ data }) => {
+            callback(data);
           });
       },
       renderRow (label) {

app/assets/javascripts/dispatcher.js

 /* eslint-disable func-names, space-before-function-paren, no-var, prefer-arrow-callback, wrap-iife, no-shadow, consistent-return, one-var, one-var-declaration-per-line, camelcase, default-case, no-new, quotes, no-duplicate-case, no-case-declarations, no-fallthrough, max-len */
-import MergeRequest from './merge_request';
 import Flash from './flash';
 import GfmAutoComplete from './gfm_auto_complete';
-import ZenMode from './zen_mode';
-import initNotes from './init_notes';
-import initIssuableSidebar from './init_issuable_sidebar';
 import { convertPermissionToBoolean } from './lib/utils/common_utils';
 import GlFieldErrors from './gl_field_errors';
 import Shortcuts from './shortcuts';
-import ShortcutsIssuable from './shortcuts_issuable';
-import Diff from './diff';
 import SearchAutocomplete from './search_autocomplete';
 var Dispatcher;
@@ -262,17 +256,10 @@ var Dispatcher;
             .catch(fail);
           break;
         case 'projects:merge_requests:show':
-          new Diff();
-          new ZenMode();
-
-          initIssuableSidebar();
-          initNotes();
-
-          const mrShowNode = document.querySelector('.merge-request');
-          window.mergeRequest = new MergeRequest({
-            action: mrShowNode.dataset.mrAction,
-          });
-          shortcut_handler = new ShortcutsIssuable(true);
+          import('./pages/projects/merge_requests/show')
+            .then(callDefault)
+            .catch(fail);
+          shortcut_handler = true;
           break;
         case 'dashboard:activity':
           import('./pages/dashboard/activity')

app/assets/javascripts/gl_form.js

-/* global autosize */
-
+import autosize from 'autosize';
 import GfmAutoComplete from './gfm_auto_complete';
 import dropzoneInput from './dropzone_input';
 import textUtils from './lib/utils/text_markdown';

app/assets/javascripts/pages/ci/lints/ci_lint_editor.js

@@ -2,11 +2,18 @@ export default class CILintEditor {
   constructor() {
     this.editor = window.ace.edit('ci-editor');
     this.textarea = document.querySelector('#content');
+    this.clearYml = document.querySelector('.clear-yml');
     this.editor.getSession().setMode('ace/mode/yaml');
     this.editor.on('input', () => {
       const content = this.editor.getSession().getValue();
       this.textarea.value = content;
     });
+
+    this.clearYml.addEventListener('click', this.clear.bind(this));
+  }
+
+  clear() {
+    this.editor.setValue('');
   }
 }

app/assets/javascripts/pages/projects/merge_requests/show/index.js

+import MergeRequest from '~/merge_request';
+import ZenMode from '~/zen_mode';
+import initNotes from '~/init_notes';
+import initIssuableSidebar from '~/init_issuable_sidebar';
+import ShortcutsIssuable from '~/shortcuts_issuable';
+import Diff from '~/diff';
+import { handleLocationHash } from '~/lib/utils/common_utils';
+
+export default () => {
+  new Diff(); // eslint-disable-line no-new
+  new ZenMode(); // eslint-disable-line no-new
+
+  initIssuableSidebar(); // eslint-disable-line no-new
+  initNotes(); // eslint-disable-line no-new
+
+  const mrShowNode = document.querySelector('.merge-request');
+
+  window.mergeRequest = new MergeRequest({
+    action: mrShowNode.dataset.mrAction,
+  });
+
+  new ShortcutsIssuable(true); // eslint-disable-line no-new
+  handleLocationHash();
+};

app/models/merge_request.rb

@@ -158,10 +158,12 @@ class MergeRequest < ActiveRecord::Base
   end
   def rebase_in_progress?
-    # The source project can be deleted
-    return false unless source_project
-    source_project.repository.rebase_in_progress?(id)
+    strong_memoize(:rebase_in_progress) do
+      # The source project can be deleted
+      next false unless source_project
+      source_project.repository.rebase_in_progress?(id)
+    end
   end
   # Use this method whenever you need to make sure the head_pipeline is synced with the

app/models/user.rb

@@ -551,7 +551,7 @@ class User < ActiveRecord::Base
     gpg_keys.each(&:update_invalid_gpg_signatures)
   end
-  # Returns the groups a user has access to
+  # Returns the groups a user has access to, either through a membership or a project authorization
   def authorized_groups
     union = Gitlab::SQL::Union
       .new([groups.select(:id), authorized_projects.select(:namespace_id)])
@@ -559,6 +559,11 @@ class User < ActiveRecord::Base
     Group.where("namespaces.id IN (#{union.to_sql})") # rubocop:disable GitlabSecurity/SqlInjection
   end
+  # Returns the groups a user is a member of, either directly or through a parent group
+  def membership_groups
+    Gitlab::GroupHierarchy.new(groups).base_and_descendants
+  end
+
   # Returns a relation of groups the user has access to, including their parent
   # and child groups (recursively).
   def all_expanded_groups

app/views/ci/lints/show.html.haml

@@ -18,6 +18,8 @@
       .col-sm-12
         .pull-left.prepend-top-10
           = submit_tag('Validate', class: 'btn btn-success submit-yml')
+        .pull-right.prepend-top-10
+          = button_tag('Clear', type: 'button', class: 'btn btn-default clear-yml')
   .row.prepend-top-20
     .col-sm-12

changelogs/unreleased/bvl-fix-concurrent-fork-network-migrations.yml

+---
+title: Avoid running `PopulateForkNetworksRange`-migration multiple times
+merge_request: 16988
+author:
+type: fixed

changelogs/unreleased/feature-26598-clear-button-ci-lint.yml

+---
+title: Added clear button to ci lint editor
+merge_request:
+author: Michael Robinson

changelogs/unreleased/feature-oidc-groups-claim.yml

+---
+title: Add groups to OpenID Connect claims
+merge_request: 16929
+author: Hassan Zamani

config/initializers/doorkeeper_openid_connect.rb

@@ -31,6 +31,7 @@ Doorkeeper::OpenidConnect.configure do
       o.claim(:website)        { |user| user.full_website_url if user.website_url? }
       o.claim(:profile)        { |user| Gitlab::Routing.url_helpers.user_url user }
       o.claim(:picture)        { |user| user.avatar_url(only_path: false) }
+      o.claim(:groups)         { |user| user.membership_groups.map(&:full_path) }
     end
   end
 end

config/locales/doorkeeper.en.yml

@@ -68,7 +68,7 @@ en:
       read_user:
         Read-only access to the user's profile information, like username, public email and full name
       openid:
-        The ability to authenticate using GitLab, and read-only access to the user's profile information
+        The ability to authenticate using GitLab, and read-only access to the user's profile information and group memberships
       sudo:
         Access to the Sudo feature, to perform API actions as any user in the system (only available for admins)
     flash:

db/migrate/20170929131201_populate_fork_networks.rb

@@ -6,22 +6,8 @@ class PopulateForkNetworks < ActiveRecord::Migration
   DOWNTIME = false
-  MIGRATION = 'PopulateForkNetworksRange'.freeze
-  BATCH_SIZE = 100
-  DELAY_INTERVAL = 15.seconds
-
-  disable_ddl_transaction!
-
-  class ForkedProjectLink < ActiveRecord::Base
-    include EachBatch
-
-    self.table_name = 'forked_project_links'
-  end
-
   def up
-    say 'Populating the `fork_networks` based on existing `forked_project_links`'
-
-    queue_background_migration_jobs_by_range_at_intervals(ForkedProjectLink, MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE)
+    say 'Fork networks will be populated in 20171205190711 - RescheduleForkNetworkCreationCaller'
   end
   def down

db/post_migrate/20171124150326_reschedule_fork_network_creation.rb

@@ -3,22 +3,8 @@ class RescheduleForkNetworkCreation < ActiveRecord::Migration
   DOWNTIME = false
-  MIGRATION = 'PopulateForkNetworksRange'.freeze
-  BATCH_SIZE = 100
-  DELAY_INTERVAL = 15.seconds
-
-  disable_ddl_transaction!
-
-  class ForkedProjectLink < ActiveRecord::Base
-    include EachBatch
-
-    self.table_name = 'forked_project_links'
-  end
-
   def up
-    say 'Populating the `fork_networks` based on existing `forked_project_links`'
-
-    queue_background_migration_jobs_by_range_at_intervals(ForkedProjectLink, MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE)
+    say 'Fork networks will be populated in 20171205190711 - RescheduleForkNetworkCreationCaller'
   end
   def down

doc/administration/repository_storage_types.md

@@ -4,50 +4,63 @@
 ## Legacy Storage
-Legacy Storage is the storage behavior prior to version 10.0. For historical reasons, GitLab replicated the same
-mapping structure from the projects URLs:
- * Project's repository: `#{namespace}/#{project_name}.git`
- * Project's wiki: `#{namespace}/#{project_name}.wiki.git`
-This structure made simple to migrate from existing solutions to GitLab and easy for Administrators to find where the
-repository is stored.
+Legacy Storage is the storage behavior prior to version 10.0. For historical
+reasons, GitLab replicated the same mapping structure from the projects URLs:
+* Project's repository: `#{namespace}/#{project_name}.git`
+* Project's wiki: `#{namespace}/#{project_name}.wiki.git`
+This structure made it simple to migrate from existing solutions to GitLab and
+easy for Administrators to find where the repository is stored.
 On the other hand this has some drawbacks:
-Storage location will concentrate huge amount of top-level namespaces. The impact can be reduced by the introduction of [multiple storage paths][storage-paths].
-Because Backups are a snapshot of the same URL mapping, if you try to recover a very old backup, you need to verify
-if any project has taken the place of an old removed project sharing the same URL. This means that `mygroup/myproject`
-from your backup may not be the same original project that is today in the same URL.
-Any change in the URL will need to be reflected on disk (when groups / users or projects are renamed). This can add a lot
-of load in big installations, and can be even worst if they are using any type of network based filesystem.
-Last, for GitLab Geo, this storage type means we have to synchronize the disk state, replicate renames in the correct
-order or we may end-up with wrong repository or missing data temporarily.
-This pattern also exists in other objects stored in GitLab, like issue Attachments, GitLab Pages artifacts,
-Docker Containers for the integrated Registry, etc.
+Storage location will concentrate huge amount of top-level namespaces. The
+impact can be reduced by the introduction of [multiple storage
+paths][storage-paths].
+Because backups are a snapshot of the same URL mapping, if you try to recover a
+very old backup, you need to verify whether any project has taken the place of
+an old removed or renamed project sharing the same URL. This means that
+`mygroup/myproject` from your backup may not be the same original project that
+is at that same URL today.
+Any change in the URL will need to be reflected on disk (when groups / users or
+projects are renamed). This can add a lot of load in big installations,
+especially if using any type of network based filesystem.
+For GitLab Geo in particular: Geo does work with legacy storage, but in some
+edge cases due to race conditions it can lead to errors when a project is
+renamed multiple times in short succession, or a project is deleted and
+recreated under the same name very quickly. We expect these race events to be
+rare, and we have not observed a race condition side-effect happening yet.
+This pattern also exists in other objects stored in GitLab, like issue
+Attachments, GitLab Pages artifacts, Docker Containers for the integrated
+Registry, etc.
 ## Hashed Storage
-Hashed Storage is the new storage behavior we are rolling out with 10.0. It's not enabled by default yet, but we
-encourage everyone to try-it and take the time to fix any script you may have that depends on the old behavior.
-Instead of coupling project URL and the folder structure where the repository will be stored on disk, we are coupling
-a hash, based on the project's ID.
-This makes the folder structure immutable, and therefore eliminates any requirement to synchronize state from URLs to
-disk structure. This means that renaming a group, user or project will cost only the database transaction, and will take
-effect immediately.
-The hash also helps to spread the repositories more evenly on the disk, so the top-level directory will contain less
-folders than the total amount of top-level namespaces.
-
-Hash format is based on hexadecimal representation of SHA256: `SHA256(project.id)`.
-Top-level folder uses first 2 characters, followed by another folder with the next 2 characters. They are both stored in
-a special folder `@hashed`, to co-exist with existing Legacy projects:
+> **Warning:** Hashed storage is in **Beta**. For the latest updates, check the
+> associated [issue](https://gitlab.com/gitlab-com/infrastructure/issues/2821)
+> and please report any problems you encounter.
+Hashed Storage is the new storage behavior we are rolling out with 10.0. Instead
+of coupling project URL and the folder structure where the repository will be
+stored on disk, we are coupling a hash, based on the project's ID. This makes
+the folder structure immutable, and therefore eliminates any requirement to
+synchronize state from URLs to disk structure. This means that renaming a group,
+user, or project will cost only the database transaction, and will take effect
+immediately.
+The hash also helps to spread the repositories more evenly on the disk, so the
+top-level directory will contain less folders than the total amount of top-level
+namespaces.
+The hash format is based on the hexadecimal representation of SHA256:
+`SHA256(project.id)`. The top-level folder uses the first 2 characters, followed
+by another folder with the next 2 characters. They are both stored in a special
+`@hashed` folder, to be able to co-exist with existing Legacy Storage projects:
 ```ruby
 # Project's repository:
@@ -57,15 +70,13 @@ a special folder `@hashed`, to co-exist with existing Legacy projects:
 "@hashed/#{hash[0..1]}/#{hash[2..3]}/#{hash}.wiki.git"
 ```
-This new format also makes possible to restore backups with confidence, as when restoring a repository from the backup,
-you will never mistakenly restore a repository in the wrong project (considering the backup is made after the migration).
-
 ### How to migrate to Hashed Storage
-In GitLab, go to **Admin > Settings**, find the **Repository Storage** section and select
-"_Create new projects using hashed storage paths_".
-To migrate your existing projects to the new storage type, check the specific [rake tasks].
+In GitLab, go to **Admin > Settings**, find the **Repository Storage** section
+and select "_Create new projects using hashed storage paths_".
+To migrate your existing projects to the new storage type, check the specific
+[rake tasks].
 [ce-28283]: https://gitlab.com/gitlab-org/gitlab-ce/issues/28283
 [rake tasks]: raketasks/storage.md#migrate-existing-projects-to-hashed-storage
@@ -73,11 +84,13 @@ To migrate your existing projects to the new storage type, check the specific [r
 ### Hashed Storage coverage
-We are incrementally moving every storable object in GitLab to the Hashed Storage pattern. You can check the current
-coverage status below.
-Note that things stored in an S3 compatible endpoint will not have the downsides mentioned earlier, if they are not
-prefixed with `#{namespace}/#{project_name}`, which is true for CI Cache and LFS Objects.
+We are incrementally moving every storable object in GitLab to the Hashed
+Storage pattern. You can check the current coverage status below (and also see
+the [issue](https://gitlab.com/gitlab-com/infrastructure/issues/2821)).
+Note that things stored in an S3 compatible endpoint will not have the downsides
+mentioned earlier, if they are not prefixed with `#{namespace}/#{project_name}`,
+which is true for CI Cache and LFS Objects.
 | Storable Object | Legacy Storage | Hashed Storage | S3 Compatible | GitLab Version |
 | --------------- | -------------- | -------------- | ------------- | -------------- |

doc/development/background_migrations.md

@@ -94,6 +94,18 @@ jobs = [['BackgroundMigrationClassName', [1]],
 BackgroundMigrationWorker.bulk_perform_in(5.minutes, jobs)
 ```
+### Rescheduling background migrations
+
+If one of the background migrations contains a bug that is fixed in a patch
+release, the background migration needs to be rescheduled so the migration would
+be repeated on systems that already performed the initial migration.
+
+When you reschedule the background migration, make sure to turn the original
+scheduling into a no-op by clearing up the `#up` and `#down` methods of the
+migration performing the scheduling. Otherwise the background migration would be
+scheduled multiple times on systems that are upgrading multiple patch releases at
+once.
+
 ## Cleaning Up
 >**Note:**

doc/development/i18n/externalization.md

@@ -126,6 +126,9 @@ strings and remove any strings that aren't used anymore. You should check this
 file in. Once the changes are on master, they will be picked up by
 [Crowdin](http://translate.gitlab.com) and be presented for translation.
+If there are merge conflicts in the `gitlab.pot` file, you can delete the file
+and regenerate it using the same command. Confirm that you are not deleting any strings accidentally by looking over the diff.
+
 The command also updates the translation files for each language: `locale/*/gitlab.po`
 These changes can be discarded, the languange files will be updated by Crowdin
 automatically.

doc/install/installation.md

@@ -299,9 +299,9 @@ sudo usermod -aG redis git
 ### Clone the Source
     # Clone GitLab repository
-    sudo -u git -H git clone https://gitlab.com/gitlab-org/gitlab-ce.git -b 10-4-stable gitlab
-**Note:** You can change `10-4-stable` to `master` if you want the *bleeding edge* version, but never install master on a production server!
+    sudo -u git -H git clone https://gitlab.com/gitlab-org/gitlab-ce.git -b 10-5-stable gitlab
+**Note:** You can change `10-5-stable` to `master` if you want the *bleeding edge* version, but never install master on a production server!
 ### Configure It