Add latest changes from gitlab-org/gitlab@master

doc/administration/packages/container_registry.md

@@ -353,10 +353,6 @@ configuring a different storage driver. By default the GitLab Container Registry
 is configured to use the filesystem driver, which makes use of [storage path](#container-registry-storage-path)
 configuration.
-NOTE: **Note:** Enabling a storage driver other than `filesystem` would mean
-that your Docker client needs to be able to access the storage backend directly.
-In that case, you must use an address that resolves and is accessible outside GitLab server. The Docker client will continue to authenticate via GitLab but data transfer will be direct to and from the storage backend.
-
 The different supported drivers are:
 | Driver     | Description                         |
@@ -425,6 +421,55 @@ storage:
 NOTE: **Note:**
 `your-s3-bucket` should only be the name of a bucket that exists, and can't include subdirectories.
+### Disable redirect for storage driver
+
+By default, users accessing a registry configured with a remote backend are redirected to the default backend for the storage driver. For example, registries can be configured using the `s3` storage driver, which redirects requests to a remote S3 bucket to alleviate load on the GitLab server.
+
+However, this behaviour is undesirable for registries used by internal hosts that usually can't access public servers. To disable redirects, set the `disable` flag to true as follows. This makes all traffic to always go through the Registry service. This results in improved security (less surface attack as the storage backend is not publicly accessible), but worse performance (all traffic is redirected via the service).
+
+**Omnibus GitLab installations**
+
+1. Edit `/etc/gitlab/gitlab.rb`:
+
+    ```ruby
+    registry['storage'] = {
+      's3' => {
+        'accesskey' => 's3-access-key',
+        'secretkey' => 's3-secret-key-for-access-key',
+        'bucket' => 'your-s3-bucket',
+        'region' => 'your-s3-region',
+        'regionendpoint' => 'your-s3-regionendpoint'
+      },
+      'redirect' => {
+        'disable' => true
+      }
+    }
+    ```
+
+1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect.
+
+**Installations from source**
+
+1. Add the `redirect` flag to your registry configuration YML file:
+
+    ```yml
+    storage:
+      s3:
+        accesskey: 'AKIAKIAKI'
+        secretkey: 'secret123'
+        bucket: 'gitlab-registry-bucket-AKIAKIAKI'
+        region: 'your-s3-region'
+        regionendpoint: 'your-s3-regionendpoint'
+      redirect:
+        disable: true
+      cache:
+        blobdescriptor: inmemory
+      delete:
+        enabled: true
+    ```
+
+1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source) for the changes to take effect.
+
 ### Storage limitations
 Currently, there is no storage limitation, which means a user can upload an

doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md

@@ -840,7 +840,7 @@ GitLab Rails console:
 ```ruby
 projects_and_size = []
-# a list of projects you want to look at, can get these however
+# You need to specify the projects that you want to look through. You can get these in any manner.
 projects = Project.last(100)
 projects.each do |p|

doc/api/graphql/reference/gitlab_schema.graphql

@@ -7355,6 +7355,11 @@ type Snippet implements Noteable {
   """
   fileName: String
+  """
+  HTTP URL to the snippet repository
+  """
+  httpUrlToRepo: String
+
   """
   Id of the snippet
   """
@@ -7395,6 +7400,11 @@ type Snippet implements Noteable {
   """
   rawUrl: String!
+  """
+  SSH URL to the snippet repository
+  """
+  sshUrlToRepo: String
+
   """
   Title of the snippet
   """

doc/api/graphql/reference/gitlab_schema.json

@@ -22213,6 +22213,20 @@
               "isDeprecated": false,
               "deprecationReason": null
             },
+            {
+              "name": "httpUrlToRepo",
+              "description": "HTTP URL to the snippet repository",
+              "args": [
+
+              ],
+              "type": {
+                "kind": "SCALAR",
+                "name": "String",
+                "ofType": null
+              },
+              "isDeprecated": false,
+              "deprecationReason": null
+            },
             {
               "name": "id",
               "description": "Id of the snippet",
@@ -22320,6 +22334,20 @@
               "isDeprecated": false,
               "deprecationReason": null
             },
+            {
+              "name": "sshUrlToRepo",
+              "description": "SSH URL to the snippet repository",
+              "args": [
+
+              ],
+              "type": {
+                "kind": "SCALAR",
+                "name": "String",
+                "ofType": null
+              },
+              "isDeprecated": false,
+              "deprecationReason": null
+            },
             {
               "name": "title",
               "description": "Title of the snippet",

doc/api/graphql/reference/index.md

@@ -1161,9 +1161,11 @@ Represents a snippet entry
 | `description` | String | Description of the snippet |
 | `descriptionHtml` | String | The GitLab Flavored Markdown rendering of `description` |
 | `fileName` | String | File Name of the snippet |
+| `httpUrlToRepo` | String | HTTP URL to the snippet repository |
 | `id` | ID! | Id of the snippet |
 | `project` | Project | The project the snippet is associated with |
 | `rawUrl` | String! | Raw URL of the snippet |
+| `sshUrlToRepo` | String | SSH URL to the snippet repository |
 | `title` | String! | Title of the snippet |
 | `updatedAt` | Time! | Timestamp this snippet was updated |
 | `userPermissions` | SnippetPermissions! | Permissions for the current user on the resource |

doc/security/README.md

@@ -19,6 +19,7 @@ type: index
 - [Send email confirmation on sign-up](user_email_confirmation.md)
 - [Security of running jobs](https://docs.gitlab.com/runner/security/)
 - [Proxying images](asset_proxy.md)
+- [CI/CD environment variables](cicd_environment_variables.md)
 ## Securing your GitLab installation

doc/security/cicd_environment_variables.md

+---
+type: reference
+---
+
+# CI/CD Environment Variables
+
+Environment variables are applied to environments via the runner and can be set from the project's **Settings > CI/CD** page.
+
+The values are encrypted using [aes-256-cbc](https://en.wikipedia.org/wiki/Advanced_Encryption_Standard) and stored in the database.
+
+This data can only be decrypted with a valid [secrets file](../raketasks/backup_restore.md#when-the-secrets-file-is-lost).

doc/user/admin_area/activating_deactivating_users.md

@@ -36,7 +36,7 @@ A user can be deactivated from the Admin Area. To do this:
 Please note that for the deactivation option to be visible to an admin, the user:
 - Must be currently active.
-- Should not have any activity in the last 180 days.
+- Must not have any signins or activity in the last 180 days.
 Users can also be deactivated using the [GitLab API](../../api/users.md#deactivate-user).

lib/api/entities/snippet.rb

@@ -10,6 +10,7 @@ module API
       expose :web_url do |snippet|
         Gitlab::UrlBuilder.build(snippet)
       end
+      expose :ssh_url_to_repo, :http_url_to_repo, if: ->(snippet) { snippet.versioned_enabled_for?(options[:current_user]) }
     end
   end
 end

lib/gitlab/metrics/dashboard/stages/project_metrics_inserter.rb

@@ -75,6 +75,7 @@ module Gitlab
           def find_metric(metrics, metric)
             return unless metrics
+            return unless metric.identifier
             metrics.find { |m| m[:id] == metric.identifier }
           end

locale/gitlab.pot

@@ -4126,6 +4126,9 @@ msgstr ""
 msgid "ClusterIntegration|Base domain"
 msgstr ""
+msgid "ClusterIntegration|Blocking mode"
+msgstr ""
+
 msgid "ClusterIntegration|CA Certificate"
 msgstr ""
@@ -4324,6 +4327,9 @@ msgstr ""
 msgid "ClusterIntegration|Gitlab Integration"
 msgstr ""
+msgid "ClusterIntegration|Global default"
+msgstr ""
+
 msgid "ClusterIntegration|Google Cloud Platform project"
 msgstr ""
@@ -4483,6 +4489,9 @@ msgstr ""
 msgid "ClusterIntegration|Loading subnetworks"
 msgstr ""
+msgid "ClusterIntegration|Logging mode"
+msgstr ""
+
 msgid "ClusterIntegration|Machine type"
 msgstr ""
@@ -4711,6 +4720,9 @@ msgstr ""
 msgid "ClusterIntegration|Set a prefix for your namespaces. If not set, defaults to your project path. If modified, existing environments will use their current namespaces until the cluster cache is cleared."
 msgstr ""
+msgid "ClusterIntegration|Set the global mode for the WAF in this cluster. This can be overridden at the environmental level."
+msgstr ""
+
 msgid "ClusterIntegration|Show"
 msgstr ""

spec/fixtures/api/schemas/cluster_status.json

@@ -38,6 +38,7 @@
         "email": { "type": ["string", "null"] },
         "stack": { "type": ["string", "null"] },
         "modsecurity_enabled": { "type": ["boolean", "null"] },
+        "modsecurity_mode": {"type": ["integer", "0"]},
         "update_available": { "type": ["boolean", "null"] },
         "can_uninstall": { "type": "boolean" },
         "available_domains": {

spec/frontend/clusters/components/ingress_modsecurity_settings_spec.js

 import { shallowMount } from '@vue/test-utils';
 import IngressModsecuritySettings from '~/clusters/components/ingress_modsecurity_settings.vue';
 import { APPLICATION_STATUS, INGRESS } from '~/clusters/constants';
-import { GlAlert, GlToggle } from '@gitlab/ui';
+import { GlAlert, GlToggle, GlDropdown } from '@gitlab/ui';
 import eventHub from '~/clusters/event_hub';
 const { UPDATING } = APPLICATION_STATUS;
@@ -13,6 +13,7 @@ describe('IngressModsecuritySettings', () => {
     modsecurity_enabled: false,
     status: 'installable',
     installed: false,
+    modsecurity_mode: 'logging',
   };
   const createComponent = (props = defaultProps) => {
@@ -29,6 +30,7 @@ describe('IngressModsecuritySettings', () => {
   const findSaveButton = () => wrapper.find('.btn-success');
   const findCancelButton = () => wrapper.find('[variant="secondary"]');
   const findModSecurityToggle = () => wrapper.find(GlToggle);
+  const findModSecurityDropdown = () => wrapper.find(GlDropdown);
   describe('when ingress is installed', () => {
     beforeEach(() => {
@@ -44,22 +46,50 @@ describe('IngressModsecuritySettings', () => {
     describe('with toggle changed by the user', () => {
       beforeEach(() => {
         findModSecurityToggle().vm.$emit('change');
+        wrapper.setProps({
+          ingress: {
+            ...defaultProps,
+            installed: true,
+            status: 'installed',
+            modsecurity_enabled: true,
+          },
+        });
       });
-      it('renders both save and cancel buttons', () => {
+      it('renders save and cancel buttons', () => {
         expect(findSaveButton().exists()).toBe(true);
         expect(findCancelButton().exists()).toBe(true);
       });
-      describe('and the save changes button is clicked', () => {
+      describe('with dropdown changed by the user', () => {
         beforeEach(() => {
-          findSaveButton().vm.$emit('click');
+          findModSecurityDropdown().vm.$children[1].$emit('click');
+          wrapper.setProps({
+            ingress: {
+              ...defaultProps,
+              installed: true,
+              status: 'installed',
+              modsecurity_enabled: true,
+              modsecurity_mode: 'blocking',
+            },
+          });
+        });
+
+        it('renders both save and cancel buttons', () => {
+          expect(findSaveButton().exists()).toBe(true);
+          expect(findCancelButton().exists()).toBe(true);
         });
-        it('triggers save event and pass current modsecurity value', () => {
-          expect(eventHub.$emit).toHaveBeenCalledWith('updateApplication', {
-            id: INGRESS,
-            params: { modsecurity_enabled: false },
+        describe('and the save changes button is clicked', () => {
+          beforeEach(() => {
+            findSaveButton().vm.$emit('click');
+          });
+
+          it('triggers save event and pass current modsecurity value', () => {
+            expect(eventHub.$emit).toHaveBeenCalledWith('updateApplication', {
+              id: INGRESS,
+              params: { modsecurity_enabled: true, modsecurity_mode: 'blocking' },
+            });
           });
         });
       });
@@ -70,7 +100,7 @@ describe('IngressModsecuritySettings', () => {
         });
         it('triggers reset event and hides both cancel and save changes button', () => {
-          expect(eventHub.$emit).toHaveBeenCalledWith('resetIngressModSecurityEnabled', INGRESS);
+          expect(eventHub.$emit).toHaveBeenCalledWith('resetIngressModSecurityChanges', INGRESS);
           expect(findSaveButton().exists()).toBe(false);
           expect(findCancelButton().exists()).toBe(false);
         });

spec/frontend/clusters/stores/clusters_store_spec.js

@@ -82,6 +82,7 @@ describe('Clusters Store', () => {
             externalHostname: null,
             installed: false,
             isEditingModSecurityEnabled: false,
+            isEditingModSecurityMode: false,
             installFailed: true,
             uninstallable: false,
             updateFailed: false,
@@ -89,6 +90,7 @@ describe('Clusters Store', () => {
             uninstallFailed: false,
             validationError: null,
             modsecurity_enabled: false,
+            modsecurity_mode: undefined,
           },
           runner: {
             title: 'GitLab Runner',

spec/graphql/types/snippet_type_spec.rb

@@ -3,12 +3,15 @@
 require 'spec_helper'
 describe GitlabSchema.types['Snippet'] do
+  let_it_be(:user) { create(:user) }
+
   it 'has the correct fields' do
     expected_fields = [:id, :title, :project, :author,
                        :file_name, :description,
                        :visibility_level, :created_at, :updated_at,
-                       :web_url, :raw_url, :notes, :discussions,
-                       :user_permissions, :description_html, :blob]
+                       :web_url, :raw_url, :ssh_url_to_repo, :http_url_to_repo,
+                       :notes, :discussions, :user_permissions,
+                       :description_html, :blob]
     expect(described_class).to have_graphql_fields(*expected_fields)
   end
@@ -17,8 +20,55 @@ describe GitlabSchema.types['Snippet'] do
     it { expect(described_class).to require_graphql_authorizations(:read_snippet) }
   end
+  shared_examples 'response without repository URLs' do
+    it 'does not respond with repository URLs' do
+      expect(response['sshUrlToRepo']).to be_nil
+      expect(response['httpUrlToRepo']).to be_nil
+    end
+  end
+
+  describe 'Repository URLs' do
+    let(:query) do
+      %(
+        {
+          snippets {
+            nodes {
+              sshUrlToRepo
+              httpUrlToRepo
+            }
+          }
+        }
+      )
+    end
+    let(:response) { subject.dig('data', 'snippets', 'nodes')[0] }
+
+    subject { GitlabSchema.execute(query, context: { current_user: user }).as_json }
+
+    context 'when snippet has repository' do
+      let!(:snippet) { create(:personal_snippet, :repository, :public, author: user) }
+
+      it 'responds with repository URLs' do
+        expect(response['sshUrlToRepo']).to eq(snippet.ssh_url_to_repo)
+        expect(response['httpUrlToRepo']).to eq(snippet.http_url_to_repo)
+      end
+
+      context 'when version_snippets feature is disabled' do
+        before do
+          stub_feature_flags(version_snippets: false)
+        end
+
+        it_behaves_like 'response without repository URLs'
+      end
+    end
+
+    context 'when snippet does not have a repository' do
+      let!(:snippet) { create(:personal_snippet, :public, author: user) }
+
+      it_behaves_like 'response without repository URLs'
+    end
+  end
+
   describe '#blob' do
-    let_it_be(:user) { create(:user) }
     let(:query_blob) { subject.dig('data', 'snippets', 'edges')[0]['node']['blob'] }
     let(:query) do
       %(

spec/lib/gitlab/metrics/dashboard/processor_spec.rb

@@ -74,6 +74,16 @@ describe Gitlab::Metrics::Dashboard::Processor do
         expect(actual_metrics_order).to eq expected_metrics_order
       end
+      context 'when the project has multiple metrics in the same group' do
+        let!(:project_response_metric) { create(:prometheus_metric, project: project, group: :response) }
+        let!(:project_response_metric_2) { create(:prometheus_metric, project: project, group: :response) }
+
+        it 'includes multiple metrics' do
+          expect(all_metrics).to include get_metric_details(project_response_metric)
+          expect(all_metrics).to include get_metric_details(project_response_metric_2)
+        end
+      end
+
       context 'when the dashboard should not include project metrics' do
         let(:sequence) do
           [

spec/models/clusters/applications/ingress_spec.rb

@@ -140,13 +140,10 @@ describe Clusters::Applications::Ingress do
   end
   describe '#values' do
-    let(:project) { build(:project) }
-    let(:cluster) { build(:cluster, projects: [project]) }
+    subject { ingress }
     context 'when modsecurity_enabled is enabled' do
       before do
-        allow(subject).to receive(:cluster).and_return(cluster)
-
         allow(subject).to receive(:modsecurity_enabled).and_return(true)
       end
@@ -154,8 +151,24 @@ describe Clusters::Applications::Ingress do
         expect(subject.values).to include("enable-modsecurity: 'true'")
       end
-      it 'includes modsecurity core ruleset enablement' do
-        expect(subject.values).to include("enable-owasp-modsecurity-crs: 'true'")
+      it 'includes modsecurity core ruleset enablement set to false' do
+        expect(subject.values).to include("enable-owasp-modsecurity-crs: 'false'")
+      end
+
+      it 'includes modsecurity snippet with information related to security rules' do
+        expect(subject.values).to include("SecRuleEngine DetectionOnly")
+        expect(subject.values).to include("Include #{described_class::MODSECURITY_OWASP_RULES_FILE}")
+      end
+
+      context 'when modsecurity_mode is set to :blocking' do
+        before do
+          subject.blocking!
+        end
+
+        it 'includes modsecurity snippet with information related to security rules' do
+          expect(subject.values).to include("SecRuleEngine On")
+          expect(subject.values).to include("Include #{described_class::MODSECURITY_OWASP_RULES_FILE}")
+        end
       end
       it 'includes modsecurity.conf content' do
@@ -176,7 +189,6 @@ describe Clusters::Applications::Ingress do
     context 'when modsecurity_enabled is disabled' do
       before do
-        allow(subject).to receive(:cluster).and_return(cluster)
         allow(subject).to receive(:modsecurity_enabled).and_return(false)
       end

spec/models/snippet_spec.rb

@@ -713,4 +713,32 @@ describe Snippet do
       it { is_expected.to eq(Gitlab.config.gitlab_shell.ssh_path_prefix + "#{snippet.project.full_path}/snippets/#{snippet.id}.git") }
     end
   end
+
+  describe '#versioned_enabled_for?' do
+    let_it_be(:user) { create(:user) }
+
+    subject { snippet.versioned_enabled_for?(user) }
+
+    context 'with repository and version_snippets enabled' do
+      let!(:snippet) { create(:personal_snippet, :repository, author: user) }
+
+      it { is_expected.to be_truthy }
+    end
+
+    context 'without repository' do
+      let!(:snippet) { create(:personal_snippet, author: user) }
+
+      it { is_expected.to be_falsy }
+    end
+
+    context 'without version_snippets feature disabled' do
+      let!(:snippet) { create(:personal_snippet, :repository, author: user) }
+
+      before do
+        stub_feature_flags(version_snippets: false)
+      end
+
+      it { is_expected.to be_falsy }
+    end
+  end
 end

spec/requests/api/project_snippets_spec.rb

@@ -85,7 +85,7 @@ describe API::ProjectSnippets do
   describe 'GET /projects/:project_id/snippets/:id' do
     let(:user) { create(:user) }
-    let(:snippet) { create(:project_snippet, :public, project: project) }
+    let(:snippet) { create(:project_snippet, :public, :repository, project: project) }
     it 'returns snippet json' do
       get api("/projects/#{project.id}/snippets/#{snippet.id}", user)
@@ -95,6 +95,18 @@ describe API::ProjectSnippets do
       expect(json_response['title']).to eq(snippet.title)
       expect(json_response['description']).to eq(snippet.description)
       expect(json_response['file_name']).to eq(snippet.file_name)
+      expect(json_response['ssh_url_to_repo']).to eq(snippet.ssh_url_to_repo)
+      expect(json_response['http_url_to_repo']).to eq(snippet.http_url_to_repo)
+    end
+
+    context 'when feature flag :version_snippets is disabled' do
+      before do
+        stub_feature_flags(version_snippets: false)
+
+        get api("/projects/#{project.id}/snippets/#{snippet.id}", user)
+      end
+
+      it_behaves_like 'snippet response without repository URLs'
     end
     it 'returns 404 for invalid snippet id' do

spec/requests/api/snippets_spec.rb

@@ -139,8 +139,8 @@ describe API::Snippets do
   describe 'GET /snippets/:id' do
     let_it_be(:admin) { create(:user, :admin) }
     let_it_be(:author) { create(:user) }
-    let_it_be(:private_snippet) { create(:personal_snippet, :private, author: author) }
-    let_it_be(:internal_snippet) { create(:personal_snippet, :internal, author: author) }
+    let_it_be(:private_snippet) { create(:personal_snippet, :repository, :private, author: author) }
+    let_it_be(:internal_snippet) { create(:personal_snippet, :repository, :internal, author: author) }
     it 'requires authentication' do
       get api("/snippets/#{private_snippet.id}", nil)
@@ -157,6 +157,18 @@ describe API::Snippets do
       expect(json_response['description']).to eq(private_snippet.description)
       expect(json_response['file_name']).to eq(private_snippet.file_name)
       expect(json_response['visibility']).to eq(private_snippet.visibility)
+      expect(json_response['ssh_url_to_repo']).to eq(private_snippet.ssh_url_to_repo)
+      expect(json_response['http_url_to_repo']).to eq(private_snippet.http_url_to_repo)
+    end
+
+    context 'when feature flag :version_snippets is disabled' do
+      before do
+        stub_feature_flags(version_snippets: false)
+
+        get api("/snippets/#{private_snippet.id}", author)
+      end
+
+      it_behaves_like 'snippet response without repository URLs'
     end
     it 'shows private snippets to an admin' do