Add latest changes from gitlab-org/gitlab@master

.gitlab/ci/docs.gitlab-ci.yml

@@ -67,7 +67,7 @@ docs lint:
     # Check the internal anchor links
     - bundle exec nanoc check internal_anchors
-graphql-docs-verify:
+graphql-reference-verify:
   extends:
     - .only-ee
     - .default-tags
@@ -82,3 +82,4 @@ graphql-docs-verify:
   needs: ["setup-test-env"]
   script:
     - bundle exec rake gitlab:graphql:check_docs
+    - bundle exec rake gitlab:graphql:check_schema

app/controllers/projects/releases_controller.rb

@@ -3,10 +3,12 @@
 class Projects::ReleasesController < Projects::ApplicationController
   # Authorize
   before_action :require_non_empty_project, except: [:index]
+  before_action :release, only: %i[edit update]
   before_action :authorize_read_release!
   before_action do
     push_frontend_feature_flag(:release_edit_page, project)
   end
+  before_action :authorize_update_release!, only: %i[edit update]
   def index
     respond_to do |format|
@@ -22,4 +24,25 @@ class Projects::ReleasesController < Projects::ApplicationController
   def releases
     ReleasesFinder.new(@project, current_user).execute
   end
+
+  def edit
+    respond_to do |format|
+      format.html { render 'edit' }
+    end
+  end
+
+  private
+
+  def authorize_update_release!
+    access_denied! unless Feature.enabled?(:release_edit_page, project)
+    access_denied! unless can?(current_user, :update_release, release)
+  end
+
+  def release
+    @release ||= project.releases.find_by_tag!(sanitized_tag_name)
+  end
+
+  def sanitized_tag_name
+    CGI.unescape(params[:tag])
+  end
 end

app/finders/releases_finder.rb

@@ -6,9 +6,11 @@ class ReleasesFinder
     @current_user = current_user
   end
-  def execute
+  def execute(preload: true)
     return Release.none unless Ability.allowed?(@current_user, :read_release, @project)
-    @project.releases.sorted
+    releases = @project.releases
+    releases = releases.preloaded if preload
+    releases.sorted
   end
 end

app/models/clusters/providers/aws.rb

@@ -3,6 +3,7 @@
 module Clusters
   module Providers
     class Aws < ApplicationRecord
+      include Gitlab::Utils::StrongMemoize
       include Clusters::Concerns::ProviderStatus
       self.table_name = 'cluster_providers_aws'
@@ -42,6 +43,18 @@ module Clusters
           session_token: nil
         )
       end
+
+      def api_client
+        strong_memoize(:api_client) do
+          ::Aws::CloudFormation::Client.new(credentials: credentials, region: region)
+        end
+      end
+
+      def credentials
+        strong_memoize(:credentials) do
+          ::Aws::Credentials.new(access_key_id, secret_access_key, session_token)
+        end
+      end
     end
   end
 end

app/models/release.rb

@@ -27,13 +27,17 @@ class Release < ApplicationRecord
   validates_associated :milestone_releases, message: -> (_, obj) { obj[:value].map(&:errors).map(&:full_messages).join(",") }
   scope :sorted, -> { order(released_at: :desc) }
-  scope :with_project_and_namespace, -> { includes(project: :namespace) }
+  scope :preloaded, -> { includes(project: :namespace) }
   delegate :repository, to: :project
   after_commit :create_evidence!, on: :create
   after_commit :notify_new_release, on: :create
+  def to_param
+    CGI.escape(tag)
+  end
+
   def commit
     strong_memoize(:commit) do
       repository.commit(actual_sha)

app/presenters/release_presenter.rb

@@ -31,6 +31,12 @@ class ReleasePresenter < Gitlab::View::Presenter::Delegated
     project_issues_url(project, params_for_issues_and_mrs)
   end
+  def edit_url
+    return unless release_edit_page_available?
+
+    edit_project_release_url(project, release)
+  end
+
   private
   def can_download_code?
@@ -44,4 +50,8 @@ class ReleasePresenter < Gitlab::View::Presenter::Delegated
   def release_mr_issue_urls_available?
     ::Feature.enabled?(:release_mr_issue_urls, project)
   end
+
+  def release_edit_page_available?
+    ::Feature.enabled?(:release_edit_page, project)
+  end
 end

app/services/clusters/aws/fetch_credentials_service.rb

+# frozen_string_literal: true
+
+module Clusters
+  module Aws
+    class FetchCredentialsService
+      attr_reader :provider
+
+      MissingRoleError = Class.new(StandardError)
+
+      def initialize(provider)
+        @provider = provider
+      end
+
+      def execute
+        raise MissingRoleError.new('AWS provisioning role not configured') unless provision_role.present?
+
+        ::Aws::AssumeRoleCredentials.new(
+          client: client,
+          role_arn: provision_role.role_arn,
+          role_session_name: session_name,
+          external_id: provision_role.role_external_id
+        ).credentials
+      end
+
+      private
+
+      def provision_role
+        provider.created_by_user.aws_role
+      end
+
+      def client
+        ::Aws::STS::Client.new(credentials: gitlab_credentials, region: provider.region)
+      end
+
+      def gitlab_credentials
+        ::Aws::Credentials.new(access_key_id, secret_access_key)
+      end
+
+      ##
+      # This setting is not yet configurable or documented as these
+      # services are not currently used. This will be addressed in
+      # https://gitlab.com/gitlab-org/gitlab/merge_requests/18307
+      def access_key_id
+        Gitlab.config.kubernetes.provisioners.aws.access_key_id
+      end
+
+      ##
+      # This setting is not yet configurable or documented as these
+      # services are not currently used. This will be addressed in
+      # https://gitlab.com/gitlab-org/gitlab/merge_requests/18307
+      def secret_access_key
+        Gitlab.config.kubernetes.provisioners.aws.secret_access_key
+      end
+
+      def session_name
+        "gitlab-eks-cluster-#{provider.cluster_id}-user-#{provider.created_by_user_id}"
+      end
+    end
+  end
+end

app/services/clusters/aws/finalize_creation_service.rb

+# frozen_string_literal: true
+
+module Clusters
+  module Aws
+    class FinalizeCreationService
+      include Gitlab::Utils::StrongMemoize
+
+      attr_reader :provider
+
+      delegate :cluster, to: :provider
+
+      def execute(provider)
+        @provider = provider
+
+        configure_provider
+        create_gitlab_service_account!
+        configure_platform_kubernetes
+        configure_node_authentication!
+
+        cluster.save!
+      rescue ::Aws::CloudFormation::Errors::ServiceError => e
+        log_service_error(e.class.name, provider.id, e.message)
+        provider.make_errored!(s_('ClusterIntegration|Failed to fetch CloudFormation stack: %{message}') % { message: e.message })
+      rescue Kubeclient::HttpError => e
+        log_service_error(e.class.name, provider.id, e.message)
+        provider.make_errored!(s_('ClusterIntegration|Failed to run Kubeclient: %{message}') % { message: e.message })
+      rescue ActiveRecord::RecordInvalid => e
+        log_service_error(e.class.name, provider.id, e.message)
+        provider.make_errored!(s_('ClusterIntegration|Failed to configure EKS provider: %{message}') % { message: e.message })
+      end
+
+      private
+
+      def create_gitlab_service_account!
+        Clusters::Kubernetes::CreateOrUpdateServiceAccountService.gitlab_creator(
+          kube_client,
+          rbac: true
+        ).execute
+      end
+
+      def configure_provider
+        provider.status_event = :make_created
+      end
+
+      def configure_platform_kubernetes
+        cluster.build_platform_kubernetes(
+          api_url: cluster_endpoint,
+          ca_cert: cluster_certificate,
+          token: request_kubernetes_token)
+      end
+
+      def request_kubernetes_token
+        Clusters::Kubernetes::FetchKubernetesTokenService.new(
+          kube_client,
+          Clusters::Kubernetes::GITLAB_ADMIN_TOKEN_NAME,
+          Clusters::Kubernetes::GITLAB_SERVICE_ACCOUNT_NAMESPACE
+        ).execute
+      end
+
+      def kube_client
+        @kube_client ||= build_kube_client!(
+          cluster_endpoint,
+          cluster_certificate
+        )
+      end
+
+      def build_kube_client!(api_url, ca_pem)
+        raise "Incomplete settings" unless api_url
+
+        Gitlab::Kubernetes::KubeClient.new(
+          api_url,
+          auth_options: kubeclient_auth_options,
+          ssl_options: kubeclient_ssl_options(ca_pem),
+          http_proxy_uri: ENV['http_proxy']
+        )
+      end
+
+      def kubeclient_auth_options
+        { bearer_token: Kubeclient::AmazonEksCredentials.token(provider.credentials, cluster.name) }
+      end
+
+      def kubeclient_ssl_options(ca_pem)
+        opts = { verify_ssl: OpenSSL::SSL::VERIFY_PEER }
+
+        if ca_pem.present?
+          opts[:cert_store] = OpenSSL::X509::Store.new
+          opts[:cert_store].add_cert(OpenSSL::X509::Certificate.new(ca_pem))
+        end
+
+        opts
+      end
+
+      def cluster_stack
+        @cluster_stack ||= provider.api_client.describe_stacks(stack_name: provider.cluster.name).stacks.first
+      end
+
+      def stack_output_value(key)
+        cluster_stack.outputs.detect { |output| output.output_key == key }.output_value
+      end
+
+      def node_instance_role_arn
+        stack_output_value('NodeInstanceRole')
+      end
+
+      def cluster_endpoint
+        strong_memoize(:cluster_endpoint) do
+          stack_output_value('ClusterEndpoint')
+        end
+      end
+
+      def cluster_certificate
+        strong_memoize(:cluster_certificate) do
+          Base64.decode64(stack_output_value('ClusterCertificate'))
+        end
+      end
+
+      def configure_node_authentication!
+        kube_client.create_config_map(node_authentication_config)
+      end
+
+      def node_authentication_config
+        Gitlab::Kubernetes::ConfigMaps::AwsNodeAuth.new(node_instance_role_arn).generate
+      end
+
+      def logger
+        @logger ||= Gitlab::Kubernetes::Logger.build
+      end
+
+      def log_service_error(exception, provider_id, message)
+        logger.error(
+          exception: exception.class.name,
+          service: self.class.name,
+          provider_id: provider_id,
+          message: message
+        )
+      end
+    end
+  end
+end

app/services/clusters/aws/provision_service.rb

+# frozen_string_literal: true
+
+module Clusters
+  module Aws
+    class ProvisionService
+      attr_reader :provider
+
+      def execute(provider)
+        @provider = provider
+
+        configure_provider_credentials
+        provision_cluster
+
+        if provider.make_creating
+          WaitForClusterCreationWorker.perform_in(
+            Clusters::Aws::VerifyProvisionStatusService::INITIAL_INTERVAL,
+            provider.cluster_id
+          )
+        else
+          provider.make_errored!("Failed to update provider record; #{provider.errors.full_messages}")
+        end
+      rescue Clusters::Aws::FetchCredentialsService::MissingRoleError
+        provider.make_errored!('Amazon role is not configured')
+      rescue ::Aws::Errors::MissingCredentialsError, Settingslogic::MissingSetting
+        provider.make_errored!('Amazon credentials are not configured')
+      rescue ::Aws::STS::Errors::ServiceError => e
+        provider.make_errored!("Amazon authentication failed; #{e.message}")
+      rescue ::Aws::CloudFormation::Errors::ServiceError => e
+        provider.make_errored!("Amazon CloudFormation request failed; #{e.message}")
+      end
+
+      private
+
+      def credentials
+        @credentials ||= Clusters::Aws::FetchCredentialsService.new(provider).execute
+      end
+
+      def configure_provider_credentials
+        provider.update!(
+          access_key_id: credentials.access_key_id,
+          secret_access_key: credentials.secret_access_key,
+          session_token: credentials.session_token
+        )
+      end
+
+      def provision_cluster
+        provider.api_client.create_stack(
+          stack_name: provider.cluster.name,
+          template_body: stack_template,
+          parameters: parameters,
+          capabilities: ["CAPABILITY_IAM"]
+        )
+      end
+
+      def parameters
+        [
+          parameter('ClusterName', provider.cluster.name),
+          parameter('ClusterRole', provider.role_arn),
+          parameter('ClusterControlPlaneSecurityGroup', provider.security_group_id),
+          parameter('VpcId', provider.vpc_id),
+          parameter('Subnets', provider.subnet_ids.join(',')),
+          parameter('NodeAutoScalingGroupDesiredCapacity', provider.num_nodes.to_s),
+          parameter('NodeInstanceType', provider.instance_type),
+          parameter('KeyName', provider.key_name)
+        ]
+      end
+
+      def parameter(key, value)
+        { parameter_key: key, parameter_value: value }
+      end
+
+      def stack_template
+        File.read(Rails.root.join('vendor', 'aws', 'cloudformation', 'eks_cluster.yaml'))
+      end
+    end
+  end
+end

app/services/clusters/aws/verify_provision_status_service.rb

+# frozen_string_literal: true
+
+module Clusters
+  module Aws
+    class VerifyProvisionStatusService
+      attr_reader :provider
+
+      INITIAL_INTERVAL = 5.minutes
+      POLL_INTERVAL = 1.minute
+      TIMEOUT = 30.minutes
+
+      def execute(provider)
+        @provider = provider
+
+        case cluster_stack.stack_status
+        when 'CREATE_IN_PROGRESS'
+          continue_creation
+        when 'CREATE_COMPLETE'
+          finalize_creation
+        else
+          provider.make_errored!("Unexpected status; #{cluster_stack.stack_status}")
+        end
+      rescue ::Aws::CloudFormation::Errors::ServiceError => e
+        provider.make_errored!("Amazon CloudFormation request failed; #{e.message}")
+      end
+
+      private
+
+      def cluster_stack
+        @cluster_stack ||= provider.api_client.describe_stacks(stack_name: provider.cluster.name).stacks.first
+      end
+
+      def continue_creation
+        if timeout_threshold.future?
+          WaitForClusterCreationWorker.perform_in(POLL_INTERVAL, provider.cluster_id)
+        else
+          provider.make_errored!(_('Kubernetes cluster creation time exceeds timeout; %{timeout}') % { timeout: TIMEOUT })
+        end
+      end
+
+      def timeout_threshold
+        cluster_stack.creation_time + TIMEOUT
+      end
+
+      def finalize_creation
+        Clusters::Aws::FinalizeCreationService.new.execute(provider)
+      end
+    end
+  end
+end

app/workers/cluster_provision_worker.rb

@@ -9,7 +9,11 @@ class ClusterProvisionWorker
   def perform(cluster_id)
     Clusters::Cluster.find_by_id(cluster_id).try do |cluster|
       cluster.provider.try do |provider|
-        Clusters::Gcp::ProvisionService.new.execute(provider) if cluster.gcp?
+        if cluster.gcp?
+          Clusters::Gcp::ProvisionService.new.execute(provider)
+        elsif cluster.aws?
+          Clusters::Aws::ProvisionService.new.execute(provider)
+        end
       end
     end
   end

app/workers/new_release_worker.rb

@@ -7,7 +7,7 @@ class NewReleaseWorker
   feature_category :release_orchestration
   def perform(release_id)
-    release = Release.with_project_and_namespace.find_by_id(release_id)
+    release = Release.preloaded.find_by_id(release_id)
     return unless release
     NotificationService.new.send_new_release_notifications(release)

app/workers/wait_for_cluster_creation_worker.rb

@@ -9,7 +9,11 @@ class WaitForClusterCreationWorker
   def perform(cluster_id)
     Clusters::Cluster.find_by_id(cluster_id).try do |cluster|
       cluster.provider.try do |provider|
-        Clusters::Gcp::VerifyProvisionStatusService.new.execute(provider) if cluster.gcp?
+        if cluster.gcp?
+          Clusters::Gcp::VerifyProvisionStatusService.new.execute(provider)
+        elsif cluster.aws?
+          Clusters::Aws::VerifyProvisionStatusService.new.execute(provider)
+        end
       end
     end
   end

config/routes/project.rb

@@ -179,7 +179,7 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do
           end
         end
-        resources :releases, only: [:index]
+        resources :releases, only: [:index, :edit], param: :tag, constraints: { tag: %r{[^/]+} }
         resources :starrers, only: [:index]
         resources :forks, only: [:index, :new, :create]
         resources :group_links, only: [:index, :create, :update, :destroy], constraints: { id: /\d+/ }

doc/api/graphql/index.md

@@ -53,6 +53,11 @@ GitLab's GraphQL reference [is available](reference/index.md).
 It is automatically generated from GitLab's GraphQL schema and embedded in a Markdown file.
+Machine-readable versions are also available:
+
+- [JSON format](reference/gitlab_schema.json)
+- [IDL format](reference/gitlab_schema.graphql)
+
 ## GraphiQL
 The API can be explored by using the GraphiQL IDE, it is available on your

doc/development/api_graphql_styleguide.md

@@ -551,7 +551,7 @@ it 'returns a successful response' do
 end
 ```
-## Documentation
-For information on generating GraphQL documentation, see
-[Rake tasks related to GraphQL](rake_tasks.md#update-graphql-documentation).
+## Documentation and Schema
+For information on generating GraphQL documentation and schema files, see
+[Rake tasks related to GraphQL](rake_tasks.md#update-graphql-documentation-and-schema-definitions).

doc/development/kubernetes.md

@@ -74,6 +74,50 @@ We have some Webmock stubs in
 [`KubernetesHelpers`](https://gitlab.com/gitlab-org/gitlab/blob/master/spec/support/helpers/kubernetes_helpers.rb)
 which can help with mocking out calls to Kubernetes API in your tests.
+### Amazon EKS integration
+
+This section outlines the process for allowing a GitLab instance to create EKS clusters.
+
+The following prerequisites are required:
+
+A `Customer` AWS account. This is the account in which the
+EKS cluster will be created. The following resources must be present:
+
+- A provisioning role that has permissions to create the cluster
+  and associated resources. It must list the `GitLab` AWS account
+  as a trusted entity.
+- A VPC, management role, security group, and subnets for use by the cluster.
+
+A `GitLab` AWS account. This is the account which performs
+the provisioning actions. The following resources must be present:
+
+- A service account with permissions to assume the provisioning
+  role in the `Customer` account above.
+- Credentials for this service account configured in GitLab via
+  the `kubernetes` section of `gitlab.yml`
+
+The process for creating a cluster is as follows:
+
+1. Using the :provision_role_external_id, GitLab assumes the role provided
+   by :provision_role_arn and stores a set of temporary credentials on the
+   provider record. By default these credentials are valid for one hour.
+1. A CloudFormation stack is created, based on the
+   [`AWS CloudFormation EKS template`](https://gitlab.com/gitlab-org/gitlab/blob/master/vendor/aws/cloudformation/eks_cluster.yaml).
+   This triggers creation of all resources required for an EKS cluster.
+1. GitLab polls the status of the stack until all resources are ready,
+   which takes somewhere between 10 and 15 minutes in most cases.
+1. When the stack is ready, GitLab stores the cluster details and generates
+   another set of temporary credentials, this time to allow connecting to
+   the cluster via Kubeclient. These credentials are valid for one minute.
+1. GitLab configures the worker nodes so that they are able to authenticate
+   to the cluster, and creates a service account for itself for future operations.
+1. Credentials that are no longer required are removed. This deletes the following
+   attributes:
+
+- `access_key_id`
+- `secret_access_key`
+- `session_token`
+
 ## Security
 ### SSRF

doc/development/rake_tasks.md

@@ -221,7 +221,7 @@ bundle exec rake db:obsolete_ignored_columns
 Feel free to remove their definitions from their `ignored_columns` definitions.
-## Update GraphQL Documentation
+## Update GraphQL Documentation and Schema definitions
 To generate GraphQL documentation based on the GitLab schema, run:
@@ -243,3 +243,13 @@ The actual renderer is at `Gitlab::Graphql::Docs::Renderer`.
 `@parsed_schema` is an instance variable that the `graphql-docs` gem expects to have available.
 `Gitlab::Graphql::Docs::Helper` defines the `object` method we currently use. This is also where you
 should implement any new methods for new types you'd like to display.
+
+### Update machine-readable schema files
+
+To generate GraphQL schema files based on the GitLab schema, run:
+
+```shell
+bundle exec rake gitlab:graphql:schema:dump
+```
+
+This uses graphql-ruby's built-in rake tasks to generate files in both [IDL](https://www.prisma.io/blog/graphql-sdl-schema-definition-language-6755bcb9ce51) and JSON formats.