Add latest changes from gitlab-org/gitlab@master

app/models/aws/role.rb

+# frozen_string_literal: true
+
+module Aws
+  class Role < ApplicationRecord
+    self.table_name = 'aws_roles'
+
+    belongs_to :user, inverse_of: :aws_role
+
+    validates :role_external_id, uniqueness: true, length: { in: 1..64 }
+    validates :role_arn,
+      length: 1..2048,
+      format: {
+        with: Gitlab::Regex.aws_arn_regex,
+        message: Gitlab::Regex.aws_arn_regex_message
+      }
+  end
+end

app/models/clusters/cluster.rb

@@ -35,6 +35,7 @@ module Clusters
     # we force autosave to happen when we save `Cluster` model
     has_one :provider_gcp, class_name: 'Clusters::Providers::Gcp', autosave: true
+    has_one :provider_aws, class_name: 'Clusters::Providers::Aws', autosave: true
     has_one :platform_kubernetes, class_name: 'Clusters::Platforms::Kubernetes', inverse_of: :cluster, autosave: true
@@ -96,14 +97,20 @@ module Clusters
     enum provider_type: {
       user: 0,
-      gcp: 1
+      gcp: 1,
+      aws: 2
     }
     scope :enabled, -> { where(enabled: true) }
     scope :disabled, -> { where(enabled: false) }
-    scope :user_provided, -> { where(provider_type: ::Clusters::Cluster.provider_types[:user]) }
-    scope :gcp_provided, -> { where(provider_type: ::Clusters::Cluster.provider_types[:gcp]) }
+
+    scope :user_provided, -> { where(provider_type: :user) }
+    scope :gcp_provided, -> { where(provider_type: :gcp) }
+    scope :aws_provided, -> { where(provider_type: :aws) }
+
     scope :gcp_installed, -> { gcp_provided.joins(:provider_gcp).merge(Clusters::Providers::Gcp.with_status(:created)) }
+    scope :aws_installed, -> { aws_provided.joins(:provider_aws).merge(Clusters::Providers::Aws.with_status(:created)) }
+
     scope :managed, -> { where(managed: true) }
     scope :default_environment, -> { where(environment_scope: DEFAULT_ENVIRONMENT) }
@@ -140,7 +147,11 @@ module Clusters
     end
     def provider
-      return provider_gcp if gcp?
+      if gcp?
+        provider_gcp
+      elsif aws?
+        provider_aws
+      end
     end
     def platform

app/models/clusters/concerns/provider_status.rb

@@ -42,6 +42,10 @@ module Clusters
         def on_creation?
           scheduled? || creating?
         end
+
+        def assign_operation_id(_)
+          # Implemented by individual providers if operation ID is supported.
+        end
       end
     end
   end

app/models/clusters/providers/aws.rb

+# frozen_string_literal: true
+
+module Clusters
+  module Providers
+    class Aws < ApplicationRecord
+      include Clusters::Concerns::ProviderStatus
+
+      self.table_name = 'cluster_providers_aws'
+
+      belongs_to :cluster, inverse_of: :provider_aws, class_name: 'Clusters::Cluster'
+      belongs_to :created_by_user, class_name: 'User'
+
+      default_value_for :region, 'us-east-1'
+      default_value_for :num_nodes, 3
+      default_value_for :instance_type, 'm5.large'
+
+      attr_encrypted :secret_access_key,
+        mode: :per_attribute_iv,
+        key: Settings.attr_encrypted_db_key_base_truncated,
+        algorithm: 'aes-256-gcm'
+
+      validates :role_arn,
+        length: 1..2048,
+        format: {
+          with: Gitlab::Regex.aws_arn_regex,
+          message: Gitlab::Regex.aws_arn_regex_message
+        }
+
+      validates :num_nodes,
+        numericality: {
+          only_integer: true,
+          greater_than: 0
+        }
+
+      validates :key_name, :region, :instance_type, :security_group_id, length: { in: 1..255 }
+      validates :subnet_ids, presence: true
+
+      def nullify_credentials
+        assign_attributes(
+          access_key_id: nil,
+          secret_access_key: nil,
+          session_token: nil
+        )
+      end
+    end
+  end
+end

app/models/project_services/jira_service.rb

@@ -122,9 +122,13 @@ class JiraService < IssueTrackerService
   end
   alias_method :original_url, :url
-
   def url
-    original_url&.chomp('/')
+    original_url&.delete_suffix('/')
+  end
+
+  alias_method :original_api_url, :api_url
+  def api_url
+    original_api_url&.delete_suffix('/')
   end
   def execute(push)

app/models/user.rb

@@ -99,6 +99,7 @@ class User < ApplicationRecord
   has_many :u2f_registrations, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
   has_many :chat_names, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
   has_one :user_synced_attributes_metadata, autosave: true
+  has_one :aws_role, class_name: 'Aws::Role'
   # Groups
   has_many :members

changelogs/unreleased/14945-fix-jira-api-url-parsing.yml

+---
+title: 'JIRA Integration API URL works having a trailing slash'
+merge_request: 18526
+author:
+type: fixed

changelogs/unreleased/46686-add-aws-cluster-data-model.yml

+---
+title: Add database tables to store AWS roles and cluster providers
+merge_request: 17057
+author:
+type: added

db/migrate/20190821040941_create_cluster_providers_aws.rb

+# frozen_string_literal: true
+
+class CreateClusterProvidersAws < ActiveRecord::Migration[5.2]
+  DOWNTIME = false
+
+  def change
+    create_table :cluster_providers_aws do |t|
+      t.references :cluster, null: false, type: :bigint, index: { unique: true }, foreign_key: { on_delete: :cascade }
+      t.references :created_by_user, type: :integer, foreign_key: { on_delete: :nullify, to_table: :users }
+
+      t.integer :num_nodes, null: false
+      t.integer :status, null: false
+
+      t.timestamps_with_timezone null: false
+
+      t.string :key_name, null: false, limit: 255
+      t.string :role_arn, null: false, limit: 2048
+      t.string :region, null: false, limit: 255
+      t.string :vpc_id, null: false, limit: 255
+      t.string :subnet_ids, null: false, array: true, default: [], limit: 255
+      t.string :security_group_id, null: false, limit: 255
+      t.string :instance_type, null: false, limit: 255
+
+      t.string :access_key_id, limit: 255
+      t.string :encrypted_secret_access_key_iv, limit: 255
+      t.text :encrypted_secret_access_key
+      t.text :session_token
+      t.text :status_reason
+
+      t.index [:cluster_id, :status]
+    end
+  end
+end

db/migrate/20191003064615_create_aws_roles.rb

+# frozen_string_literal: true
+
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class CreateAwsRoles < ActiveRecord::Migration[5.2]
+  DOWNTIME = false
+
+  def change
+    create_table :aws_roles, id: false do |t|
+      t.references :user, primary_key: true, default: nil, type: :integer, index: { unique: true }, foreign_key: { on_delete: :cascade }
+
+      t.timestamps_with_timezone null: false
+
+      t.string :role_arn, null: false, limit: 2048
+      t.string :role_external_id, null: false, limit: 64
+
+      t.index :role_external_id, unique: true
+    end
+  end
+end

db/schema.rb

@@ -467,6 +467,15 @@ ActiveRecord::Schema.define(version: 2019_10_16_072826) do
     t.index ["user_id", "name"], name: "index_award_emoji_on_user_id_and_name"
   end
+  create_table "aws_roles", primary_key: "user_id", id: :integer, default: nil, force: :cascade do |t|
+    t.datetime_with_timezone "created_at", null: false
+    t.datetime_with_timezone "updated_at", null: false
+    t.string "role_arn", limit: 2048, null: false
+    t.string "role_external_id", limit: 64, null: false
+    t.index ["role_external_id"], name: "index_aws_roles_on_role_external_id", unique: true
+    t.index ["user_id"], name: "index_aws_roles_on_user_id", unique: true
+  end
+
   create_table "badges", id: :serial, force: :cascade do |t|
     t.string "link_url", null: false
     t.string "image_url", null: false
@@ -968,6 +977,30 @@ ActiveRecord::Schema.define(version: 2019_10_16_072826) do
     t.index ["project_id"], name: "index_cluster_projects_on_project_id"
   end
+  create_table "cluster_providers_aws", force: :cascade do |t|
+    t.bigint "cluster_id", null: false
+    t.integer "created_by_user_id"
+    t.integer "num_nodes", null: false
+    t.integer "status", null: false
+    t.datetime_with_timezone "created_at", null: false
+    t.datetime_with_timezone "updated_at", null: false
+    t.string "key_name", limit: 255, null: false
+    t.string "role_arn", limit: 2048, null: false
+    t.string "region", limit: 255, null: false
+    t.string "vpc_id", limit: 255, null: false
+    t.string "subnet_ids", limit: 255, default: [], null: false, array: true
+    t.string "security_group_id", limit: 255, null: false
+    t.string "instance_type", limit: 255, null: false
+    t.string "access_key_id", limit: 255
+    t.string "encrypted_secret_access_key_iv", limit: 255
+    t.text "encrypted_secret_access_key"
+    t.text "session_token"
+    t.text "status_reason"
+    t.index ["cluster_id", "status"], name: "index_cluster_providers_aws_on_cluster_id_and_status"
+    t.index ["cluster_id"], name: "index_cluster_providers_aws_on_cluster_id", unique: true
+    t.index ["created_by_user_id"], name: "index_cluster_providers_aws_on_created_by_user_id"
+  end
+
   create_table "cluster_providers_gcp", id: :serial, force: :cascade do |t|
     t.integer "cluster_id", null: false
     t.integer "status"
@@ -3933,6 +3966,7 @@ ActiveRecord::Schema.define(version: 2019_10_16_072826) do
   add_foreign_key "approval_project_rules_users", "users", on_delete: :cascade
   add_foreign_key "approvals", "merge_requests", name: "fk_310d714958", on_delete: :cascade
   add_foreign_key "approver_groups", "namespaces", column: "group_id", on_delete: :cascade
+  add_foreign_key "aws_roles", "users", on_delete: :cascade
   add_foreign_key "badges", "namespaces", column: "group_id", on_delete: :cascade
   add_foreign_key "badges", "projects", on_delete: :cascade
   add_foreign_key "board_assignees", "boards", on_delete: :cascade
@@ -3996,6 +4030,8 @@ ActiveRecord::Schema.define(version: 2019_10_16_072826) do
   add_foreign_key "cluster_platforms_kubernetes", "clusters", on_delete: :cascade
   add_foreign_key "cluster_projects", "clusters", on_delete: :cascade
   add_foreign_key "cluster_projects", "projects", on_delete: :cascade
+  add_foreign_key "cluster_providers_aws", "clusters", on_delete: :cascade
+  add_foreign_key "cluster_providers_aws", "users", column: "created_by_user_id", on_delete: :nullify
   add_foreign_key "cluster_providers_gcp", "clusters", on_delete: :cascade
   add_foreign_key "clusters", "projects", column: "management_project_id", name: "fk_f05c5e5a42", on_delete: :nullify
   add_foreign_key "clusters", "users", on_delete: :nullify

lib/gitlab/danger/roulette.rb

@@ -50,7 +50,7 @@ module Gitlab
       # @param [Teammate] person
       # @return [Boolean]
       def valid_person?(person)
-        !mr_author?(person) && !person.out_of_office?
+        !mr_author?(person) && person.available?
       end
       # @param [Teammate] person

lib/gitlab/danger/teammate.rb

@@ -43,12 +43,22 @@ module Gitlab
         nil # better no status than a crashing Danger
       end
+      # @return [Boolean]
+      def available?
+        !out_of_office? && has_capacity?
+      end
+
+      private
+
       # @return [Boolean]
       def out_of_office?
         status&.dig("message")&.match?(/OOO/i) || false
       end
-      private
+      # @return [Boolean]
+      def has_capacity?
+        status&.dig("emoji") != 'red_circle'
+      end
       def has_capability?(project, category, kind, labels)
         case category

lib/gitlab/regex.rb

@@ -119,6 +119,15 @@ module Gitlab
     def breakline_regex
       @breakline_regex ||= /\r\n|\r|\n/
     end
+
+    # https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html
+    def aws_arn_regex
+      /\Aarn:\S+\z/
+    end
+
+    def aws_arn_regex_message
+      "must be a valid Amazon Resource Name"
+    end
   end
 end

spec/db/schema_spec.rb

@@ -19,6 +19,7 @@ describe 'Database schema' do
     approver_groups: %w[target_id],
     audit_events: %w[author_id entity_id],
     award_emoji: %w[awardable_id user_id],
+    aws_roles: %w[role_external_id],
     boards: %w[milestone_id],
     chat_names: %w[chat_id service_id team_id user_id],
     chat_teams: %w[team_id],
@@ -26,6 +27,7 @@ describe 'Database schema' do
     ci_pipelines: %w[user_id],
     ci_runner_projects: %w[runner_id],
     ci_trigger_requests: %w[commit_id],
+    cluster_providers_aws: %w[security_group_id vpc_id access_key_id],
     cluster_providers_gcp: %w[gcp_project_id operation_id],
     deploy_keys_projects: %w[deploy_key_id],
     deployments: %w[deployable_id environment_id user_id],

spec/factories/aws/roles.rb

+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :aws_role, class: Aws::Role do
+    user
+
+    role_arn { 'arn:aws:iam::123456789012:role/role-name' }
+    sequence(:role_external_id) { |n| "external-id-#{n}" }
+  end
+end

spec/factories/clusters/clusters.rb

@@ -53,6 +53,14 @@ FactoryBot.define do
       platform_kubernetes factory: [:cluster_platform_kubernetes, :configured]
     end
+    trait :provided_by_aws do
+      provider_type { :aws }
+      platform_type { :kubernetes }
+
+      provider_aws factory: [:cluster_provider_aws, :created]
+      platform_kubernetes factory: [:cluster_platform_kubernetes, :configured]
+    end
+
     trait :providing_by_gcp do
       provider_type { :gcp }
       provider_gcp factory: [:cluster_provider_gcp, :creating]

spec/factories/clusters/providers/aws.rb

+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :cluster_provider_aws, class: Clusters::Providers::Aws do
+    cluster
+    created_by_user factory: :user
+
+    role_arn { 'arn:aws:iam::123456789012:role/role-name' }
+    vpc_id { 'vpc-00000000000000000' }
+    subnet_ids { %w(subnet-00000000000000000 subnet-11111111111111111) }
+    security_group_id { 'sg-00000000000000000' }
+    key_name { 'user' }
+
+    trait :scheduled do
+      access_key_id { 'access_key_id' }
+      secret_access_key { 'secret_access_key' }
+      session_token { 'session_token' }
+    end
+
+    trait :creating do
+      after(:build) do |provider|
+        provider.make_creating
+      end
+    end
+
+    trait :created do
+      after(:build) do |provider|
+        provider.make_created
+      end
+    end
+
+    trait :errored do
+      after(:build) do |provider|
+        provider.make_errored('An error occurred')
+      end
+    end
+  end
+end

spec/lib/gitlab/danger/roulette_spec.rb

@@ -104,11 +104,13 @@ describe Gitlab::Danger::Roulette do
     let(:person2) { Gitlab::Danger::Teammate.new('username' => 'godfat') }
     let(:author) { Gitlab::Danger::Teammate.new('username' => 'filipa') }
     let(:ooo) { Gitlab::Danger::Teammate.new('username' => 'jacopo-beschi') }
+    let(:no_capacity) { Gitlab::Danger::Teammate.new('username' => 'uncharged') }
     before do
-      stub_person_message(person1, 'making GitLab magic')
-      stub_person_message(person2, 'making GitLab magic')
-      stub_person_message(ooo, 'OOO till 15th')
+      stub_person_status(person1, message: 'making GitLab magic')
+      stub_person_status(person2, message: 'making GitLab magic')
+      stub_person_status(ooo, message: 'OOO till 15th')
+      stub_person_status(no_capacity, message: 'At capacity for the next few days', emoji: 'red_circle')
       # we don't stub Filipa, as she is the author and
       # we should not fire request checking for her
@@ -131,10 +133,14 @@ describe Gitlab::Danger::Roulette do
       expect(subject.spin_for_person([author], random: Random.new)).to be_nil
     end
+    it 'excludes person with no capacity' do
+      expect(subject.spin_for_person([no_capacity], random: Random.new)).to be_nil
+    end
+
     private
-    def stub_person_message(person, message)
-      body = { message: message }.to_json
+    def stub_person_status(person, message: 'dummy message', emoji: 'unicorn')
+      body = { message: message, emoji: emoji }.to_json
       WebMock
         .stub_request(:get, "https://gitlab.com/api/v4/users/#{person.username}/status")

spec/lib/gitlab/danger/teammate_spec.rb

@@ -135,17 +135,17 @@ describe Gitlab::Danger::Teammate do
     end
   end
-  describe '#out_of_office?' do
+  describe '#available?' do
     using RSpec::Parameterized::TableSyntax
     let(:capabilities) { ['dry head'] }
     where(:status, :result) do
-      nil                              | false
-      {}                               | false
-      { message: 'dear reader' }       | false
-      { message: 'OOO: massage' }      | true
-      { message: 'love it SOOO much' } | true
+      {}                               | true
+      { message: 'dear reader' }       | true
+      { message: 'OOO: massage' }      | false
+      { message: 'love it SOOO much' } | false
+      { emoji: 'red_circle' }          | false
     end
     with_them do
@@ -154,7 +154,15 @@ describe Gitlab::Danger::Teammate do
                                                      .and_return(status&.stringify_keys)
       end
-      it { expect(subject.out_of_office?).to be result }
+      it { expect(subject.available?).to be result }
+    end
+
+    it 'returns true if request fails' do
+      expect(Gitlab::Danger::RequestHelper).to receive(:http_get_json)
+                                                   .exactly(2).times
+                                                   .and_raise(Gitlab::Danger::RequestHelper::HTTPError.new)
+
+      expect(subject.available?).to be true
     end
   end
 end