Add pages domains acme orders

Extract acme double to helper

Create ACME challanges for pages domains

* Create order & challange through API
* save them to database
* request challenge validation

We're saving order and challenge as one entity,
that wouldn't be correct if we would order certificates for
several domains simultaneously, but we always order certificate
per domain

Add controller for processing acme challenges redirected from pages

Don't save acme challenge url - we don't use it

Validate acme challenge attributes

Encrypt private_key in acme orders

app/controllers/acme_challenges_controller.rb

+# frozen_string_literal: true
+
+class AcmeChallengesController < ActionController::Base
+  def show
+    if acme_order
+      render plain: acme_order.challenge_file_content, content_type: 'text/plain'
+    else
+      head :not_found
+    end
+  end
+
+  private
+
+  def acme_order
+    @acme_order ||= PagesDomainAcmeOrder.find_by_domain_and_token(params[:domain], params[:token])
+  end
+end

app/models/pages_domain.rb

@@ -5,6 +5,7 @@ class PagesDomain < ApplicationRecord
   VERIFICATION_THRESHOLD = 3.days.freeze
   belongs_to :project
+  has_many :acme_orders, class_name: "PagesDomainAcmeOrder"
   validates :domain, hostname: { allow_numeric_hostname: true }
   validates :domain, uniqueness: { case_sensitive: false }

app/models/pages_domain_acme_order.rb

+# frozen_string_literal: true
+
+class PagesDomainAcmeOrder < ApplicationRecord
+  belongs_to :pages_domain
+
+  scope :expired, -> { where("expires_at < ?", Time.now) }
+
+  validates :pages_domain, presence: true
+  validates :expires_at, presence: true
+  validates :url, presence: true
+  validates :challenge_token, presence: true
+  validates :challenge_file_content, presence: true
+  validates :private_key, presence: true
+
+  attr_encrypted :private_key,
+                 mode: :per_attribute_iv,
+                 key: Settings.attr_encrypted_db_key_base_truncated,
+                 algorithm: 'aes-256-gcm',
+                 encode: true
+
+  def self.find_by_domain_and_token(domain_name, challenge_token)
+    joins(:pages_domain).find_by(pages_domains: { domain: domain_name }, challenge_token: challenge_token)
+  end
+end

app/services/pages_domains/create_acme_order_service.rb

+# frozen_string_literal: true
+
+module PagesDomains
+  class CreateAcmeOrderService
+    attr_reader :pages_domain
+
+    def initialize(pages_domain)
+      @pages_domain = pages_domain
+    end
+
+    def execute
+      lets_encrypt_client = Gitlab::LetsEncrypt::Client.new
+      order = lets_encrypt_client.new_order(pages_domain.domain)
+
+      challenge = order.new_challenge
+
+      private_key = OpenSSL::PKey::RSA.new(4096)
+      saved_order = pages_domain.acme_orders.create!(
+        url: order.url,
+        expires_at: order.expires,
+        private_key: private_key.to_pem,
+
+        challenge_token: challenge.token,
+        challenge_file_content: challenge.file_content
+      )
+
+      challenge.request_validation
+      saved_order
+    end
+  end
+end

app/services/pages_domains/obtain_lets_encrypt_certificate_service.rb

+# frozen_string_literal: true
+
+module PagesDomains
+  class ObtainLetsEncryptCertificateService
+    attr_reader :pages_domain
+
+    def initialize(pages_domain)
+      @pages_domain = pages_domain
+    end
+
+    def execute
+      pages_domain.acme_orders.expired.delete_all
+      acme_order = pages_domain.acme_orders.first
+
+      unless acme_order
+        ::PagesDomains::CreateAcmeOrderService.new(pages_domain).execute
+        return
+      end
+
+      api_order = ::Gitlab::LetsEncrypt::Client.new.load_order(acme_order.url)
+
+      # https://tools.ietf.org/html/rfc8555#section-7.1.6 - statuses diagram
+      case api_order.status
+      when 'ready'
+        api_order.request_certificate(private_key: acme_order.private_key, domain: pages_domain.domain)
+      when 'valid'
+        save_certificate(acme_order.private_key, api_order)
+        acme_order.destroy!
+        # when 'invalid'
+        # TODO: implement error handling
+      end
+    end
+
+    private
+
+    def save_certificate(private_key, api_order)
+      certificate = api_order.certificate
+      pages_domain.update!(key: private_key, certificate: certificate)
+    end
+  end
+end

config/routes.rb

@@ -75,6 +75,8 @@ Rails.application.routes.draw do
       resources :issues, module: :boards, only: [:index, :update]
     end
+    get 'acme-challenge/' => 'acme_challenges#show'
+
     # UserCallouts
     resources :user_callouts, only: [:create]

db/migrate/20190429082448_create_pages_domain_acme_orders.rb

+# frozen_string_literal: true
+
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class CreatePagesDomainAcmeOrders < ActiveRecord::Migration[5.1]
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  def change
+    create_table :pages_domain_acme_orders do |t|
+      t.references :pages_domain, null: false, index: true, foreign_key: { on_delete: :cascade }, type: :integer
+
+      t.datetime_with_timezone :expires_at, null: false
+      t.timestamps_with_timezone null: false
+
+      t.string :url, null: false
+
+      t.string :challenge_token, null: false, index: true
+      t.text :challenge_file_content, null: false
+
+      t.text :encrypted_private_key, null: false
+      t.text :encrypted_private_key_iv, null: false
+    end
+  end
+end

db/schema.rb

@@ -1571,6 +1571,20 @@ ActiveRecord::Schema.define(version: 20190530154715) do
     t.index ["access_grant_id"], name: "index_oauth_openid_requests_on_access_grant_id", using: :btree
   end
+  create_table "pages_domain_acme_orders", force: :cascade do |t|
+    t.integer "pages_domain_id", null: false
+    t.datetime_with_timezone "expires_at", null: false
+    t.datetime_with_timezone "created_at", null: false
+    t.datetime_with_timezone "updated_at", null: false
+    t.string "url", null: false
+    t.string "challenge_token", null: false
+    t.text "challenge_file_content", null: false
+    t.text "encrypted_private_key", null: false
+    t.text "encrypted_private_key_iv", null: false
+    t.index ["challenge_token"], name: "index_pages_domain_acme_orders_on_challenge_token", using: :btree
+    t.index ["pages_domain_id"], name: "index_pages_domain_acme_orders_on_pages_domain_id", using: :btree
+  end
+
   create_table "pages_domains", id: :serial, force: :cascade do |t|
     t.integer "project_id"
     t.text "certificate"
@@ -2560,6 +2574,7 @@ ActiveRecord::Schema.define(version: 20190530154715) do
   add_foreign_key "notes", "projects", name: "fk_99e097b079", on_delete: :cascade
   add_foreign_key "notification_settings", "users", name: "fk_0c95e91db7", on_delete: :cascade
   add_foreign_key "oauth_openid_requests", "oauth_access_grants", column: "access_grant_id", name: "fk_oauth_openid_requests_oauth_access_grants_access_grant_id"
+  add_foreign_key "pages_domain_acme_orders", "pages_domains", on_delete: :cascade
   add_foreign_key "pages_domains", "projects", name: "fk_ea2f6dfc6f", on_delete: :cascade
   add_foreign_key "personal_access_tokens", "users"
   add_foreign_key "pool_repositories", "projects", column: "source_project_id", on_delete: :nullify

lib/gitlab/lets_encrypt/challenge.rb

@@ -7,7 +7,7 @@ module Gitlab
         @acme_challenge = acme_challenge
       end
-      delegate :url, :token, :file_content, :status, :request_validation, to: :acme_challenge
+      delegate :token, :file_content, :status, :request_validation, to: :acme_challenge
       private

lib/gitlab/lets_encrypt/order.rb

@@ -13,7 +13,16 @@ module Gitlab
         ::Gitlab::LetsEncrypt::Challenge.new(challenge)
       end
-      delegate :url, :status, to: :acme_order
+      def request_certificate(domain:, private_key:)
+        csr = ::Acme::Client::CertificateRequest.new(
+          private_key: OpenSSL::PKey.read(private_key),
+          subject: { common_name: domain }
+        )
+
+        acme_order.finalize(csr: csr)
+      end
+
+      delegate :url, :status, :expires, :certificate, to: :acme_order
       private

spec/controllers/acme_challenges_controller_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe AcmeChallengesController do
+  describe '#show' do
+    let!(:acme_order) { create(:pages_domain_acme_order) }
+
+    def make_request(domain, token)
+      get(:show, params: { domain: domain, token: token })
+    end
+
+    before do
+      make_request(domain, token)
+    end
+
+    context 'with right domain and token' do
+      let(:domain) { acme_order.pages_domain.domain }
+      let(:token) { acme_order.challenge_token }
+
+      it 'renders acme challenge file content' do
+        expect(response.body).to eq(acme_order.challenge_file_content)
+      end
+    end
+
+    context 'when domain is invalid' do
+      let(:domain) { 'somewrongdomain.com' }
+      let(:token) { acme_order.challenge_token }
+
+      it 'renders not found' do
+        expect(response).to have_gitlab_http_status(404)
+      end
+    end
+
+    context 'when token is invalid' do
+      let(:domain) { acme_order.pages_domain.domain }
+      let(:token) { 'wrongtoken' }
+
+      it 'renders not found' do
+        expect(response).to have_gitlab_http_status(404)
+      end
+    end
+  end
+end

spec/factories/pages_domain_acme_orders.rb

+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :pages_domain_acme_order do
+    pages_domain
+    url { 'https://example.com/' }
+    expires_at { 1.day.from_now }
+    challenge_token { 'challenge_token' }
+    challenge_file_content { 'filecontent' }
+
+    private_key { OpenSSL::PKey::RSA.new(4096).to_pem }
+
+    trait :expired do
+      expires_at { 1.day.ago }
+    end
+  end
+end

spec/lib/gitlab/lets_encrypt/challenge_spec.rb

@@ -3,23 +3,11 @@
 require 'spec_helper'
 describe ::Gitlab::LetsEncrypt::Challenge do
-  delegated_methods = {
-    url: 'https://example.com/',
-    status: 'pending',
-    token: 'tokenvalue',
-    file_content: 'hereisfilecontent',
-    request_validation: true
-  }
-  let(:acme_challenge) do
-    acme_challenge = instance_double('Acme::Client::Resources::Challenge')
-    allow(acme_challenge).to receive_messages(delegated_methods)
-    acme_challenge
-  end
-
-  let(:challenge) { described_class.new(acme_challenge) }
-  delegated_methods.each do |method, value|
+  include LetsEncryptHelpers
+  let(:challenge) { described_class.new(acme_challenge_double) }
+  LetsEncryptHelpers::ACME_CHALLENGE_METHODS.each do |method, value|
     describe "##{method}" do
       it 'delegates to Acme::Client::Resources::Challenge' do
         expect(challenge.public_send(method)).to eq(value)

spec/lib/gitlab/lets_encrypt/order_spec.rb

@@ -3,20 +3,13 @@
 require 'spec_helper'
 describe ::Gitlab::LetsEncrypt::Order do
-  delegated_methods = {
-    url: 'https://example.com/',
-    status: 'valid'
-  }
-
-  let(:acme_order) do
-    acme_order = instance_double('Acme::Client::Resources::Order')
-    allow(acme_order).to receive_messages(delegated_methods)
-    acme_order
-  end
+  include LetsEncryptHelpers
+
+  let(:acme_order) { acme_order_double }
   let(:order) { described_class.new(acme_order) }
-  delegated_methods.each do |method, value|
+  LetsEncryptHelpers::ACME_ORDER_METHODS.each do |method, value|
     describe "##{method}" do
       it 'delegates to Acme::Client::Resources::Order' do
         expect(order.public_send(method)).to eq(value)
@@ -25,15 +18,24 @@ describe ::Gitlab::LetsEncrypt::Order do
   end
   describe '#new_challenge' do
-    before do
-      challenge = instance_double('Acme::Client::Resources::Challenges::HTTP01')
-      authorization = instance_double('Acme::Client::Resources::Authorization')
-      allow(authorization).to receive(:http).and_return(challenge)
-      allow(acme_order).to receive(:authorizations).and_return([authorization])
-    end
-
     it 'returns challenge' do
       expect(order.new_challenge).to be_a(::Gitlab::LetsEncrypt::Challenge)
     end
   end
+
+  describe '#request_certificate' do
+    let(:private_key) do
+      OpenSSL::PKey::RSA.new(4096).to_pem
+    end
+
+    it 'generates csr and finalizes order' do
+      expect(acme_order).to receive(:finalize) do |csr:|
+        expect do
+          csr.csr # it's being evaluated lazily
+        end.not_to raise_error
+      end
+
+      order.request_certificate(domain: 'example.com', private_key: private_key)
+    end
+  end
 end

spec/models/pages_domain_acme_order_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe PagesDomainAcmeOrder do
+  using RSpec::Parameterized::TableSyntax
+
+  describe '.expired' do
+    let!(:not_expired_order) { create(:pages_domain_acme_order) }
+    let!(:expired_order) { create(:pages_domain_acme_order, :expired) }
+
+    it 'returns only expired orders' do
+      expect(described_class.count).to eq(2)
+      expect(described_class.expired).to eq([expired_order])
+    end
+  end
+
+  describe '.find_by_domain_and_token' do
+    let!(:domain) { create(:pages_domain, domain: 'test.com') }
+    let!(:acme_order) { create(:pages_domain_acme_order, challenge_token: 'righttoken', pages_domain: domain) }
+
+    where(:domain_name, :challenge_token, :present) do
+      'test.com' | 'righttoken' | true
+      'test.com' | 'wrongtoken' | false
+      'test.org' | 'righttoken' | false
+    end
+
+    with_them do
+      subject { described_class.find_by_domain_and_token(domain_name, challenge_token).present? }
+
+      it { is_expected.to eq(present) }
+    end
+  end
+
+  subject { create(:pages_domain_acme_order) }
+
+  describe 'associations' do
+    it { is_expected.to belong_to(:pages_domain) }
+  end
+
+  describe 'validations' do
+    it { is_expected.to validate_presence_of(:pages_domain) }
+    it { is_expected.to validate_presence_of(:expires_at) }
+    it { is_expected.to validate_presence_of(:url) }
+    it { is_expected.to validate_presence_of(:challenge_token) }
+    it { is_expected.to validate_presence_of(:challenge_file_content) }
+    it { is_expected.to validate_presence_of(:private_key) }
+  end
+end

spec/services/pages_domains/create_acme_order_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe PagesDomains::CreateAcmeOrderService do
+  include LetsEncryptHelpers
+
+  let(:pages_domain) { create(:pages_domain) }
+
+  let(:challenge) { ::Gitlab::LetsEncrypt::Challenge.new(acme_challenge_double) }
+
+  let(:order_double) do
+    Gitlab::LetsEncrypt::Order.new(acme_order_double).tap do |order|
+      allow(order).to receive(:new_challenge).and_return(challenge)
+    end
+  end
+
+  let(:lets_encrypt_client) do
+    instance_double('Gitlab::LetsEncrypt::Client').tap do |client|
+      allow(client).to receive(:new_order).with(pages_domain.domain)
+        .and_return(order_double)
+    end
+  end
+
+  let(:service) { described_class.new(pages_domain) }
+
+  before do
+    allow(::Gitlab::LetsEncrypt::Client).to receive(:new).and_return(lets_encrypt_client)
+  end
+
+  it 'saves order to database before requesting validation' do
+    allow(pages_domain.acme_orders).to receive(:create!).and_call_original
+    allow(challenge).to receive(:request_validation).and_call_original
+
+    service.execute
+
+    expect(pages_domain.acme_orders).to have_received(:create!).ordered
+    expect(challenge).to have_received(:request_validation).ordered
+  end
+
+  it 'generates and saves private key' do
+    service.execute
+
+    saved_order = PagesDomainAcmeOrder.last
+    expect { OpenSSL::PKey::RSA.new(saved_order.private_key) }.not_to raise_error
+  end
+
+  it 'properly saves order attributes' do
+    service.execute
+
+    saved_order = PagesDomainAcmeOrder.last
+    expect(saved_order.url).to eq(order_double.url)
+    expect(saved_order.expires_at).to be_like_time(order_double.expires)
+  end
+
+  it 'properly saves challenge attributes' do
+    service.execute
+
+    saved_order = PagesDomainAcmeOrder.last
+    expect(saved_order.challenge_token).to eq(challenge.token)
+    expect(saved_order.challenge_file_content).to eq(challenge.file_content)
+  end
+end

spec/services/pages_domains/obtain_lets_encrypt_certificate_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe PagesDomains::ObtainLetsEncryptCertificateService do
+  include LetsEncryptHelpers
+
+  let(:pages_domain) { create(:pages_domain, :without_certificate, :without_key) }
+  let(:service) { described_class.new(pages_domain) }
+
+  before do
+    stub_lets_encrypt_settings
+  end
+
+  def expect_to_create_acme_challenge
+    expect(::PagesDomains::CreateAcmeOrderService).to receive(:new).with(pages_domain)
+      .and_wrap_original do |m, *args|
+      create_service = m.call(*args)
+
+      expect(create_service).to receive(:execute)
+
+      create_service
+    end
+  end
+
+  def stub_lets_encrypt_order(url, status)
+    order = ::Gitlab::LetsEncrypt::Order.new(acme_order_double(status: status))
+
+    allow_any_instance_of(::Gitlab::LetsEncrypt::Client).to(
+      receive(:load_order).with(url).and_return(order)
+    )
+
+    order
+  end
+
+  context 'when there is no acme order' do
+    it 'creates acme order' do
+      expect_to_create_acme_challenge
+
+      service.execute
+    end
+  end
+
+  context 'when there is expired acme order' do
+    let!(:existing_order) do
+      create(:pages_domain_acme_order, :expired, pages_domain: pages_domain)
+    end
+
+    it 'removes acme order and creates new one' do
+      expect_to_create_acme_challenge
+
+      service.execute
+
+      expect(PagesDomainAcmeOrder.find_by_id(existing_order.id)).to be_nil
+    end
+  end
+
+  %w(pending processing).each do |status|
+    context "there is an order in '#{status}' status" do
+      let(:existing_order) do
+        create(:pages_domain_acme_order, pages_domain: pages_domain)
+      end
+
+      before do
+        stub_lets_encrypt_order(existing_order.url, status)
+      end
+
+      it 'does not raise errors' do
+        expect do
+          service.execute
+        end.not_to raise_error
+      end
+    end
+  end
+
+  context 'when order is ready' do
+    let(:existing_order) do
+      create(:pages_domain_acme_order, pages_domain: pages_domain)
+    end
+
+    let!(:api_order) do
+      stub_lets_encrypt_order(existing_order.url, 'ready')
+    end
+
+    it 'request certificate' do
+      expect(api_order).to receive(:request_certificate).and_call_original
+
+      service.execute
+    end
+  end
+
+  context 'when order is valid' do
+    let(:existing_order) do
+      create(:pages_domain_acme_order, pages_domain: pages_domain)
+    end
+
+    let!(:api_order) do
+      stub_lets_encrypt_order(existing_order.url, 'valid')
+    end
+
+    let(:certificate) do
+      key = OpenSSL::PKey.read(existing_order.private_key)
+
+      subject = "/C=BE/O=Test/OU=Test/CN=#{pages_domain.domain}"
+
+      cert = OpenSSL::X509::Certificate.new
+      cert.subject = cert.issuer = OpenSSL::X509::Name.parse(subject)
+      cert.not_before = Time.now
+      cert.not_after = 1.year.from_now
+      cert.public_key = key.public_key
+      cert.serial = 0x0
+      cert.version = 2
+
+      ef = OpenSSL::X509::ExtensionFactory.new
+      ef.subject_certificate = cert
+      ef.issuer_certificate = cert
+      cert.extensions = [
+        ef.create_extension("basicConstraints", "CA:TRUE", true),
+        ef.create_extension("subjectKeyIdentifier", "hash")
+      ]
+      cert.add_extension ef.create_extension("authorityKeyIdentifier",
+                                             "keyid:always,issuer:always")
+
+      cert.sign key, OpenSSL::Digest::SHA1.new
+
+      cert.to_pem
+    end
+
+    before do
+      expect(api_order).to receive(:certificate) { certificate }
+    end
+
+    it 'saves private_key and certificate for domain' do
+      service.execute
+
+      expect(pages_domain.key).to be_present
+      expect(pages_domain.certificate).to eq(certificate)
+    end
+
+    it 'removes order from database' do
+      service.execute
+
+      expect(PagesDomainAcmeOrder.find_by_id(existing_order.id)).to be_nil
+    end
+  end
+end

spec/support/helpers/lets_encrypt_helpers.rb

 # frozen_string_literal: true
 module LetsEncryptHelpers
+  ACME_ORDER_METHODS = {
+    url: 'https://example.com/',
+    status: 'valid',
+    expires: 2.days.from_now
+  }.freeze
+
+  ACME_CHALLENGE_METHODS = {
+    status: 'pending',
+    token: 'tokenvalue',
+    file_content: 'hereisfilecontent',
+    request_validation: true
+  }.freeze
+
+  def stub_lets_encrypt_settings
+    stub_application_setting(
+      lets_encrypt_notification_email: 'myemail@test.example.com',
+      lets_encrypt_terms_of_service_accepted: true
+    )
+  end
+
   def stub_lets_encrypt_client
     client = instance_double('Acme::Client')
@@ -16,4 +36,24 @@ module LetsEncryptHelpers
     client
   end
+
+  def acme_challenge_double
+    challenge = instance_double('Acme::Client::Resources::Challenges::HTTP01')
+    allow(challenge).to receive_messages(ACME_CHALLENGE_METHODS)
+    challenge
+  end
+
+  def acme_authorization_double
+    authorization = instance_double('Acme::Client::Resources::Authorization')
+    allow(authorization).to receive(:http).and_return(acme_challenge_double)
+    authorization
+  end
+
+  def acme_order_double(attributes = {})
+    acme_order = instance_double('Acme::Client::Resources::Order')
+    allow(acme_order).to receive_messages(ACME_ORDER_METHODS.merge(attributes))
+    allow(acme_order).to receive(:authorizations).and_return([acme_authorization_double])
+    allow(acme_order).to receive(:finalize)
+    acme_order
+  end
 end